traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Generalize precedence datatype to include strings

Browse source 
Right now, the strings are not used in comparisons, but they
are passed through the grammar processing pipeline, and are
available to the parse table construction algorithm.

This also cleans up a confusing aspect of the parse table
construction, in which precedences and associativities were
temporarily stored in the parse table data structure itself.
This commit is contained in:
Max Brunsfeld 2021-02-23 20:48:32 -08:00
parent 2f28a35e1b
commit d40f118370
10 changed files with 235 additions and 162 deletions
Download patch file Download diff file Expand all files Collapse all files

156
cli/src/generate/build_tables/build_parse_table.rs
View file

@ -5,16 +5,15 @@ use crate::generate::grammars::{
InlinedProductionMap, LexicalGrammar, SyntaxGrammar, VariableType,
};
use crate::generate::node_types::VariableInfo;
use crate::generate::rules::{Associativity, Symbol, SymbolType, TokenSet};
use crate::generate::rules::{Associativity, Precedence, Symbol, SymbolType, TokenSet};
use crate::generate::tables::{
FieldLocation, GotoAction, ParseAction, ParseState, ParseStateId, ParseTable, ParseTableEntry,
ProductionInfo, ProductionInfoId,
};
use core::ops::Range;
use std::collections::hash_map::Entry;
use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
use std::fmt::Write;
use std::u32;
use std::{cmp::Ordering, collections::hash_map::Entry};
// For conflict reporting, each parse state is associated with an example
// sequence of symbols that could lead to that parse state.
@ -29,6 +28,14 @@ struct AuxiliarySymbolInfo {
parent_symbols: Vec<Symbol>,
}
#[derive(Debug, Default)]
struct ReductionInfo {
precedence: Precedence,
has_left_assoc: bool,
has_right_assoc: bool,
has_non_assoc: bool,
}
struct ParseStateQueueEntry {
state_id: ParseStateId,
preceding_auxiliary_symbols: AuxiliarySymbolSequence,
@ -118,8 +125,6 @@ impl<'a> ParseTableBuilder<'a> {
)?;
}
self.remove_precedences();
Ok((self.parse_table, self.parse_state_info_by_id))
}
@ -179,6 +184,7 @@ impl<'a> ParseTableBuilder<'a> {
let mut terminal_successors = BTreeMap::new();
let mut non_terminal_successors = BTreeMap::new();
let mut lookaheads_with_conflicts = TokenSet::new();
let mut reduction_infos = HashMap::<Symbol, ReductionInfo>::new();
// Each item in the item set contributes to either or a Shift action or a Reduce
// action in this state.
@ -217,31 +223,50 @@ impl<'a> ParseTableBuilder<'a> {
ParseAction::Reduce {
symbol: Symbol::non_terminal(item.variable_index as usize),
child_count: item.step_index as usize,
precedence: item.precedence(),
associativity: item.associativity(),
dynamic_precedence: item.production.dynamic_precedence,
production_id: self.get_production_id(item),
}
};
let precedence = item.precedence();
let associativity = item.associativity();
for lookahead in lookaheads.iter() {
let entry = self.parse_table.states[state_id]
let table_entry = self.parse_table.states[state_id]
.terminal_entries
.entry(lookahead);
let entry = entry.or_insert_with(|| ParseTableEntry::new());
.entry(lookahead)
.or_insert_with(|| ParseTableEntry::new());
let reduction_info = reduction_infos.entry(lookahead).or_default();
// While inserting Reduce actions, eagerly resolve conflicts related
// to precedence: avoid inserting lower-precedence reductions, and
// clear the action list when inserting higher-precedence reductions.
if entry.actions.is_empty() {
entry.actions.push(action);
} else if action.precedence() > entry.actions[0].precedence() {
entry.actions.clear();
entry.actions.push(action);
lookaheads_with_conflicts.remove(&lookahead);
} else if action.precedence() == entry.actions[0].precedence() {
entry.actions.push(action);
lookaheads_with_conflicts.insert(lookahead);
if table_entry.actions.is_empty() {
table_entry.actions.push(action);
} else {
match Self::compare_precedence(
&self.syntax_grammar,
precedence,
&reduction_info.precedence,
) {
Ordering::Greater => {
table_entry.actions.clear();
table_entry.actions.push(action);
lookaheads_with_conflicts.remove(&lookahead);
*reduction_info = ReductionInfo::default();
}
Ordering::Equal => {
table_entry.actions.push(action);
lookaheads_with_conflicts.insert(lookahead);
}
Ordering::Less => continue,
}
}
reduction_info.precedence = precedence.clone();
match associativity {
Some(Associativity::Left) => reduction_info.has_left_assoc = true,
Some(Associativity::Right) => reduction_info.has_right_assoc = true,
None => reduction_info.has_non_assoc = true,
}
}
}
@ -302,6 +327,7 @@ impl<'a> ParseTableBuilder<'a> {
&preceding_symbols,
&preceding_auxiliary_symbols,
symbol,
reduction_infos.get(&symbol).unwrap(),
)?;
}
@ -381,6 +407,7 @@ impl<'a> ParseTableBuilder<'a> {
preceding_symbols: &SymbolSequence,
preceding_auxiliary_symbols: &Vec<AuxiliarySymbolInfo>,
conflicting_lookahead: Symbol,
reduction_info: &ReductionInfo,
) -> Result<()> {
let entry = self.parse_table.states[state_id]
.terminal_entries
@ -393,9 +420,8 @@ impl<'a> ParseTableBuilder<'a> {
// sorted out ahead of time in `add_actions`. But there can still be
// REDUCE-REDUCE conflicts where all actions have the *same*
// precedence, and there can still be SHIFT/REDUCE conflicts.
let reduce_precedence = entry.actions[0].precedence();
let mut considered_associativity = false;
let mut shift_precedence: Option<Range<i32>> = None;
let mut shift_precedence: Vec<&Precedence> = Vec::new();
let mut conflicting_items = HashSet::new();
for (item, lookaheads) in &item_set.entries {
if let Some(step) = item.step() {
@ -409,15 +435,9 @@ impl<'a> ParseTableBuilder<'a> {
conflicting_items.insert(item);
}
let precedence = item.precedence();
if let Some(range) = &mut shift_precedence {
if precedence < range.start {
range.start = precedence;
} else if precedence > range.end {
range.end = precedence;
}
} else {
shift_precedence = Some(precedence..precedence);
let p = item.precedence();
if let Err(i) = shift_precedence.binary_search(&p) {
shift_precedence.insert(i, p);
}
}
}
@ -429,8 +449,6 @@ impl<'a> ParseTableBuilder<'a> {
}
if let ParseAction::Shift { is_repetition, .. } = entry.actions.last_mut().unwrap() {
let shift_precedence = shift_precedence.unwrap_or(0..0);
// If all of the items in the conflict have the same parent symbol,
// and that parent symbols is auxiliary, then this is just the intentional
// ambiguity associated with a repeat rule. Resolve that class of ambiguity
@ -448,40 +466,37 @@ impl<'a> ParseTableBuilder<'a> {
}
// If the SHIFT action has higher precedence, remove all the REDUCE actions.
if shift_precedence.start > reduce_precedence
|| (shift_precedence.start == reduce_precedence
&& shift_precedence.end > reduce_precedence)
{
let mut shift_is_less = false;
let mut shift_is_more = false;
for p in shift_precedence {
match Self::compare_precedence(&self.syntax_grammar, p, &reduction_info.precedence)
{
Ordering::Greater => shift_is_more = true,
Ordering::Less => shift_is_less = true,
Ordering::Equal => {}
}
}
if shift_is_more && !shift_is_less {
entry.actions.drain(0..entry.actions.len() - 1);
}
// If the REDUCE actions have higher precedence, remove the SHIFT action.
else if shift_precedence.end < reduce_precedence
|| (shift_precedence.end == reduce_precedence
&& shift_precedence.start < reduce_precedence)
{
else if shift_is_less && !shift_is_more {
entry.actions.pop();
conflicting_items.retain(|item| item.is_done());
}
// If the SHIFT and REDUCE actions have the same predence, consider
// the REDUCE actions' associativity.
else if shift_precedence == (reduce_precedence..reduce_precedence) {
else if !shift_is_less && !shift_is_more {
considered_associativity = true;
let mut has_left = false;
let mut has_right = false;
let mut has_non = false;
for action in &entry.actions {
if let ParseAction::Reduce { associativity, .. } = action {
match associativity {
Some(Associativity::Left) => has_left = true,
Some(Associativity::Right) => has_right = true,
None => has_non = true,
}
}
}
// If all Reduce actions are left associative, remove the SHIFT action.
// If all Reduce actions are right associative, remove the REDUCE actions.
match (has_left, has_non, has_right) {
match (
reduction_info.has_left_assoc,
reduction_info.has_non_assoc,
reduction_info.has_right_assoc,
) {
(true, false, false) => {
entry.actions.pop();
conflicting_items.retain(|item| item.is_done());
@ -595,7 +610,7 @@ impl<'a> ParseTableBuilder<'a> {
"(precedence: {}, associativity: {:?})",
precedence, associativity
))
} else if precedence != 0 {
} else if !precedence.is_none() {
Some(format!("(precedence: {})", precedence))
} else {
None
@ -714,6 +729,17 @@ impl<'a> ParseTableBuilder<'a> {
Err(Error::new(msg))
}
fn compare_precedence(
_grammar: &SyntaxGrammar,
left: &Precedence,
right: &Precedence,
) -> Ordering {
// TODO - compare named precedence
let left_integer = left.as_integer();
let right_integer = right.as_integer();
left_integer.cmp(&right_integer)
}
fn get_auxiliary_node_info(
&self,
item_set: &ParseItemSet,
@ -739,26 +765,6 @@ impl<'a> ParseTableBuilder<'a> {
}
}
fn remove_precedences(&mut self) {
for state in self.parse_table.states.iter_mut() {
for (_, entry) in state.terminal_entries.iter_mut() {
for action in entry.actions.iter_mut() {
match action {
ParseAction::Reduce {
precedence,
associativity,
..
} => {
*precedence = 0;
*associativity = None;
}
_ => {}
}
}
}
}
}
fn get_production_id(&mut self, item: &ParseItem) -> ProductionInfoId {
let mut production_info = ProductionInfo {
alias_sequence: Vec::new(),

24
cli/src/generate/build_tables/item.rs
View file

@ -1,8 +1,5 @@
use crate::generate::grammars::{
LexicalGrammar, Production, ProductionStep, SyntaxGrammar,
};
use crate::generate::rules::Associativity;
use crate::generate::rules::{Symbol, SymbolType, TokenSet};
use crate::generate::grammars::{LexicalGrammar, Production, ProductionStep, SyntaxGrammar};
use crate::generate::rules::{Associativity, Precedence, Symbol, SymbolType, TokenSet};
use lazy_static::lazy_static;
use std::cmp::Ordering;
use std::fmt;
@ -17,7 +14,7 @@ lazy_static! {
index: 0,
kind: SymbolType::NonTerminal,
},
precedence: 0,
precedence: Precedence::None,
associativity: None,
alias: None,
field_name: None,
@ -82,8 +79,9 @@ impl<'a> ParseItem<'a> {
self.prev_step().and_then(|step| step.associativity)
}
pub fn precedence(&self) -> i32 {
self.prev_step().map_or(0, |step| step.precedence)
pub fn precedence(&self) -> &Precedence {
self.prev_step()
.map_or(&Precedence::None, |step| &step.precedence)
}
pub fn prev_step(&self) -> Option<&'a ProductionStep> {
@ -165,12 +163,12 @@ impl<'a> fmt::Display for ParseItemDisplay<'a> {
if i == self.0.step_index as usize {
write!(f, " •")?;
if let Some(associativity) = step.associativity {
if step.precedence != 0 {
if !step.precedence.is_none() {
write!(f, " ({} {:?})", step.precedence, associativity)?;
} else {
write!(f, " ({:?})", associativity)?;
}
} else if step.precedence != 0 {
} else if !step.precedence.is_none() {
write!(f, " ({})", step.precedence)?;
}
}
@ -197,12 +195,12 @@ impl<'a> fmt::Display for ParseItemDisplay<'a> {
write!(f, " •")?;
if let Some(step) = self.0.production.steps.last() {
if let Some(associativity) = step.associativity {
if step.precedence != 0 {
if !step.precedence.is_none() {
write!(f, " ({} {:?})", step.precedence, associativity)?;
} else {
write!(f, " ({:?})", associativity)?;
}
} else if step.precedence != 0 {
} else if !step.precedence.is_none() {
write!(f, " ({})", step.precedence)?;
}
}
@ -257,7 +255,7 @@ impl<'a> Hash for ParseItem<'a> {
hasher.write_u32(self.step_index);
hasher.write_i32(self.production.dynamic_precedence);
hasher.write_usize(self.production.steps.len());
hasher.write_i32(self.precedence());
self.precedence().hash(hasher);
self.associativity().hash(hasher);
// When comparing two parse items, the symbols that were already consumed by

4
cli/src/generate/build_tables/token_conflicts.rs
View file

@ -380,7 +380,7 @@ mod tests {
use super::*;
use crate::generate::grammars::{Variable, VariableType};
use crate::generate::prepare_grammar::{expand_tokens, ExtractedLexicalGrammar};
use crate::generate::rules::{Rule, Symbol};
use crate::generate::rules::{Precedence, Rule, Symbol};
#[test]
fn test_starting_characters() {
@ -508,7 +508,7 @@ mod tests {
Variable {
name: "anything".to_string(),
kind: VariableType::Named,
rule: Rule::prec(-1, Rule::pattern(".*")),
rule: Rule::prec(Precedence::Integer(-1), Rule::pattern(".*")),
},
],
})

12
cli/src/generate/grammars.rs
View file

@ -1,5 +1,5 @@
use super::nfa::Nfa;
use super::rules::{Alias, Associativity, Rule, Symbol};
use super::rules::{Alias, Associativity, Precedence, Rule, Symbol};
use std::collections::HashMap;
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
@ -52,7 +52,7 @@ pub(crate) struct LexicalGrammar {
#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub(crate) struct ProductionStep {
pub symbol: Symbol,
pub precedence: i32,
pub precedence: Precedence,
pub associativity: Option<Associativity>,
pub alias: Option<Alias>,
pub field_name: Option<String>,
@ -100,14 +100,18 @@ impl ProductionStep {
pub(crate) fn new(symbol: Symbol) -> Self {
Self {
symbol,
precedence: 0,
precedence: Precedence::None,
associativity: None,
alias: None,
field_name: None,
}
}
pub(crate) fn with_prec(self, precedence: i32, associativity: Option<Associativity>) -> Self {
pub(crate) fn with_prec(
self,
precedence: Precedence,
associativity: Option<Associativity>,
) -> Self {
Self {
symbol: self.symbol,
precedence,

34
cli/src/generate/parse_grammar.rs
View file

@ -1,5 +1,5 @@
use super::grammars::{InputGrammar, Variable, VariableType};
use super::rules::Rule;
use super::rules::{Precedence, Rule};
use crate::error::Result;
use serde_derive::Deserialize;
use serde_json::{Map, Value};
@ -44,15 +44,15 @@ enum RuleJSON {
content: Box<RuleJSON>,
},
PREC_LEFT {
value: i32,
value: PrecedenceJSON,
content: Box<RuleJSON>,
},
PREC_RIGHT {
value: i32,
value: PrecedenceJSON,
content: Box<RuleJSON>,
},
PREC {
value: i32,
value: PrecedenceJSON,
content: Box<RuleJSON>,
},
TOKEN {
@ -63,6 +63,13 @@ enum RuleJSON {
},
}
#[derive(Deserialize)]
#[serde(untagged)]
enum PrecedenceJSON {
Integer(i32),
Name(String),
}
#[derive(Deserialize)]
pub(crate) struct GrammarJSON {
pub(crate) name: String,
@ -133,9 +140,13 @@ fn parse_rule(json: RuleJSON) -> Rule {
RuleJSON::REPEAT { content } => {
Rule::choice(vec![Rule::repeat(parse_rule(*content)), Rule::Blank])
}
RuleJSON::PREC { value, content } => Rule::prec(value, parse_rule(*content)),
RuleJSON::PREC_LEFT { value, content } => Rule::prec_left(value, parse_rule(*content)),
RuleJSON::PREC_RIGHT { value, content } => Rule::prec_right(value, parse_rule(*content)),
RuleJSON::PREC { value, content } => Rule::prec(value.into(), parse_rule(*content)),
RuleJSON::PREC_LEFT { value, content } => {
Rule::prec_left(value.into(), parse_rule(*content))
}
RuleJSON::PREC_RIGHT { value, content } => {
Rule::prec_right(value.into(), parse_rule(*content))
}
RuleJSON::PREC_DYNAMIC { value, content } => {
Rule::prec_dynamic(value, parse_rule(*content))
}
@ -144,6 +155,15 @@ fn parse_rule(json: RuleJSON) -> Rule {
}
}
impl Into<Precedence> for PrecedenceJSON {
fn into(self) -> Precedence {
match self {
PrecedenceJSON::Integer(i) => Precedence::Integer(i),
PrecedenceJSON::Name(i) => Precedence::Name(i),
}
}
}
#[cfg(test)]
mod tests {
use super::*;

35
cli/src/generate/prepare_grammar/expand_tokens.rs
View file

@ -1,8 +1,11 @@
use super::ExtractedLexicalGrammar;
use crate::error::{Error, Result};
use crate::generate::grammars::{LexicalGrammar, LexicalVariable};
use crate::generate::nfa::{CharacterSet, Nfa, NfaState};
use crate::generate::rules::Rule;
use crate::{
error::{Error, Result},
generate::rules::Precedence,
};
use lazy_static::lazy_static;
use regex::Regex;
use regex_syntax::ast::{
@ -46,10 +49,12 @@ fn get_implicit_precedence(rule: &Rule) -> i32 {
}
fn get_completion_precedence(rule: &Rule) -> i32 {
match rule {
Rule::Metadata { params, .. } => params.precedence.unwrap_or(0),
_ => 0,
if let Rule::Metadata { params, .. } = rule {
if let Precedence::Integer(p) = params.precedence {
return p;
}
}
0
}
fn preprocess_regex(content: &str) -> String {
@ -187,11 +192,14 @@ impl NfaBuilder {
}
}
Rule::Metadata { rule, params } => {
if let Some(precedence) = params.precedence {
self.precedence_stack.push(precedence);
}
let has_precedence = if let Precedence::Integer(precedence) = &params.precedence {
self.precedence_stack.push(*precedence);
true
} else {
false
};
let result = self.expand_rule(rule, next_state_id);
if params.precedence.is_some() {
if has_precedence {
self.precedence_stack.pop();
}
result
@ -626,8 +634,8 @@ mod tests {
// shorter tokens with higher precedence
Row {
rules: vec![
Rule::prec(2, Rule::pattern("abc")),
Rule::prec(1, Rule::pattern("ab[cd]e")),
Rule::prec(Precedence::Integer(2), Rule::pattern("abc")),
Rule::prec(Precedence::Integer(1), Rule::pattern("ab[cd]e")),
Rule::pattern("[a-e]+"),
],
separators: vec![Rule::string("\\\n"), Rule::pattern("\\s")],
@ -640,8 +648,11 @@ mod tests {
// immediate tokens with higher precedence
Row {
rules: vec![
Rule::prec(1, Rule::pattern("[^a]+")),
Rule::immediate_token(Rule::prec(2, Rule::pattern("[^ab]+"))),
Rule::prec(Precedence::Integer(1), Rule::pattern("[^a]+")),
Rule::immediate_token(Rule::prec(
Precedence::Integer(2),
Rule::pattern("[^ab]+"),
)),
],
separators: vec![Rule::pattern("\\s")],
examples: vec![("cccb", Some((1, "ccc")))],

49
cli/src/generate/prepare_grammar/flatten_grammar.rs
View file

@ -3,12 +3,11 @@ use crate::error::{Error, Result};
use crate::generate::grammars::{
Production, ProductionStep, SyntaxGrammar, SyntaxVariable, Variable,
};
use crate::generate::rules::Symbol;
use crate::generate::rules::{Alias, Associativity, Rule};
use crate::generate::rules::{Alias, Associativity, Precedence, Rule, Symbol};
struct RuleFlattener {
production: Production,
precedence_stack: Vec<i32>,
precedence_stack: Vec<Precedence>,
associativity_stack: Vec<Associativity>,
alias_stack: Vec<Alias>,
field_name_stack: Vec<String>,
@ -45,9 +44,9 @@ impl RuleFlattener {
}
Rule::Metadata { rule, params } => {
let mut has_precedence = false;
if let Some(precedence) = params.precedence {
if !params.precedence.is_none() {
has_precedence = true;
self.precedence_stack.push(precedence);
self.precedence_stack.push(params.precedence);
}
let mut has_associativity = false;
@ -77,8 +76,11 @@ impl RuleFlattener {
if has_precedence {
self.precedence_stack.pop();
if did_push && !at_end {
self.production.steps.last_mut().unwrap().precedence =
self.precedence_stack.last().cloned().unwrap_or(0);
self.production.steps.last_mut().unwrap().precedence = self
.precedence_stack
.last()
.cloned()
.unwrap_or(Precedence::None);
}
}
@ -103,7 +105,11 @@ impl RuleFlattener {
Rule::Symbol(symbol) => {
self.production.steps.push(ProductionStep {
symbol,
precedence: self.precedence_stack.last().cloned().unwrap_or(0),
precedence: self
.precedence_stack
.last()
.cloned()
.unwrap_or(Precedence::None),
associativity: self.associativity_stack.last().cloned(),
alias: self.alias_stack.last().cloned(),
field_name: self.field_name_stack.last().cloned(),
@ -223,12 +229,12 @@ mod tests {
rule: Rule::seq(vec![
Rule::non_terminal(1),
Rule::prec_left(
101,
Precedence::Integer(101),
Rule::seq(vec![
Rule::non_terminal(2),
Rule::choice(vec![
Rule::prec_right(
102,
Precedence::Integer(102),
Rule::seq(vec![Rule::non_terminal(3), Rule::non_terminal(4)]),
),
Rule::non_terminal(5),
@ -249,11 +255,11 @@ mod tests {
steps: vec![
ProductionStep::new(Symbol::non_terminal(1)),
ProductionStep::new(Symbol::non_terminal(2))
.with_prec(101, Some(Associativity::Left)),
.with_prec(Precedence::Integer(101), Some(Associativity::Left)),
ProductionStep::new(Symbol::non_terminal(3))
.with_prec(102, Some(Associativity::Right)),
.with_prec(Precedence::Integer(102), Some(Associativity::Right)),
ProductionStep::new(Symbol::non_terminal(4))
.with_prec(101, Some(Associativity::Left)),
.with_prec(Precedence::Integer(101), Some(Associativity::Left)),
ProductionStep::new(Symbol::non_terminal(6)),
ProductionStep::new(Symbol::non_terminal(7)),
]
@ -263,9 +269,9 @@ mod tests {
steps: vec![
ProductionStep::new(Symbol::non_terminal(1)),
ProductionStep::new(Symbol::non_terminal(2))
.with_prec(101, Some(Associativity::Left)),
.with_prec(Precedence::Integer(101), Some(Associativity::Left)),
ProductionStep::new(Symbol::non_terminal(5))
.with_prec(101, Some(Associativity::Left)),
.with_prec(Precedence::Integer(101), Some(Associativity::Left)),
ProductionStep::new(Symbol::non_terminal(6)),
ProductionStep::new(Symbol::non_terminal(7)),
]
@ -334,7 +340,7 @@ mod tests {
name: "test".to_string(),
kind: VariableType::Named,
rule: Rule::prec_left(
101,
Precedence::Integer(101),
Rule::seq(vec![Rule::non_terminal(1), Rule::non_terminal(2)]),
),
})
@ -346,9 +352,9 @@ mod tests {
dynamic_precedence: 0,
steps: vec![
ProductionStep::new(Symbol::non_terminal(1))
.with_prec(101, Some(Associativity::Left)),
.with_prec(Precedence::Integer(101), Some(Associativity::Left)),
ProductionStep::new(Symbol::non_terminal(2))
.with_prec(101, Some(Associativity::Left)),
.with_prec(Precedence::Integer(101), Some(Associativity::Left)),
]
}]
);
@ -356,7 +362,10 @@ mod tests {
let result = flatten_variable(Variable {
name: "test".to_string(),
kind: VariableType::Named,
rule: Rule::prec_left(101, Rule::seq(vec![Rule::non_terminal(1)])),
rule: Rule::prec_left(
Precedence::Integer(101),
Rule::seq(vec![Rule::non_terminal(1)]),
),
})
.unwrap();
@ -365,7 +374,7 @@ mod tests {
vec![Production {
dynamic_precedence: 0,
steps: vec![ProductionStep::new(Symbol::non_terminal(1))
.with_prec(101, Some(Associativity::Left)),]
.with_prec(Precedence::Integer(101), Some(Associativity::Left)),]
}]
);
}

16
cli/src/generate/prepare_grammar/process_inlines.rs
View file

@ -120,7 +120,7 @@ impl InlinedProductionMapBuilder {
}
}
if let Some(last_inserted_step) = inserted_steps.last_mut() {
if last_inserted_step.precedence == 0 {
if last_inserted_step.precedence.is_none() {
last_inserted_step.precedence = removed_step.precedence;
}
if last_inserted_step.associativity == None {
@ -193,7 +193,7 @@ pub(super) fn process_inlines(grammar: &SyntaxGrammar) -> InlinedProductionMap {
mod tests {
use super::*;
use crate::generate::grammars::{ProductionStep, SyntaxVariable, VariableType};
use crate::generate::rules::{Associativity, Symbol};
use crate::generate::rules::{Associativity, Precedence, Symbol};
#[test]
fn test_basic_inlining() {
@ -401,7 +401,7 @@ mod tests {
steps: vec![
// inlined
ProductionStep::new(Symbol::non_terminal(1))
.with_prec(1, Some(Associativity::Left)),
.with_prec(Precedence::Integer(1), Some(Associativity::Left)),
ProductionStep::new(Symbol::terminal(10)),
// inlined
ProductionStep::new(Symbol::non_terminal(2))
@ -416,7 +416,7 @@ mod tests {
dynamic_precedence: 0,
steps: vec![
ProductionStep::new(Symbol::terminal(11))
.with_prec(2, None)
.with_prec(Precedence::Integer(2), None)
.with_alias("inner_alias", true),
ProductionStep::new(Symbol::terminal(12)),
],
@ -453,12 +453,12 @@ mod tests {
// The first step in the inlined production retains its precedence
// and alias.
ProductionStep::new(Symbol::terminal(11))
.with_prec(2, None)
.with_prec(Precedence::Integer(2), None)
.with_alias("inner_alias", true),
// The final step of the inlined production inherits the precedence of
// the inlined step.
ProductionStep::new(Symbol::terminal(12))
.with_prec(1, Some(Associativity::Left)),
.with_prec(Precedence::Integer(1), Some(Associativity::Left)),
ProductionStep::new(Symbol::terminal(10)),
ProductionStep::new(Symbol::non_terminal(2)).with_alias("outer_alias", true),
]
@ -475,10 +475,10 @@ mod tests {
dynamic_precedence: 0,
steps: vec![
ProductionStep::new(Symbol::terminal(11))
.with_prec(2, None)
.with_prec(Precedence::Integer(2), None)
.with_alias("inner_alias", true),
ProductionStep::new(Symbol::terminal(12))
.with_prec(1, Some(Associativity::Left)),
.with_prec(Precedence::Integer(1), Some(Associativity::Left)),
ProductionStep::new(Symbol::terminal(10)),
// All steps of the inlined production inherit their alias from the
// inlined step.

53
cli/src/generate/rules.rs
View file

@ -1,7 +1,7 @@
use super::grammars::VariableType;
use smallbitvec::SmallBitVec;
use std::collections::HashMap;
use std::iter::FromIterator;
use std::{collections::HashMap, fmt};
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub(crate) enum SymbolType {
@ -24,11 +24,18 @@ pub(crate) struct Alias {
pub is_named: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub enum Precedence {
None,
Integer(i32),
Name(String),
}
pub(crate) type AliasMap = HashMap<Symbol, Alias>;
#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)]
pub(crate) struct MetadataParams {
pub precedence: Option<i32>,
pub precedence: Precedence,
pub dynamic_precedence: i32,
pub associativity: Option<Associativity>,
pub is_token: bool,
@ -99,23 +106,23 @@ impl Rule {
})
}
pub fn prec(value: i32, content: Rule) -> Self {
pub fn prec(value: Precedence, content: Rule) -> Self {
add_metadata(content, |params| {
params.precedence = Some(value);
params.precedence = value;
})
}
pub fn prec_left(value: i32, content: Rule) -> Self {
pub fn prec_left(value: Precedence, content: Rule) -> Self {
add_metadata(content, |params| {
params.associativity = Some(Associativity::Left);
params.precedence = Some(value);
params.precedence = value;
})
}
pub fn prec_right(value: i32, content: Rule) -> Self {
pub fn prec_right(value: Precedence, content: Rule) -> Self {
add_metadata(content, |params| {
params.associativity = Some(Associativity::Right);
params.precedence = Some(value);
params.precedence = value;
})
}
@ -152,6 +159,20 @@ impl Alias {
}
}
impl Precedence {
pub fn as_integer(&self) -> i32 {
if let Precedence::Integer(i) = self {
*i
} else {
0
}
}
pub fn is_none(&self) -> bool {
matches!(self, Precedence::None)
}
}
#[cfg(test)]
impl Rule {
pub fn terminal(index: usize) -> Self {
@ -437,3 +458,19 @@ fn choice_helper(result: &mut Vec<Rule>, rule: Rule) {
}
}
}
impl fmt::Display for Precedence {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Precedence::Integer(i) => write!(f, "{}", i),
Precedence::Name(s) => write!(f, "'{}'", s),
Precedence::None => write!(f, "none"),
}
}
}
impl Default for Precedence {
fn default() -> Self {
Precedence::None
}
}

14
cli/src/generate/tables.rs
View file

@ -1,5 +1,5 @@
use super::nfa::CharacterSet;
use super::rules::{Alias, Associativity, Symbol, TokenSet};
use super::rules::{Alias, Symbol, TokenSet};
use std::collections::{BTreeMap, HashMap};
pub(crate) type ProductionInfoId = usize;
pub(crate) type ParseStateId = usize;
@ -17,9 +17,7 @@ pub(crate) enum ParseAction {
Reduce {
symbol: Symbol,
child_count: usize,
precedence: i32,
dynamic_precedence: i32,
associativity: Option<Associativity>,
production_id: ProductionInfoId,
},
}
@ -163,13 +161,3 @@ impl ParseState {
}
}
}
impl ParseAction {
pub fn precedence(&self) -> i32 {
if let ParseAction::Reduce { precedence, .. } = self {
*precedence
} else {
0
}
}
}