traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(generate): improve error message when a duplicate token is used as the word token

Browse source
This commit is contained in:
Amaan Qureshi 2025-01-10 02:23:24 -05:00
parent 4d6740980c
commit d38aa596e1
1 changed files with 11 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

14
cli/generate/src/prepare_grammar/extract_tokens.rs
View file

@ -26,8 +26,8 @@ unless they are used only as the grammar's start rule.
ExternalTokenNonTerminal(String),
#[error("Non-symbol rules cannot be used as external tokens")]
NonSymbolExternalToken,
#[error("Non-terminal symbol '{0}' cannot be used as the word token")]
NonTerminalWordToken(String),
#[error("Non-terminal symbol '{0}' cannot be used as the word token, because its rule is duplicated in '{1}'")]
NonTerminalWordToken(String, String),
#[error("Reserved words must be tokens")]
NonTokenReservedWord,
}
@ -161,8 +161,16 @@ pub(super) fn extract_tokens(
if let Some(token) = grammar.word_token {
let token = symbol_replacer.replace_symbol(token);
if token.is_non_terminal() {
let word_token_variable = &variables[token.index];
let conflicting_variable = variables
.iter()
.enumerate()
.find(|(i, v)| *i != token.index && v.rule == word_token_variable.rule)
.expect("Failed to find a variable with the same rule as the word token");
Err(ExtractTokensError::NonTerminalWordToken(
variables[token.index].name.clone(),
word_token_variable.name.clone(),
conflicting_variable.1.name.clone(),
))?;
}
word_token = Some(token);