traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Revert "feat(generate): allow more characters for keywords"

Browse source 
This reverts commit 0269357c5a.
This commit is contained in:
Amaan Qureshi 2025-09-17 01:44:57 -04:00 • committed by Amaan Qureshi
parent 04cfee5664
commit 317e2e74c2
1 changed files with 3 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

11
crates/generate/src/build_tables.rs
View file

@ -334,7 +334,7 @@ fn identify_keywords(
.enumerate()
.filter_map(|(i, variable)| {
cursor.reset(vec![variable.start_state]);
if all_chars_are_valid_for_keywords(&cursor)
if all_chars_are_alphabetical(&cursor)
&& token_conflict_map.does_match_same_string(i, word_token.index)
&& !token_conflict_map.does_match_different_string(i, word_token.index)
{
@ -531,17 +531,12 @@ fn report_state_info<'a>(
}
}
/// This definition should match the set of characters that are typically
/// allowed in programming language keywords. Note that it is provisional,
/// and can be adjusted if necessary.
fn all_chars_are_valid_for_keywords(cursor: &NfaCursor) -> bool {
fn all_chars_are_alphabetical(cursor: &NfaCursor) -> bool {
cursor.transition_chars().all(|(chars, is_sep)| {
if is_sep {
true
} else {
chars
.chars()
.all(|c| c.is_alphanumeric() || "_!@#$-:.?/`".contains(c))
chars.chars().all(|c| c.is_alphabetic() || c == '_')
}
})
}