traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #319 from ervinoro/master

Browse source 
Allow hex characters in unicode code points
This commit is contained in:
Max Brunsfeld 2019-04-09 12:13:57 -07:00 • committed by GitHub
commit 82ccc0e56d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 5 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

7
cli/src/generate/prepare_grammar/expand_tokens.rs
View file

@ -12,7 +12,7 @@ use std::i32;
lazy_static! {
static ref CURLY_BRACE_REGEX: Regex =
Regex::new(r#"(^|[^\\])\{([^}]*[^0-9,}][^}]*)\}"#).unwrap();
Regex::new(r#"(^|[^\\])\{([^}]*[^0-9A-F,}][^}]*)\}"#).unwrap();
}
const ALLOWED_REDUNDANT_ESCAPED_CHARS: [char; 4] = ['!', '\'', '"', '/'];
@ -429,7 +429,7 @@ mod tests {
.find(|t| t.characters.contains(c) && t.precedence >= result_precedence)
{
cursor.reset(states);
end_char += 1;
end_char += c.len_utf8();
if is_separator {
start_char = end_char;
}
@ -651,11 +651,14 @@ mod tests {
Rule::pattern(r#"u{[0-9a-fA-F]+}"#),
// Already-escaped curly braces
Rule::pattern(r#"\{[ab]{3}\}"#),
// Unicode codepoints
Rule::pattern(r#"\u{1000A}"#),
],
separators: vec![],
examples: vec![
("u{1234} ok", Some((0, "u{1234}"))),
("{aba}}", Some((1, "{aba}"))),
("\u{1000A}", Some((2, "\u{1000A}"))),
],
},
];