traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: disallow tokens that match the empty string

Browse source
This commit is contained in:
Amaan Qureshi 2024-11-02 00:53:21 -04:00
parent 8c802da174
commit 310a9f0704
5 changed files with 140 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

113
cli/src/tests/parser_test.rs
View file

@ -1679,7 +1679,6 @@ fn test_decode_utf32() {
)
}
} else {
println!("bad decode: {bytes:?}");
(0, 0)
}
}
@ -1816,6 +1815,118 @@ fn test_decode_utf24le() {
);
}
#[test]
fn test_grammars_that_should_not_compile() {
assert!(generate_parser_for_grammar(
r#"
{
"name": "issue_1111",
"rules": {
"source_file": { "type": "STRING", "value": "" }
},
}
"#
)
.is_err());
assert!(generate_parser_for_grammar(
r#"
{
"name": "issue_1271",
"rules": {
"source_file": { "type": "SYMBOL", "name": "identifier" },
"identifier": {
"type": "TOKEN",
"content": {
"type": "REPEAT",
"content": { "type": "PATTERN", "value": "a" }
}
}
},
}
"#,
)
.is_err());
assert!(generate_parser_for_grammar(
r#"
{
"name": "issue_1156_expl_1",
"rules": {
"source_file": {
"type": "TOKEN",
"content": {
"type": "REPEAT",
"content": { "type": "STRING", "value": "c" }
}
}
},
}
"#
)
.is_err());
assert!(generate_parser_for_grammar(
r#"
{
"name": "issue_1156_expl_2",
"rules": {
"source_file": {
"type": "TOKEN",
"content": {
"type": "CHOICE",
"members": [
{ "type": "STRING", "value": "e" },
{ "type": "BLANK" }
]
}
}
},
}
"#
)
.is_err());
assert!(generate_parser_for_grammar(
r#"
{
"name": "issue_1156_expl_3",
"rules": {
"source_file": {
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "REPEAT",
"content": { "type": "STRING", "value": "p" }
}
}
},
}
"#
)
.is_err());
assert!(generate_parser_for_grammar(
r#"
{
"name": "issue_1156_expl_4",
"rules": {
"source_file": {
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "CHOICE",
"members": [
{ "type": "STRING", "value": "r" },
{ "type": "BLANK" }
]
}
}
},
}
"#
)
.is_err());
}
const fn simple_range(start: usize, end: usize) -> Range {
Range {
start_byte: start,