traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: disallow tokens that match the empty string

Browse source
This commit is contained in:
Amaan Qureshi 2024-11-02 00:53:21 -04:00
parent 8c802da174
commit 310a9f0704
5 changed files with 140 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

6
cli/src/tests/async_context_test.rs
View file

@ -22,7 +22,6 @@ fn test_node_in_fut() {
let root_ref = &root;
let fut_val_fn = || async {
// eprintln!("fut_val_fn: {}", root.child(0).unwrap().kind());
yield_now().await;
root.child(0).unwrap().kind()
};
@ -30,7 +29,6 @@ fn test_node_in_fut() {
yield_now().await;
let fut_ref_fn = || async {
// eprintln!("fut_ref_fn: {}", root_ref.child(0).unwrap().kind());
yield_now().await;
root_ref.child(0).unwrap().kind()
};
@ -40,13 +38,11 @@ fn test_node_in_fut() {
assert_eq!(f1, f2);
let fut_val = async {
// eprintln!("fut_val: {}", root.child(0).unwrap().kind());
yield_now().await;
root.child(0).unwrap().kind()
};
let fut_ref = async {
// eprintln!("fut_ref: {}", root_ref.child(0).unwrap().kind());
yield_now().await;
root_ref.child(0).unwrap().kind()
};
@ -58,7 +54,6 @@ fn test_node_in_fut() {
f1
})
.join();
// eprintln!("pended: {pended:?}");
assert_eq!(ret, "comment");
assert_eq!(pended, 5);
}
@ -215,7 +210,6 @@ where
match future.as_mut().poll(&mut cx) {
Poll::Pending => pending += 1,
Poll::Ready(r) => {
// eprintln!("ready, pended: {pending}");
break r;
}
}

113
cli/src/tests/parser_test.rs
View file

@ -1679,7 +1679,6 @@ fn test_decode_utf32() {
)
}
} else {
println!("bad decode: {bytes:?}");
(0, 0)
}
}
@ -1816,6 +1815,118 @@ fn test_decode_utf24le() {
);
}
#[test]
fn test_grammars_that_should_not_compile() {
assert!(generate_parser_for_grammar(
r#"
{
"name": "issue_1111",
"rules": {
"source_file": { "type": "STRING", "value": "" }
},
}
"#
)
.is_err());
assert!(generate_parser_for_grammar(
r#"
{
"name": "issue_1271",
"rules": {
"source_file": { "type": "SYMBOL", "name": "identifier" },
"identifier": {
"type": "TOKEN",
"content": {
"type": "REPEAT",
"content": { "type": "PATTERN", "value": "a" }
}
}
},
}
"#,
)
.is_err());
assert!(generate_parser_for_grammar(
r#"
{
"name": "issue_1156_expl_1",
"rules": {
"source_file": {
"type": "TOKEN",
"content": {
"type": "REPEAT",
"content": { "type": "STRING", "value": "c" }
}
}
},
}
"#
)
.is_err());
assert!(generate_parser_for_grammar(
r#"
{
"name": "issue_1156_expl_2",
"rules": {
"source_file": {
"type": "TOKEN",
"content": {
"type": "CHOICE",
"members": [
{ "type": "STRING", "value": "e" },
{ "type": "BLANK" }
]
}
}
},
}
"#
)
.is_err());
assert!(generate_parser_for_grammar(
r#"
{
"name": "issue_1156_expl_3",
"rules": {
"source_file": {
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "REPEAT",
"content": { "type": "STRING", "value": "p" }
}
}
},
}
"#
)
.is_err());
assert!(generate_parser_for_grammar(
r#"
{
"name": "issue_1156_expl_4",
"rules": {
"source_file": {
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "CHOICE",
"members": [
{ "type": "STRING", "value": "r" },
{ "type": "BLANK" }
]
}
}
},
}
"#
)
.is_err());
}
const fn simple_range(start: usize, end: usize) -> Range {
Range {
start_byte: start,