Fix cases where error recovery could infinite loop (#4257) (#4262)

* Rename corpus test functions to allow easy filtering by language

* Use usize for seed argument

* Avoid retaining useless stack versions when reductions merge

We found this problem when debugging an infinite loop that happened
during error recovery when using the Zig grammar. The large number of
unnecessary paused stack versions were preventing the correct recovery
strategy from being tried.

* Fix leaked lookahead token when reduction results in a merged stack

* Enable running PHP tests in CI

* Fix possible infinite loop during error recovery at EOF

* Account for external scanner state changes when detecting changed ranges in subtrees

(cherry picked from commit 066fd77d39)

Co-authored-by: Max Brunsfeld <maxbrunsfeld@gmail.com>

cli/src/fuzz/mod.rs

@@ -56,7 +56,9 @@ fn regex_env_var(name: &'static str) -> Option<Regex> {
 pub fn new_seed() -> usize {
     int_env_var("TREE_SITTER_SEED").unwrap_or_else(|| {
         let mut rng = rand::thread_rng();
-        rng.gen::<usize>()
+        let seed = rng.gen::<usize>();
+        eprintln!("Seed: {seed}");
+        seed
     })
 }
 

cli/src/tests/corpus_test.rs

@@ -23,7 +23,7 @@ use crate::{
 };
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_bash(seed: usize) {
+fn test_corpus_for_bash_language(seed: usize) {
     test_language_corpus(
         "bash",
         seed,
@@ -39,73 +39,77 @@ fn test_corpus_for_bash(seed: usize) {
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_c(seed: usize) {
+fn test_corpus_for_c_language(seed: usize) {
     test_language_corpus("c", seed, None, None);
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_cpp(seed: usize) {
+fn test_corpus_for_cpp_language(seed: usize) {
     test_language_corpus("cpp", seed, None, None);
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_embedded_template(seed: usize) {
+fn test_corpus_for_embedded_template_language(seed: usize) {
     test_language_corpus("embedded-template", seed, None, None);
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_go(seed: usize) {
+fn test_corpus_for_go_language(seed: usize) {
     test_language_corpus("go", seed, None, None);
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_html(seed: usize) {
+fn test_corpus_for_html_language(seed: usize) {
     test_language_corpus("html", seed, None, None);
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_java(seed: usize) {
-    test_language_corpus("java", seed, None, None);
+fn test_corpus_for_java_language(seed: usize) {
+    test_language_corpus(
+        "java",
+        seed,
+        Some(&["java - corpus - expressions - switch with unnamed pattern variable"]),
+        None,
+    );
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_javascript(seed: usize) {
+fn test_corpus_for_javascript_language(seed: usize) {
     test_language_corpus("javascript", seed, None, None);
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_json(seed: usize) {
+fn test_corpus_for_json_language(seed: usize) {
     test_language_corpus("json", seed, None, None);
 }
 
-#[ignore]
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_php(seed: usize) {
-    test_language_corpus("php", seed, None, None);
+fn test_corpus_for_php_language(seed: usize) {
+    test_language_corpus("php", seed, None, Some("php"));
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_python(seed: usize) {
+fn test_corpus_for_python_language(seed: usize) {
     test_language_corpus("python", seed, None, None);
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_ruby(seed: usize) {
+fn test_corpus_for_ruby_language(seed: usize) {
     test_language_corpus("ruby", seed, None, None);
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_rust(seed: usize) {
+fn test_corpus_for_rust_language(seed: usize) {
     test_language_corpus("rust", seed, None, None);
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_typescript(seed: usize) {
+fn test_corpus_for_typescript_language(seed: usize) {
     test_language_corpus("typescript", seed, None, Some("typescript"));
 }
 
 #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
-fn test_corpus_for_tsx(seed: usize) {
+fn test_corpus_for_tsx_language(seed: usize) {
     test_language_corpus("typescript", seed, None, Some("tsx"));
 }