fix: rework parser.h includes for test grammars and multi-grammar repos

cli/benches/benchmark.rs

@@ -209,7 +209,7 @@ fn parse(path: &Path, max_path_length: usize, mut action: impl FnMut(&[u8])) ->
 fn get_language(path: &Path) -> Language {
     let src_dir = GRAMMARS_DIR.join(path).join("src");
     TEST_LOADER
-        .load_language_at_path(&src_dir, &src_dir)
+        .load_language_at_path(&src_dir, &[&src_dir])
         .with_context(|| format!("Failed to load language at path {:?}", src_dir))
         .unwrap()
 }

cli/loader/src/lib.rs

@@ -322,12 +322,16 @@ impl Loader {
         language
             .get_or_try_init(|| {
                 let src_path = path.join("src");
-                self.load_language_at_path(&src_path, &src_path)
+                self.load_language_at_path(&src_path, &[&src_path])
             })
             .cloned()
     }
 
-    pub fn load_language_at_path(&self, src_path: &Path, header_path: &Path) -> Result<Language> {
+    pub fn load_language_at_path(
+        &self,
+        src_path: &Path,
+        header_paths: &[&Path],
+    ) -> Result<Language> {
         let grammar_path = src_path.join("grammar.json");
 
         #[derive(Deserialize)]
@@ -339,13 +343,13 @@ impl Loader {
         let grammar_json: GrammarJSON = serde_json::from_reader(BufReader::new(&mut grammar_file))
             .with_context(|| "Failed to parse grammar.json")?;
 
-        self.load_language_at_path_with_name(src_path, &header_path, &grammar_json.name)
+        self.load_language_at_path_with_name(src_path, header_paths, &grammar_json.name)
     }
 
     pub fn load_language_at_path_with_name(
         &self,
         src_path: &Path,
-        header_path: &Path,
+        header_paths: &[&Path],
         name: &str,
     ) -> Result<Language> {
         let mut lib_name = name.to_string();
@@ -391,7 +395,7 @@ impl Loader {
         {
             if recompile {
                 self.compile_parser_to_dylib(
-                    header_path,
+                    header_paths,
                     &parser_path,
                     &scanner_path,
                     &library_path,
@@ -413,7 +417,7 @@ impl Loader {
 
     fn compile_parser_to_dylib(
         &self,
-        header_path: &Path,
+        header_paths: &[&Path],
         parser_path: &Path,
         scanner_path: &Option<PathBuf>,
         library_path: &PathBuf,
@@ -433,7 +437,10 @@ impl Loader {
         }
 
         if compiler.is_like_msvc() {
-            command.args(&["/nologo", "/LD", "/I"]).arg(header_path);
+            command.args(&["/nologo", "/LD"]);
+            header_paths.iter().for_each(|path| {
+                command.arg(format!("/I{}", path.to_string_lossy()));
+            });
             if self.debug_build {
                 command.arg("/Od");
             } else {
@@ -451,11 +458,13 @@ impl Loader {
                 .arg("-shared")
                 .arg("-fno-exceptions")
                 .arg("-g")
-                .arg("-I")
-                .arg(header_path)
                 .arg("-o")
                 .arg(&library_path);
 
+            header_paths.iter().for_each(|path| {
+                command.arg(format!("-I{}", path.to_string_lossy()));
+            });
+
             if !cfg!(windows) {
                 command.arg("-fPIC");
             }

cli/src/tests/helpers/fixtures.rs

@@ -1,3 +1,4 @@
+use anyhow::Context;
 use lazy_static::lazy_static;
 use std::path::{Path, PathBuf};
 use std::{env, fs};
@@ -28,7 +29,10 @@ pub fn fixtures_dir<'a>() -> &'static Path {
 
 pub fn get_language(name: &str) -> Language {
     TEST_LOADER
-        .load_language_at_path(&GRAMMARS_DIR.join(name).join("src"), &HEADER_DIR)
+        .load_language_at_path(
+            &GRAMMARS_DIR.join(name).join("src"),
+            &[&HEADER_DIR, &GRAMMARS_DIR.join(name).join("src")],
+        )
         .unwrap()
 }
 
@@ -93,7 +97,18 @@ pub fn get_test_language(name: &str, parser_code: &str, path: Option<&Path>) ->
         }
     }
 
+    let header_path = src_dir.join("tree_sitter");
+    fs::create_dir_all(&header_path).unwrap();
+    fs::write(&header_path.join("parser.h"), tree_sitter::PARSER_HEADER)
+        .with_context(|| {
+            format!(
+                "Failed to write {:?}",
+                header_path.join("parser.h").file_name().unwrap()
+            )
+        })
+        .unwrap();
+
     TEST_LOADER
-        .load_language_at_path_with_name(&src_dir, &HEADER_DIR, name)
+        .load_language_at_path_with_name(&src_dir, &[&HEADER_DIR], name)
         .unwrap()
 }

cli/src/tests/parser_test.rs

@@ -432,7 +432,7 @@ fn test_parsing_after_editing_tree_that_depends_on_column_values() {
     let dir = fixtures_dir()
         .join("test_grammars")
         .join("uses_current_column");
-    let grammar = fs::read_to_string(&dir.join("grammar.json")).unwrap();
+    let grammar = fs::read_to_string(dir.join("grammar.json")).unwrap();
     let (grammar_name, parser_code) = generate_parser_for_grammar(&grammar).unwrap();
 
     let mut parser = Parser::new();

test/fixtures/test_grammars/epsilon_external_tokens/scanner.c

@@ -1,4 +1,4 @@
-#include <tree_sitter/parser.h>
+#include "tree_sitter/parser.h"
 
 enum TokenType {
   ZERO_WIDTH_TOKEN

test/fixtures/test_grammars/external_and_internal_anonymous_tokens/scanner.c

@@ -1,4 +1,4 @@
-#include <tree_sitter/parser.h>
+#include "tree_sitter/parser.h"
 
 enum {
   STRING,

test/fixtures/test_grammars/external_and_internal_tokens/scanner.c

@@ -1,4 +1,4 @@
-#include <tree_sitter/parser.h>
+#include "tree_sitter/parser.h"
 
 enum {
   STRING,

test/fixtures/test_grammars/external_extra_tokens/scanner.c

@@ -1,4 +1,4 @@
-#include <tree_sitter/parser.h>
+#include "tree_sitter/parser.h"
 
 enum {
   COMMENT,

test/fixtures/test_grammars/external_tokens/scanner.c

@@ -1,4 +1,4 @@
-#include <tree_sitter/parser.h>
+#include "tree_sitter/parser.h"
 
 enum {
   percent_string,

test/fixtures/test_grammars/external_unicode_column_alignment/scanner.c

@@ -1,4 +1,5 @@
-#include <tree_sitter/parser.h>
+#include "tree_sitter/parser.h"
+
 #include <wctype.h>
 #include <string.h>
 

test/fixtures/test_grammars/get_col_should_hang_not_crash/scanner.c

@@ -1,4 +1,4 @@
-#include <tree_sitter/parser.h>
+#include "tree_sitter/parser.h"
 
 unsigned tree_sitter_get_col_should_hang_not_crash_external_scanner_serialize() { return 0; }
 

test/fixtures/test_grammars/inverted_external_token/scanner.c

@@ -1,4 +1,4 @@
-#include <tree_sitter/parser.h>
+#include "tree_sitter/parser.h"
 
 enum {
   LINE_BREAK

test/fixtures/test_grammars/uses_current_column/scanner.c

@@ -1,6 +1,7 @@
+#include "tree_sitter/parser.h"
+
 #include <stdlib.h>
 #include <wctype.h>
-#include <tree_sitter/parser.h>
 
 enum TokenType {
   INDENT,