From 9f2dc9d6b55b26f0dc3ea51bae632c68dc2f833e Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Fri, 2 Feb 2024 10:26:18 -0500 Subject: [PATCH] fix: rework parser.h includes for test grammars and multi-grammar repos --- cli/benches/benchmark.rs | 2 +- cli/loader/src/lib.rs | 27 ++++++++++++------- cli/src/tests/helpers/fixtures.rs | 19 +++++++++++-- cli/src/tests/parser_test.rs | 2 +- .../epsilon_external_tokens/scanner.c | 2 +- .../scanner.c | 2 +- .../external_and_internal_tokens/scanner.c | 2 +- .../external_extra_tokens/scanner.c | 2 +- .../test_grammars/external_tokens/scanner.c | 2 +- .../scanner.c | 3 ++- .../get_col_should_hang_not_crash/scanner.c | 2 +- .../inverted_external_token/scanner.c | 2 +- .../uses_current_column/scanner.c | 3 ++- 13 files changed, 48 insertions(+), 22 deletions(-) diff --git a/cli/benches/benchmark.rs b/cli/benches/benchmark.rs index 51246f03..1dfc8ece 100644 --- a/cli/benches/benchmark.rs +++ b/cli/benches/benchmark.rs @@ -209,7 +209,7 @@ fn parse(path: &Path, max_path_length: usize, mut action: impl FnMut(&[u8])) -> fn get_language(path: &Path) -> Language { let src_dir = GRAMMARS_DIR.join(path).join("src"); TEST_LOADER - .load_language_at_path(&src_dir, &src_dir) + .load_language_at_path(&src_dir, &[&src_dir]) .with_context(|| format!("Failed to load language at path {:?}", src_dir)) .unwrap() } diff --git a/cli/loader/src/lib.rs b/cli/loader/src/lib.rs index 8febf222..53af492e 100644 --- a/cli/loader/src/lib.rs +++ b/cli/loader/src/lib.rs @@ -322,12 +322,16 @@ impl Loader { language .get_or_try_init(|| { let src_path = path.join("src"); - self.load_language_at_path(&src_path, &src_path) + self.load_language_at_path(&src_path, &[&src_path]) }) .cloned() } - pub fn load_language_at_path(&self, src_path: &Path, header_path: &Path) -> Result { + pub fn load_language_at_path( + &self, + src_path: &Path, + header_paths: &[&Path], + ) -> Result { let grammar_path = src_path.join("grammar.json"); #[derive(Deserialize)] @@ -339,13 +343,13 @@ impl Loader { let grammar_json: GrammarJSON = serde_json::from_reader(BufReader::new(&mut grammar_file)) .with_context(|| "Failed to parse grammar.json")?; - self.load_language_at_path_with_name(src_path, &header_path, &grammar_json.name) + self.load_language_at_path_with_name(src_path, header_paths, &grammar_json.name) } pub fn load_language_at_path_with_name( &self, src_path: &Path, - header_path: &Path, + header_paths: &[&Path], name: &str, ) -> Result { let mut lib_name = name.to_string(); @@ -391,7 +395,7 @@ impl Loader { { if recompile { self.compile_parser_to_dylib( - header_path, + header_paths, &parser_path, &scanner_path, &library_path, @@ -413,7 +417,7 @@ impl Loader { fn compile_parser_to_dylib( &self, - header_path: &Path, + header_paths: &[&Path], parser_path: &Path, scanner_path: &Option, library_path: &PathBuf, @@ -433,7 +437,10 @@ impl Loader { } if compiler.is_like_msvc() { - command.args(&["/nologo", "/LD", "/I"]).arg(header_path); + command.args(&["/nologo", "/LD"]); + header_paths.iter().for_each(|path| { + command.arg(format!("/I{}", path.to_string_lossy())); + }); if self.debug_build { command.arg("/Od"); } else { @@ -451,11 +458,13 @@ impl Loader { .arg("-shared") .arg("-fno-exceptions") .arg("-g") - .arg("-I") - .arg(header_path) .arg("-o") .arg(&library_path); + header_paths.iter().for_each(|path| { + command.arg(format!("-I{}", path.to_string_lossy())); + }); + if !cfg!(windows) { command.arg("-fPIC"); } diff --git a/cli/src/tests/helpers/fixtures.rs b/cli/src/tests/helpers/fixtures.rs index 7cb981ae..da4f305b 100644 --- a/cli/src/tests/helpers/fixtures.rs +++ b/cli/src/tests/helpers/fixtures.rs @@ -1,3 +1,4 @@ +use anyhow::Context; use lazy_static::lazy_static; use std::path::{Path, PathBuf}; use std::{env, fs}; @@ -28,7 +29,10 @@ pub fn fixtures_dir<'a>() -> &'static Path { pub fn get_language(name: &str) -> Language { TEST_LOADER - .load_language_at_path(&GRAMMARS_DIR.join(name).join("src"), &HEADER_DIR) + .load_language_at_path( + &GRAMMARS_DIR.join(name).join("src"), + &[&HEADER_DIR, &GRAMMARS_DIR.join(name).join("src")], + ) .unwrap() } @@ -93,7 +97,18 @@ pub fn get_test_language(name: &str, parser_code: &str, path: Option<&Path>) -> } } + let header_path = src_dir.join("tree_sitter"); + fs::create_dir_all(&header_path).unwrap(); + fs::write(&header_path.join("parser.h"), tree_sitter::PARSER_HEADER) + .with_context(|| { + format!( + "Failed to write {:?}", + header_path.join("parser.h").file_name().unwrap() + ) + }) + .unwrap(); + TEST_LOADER - .load_language_at_path_with_name(&src_dir, &HEADER_DIR, name) + .load_language_at_path_with_name(&src_dir, &[&HEADER_DIR], name) .unwrap() } diff --git a/cli/src/tests/parser_test.rs b/cli/src/tests/parser_test.rs index ff8818fe..511ef02a 100644 --- a/cli/src/tests/parser_test.rs +++ b/cli/src/tests/parser_test.rs @@ -432,7 +432,7 @@ fn test_parsing_after_editing_tree_that_depends_on_column_values() { let dir = fixtures_dir() .join("test_grammars") .join("uses_current_column"); - let grammar = fs::read_to_string(&dir.join("grammar.json")).unwrap(); + let grammar = fs::read_to_string(dir.join("grammar.json")).unwrap(); let (grammar_name, parser_code) = generate_parser_for_grammar(&grammar).unwrap(); let mut parser = Parser::new(); diff --git a/test/fixtures/test_grammars/epsilon_external_tokens/scanner.c b/test/fixtures/test_grammars/epsilon_external_tokens/scanner.c index 85bc7c60..d25c1ff3 100644 --- a/test/fixtures/test_grammars/epsilon_external_tokens/scanner.c +++ b/test/fixtures/test_grammars/epsilon_external_tokens/scanner.c @@ -1,4 +1,4 @@ -#include +#include "tree_sitter/parser.h" enum TokenType { ZERO_WIDTH_TOKEN diff --git a/test/fixtures/test_grammars/external_and_internal_anonymous_tokens/scanner.c b/test/fixtures/test_grammars/external_and_internal_anonymous_tokens/scanner.c index 644bbd0a..187c675e 100644 --- a/test/fixtures/test_grammars/external_and_internal_anonymous_tokens/scanner.c +++ b/test/fixtures/test_grammars/external_and_internal_anonymous_tokens/scanner.c @@ -1,4 +1,4 @@ -#include +#include "tree_sitter/parser.h" enum { STRING, diff --git a/test/fixtures/test_grammars/external_and_internal_tokens/scanner.c b/test/fixtures/test_grammars/external_and_internal_tokens/scanner.c index 43a4bc9d..55454f29 100644 --- a/test/fixtures/test_grammars/external_and_internal_tokens/scanner.c +++ b/test/fixtures/test_grammars/external_and_internal_tokens/scanner.c @@ -1,4 +1,4 @@ -#include +#include "tree_sitter/parser.h" enum { STRING, diff --git a/test/fixtures/test_grammars/external_extra_tokens/scanner.c b/test/fixtures/test_grammars/external_extra_tokens/scanner.c index ac6d8407..95d58426 100644 --- a/test/fixtures/test_grammars/external_extra_tokens/scanner.c +++ b/test/fixtures/test_grammars/external_extra_tokens/scanner.c @@ -1,4 +1,4 @@ -#include +#include "tree_sitter/parser.h" enum { COMMENT, diff --git a/test/fixtures/test_grammars/external_tokens/scanner.c b/test/fixtures/test_grammars/external_tokens/scanner.c index 490100d4..163ae51b 100644 --- a/test/fixtures/test_grammars/external_tokens/scanner.c +++ b/test/fixtures/test_grammars/external_tokens/scanner.c @@ -1,4 +1,4 @@ -#include +#include "tree_sitter/parser.h" enum { percent_string, diff --git a/test/fixtures/test_grammars/external_unicode_column_alignment/scanner.c b/test/fixtures/test_grammars/external_unicode_column_alignment/scanner.c index a9e98735..3380bba2 100644 --- a/test/fixtures/test_grammars/external_unicode_column_alignment/scanner.c +++ b/test/fixtures/test_grammars/external_unicode_column_alignment/scanner.c @@ -1,4 +1,5 @@ -#include +#include "tree_sitter/parser.h" + #include #include diff --git a/test/fixtures/test_grammars/get_col_should_hang_not_crash/scanner.c b/test/fixtures/test_grammars/get_col_should_hang_not_crash/scanner.c index d21ec6d4..000647af 100644 --- a/test/fixtures/test_grammars/get_col_should_hang_not_crash/scanner.c +++ b/test/fixtures/test_grammars/get_col_should_hang_not_crash/scanner.c @@ -1,4 +1,4 @@ -#include +#include "tree_sitter/parser.h" unsigned tree_sitter_get_col_should_hang_not_crash_external_scanner_serialize() { return 0; } diff --git a/test/fixtures/test_grammars/inverted_external_token/scanner.c b/test/fixtures/test_grammars/inverted_external_token/scanner.c index 260994c5..33fde6d1 100644 --- a/test/fixtures/test_grammars/inverted_external_token/scanner.c +++ b/test/fixtures/test_grammars/inverted_external_token/scanner.c @@ -1,4 +1,4 @@ -#include +#include "tree_sitter/parser.h" enum { LINE_BREAK diff --git a/test/fixtures/test_grammars/uses_current_column/scanner.c b/test/fixtures/test_grammars/uses_current_column/scanner.c index 62b16392..b2b59281 100644 --- a/test/fixtures/test_grammars/uses_current_column/scanner.c +++ b/test/fixtures/test_grammars/uses_current_column/scanner.c @@ -1,6 +1,7 @@ +#include "tree_sitter/parser.h" + #include #include -#include enum TokenType { INDENT,