From a8292f4fe99d87dfee886e146307da0a8beb2a9c Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 15 Jan 2019 10:27:39 -0800 Subject: [PATCH] Load all fixture grammars dynamically This way the build doesn't take forever any time a single grammar has been regenerated. --- cli/src/generate/nfa.rs | 5 +-- cli/src/generate/properties.rs | 3 +- cli/src/loader.rs | 49 ++++++++++++++++++-------- cli/src/tests/corpuses.rs | 64 ++++++++++------------------------ cli/src/tests/fixtures.rs | 51 +++++++++++++++++++++++++++ cli/src/tests/languages.rs | 21 ----------- cli/src/tests/mod.rs | 2 +- cli/src/tests/parser_api.rs | 8 +++-- lib/build.rs | 53 ---------------------------- script/regenerate-fixtures | 6 ++++ 10 files changed, 119 insertions(+), 143 deletions(-) create mode 100644 cli/src/tests/fixtures.rs delete mode 100644 cli/src/tests/languages.rs diff --git a/cli/src/generate/nfa.rs b/cli/src/generate/nfa.rs index 54e34814..674391ff 100644 --- a/cli/src/generate/nfa.rs +++ b/cli/src/generate/nfa.rs @@ -55,10 +55,6 @@ impl CharacterSet { CharacterSet::Include(Vec::new()) } - pub fn all() -> Self { - CharacterSet::Exclude(Vec::new()) - } - pub fn negate(self) -> CharacterSet { match self { CharacterSet::Include(chars) => CharacterSet::Exclude(chars), @@ -182,6 +178,7 @@ impl CharacterSet { } } + #[cfg(test)] pub fn contains(&self, c: char) -> bool { match self { CharacterSet::Include(chars) => chars.contains(&c), diff --git a/cli/src/generate/properties.rs b/cli/src/generate/properties.rs index e1492d6f..b16e698a 100644 --- a/cli/src/generate/properties.rs +++ b/cli/src/generate/properties.rs @@ -464,7 +464,8 @@ fn parse_property_sheet(path: &Path, css: &str) -> Result> { rsass::Item::AtRule { name, args, .. } => match name.as_str() { "schema" => { if let Some(s) = get_sass_string(args) { - let schema_path = resolve_path(path, s)?; + // TODO - use schema + let _schema_path = resolve_path(path, s)?; items.remove(i); continue; } else { diff --git a/cli/src/loader.rs b/cli/src/loader.rs index af1ab7be..83b878b9 100644 --- a/cli/src/loader.rs +++ b/cli/src/loader.rs @@ -23,11 +23,11 @@ struct LanguageRepo { } pub struct LanguageConfiguration { - name: String, - content_regex: Option, - first_line_regex: Option, + _name: String, + _content_regex: Option, + _first_line_regex: Option, file_types: Vec, - highlight_property_sheet: Option>, + _highlight_property_sheet: Option>, } pub struct Loader { @@ -108,16 +108,21 @@ impl Loader { let language = if let Some(language) = repo.language { language } else { - let language = self.load_language_at_path(&repo.name, &repo.path)?; + let src_path = repo.path.join("src"); + let language = self.load_language_at_path(&repo.name, &src_path, &src_path)?; self.language_repos[id].language = Some(language); language }; Ok((language, &self.language_repos[id].configurations)) } - fn load_language_at_path(&self, name: &str, language_path: &Path) -> io::Result { - let src_path = language_path.join("src"); - let parser_c_path = src_path.join("parser.c"); + pub fn load_language_at_path( + &self, + name: &str, + src_path: &Path, + header_path: &Path, + ) -> io::Result { + let parser_path = src_path.join("parser.c"); let scanner_path; let scanner_c_path = src_path.join("scanner.c"); @@ -132,7 +137,7 @@ impl Loader { } } - self.load_language_from_sources(name, &src_path, &parser_c_path, &scanner_path) + self.load_language_from_sources(name, &header_path, &parser_path, &scanner_path) } pub fn load_language_from_sources( @@ -148,6 +153,7 @@ impl Loader { if needs_recompile(&library_path, &parser_path, &scanner_path)? { let mut config = cc::Build::new(); config + .cpp(true) .opt_level(2) .cargo_metadata(false) .target(env!("BUILD_TARGET")) @@ -197,13 +203,14 @@ impl Loader { "Parser compilation failed.\nStdout: {}\nStderr: {}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) - ).as_str(), + ) + .as_str(), )); } } let library = Library::new(library_path)?; - let language_fn_name = format!("tree_sitter_{}", name); + let language_fn_name = format!("tree_sitter_{}", replace_dashes_with_underscores(name)); let language = unsafe { let language_fn: Symbol Language> = library.get(language_fn_name.as_bytes())?; @@ -248,15 +255,15 @@ impl Loader { configurations .into_iter() .map(|conf| LanguageConfiguration { - name: conf.name, + _name: conf.name, file_types: conf.file_types.unwrap_or(Vec::new()), - content_regex: conf + _content_regex: conf .content_regex .and_then(|r| RegexBuilder::new(&r).multi_line(true).build().ok()), - first_line_regex: conf + _first_line_regex: conf .first_line_regex .and_then(|r| RegexBuilder::new(&r).multi_line(true).build().ok()), - highlight_property_sheet: conf.highlights.map(|d| Err(d.into())), + _highlight_property_sheet: conf.highlights.map(|d| Err(d.into())), }) .collect() }); @@ -304,3 +311,15 @@ fn needs_recompile( fn mtime(path: &Path) -> io::Result { Ok(fs::metadata(path)?.modified()?) } + +fn replace_dashes_with_underscores(name: &str) -> String { + let mut result = String::with_capacity(name.len()); + for c in name.chars() { + if c == '-' { + result.push('_'); + } else { + result.push(c); + } + } + result +} diff --git a/cli/src/tests/corpuses.rs b/cli/src/tests/corpuses.rs index e1fe9189..e55c8f57 100644 --- a/cli/src/tests/corpuses.rs +++ b/cli/src/tests/corpuses.rs @@ -1,27 +1,21 @@ -use super::languages; +use super::fixtures::{get_language, get_test_language, fixtures_dir}; use crate::generate; -use crate::loader::Loader; use crate::test::{parse_tests, TestEntry}; use crate::util; use std::fs; -use std::path::PathBuf; -use tree_sitter::{Language, Parser, LogType}; +use tree_sitter::{LogType, Parser}; + +const LANGUAGES: &'static [&'static str] = &[ + "bash", + "c", + "cpp", + "embedded-template", + "go", + "html", + "javascript", +]; lazy_static! { - static ref LANGUAGES: [(&'static str, Language); 7] = [ - ("bash", languages::bash()), - ("c", languages::c()), - ("cpp", languages::cpp()), - ("embedded-template", languages::embedded_template()), - ("go", languages::go()), - ("html", languages::html()), - ("javascript", languages::javascript()), - ]; - static ref ROOT_DIR: PathBuf = [env!("CARGO_MANIFEST_DIR"), ".."].iter().collect(); - static ref HEADER_DIR: PathBuf = ROOT_DIR.join("lib").join("include"); - static ref SCRATCH_DIR: PathBuf = ROOT_DIR.join("target").join("scratch"); - static ref FIXTURES_DIR: PathBuf = ROOT_DIR.join("test").join("fixtures"); - static ref EXEC_PATH: PathBuf = std::env::current_exe().unwrap(); static ref LANGUAGE_FILTER: Option = std::env::var("TREE_SITTER_TEST_LANGUAGE_FILTER").ok(); static ref EXAMPLE_FILTER: Option = @@ -34,7 +28,7 @@ lazy_static! { fn test_real_language_corpus_files() { let mut log_session = None; let mut parser = Parser::new(); - let grammars_dir = FIXTURES_DIR.join("grammars"); + let grammars_dir = fixtures_dir().join("grammars"); if *LOG_ENABLED { parser.set_logger(Some(Box::new(|log_type, msg| { @@ -48,7 +42,7 @@ fn test_real_language_corpus_files() { log_session = Some(util::log_graphs(&mut parser, "log.html").unwrap()); } - for (language_name, language) in LANGUAGES.iter().cloned() { + for language_name in LANGUAGES.iter().cloned() { if let Some(filter) = LANGUAGE_FILTER.as_ref() { if !language_name.contains(filter.as_str()) { continue; @@ -57,6 +51,7 @@ fn test_real_language_corpus_files() { eprintln!("language: {:?}", language_name); + let language = get_language(language_name); let corpus_dir = grammars_dir.join(language_name).join("corpus"); let test = parse_tests(&corpus_dir).unwrap(); parser.set_language(language).unwrap(); @@ -69,12 +64,9 @@ fn test_real_language_corpus_files() { #[test] fn test_feature_corpus_files() { - fs::create_dir_all(SCRATCH_DIR.as_path()).unwrap(); - - let loader = Loader::new(SCRATCH_DIR.clone()); let mut log_session = None; let mut parser = Parser::new(); - let test_grammars_dir = FIXTURES_DIR.join("test_grammars"); + let test_grammars_dir = fixtures_dir().join("test_grammars"); if *LOG_ENABLED { parser.set_logger(Some(Box::new(|log_type, msg| { @@ -128,27 +120,7 @@ fn test_feature_corpus_files() { } else { let corpus_path = test_path.join("corpus.txt"); let c_code = generate_result.unwrap(); - let parser_c_path = SCRATCH_DIR.join(&format!("{}-parser.c", language_name)); - if !fs::read_to_string(&parser_c_path) - .map(|content| content == c_code) - .unwrap_or(false) - { - fs::write(&parser_c_path, c_code).unwrap(); - } - let scanner_path = test_path.join("scanner.c"); - let scanner_path = if scanner_path.exists() { - Some(scanner_path) - } else { - None - }; - let language = loader - .load_language_from_sources( - language_name, - &HEADER_DIR, - &parser_c_path, - &scanner_path, - ) - .unwrap(); + let language = get_test_language(language_name, c_code, &test_path); let test = parse_tests(&corpus_path).unwrap(); parser.set_language(language).unwrap(); run_mutation_tests(&mut parser, test); @@ -180,7 +152,7 @@ fn run_mutation_tests(parser: &mut Parser, test: TestEntry) { let actual = tree.root_node().to_sexp(); assert_eq!(actual, output); } - TestEntry::Group { name, children } => { + TestEntry::Group { children, .. } => { for child in children { run_mutation_tests(parser, child); } diff --git a/cli/src/tests/fixtures.rs b/cli/src/tests/fixtures.rs new file mode 100644 index 00000000..978a1212 --- /dev/null +++ b/cli/src/tests/fixtures.rs @@ -0,0 +1,51 @@ +use crate::loader::Loader; +use std::path::{Path, PathBuf}; +use tree_sitter::Language; +use std::fs; + +lazy_static! { + static ref ROOT_DIR: PathBuf = [env!("CARGO_MANIFEST_DIR"), ".."].iter().collect(); + static ref FIXTURES_DIR: PathBuf = ROOT_DIR.join("test").join("fixtures"); + static ref HEADER_DIR: PathBuf = ROOT_DIR.join("lib").join("include"); + static ref GRAMMARS_DIR: PathBuf = ROOT_DIR.join("test").join("fixtures").join("grammars"); + static ref SCRATCH_DIR: PathBuf = { + let result = ROOT_DIR.join("target").join("scratch"); + fs::create_dir_all(&result).unwrap(); + result + }; + static ref TEST_LOADER: Loader = Loader::new(SCRATCH_DIR.clone()); +} + +pub fn fixtures_dir<'a>() -> &'static Path { + &FIXTURES_DIR +} + +pub fn get_language(name: &str) -> Language { + TEST_LOADER + .load_language_at_path(name, &GRAMMARS_DIR.join(name).join("src"), &HEADER_DIR) + .unwrap() +} + +pub fn get_test_language(name: &str, parser_code: String, path: &Path) -> Language { + let parser_c_path = SCRATCH_DIR.join(&format!("{}-parser.c", name)); + if !fs::read_to_string(&parser_c_path) + .map(|content| content == parser_code) + .unwrap_or(false) + { + fs::write(&parser_c_path, parser_code).unwrap(); + } + let scanner_path = path.join("scanner.c"); + let scanner_path = if scanner_path.exists() { + Some(scanner_path) + } else { + None + }; + TEST_LOADER + .load_language_from_sources( + name, + &HEADER_DIR, + &parser_c_path, + &scanner_path, + ) + .unwrap() +} diff --git a/cli/src/tests/languages.rs b/cli/src/tests/languages.rs deleted file mode 100644 index e093d218..00000000 --- a/cli/src/tests/languages.rs +++ /dev/null @@ -1,21 +0,0 @@ -use tree_sitter::Language; - -extern "C" { - fn tree_sitter_bash() -> Language; - fn tree_sitter_c() -> Language; - fn tree_sitter_cpp() -> Language; - fn tree_sitter_embedded_template() -> Language; - fn tree_sitter_go() -> Language; - fn tree_sitter_html() -> Language; - fn tree_sitter_javascript() -> Language; - fn tree_sitter_rust() -> Language; -} - -pub fn bash() -> Language { unsafe { tree_sitter_bash() } } -pub fn c() -> Language { unsafe { tree_sitter_c() } } -pub fn cpp() -> Language { unsafe { tree_sitter_cpp() } } -pub fn embedded_template() -> Language { unsafe { tree_sitter_embedded_template() } } -pub fn go() -> Language { unsafe { tree_sitter_go() } } -pub fn html() -> Language { unsafe { tree_sitter_html() } } -pub fn javascript() -> Language { unsafe { tree_sitter_javascript() } } -pub fn rust() -> Language { unsafe { tree_sitter_rust() } } diff --git a/cli/src/tests/mod.rs b/cli/src/tests/mod.rs index bc199616..c9f1dda4 100644 --- a/cli/src/tests/mod.rs +++ b/cli/src/tests/mod.rs @@ -1,3 +1,3 @@ -mod languages; +mod fixtures; mod corpuses; mod parser_api; diff --git a/cli/src/tests/parser_api.rs b/cli/src/tests/parser_api.rs index d717bfab..a399bf38 100644 --- a/cli/src/tests/parser_api.rs +++ b/cli/src/tests/parser_api.rs @@ -1,6 +1,10 @@ -use super::languages::rust; +use super::fixtures::get_language; use std::thread; -use tree_sitter::{InputEdit, LogType, Parser, Point, PropertySheet}; +use tree_sitter::{InputEdit, LogType, Parser, Point, PropertySheet, Language}; + +fn rust() -> Language { + get_language("rust") +} #[test] fn test_basic_parsing() { diff --git a/lib/build.rs b/lib/build.rs index e4d1f91a..eb6fea8b 100644 --- a/lib/build.rs +++ b/lib/build.rs @@ -2,61 +2,8 @@ extern crate cc; use std::env; use std::path::{Path, PathBuf}; -use std::fs; fn main() { - println!("cargo:rerun-if-env-changed=TREE_SITTER_TEST"); - if env::var("TREE_SITTER_TEST").is_ok() { - let mut parser_config = cc::Build::new(); - parser_config - .opt_level(0) - .flag_if_supported("-Wno-unused-parameter"); - - let mut scanner_c_config = cc::Build::new(); - scanner_c_config - .flag_if_supported("-std=c99") - .flag_if_supported("-Wno-unused-parameter"); - - let mut scanner_cxx_config = cc::Build::new(); - scanner_cxx_config - .cpp(true) - .flag_if_supported("-Wno-unused-parameter"); - - let grammars_dir: PathBuf = ["..", "test", "fixtures", "grammars"].iter().collect(); - for entry in fs::read_dir(&grammars_dir).expect("Failed to list grammar directory") { - let entry = entry.expect("Failed to load grammars directory entry"); - if !entry.path().is_dir() { - continue; - } - let parser_dir_path = entry.path(); - let parser_src_path = parser_dir_path.join("src"); - let parser_c_path = parser_src_path.join("parser.c"); - let scanner_c_path = parser_src_path.join("scanner.c"); - let scanner_cc_path = parser_src_path.join("scanner.cc"); - - println!("cargo:rerun-if-changed={}", parser_c_path.to_str().unwrap()); - parser_config - .include(&parser_src_path) - .opt_level(0) - .file(&parser_c_path); - if scanner_cc_path.exists() { - println!("cargo:rerun-if-changed={}", scanner_cc_path.to_str().unwrap()); - scanner_cxx_config - .include(&parser_src_path) - .file(&scanner_cc_path); - } else if scanner_c_path.exists() { - println!("cargo:rerun-if-changed={}", scanner_c_path.to_str().unwrap()); - scanner_c_config - .include(&parser_src_path) - .file(&scanner_c_path); - } - } - - parser_config.compile("fixture-parsers"); - scanner_c_config.compile("fixture-scanners-c"); - scanner_cxx_config.compile("fixture-scanners-cxx"); - } - println!("cargo:rerun-if-env-changed=TREE_SITTER_STATIC_ANALYSIS"); if env::var("TREE_SITTER_STATIC_ANALYSIS").is_ok() { let clang_path = which("clang").unwrap(); diff --git a/script/regenerate-fixtures b/script/regenerate-fixtures index 15e3c09d..c47c53f9 100755 --- a/script/regenerate-fixtures +++ b/script/regenerate-fixtures @@ -2,6 +2,8 @@ set -e +cargo build --release + root_dir=$PWD tree_sitter=${root_dir}/target/release/tree-sitter grammars_dir=${root_dir}/test/fixtures/grammars @@ -19,6 +21,10 @@ grammar_names=( rust ) +if [[ "$#" > 0 ]]; then + grammar_names=($1) +fi + for grammar_name in "${grammar_names[@]}"; do echo "Regenerating ${grammar_name} parser" cd ${grammars_dir}/${grammar_name}