traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Load all fixture grammars dynamically

Browse source 
This way the build doesn't take forever any time a single grammar has 
been regenerated.
This commit is contained in:
Max Brunsfeld 2019-01-15 10:27:39 -08:00
parent 5c3c1dd0bd
commit a8292f4fe9
10 changed files with 119 additions and 143 deletions
Download patch file Download diff file Expand all files Collapse all files

5
cli/src/generate/nfa.rs
View file

@ -55,10 +55,6 @@ impl CharacterSet {
CharacterSet::Include(Vec::new())
}
pub fn all() -> Self {
CharacterSet::Exclude(Vec::new())
}
pub fn negate(self) -> CharacterSet {
match self {
CharacterSet::Include(chars) => CharacterSet::Exclude(chars),
@ -182,6 +178,7 @@ impl CharacterSet {
}
}
#[cfg(test)]
pub fn contains(&self, c: char) -> bool {
match self {
CharacterSet::Include(chars) => chars.contains(&c),

3
cli/src/generate/properties.rs
View file

@ -464,7 +464,8 @@ fn parse_property_sheet(path: &Path, css: &str) -> Result<Vec<Rule>> {
rsass::Item::AtRule { name, args, .. } => match name.as_str() {
"schema" => {
if let Some(s) = get_sass_string(args) {
let schema_path = resolve_path(path, s)?;
// TODO - use schema
let _schema_path = resolve_path(path, s)?;
items.remove(i);
continue;
} else {

49
cli/src/loader.rs
View file

@ -23,11 +23,11 @@ struct LanguageRepo {
}
pub struct LanguageConfiguration {
name: String,
content_regex: Option<Regex>,
first_line_regex: Option<Regex>,
_name: String,
_content_regex: Option<Regex>,
_first_line_regex: Option<Regex>,
file_types: Vec<String>,
highlight_property_sheet: Option<Result<PropertySheet, PathBuf>>,
_highlight_property_sheet: Option<Result<PropertySheet, PathBuf>>,
}
pub struct Loader {
@ -108,16 +108,21 @@ impl Loader {
let language = if let Some(language) = repo.language {
language
} else {
let language = self.load_language_at_path(&repo.name, &repo.path)?;
let src_path = repo.path.join("src");
let language = self.load_language_at_path(&repo.name, &src_path, &src_path)?;
self.language_repos[id].language = Some(language);
language
};
Ok((language, &self.language_repos[id].configurations))
}
fn load_language_at_path(&self, name: &str, language_path: &Path) -> io::Result<Language> {
let src_path = language_path.join("src");
let parser_c_path = src_path.join("parser.c");
pub fn load_language_at_path(
&self,
name: &str,
src_path: &Path,
header_path: &Path,
) -> io::Result<Language> {
let parser_path = src_path.join("parser.c");
let scanner_path;
let scanner_c_path = src_path.join("scanner.c");
@ -132,7 +137,7 @@ impl Loader {
}
}
self.load_language_from_sources(name, &src_path, &parser_c_path, &scanner_path)
self.load_language_from_sources(name, &header_path, &parser_path, &scanner_path)
}
pub fn load_language_from_sources(
@ -148,6 +153,7 @@ impl Loader {
if needs_recompile(&library_path, &parser_path, &scanner_path)? {
let mut config = cc::Build::new();
config
.cpp(true)
.opt_level(2)
.cargo_metadata(false)
.target(env!("BUILD_TARGET"))
@ -197,13 +203,14 @@ impl Loader {
"Parser compilation failed.\nStdout: {}\nStderr: {}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
).as_str(),
)
.as_str(),
));
}
}
let library = Library::new(library_path)?;
let language_fn_name = format!("tree_sitter_{}", name);
let language_fn_name = format!("tree_sitter_{}", replace_dashes_with_underscores(name));
let language = unsafe {
let language_fn: Symbol<unsafe extern "C" fn() -> Language> =
library.get(language_fn_name.as_bytes())?;
@ -248,15 +255,15 @@ impl Loader {
configurations
.into_iter()
.map(|conf| LanguageConfiguration {
name: conf.name,
_name: conf.name,
file_types: conf.file_types.unwrap_or(Vec::new()),
content_regex: conf
_content_regex: conf
.content_regex
.and_then(|r| RegexBuilder::new(&r).multi_line(true).build().ok()),
first_line_regex: conf
_first_line_regex: conf
.first_line_regex
.and_then(|r| RegexBuilder::new(&r).multi_line(true).build().ok()),
highlight_property_sheet: conf.highlights.map(|d| Err(d.into())),
_highlight_property_sheet: conf.highlights.map(|d| Err(d.into())),
})
.collect()
});
@ -304,3 +311,15 @@ fn needs_recompile(
fn mtime(path: &Path) -> io::Result<SystemTime> {
Ok(fs::metadata(path)?.modified()?)
}
fn replace_dashes_with_underscores(name: &str) -> String {
let mut result = String::with_capacity(name.len());
for c in name.chars() {
if c == '-' {
result.push('_');
} else {
result.push(c);
}
}
result
}

64
cli/src/tests/corpuses.rs
View file

@ -1,27 +1,21 @@
use super::languages;
use super::fixtures::{get_language, get_test_language, fixtures_dir};
use crate::generate;
use crate::loader::Loader;
use crate::test::{parse_tests, TestEntry};
use crate::util;
use std::fs;
use std::path::PathBuf;
use tree_sitter::{Language, Parser, LogType};
use tree_sitter::{LogType, Parser};
const LANGUAGES: &'static [&'static str] = &[
"bash",
"c",
"cpp",
"embedded-template",
"go",
"html",
"javascript",
];
lazy_static! {
static ref LANGUAGES: [(&'static str, Language); 7] = [
("bash", languages::bash()),
("c", languages::c()),
("cpp", languages::cpp()),
("embedded-template", languages::embedded_template()),
("go", languages::go()),
("html", languages::html()),
("javascript", languages::javascript()),
];
static ref ROOT_DIR: PathBuf = [env!("CARGO_MANIFEST_DIR"), ".."].iter().collect();
static ref HEADER_DIR: PathBuf = ROOT_DIR.join("lib").join("include");
static ref SCRATCH_DIR: PathBuf = ROOT_DIR.join("target").join("scratch");
static ref FIXTURES_DIR: PathBuf = ROOT_DIR.join("test").join("fixtures");
static ref EXEC_PATH: PathBuf = std::env::current_exe().unwrap();
static ref LANGUAGE_FILTER: Option<String> =
std::env::var("TREE_SITTER_TEST_LANGUAGE_FILTER").ok();
static ref EXAMPLE_FILTER: Option<String> =
@ -34,7 +28,7 @@ lazy_static! {
fn test_real_language_corpus_files() {
let mut log_session = None;
let mut parser = Parser::new();
let grammars_dir = FIXTURES_DIR.join("grammars");
let grammars_dir = fixtures_dir().join("grammars");
if *LOG_ENABLED {
parser.set_logger(Some(Box::new(|log_type, msg| {
@ -48,7 +42,7 @@ fn test_real_language_corpus_files() {
log_session = Some(util::log_graphs(&mut parser, "log.html").unwrap());
}
for (language_name, language) in LANGUAGES.iter().cloned() {
for language_name in LANGUAGES.iter().cloned() {
if let Some(filter) = LANGUAGE_FILTER.as_ref() {
if !language_name.contains(filter.as_str()) {
continue;
@ -57,6 +51,7 @@ fn test_real_language_corpus_files() {
eprintln!("language: {:?}", language_name);
let language = get_language(language_name);
let corpus_dir = grammars_dir.join(language_name).join("corpus");
let test = parse_tests(&corpus_dir).unwrap();
parser.set_language(language).unwrap();
@ -69,12 +64,9 @@ fn test_real_language_corpus_files() {
#[test]
fn test_feature_corpus_files() {
fs::create_dir_all(SCRATCH_DIR.as_path()).unwrap();
let loader = Loader::new(SCRATCH_DIR.clone());
let mut log_session = None;
let mut parser = Parser::new();
let test_grammars_dir = FIXTURES_DIR.join("test_grammars");
let test_grammars_dir = fixtures_dir().join("test_grammars");
if *LOG_ENABLED {
parser.set_logger(Some(Box::new(|log_type, msg| {
@ -128,27 +120,7 @@ fn test_feature_corpus_files() {
} else {
let corpus_path = test_path.join("corpus.txt");
let c_code = generate_result.unwrap();
let parser_c_path = SCRATCH_DIR.join(&format!("{}-parser.c", language_name));
if !fs::read_to_string(&parser_c_path)
.map(|content| content == c_code)
.unwrap_or(false)
{
fs::write(&parser_c_path, c_code).unwrap();
}
let scanner_path = test_path.join("scanner.c");
let scanner_path = if scanner_path.exists() {
Some(scanner_path)
} else {
None
};
let language = loader
.load_language_from_sources(
language_name,
&HEADER_DIR,
&parser_c_path,
&scanner_path,
)
.unwrap();
let language = get_test_language(language_name, c_code, &test_path);
let test = parse_tests(&corpus_path).unwrap();
parser.set_language(language).unwrap();
run_mutation_tests(&mut parser, test);
@ -180,7 +152,7 @@ fn run_mutation_tests(parser: &mut Parser, test: TestEntry) {
let actual = tree.root_node().to_sexp();
assert_eq!(actual, output);
}
TestEntry::Group { name, children } => {
TestEntry::Group { children, .. } => {
for child in children {
run_mutation_tests(parser, child);
}

51
cli/src/tests/fixtures.rs Normal file
View file

@ -0,0 +1,51 @@
use crate::loader::Loader;
use std::path::{Path, PathBuf};
use tree_sitter::Language;
use std::fs;
lazy_static! {
static ref ROOT_DIR: PathBuf = [env!("CARGO_MANIFEST_DIR"), ".."].iter().collect();
static ref FIXTURES_DIR: PathBuf = ROOT_DIR.join("test").join("fixtures");
static ref HEADER_DIR: PathBuf = ROOT_DIR.join("lib").join("include");
static ref GRAMMARS_DIR: PathBuf = ROOT_DIR.join("test").join("fixtures").join("grammars");
static ref SCRATCH_DIR: PathBuf = {
let result = ROOT_DIR.join("target").join("scratch");
fs::create_dir_all(&result).unwrap();
result
};
static ref TEST_LOADER: Loader = Loader::new(SCRATCH_DIR.clone());
}
pub fn fixtures_dir<'a>() -> &'static Path {
&FIXTURES_DIR
}
pub fn get_language(name: &str) -> Language {
TEST_LOADER
.load_language_at_path(name, &GRAMMARS_DIR.join(name).join("src"), &HEADER_DIR)
.unwrap()
}
pub fn get_test_language(name: &str, parser_code: String, path: &Path) -> Language {
let parser_c_path = SCRATCH_DIR.join(&format!("{}-parser.c", name));
if !fs::read_to_string(&parser_c_path)
.map(|content| content == parser_code)
.unwrap_or(false)
{
fs::write(&parser_c_path, parser_code).unwrap();
}
let scanner_path = path.join("scanner.c");
let scanner_path = if scanner_path.exists() {
Some(scanner_path)
} else {
None
};
TEST_LOADER
.load_language_from_sources(
name,
&HEADER_DIR,
&parser_c_path,
&scanner_path,
)
.unwrap()
}

21
cli/src/tests/languages.rs
View file

@ -1,21 +0,0 @@
use tree_sitter::Language;
extern "C" {
fn tree_sitter_bash() -> Language;
fn tree_sitter_c() -> Language;
fn tree_sitter_cpp() -> Language;
fn tree_sitter_embedded_template() -> Language;
fn tree_sitter_go() -> Language;
fn tree_sitter_html() -> Language;
fn tree_sitter_javascript() -> Language;
fn tree_sitter_rust() -> Language;
}
pub fn bash() -> Language { unsafe { tree_sitter_bash() } }
pub fn c() -> Language { unsafe { tree_sitter_c() } }
pub fn cpp() -> Language { unsafe { tree_sitter_cpp() } }
pub fn embedded_template() -> Language { unsafe { tree_sitter_embedded_template() } }
pub fn go() -> Language { unsafe { tree_sitter_go() } }
pub fn html() -> Language { unsafe { tree_sitter_html() } }
pub fn javascript() -> Language { unsafe { tree_sitter_javascript() } }
pub fn rust() -> Language { unsafe { tree_sitter_rust() } }

2
cli/src/tests/mod.rs
View file

@ -1,3 +1,3 @@
mod languages;
mod fixtures;
mod corpuses;
mod parser_api;

8
cli/src/tests/parser_api.rs
View file

@ -1,6 +1,10 @@
use super::languages::rust;
use super::fixtures::get_language;
use std::thread;
use tree_sitter::{InputEdit, LogType, Parser, Point, PropertySheet};
use tree_sitter::{InputEdit, LogType, Parser, Point, PropertySheet, Language};
fn rust() -> Language {
get_language("rust")
}
#[test]
fn test_basic_parsing() {

53
lib/build.rs
View file

@ -2,61 +2,8 @@ extern crate cc;
use std::env;
use std::path::{Path, PathBuf};
use std::fs;
fn main() {
println!("cargo:rerun-if-env-changed=TREE_SITTER_TEST");
if env::var("TREE_SITTER_TEST").is_ok() {
let mut parser_config = cc::Build::new();
parser_config
.opt_level(0)
.flag_if_supported("-Wno-unused-parameter");
let mut scanner_c_config = cc::Build::new();
scanner_c_config
.flag_if_supported("-std=c99")
.flag_if_supported("-Wno-unused-parameter");
let mut scanner_cxx_config = cc::Build::new();
scanner_cxx_config
.cpp(true)
.flag_if_supported("-Wno-unused-parameter");
let grammars_dir: PathBuf = ["..", "test", "fixtures", "grammars"].iter().collect();
for entry in fs::read_dir(&grammars_dir).expect("Failed to list grammar directory") {
let entry = entry.expect("Failed to load grammars directory entry");
if !entry.path().is_dir() {
continue;
}
let parser_dir_path = entry.path();
let parser_src_path = parser_dir_path.join("src");
let parser_c_path = parser_src_path.join("parser.c");
let scanner_c_path = parser_src_path.join("scanner.c");
let scanner_cc_path = parser_src_path.join("scanner.cc");
println!("cargo:rerun-if-changed={}", parser_c_path.to_str().unwrap());
parser_config
.include(&parser_src_path)
.opt_level(0)
.file(&parser_c_path);
if scanner_cc_path.exists() {
println!("cargo:rerun-if-changed={}", scanner_cc_path.to_str().unwrap());
scanner_cxx_config
.include(&parser_src_path)
.file(&scanner_cc_path);
} else if scanner_c_path.exists() {
println!("cargo:rerun-if-changed={}", scanner_c_path.to_str().unwrap());
scanner_c_config
.include(&parser_src_path)
.file(&scanner_c_path);
}
}
parser_config.compile("fixture-parsers");
scanner_c_config.compile("fixture-scanners-c");
scanner_cxx_config.compile("fixture-scanners-cxx");
}
println!("cargo:rerun-if-env-changed=TREE_SITTER_STATIC_ANALYSIS");
if env::var("TREE_SITTER_STATIC_ANALYSIS").is_ok() {
let clang_path = which("clang").unwrap();

6
script/regenerate-fixtures
View file

@ -2,6 +2,8 @@
set -e
cargo build --release
root_dir=$PWD
tree_sitter=${root_dir}/target/release/tree-sitter
grammars_dir=${root_dir}/test/fixtures/grammars
@ -19,6 +21,10 @@ grammar_names=(
rust
)
if [[ "$#" > 0 ]]; then
grammar_names=($1)
fi
for grammar_name in "${grammar_names[@]}"; do
echo "Regenerating ${grammar_name} parser"
cd ${grammars_dir}/${grammar_name}