traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Start work on running test corpus tests

Browse source
This commit is contained in:
Max Brunsfeld 2019-01-11 13:30:45 -08:00
parent 0d85a1ef53
commit e64f7a64a1
15 changed files with 328 additions and 168 deletions
Download patch file Download diff file Expand all files Collapse all files

226
cli/src/loader.rs
View file

@ -6,6 +6,7 @@ use std::io;
use std::mem;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::SystemTime;
use tree_sitter::{Language, PropertySheet};
const PACKAGE_JSON_PATH: &'static str = "package.json";
@ -37,7 +38,7 @@ pub struct LanguageConfiguration {
pub struct Loader {
parser_lib_path: PathBuf,
language_repos: Vec<LanguageRepo>,
language_configuration_indices_by_file_type: HashMap<String, Vec<(usize, usize)>>,
language_configuration_ids_by_file_type: HashMap<String, Vec<(usize, usize)>>,
}
unsafe impl Send for Loader {}
@ -48,19 +49,20 @@ impl Loader {
Loader {
parser_lib_path,
language_repos: Vec::new(),
language_configuration_indices_by_file_type: HashMap::new(),
language_configuration_ids_by_file_type: HashMap::new(),
}
}
pub fn find_parsers(&mut self, parser_src_paths: &Vec<PathBuf>) -> io::Result<()> {
pub fn find_all_languages(&mut self, parser_src_paths: &Vec<PathBuf>) -> io::Result<()> {
for parser_container_dir in parser_src_paths.iter() {
for entry in fs::read_dir(parser_container_dir)? {
let entry = entry?;
if let Some(parser_dir_name) = entry.file_name().to_str() {
if parser_dir_name.starts_with("tree-sitter-") {
if self.load_language_configurations(
&parser_container_dir.join(parser_dir_name),
).is_err() {
if self
.find_language_at_path(&parser_container_dir.join(parser_dir_name))
.is_err()
{
eprintln!("Error loading {}", parser_dir_name);
}
}
@ -70,90 +72,126 @@ impl Loader {
Ok(())
}
pub fn language_configuration_at_path(
&mut self,
path: &Path,
) -> io::Result<Option<(Language, &LanguageConfiguration)>> {
let repo_index = self.load_language_configurations(path)?;
self.load_language_from_repo(repo_index, 0)
}
pub fn language_for_file_name(
&mut self,
path: &Path,
) -> io::Result<Option<(Language, &LanguageConfiguration)>> {
let indices = path
.file_name()
.and_then(|n| n.to_str())
.and_then(|file_name| {
self.language_configuration_indices_by_file_type
.get(file_name)
})
.or_else(|| {
path.extension()
.and_then(|extension| extension.to_str())
.and_then(|extension| {
self.language_configuration_indices_by_file_type
.get(extension)
})
});
if let Some(indices) = indices {
// TODO use `content-regex` to pick one
for (repo_index, conf_index) in indices {
return self.load_language_from_repo(*repo_index, *conf_index);
}
}
Ok(None)
}
fn load_language_from_repo(
&mut self,
repo_index: usize,
conf_index: usize,
) -> io::Result<Option<(Language, &LanguageConfiguration)>> {
let repo = &self.language_repos[repo_index];
let language = if let Some(language) = repo.language {
language
} else {
let language = self.load_language_at_path(&repo.name, &repo.path)?;
self.language_repos[repo_index].language = Some(language);
language
};
if let Some(configuration) = self.language_repos[repo_index]
.configurations
.get(conf_index)
{
Ok(Some((language, configuration)))
pub fn language_at_path(&mut self, path: &Path) -> io::Result<Option<Language>> {
if let Ok(id) = self.find_language_at_path(path) {
Ok(Some(self.language_configuration_for_id(id)?.0))
} else {
Ok(None)
}
}
pub fn language_configuration_for_file_name(
&mut self,
path: &Path,
) -> io::Result<Option<(Language, &LanguageConfiguration)>> {
let ids = path
.file_name()
.and_then(|n| n.to_str())
.and_then(|file_name| self.language_configuration_ids_by_file_type.get(file_name))
.or_else(|| {
path.extension()
.and_then(|extension| extension.to_str())
.and_then(|extension| {
self.language_configuration_ids_by_file_type.get(extension)
})
});
if let Some(ids) = ids {
// TODO use `content-regex` to pick one
for (repo_id, configuration_id) in ids.iter().cloned() {
let (language, configurations) = self.language_configuration_for_id(repo_id)?;
return Ok(Some((language, &configurations[configuration_id])));
}
}
Ok(None)
}
fn language_configuration_for_id(
&mut self,
id: usize,
) -> io::Result<(Language, &Vec<LanguageConfiguration>)> {
let repo = &self.language_repos[id];
let language = if let Some(language) = repo.language {
language
} else {
let language = self.load_language_at_path(&repo.name, &repo.path)?;
self.language_repos[id].language = Some(language);
language
};
Ok((language, &self.language_repos[id].configurations))
}
fn load_language_at_path(&self, name: &str, language_path: &Path) -> io::Result<Language> {
let parser_c_path = language_path.join(PARSER_C_PATH);
let src_path = language_path.join("src");
let parser_c_path = src_path.join("parser.c");
let scanner_path;
let scanner_c_path = src_path.join("scanner.c");
if scanner_c_path.exists() {
scanner_path = Some(scanner_c_path);
} else {
let scanner_cc_path = src_path.join("scanner.cc");
if scanner_cc_path.exists() {
scanner_path = Some(scanner_cc_path);
} else {
scanner_path = None;
}
}
self.load_language_from_sources(name, &src_path, &parser_c_path, &scanner_path)
}
pub fn load_language_from_sources(
&self,
name: &str,
header_path: &Path,
parser_path: &Path,
scanner_path: &Option<PathBuf>,
) -> io::Result<Language> {
let mut library_path = self.parser_lib_path.join(name);
library_path.set_extension(DYLIB_EXTENSION);
if !library_path.exists() || was_modified_more_recently(&parser_c_path, &library_path)? {
let compiler_name = std::env::var("CXX").unwrap_or("c++".to_owned());
let mut command = Command::new(compiler_name);
command
.arg("-shared")
.arg("-fPIC")
.arg("-I")
.arg(language_path.join("src"))
.arg("-o")
.arg(&library_path)
.arg("-xc")
.arg(parser_c_path);
let scanner_c_path = language_path.join(SCANNER_C_PATH);
let scanner_cc_path = language_path.join(SCANNER_CC_PATH);
if scanner_c_path.exists() {
command.arg("-xc").arg(scanner_c_path);
} else if scanner_cc_path.exists() {
command.arg("-xc++").arg(scanner_cc_path);
if needs_recompile(&library_path, &parser_path, &scanner_path)? {
let mut config = cc::Build::new();
config
.opt_level(2)
.cargo_metadata(false)
.target(env!("BUILD_TARGET"))
.host(env!("BUILD_TARGET"));
let compiler = config.get_compiler();
let compiler_path = compiler.path();
let mut command = Command::new(compiler_path);
if cfg!(windows) {
command
.args(&["/nologo", "/LD", "/I"])
.arg(header_path)
.arg("/Od")
.arg(parser_path);
if let Some(scanner_path) = scanner_path.as_ref() {
command.arg(scanner_path);
}
command
.arg("/link")
.arg(format!("/out:{}", library_path.to_str().unwrap()));
} else {
command
.arg("-shared")
.arg("-fPIC")
.arg("-I")
.arg(header_path)
.arg("-o")
.arg(&library_path)
.arg("-xc")
.arg(parser_path);
if let Some(scanner_path) = scanner_path.as_ref() {
if scanner_path.extension() == Some("c".as_ref()) {
command.arg(scanner_path);
} else {
command.arg("-xc++").arg(scanner_path);
}
}
}
command.output()?;
}
@ -168,7 +206,7 @@ impl Loader {
Ok(language)
}
fn load_language_configurations<'a>(&'a mut self, parser_path: &Path) -> io::Result<usize> {
fn find_language_at_path<'a>(&'a mut self, parser_path: &Path) -> io::Result<usize> {
let name = parser_path
.file_name()
.unwrap()
@ -218,7 +256,7 @@ impl Loader {
for (i, configuration) in configurations.iter().enumerate() {
for file_type in &configuration.file_types {
self.language_configuration_indices_by_file_type
self.language_configuration_ids_by_file_type
.entry(file_type.to_string())
.or_insert(Vec::new())
.push((self.language_repos.len(), i));
@ -236,6 +274,26 @@ impl Loader {
}
}
fn was_modified_more_recently(a: &Path, b: &Path) -> io::Result<bool> {
Ok(fs::metadata(a)?.modified()? > fs::metadata(b)?.modified()?)
fn needs_recompile(
lib_path: &Path,
parser_c_path: &Path,
scanner_path: &Option<PathBuf>,
) -> io::Result<bool> {
if !lib_path.exists() {
return Ok(true);
}
let lib_mtime = mtime(lib_path)?;
if mtime(parser_c_path)? > lib_mtime {
return Ok(true);
}
if let Some(scanner_path) = scanner_path {
if mtime(scanner_path)? > lib_mtime {
return Ok(true);
}
}
Ok(false)
}
fn mtime(path: &Path) -> io::Result<SystemTime> {
Ok(fs::metadata(path)?.modified()?)
}