traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Get logging flags working properly with test script

Browse source
This commit is contained in:
Max Brunsfeld 2019-01-14 17:19:46 -08:00
parent b1fa49448d
commit 5c3c1dd0bd
19 changed files with 140 additions and 225 deletions
Download patch file Download diff file Expand all files Collapse all files

9
cli/src/parse.rs
View file

@ -11,14 +11,13 @@ pub fn parse_file_at_path(
debug: bool,
debug_graph: bool,
) -> Result<()> {
let mut log_session = None;
let mut parser = Parser::new();
parser.set_language(language)?;
let source_code = fs::read_to_string(path)?;
let mut log_session = None;
if debug_graph {
log_session = Some(util::start_logging_graphs(&mut parser, "log.html")?);
log_session = Some(util::log_graphs(&mut parser, "log.html")?);
} else if debug {
parser.set_logger(Some(Box::new(|log_type, message| {
if log_type == LogType::Lex {
@ -32,9 +31,7 @@ pub fn parse_file_at_path(
.parse_str(&source_code, None)
.expect("Incompatible language version");
if let Some(log_session) = log_session {
util::stop_logging_graphs(&mut parser, log_session)?;
}
drop(log_session);
let stdout = io::stdout();
let mut stdout = stdout.lock();

10
cli/src/test.rs
View file

@ -44,13 +44,12 @@ pub fn run_tests_at_path(
filter: Option<&str>,
) -> Result<()> {
let test_entry = parse_tests(path)?;
let mut log_session = None;
let mut parser = Parser::new();
parser.set_language(language)?;
let mut log_session = None;
if debug_graph {
log_session = Some(util::start_logging_graphs(&mut parser, "log.html")?);
log_session = Some(util::log_graphs(&mut parser, "log.html")?);
} else if debug {
parser.set_logger(Some(Box::new(|log_type, message| {
if log_type == LogType::Lex {
@ -103,10 +102,7 @@ pub fn run_tests_at_path(
}
}
if let Some(log_session) = log_session {
util::stop_logging_graphs(&mut parser, log_session)?;
}
drop(log_session);
Ok(())
}

95
cli/src/tests/corpuses.rs
View file

@ -2,12 +2,14 @@ use super::languages;
use crate::generate;
use crate::loader::Loader;
use crate::test::{parse_tests, TestEntry};
use crate::util;
use std::fs;
use std::path::PathBuf;
use tree_sitter::{Language, Parser};
use tree_sitter::{Language, Parser, LogType};
lazy_static! {
static ref LANGUAGES: [(&'static str, Language); 6] = [
static ref LANGUAGES: [(&'static str, Language); 7] = [
("bash", languages::bash()),
("c", languages::c()),
("cpp", languages::cpp()),
("embedded-template", languages::embedded_template()),
@ -20,45 +22,87 @@ lazy_static! {
static ref SCRATCH_DIR: PathBuf = ROOT_DIR.join("target").join("scratch");
static ref FIXTURES_DIR: PathBuf = ROOT_DIR.join("test").join("fixtures");
static ref EXEC_PATH: PathBuf = std::env::current_exe().unwrap();
static ref LANGUAGE_FILTER: Option<String> =
std::env::var("TREE_SITTER_TEST_LANGUAGE_FILTER").ok();
static ref EXAMPLE_FILTER: Option<String> =
std::env::var("TREE_SITTER_TEST_EXAMPLE_FILTER").ok();
static ref LOG_ENABLED: bool = std::env::var("TREE_SITTER_ENABLE_LOG").is_ok();
static ref LOG_GRAPH_ENABLED: bool = std::env::var("TREE_SITTER_ENABLE_LOG_GRAPHS").is_ok();
}
#[test]
fn test_real_language_corpus_files() {
let mut log_session = None;
let mut parser = Parser::new();
let grammars_dir = FIXTURES_DIR.join("grammars");
for (name, language) in LANGUAGES.iter().cloned() {
let corpus_dir = grammars_dir.join(name).join("corpus");
if *LOG_ENABLED {
parser.set_logger(Some(Box::new(|log_type, msg| {
if log_type == LogType::Lex {
eprintln!(" {}", msg);
} else {
eprintln!("{}", msg);
}
})));
} else if *LOG_GRAPH_ENABLED {
log_session = Some(util::log_graphs(&mut parser, "log.html").unwrap());
}
for (language_name, language) in LANGUAGES.iter().cloned() {
if let Some(filter) = LANGUAGE_FILTER.as_ref() {
if !language_name.contains(filter.as_str()) {
continue;
}
}
eprintln!("language: {:?}", language_name);
let corpus_dir = grammars_dir.join(language_name).join("corpus");
let test = parse_tests(&corpus_dir).unwrap();
parser.set_language(language).unwrap();
run_mutation_tests(&mut parser, test);
}
drop(parser);
drop(log_session);
}
#[test]
fn test_feature_corpus_files() {
fs::create_dir_all(SCRATCH_DIR.as_path()).unwrap();
let filter = std::env::var("TREE_SITTER_TEST_FILTER").ok();
let mut loader = Loader::new(SCRATCH_DIR.clone());
let loader = Loader::new(SCRATCH_DIR.clone());
let mut log_session = None;
let mut parser = Parser::new();
let test_grammars_dir = FIXTURES_DIR.join("test_grammars");
if *LOG_ENABLED {
parser.set_logger(Some(Box::new(|log_type, msg| {
if log_type == LogType::Lex {
eprintln!(" {}", msg);
} else {
eprintln!("{}", msg);
}
})));
} else if *LOG_GRAPH_ENABLED {
log_session = Some(util::log_graphs(&mut parser, "log.html").unwrap());
}
for entry in fs::read_dir(&test_grammars_dir).unwrap() {
let entry = entry.unwrap();
if !entry.metadata().unwrap().is_dir() {
continue;
}
let test_name = entry.file_name();
let test_name = test_name.to_str().unwrap();
let language_name = entry.file_name();
let language_name = language_name.to_str().unwrap();
if let Some(filter) = filter.as_ref() {
if !test_name.contains(filter.as_str()) {
if let Some(filter) = LANGUAGE_FILTER.as_ref() {
if !language_name.contains(filter.as_str()) {
continue;
}
}
eprintln!("test: {:?}", test_name);
eprintln!("test language: {:?}", language_name);
let test_path = entry.path();
let grammar_path = test_path.join("grammar.json");
@ -78,13 +122,13 @@ fn test_feature_corpus_files() {
} else {
panic!(
"Expected error message but got none for test grammar '{}'",
test_name
language_name
);
}
} else {
let corpus_path = test_path.join("corpus.txt");
let c_code = generate_result.unwrap();
let parser_c_path = SCRATCH_DIR.join(&format!("{}-parser.c", test_name));
let parser_c_path = SCRATCH_DIR.join(&format!("{}-parser.c", language_name));
if !fs::read_to_string(&parser_c_path)
.map(|content| content == c_code)
.unwrap_or(false)
@ -98,18 +142,21 @@ fn test_feature_corpus_files() {
None
};
let language = loader
.load_language_from_sources(test_name, &HEADER_DIR, &parser_c_path, &scanner_path)
.load_language_from_sources(
language_name,
&HEADER_DIR,
&parser_c_path,
&scanner_path,
)
.unwrap();
let test = parse_tests(&corpus_path).unwrap();
parser.set_language(language).unwrap();
run_mutation_tests(&mut parser, test);
}
}
// for (name, language) in LANGUAGES.iter().cloned() {
// let corpus_dir = grammars_dir.join(name).join("corpus");
// let test = parse_tests(&corpus_dir).unwrap();
// parser.set_language(language).unwrap();
// run_mutation_tests(&mut parser, test);
// }
drop(parser);
drop(log_session);
}
fn run_mutation_tests(parser: &mut Parser, test: TestEntry) {
@ -119,6 +166,14 @@ fn run_mutation_tests(parser: &mut Parser, test: TestEntry) {
input,
output,
} => {
if let Some(filter) = EXAMPLE_FILTER.as_ref() {
if !name.contains(filter.as_str()) {
return;
}
}
eprintln!(" example: {:?}", name);
let tree = parser
.parse_utf8(&mut |byte_offset, _| &input[byte_offset..], None)
.unwrap();

2
cli/src/tests/languages.rs
View file

@ -1,6 +1,7 @@
use tree_sitter::Language;
extern "C" {
fn tree_sitter_bash() -> Language;
fn tree_sitter_c() -> Language;
fn tree_sitter_cpp() -> Language;
fn tree_sitter_embedded_template() -> Language;
@ -10,6 +11,7 @@ extern "C" {
fn tree_sitter_rust() -> Language;
}
pub fn bash() -> Language { unsafe { tree_sitter_bash() } }
pub fn c() -> Language { unsafe { tree_sitter_c() } }
pub fn cpp() -> Language { unsafe { tree_sitter_cpp() } }
pub fn embedded_template() -> Language { unsafe { tree_sitter_embedded_template() } }

4
cli/src/tests/parser_api.rs
View file

@ -324,10 +324,6 @@ fn test_custom_utf16_input() {
let mut parser = Parser::new();
parser.set_language(rust()).unwrap();
parser.set_logger(Some(Box::new(|t, message| {
println!("log: {:?} {}", t, message);
})));
let lines: Vec<Vec<u16>> = ["pub fn foo() {", " 1", "}"]
.iter()
.map(|s| s.encode_utf16().collect())

55
cli/src/util.rs
View file

@ -1,29 +1,28 @@
use std::fs::File;
use std::io::{Result, Write};
#[cfg(unix)]
use std::path::PathBuf;
#[cfg(unix)]
use std::process::{Child, ChildStdin, Command, Stdio};
use std::str;
use tree_sitter::Parser;
const HTML_HEADER: &[u8] = b"<!DOCTYPE html>\n<style>svg { width: 100%; }</style>\n\n";
#[cfg(windows)]
pub(crate) struct LogSession();
#[cfg(unix)]
pub(crate) struct LogSession(PathBuf, Option<Child>, Option<ChildStdin>);
#[cfg(windows)]
pub(crate) fn start_logging_graphs(parser: &mut Parser, path: &str) -> Result<LogSession> {
pub(crate) fn log_graphs(parser: &mut Parser, path: &str) -> std::io::Result<LogSession> {
Ok(LogSession())
}
#[cfg(windows)]
pub(crate) fn stop_logging_graphs(parser: &mut Parser, mut session: LogSession) -> Result<()> {
Ok(())
}
#[cfg(unix)]
pub(crate) struct LogSession(Child, ChildStdin);
pub(crate) fn log_graphs(parser: &mut Parser, path: &str) -> std::io::Result<LogSession> {
use std::io::Write;
#[cfg(unix)]
pub(crate) fn start_logging_graphs(parser: &mut Parser, path: &str) -> Result<LogSession> {
let mut dot_file = File::create(path)?;
dot_file.write(b"<!DOCTYPE html>\n<style>svg { width: 100%; }</style>\n\n")?;
let mut dot_file = std::fs::File::create(path)?;
dot_file.write(HTML_HEADER)?;
let mut dot_process = Command::new("dot")
.arg("-Tsvg")
.stdin(Stdio::piped())
@ -34,25 +33,23 @@ pub(crate) fn start_logging_graphs(parser: &mut Parser, path: &str) -> Result<Lo
.stdin
.take()
.expect("Failed to open stdin for Dot");
parser.print_dot_graphs(&dot_stdin);
Ok(LogSession(dot_process, dot_stdin))
Ok(LogSession(PathBuf::from(path), Some(dot_process), Some(dot_stdin)))
}
#[cfg(unix)]
pub(crate) fn stop_logging_graphs(parser: &mut Parser, mut session: LogSession) -> Result<()> {
drop(session.1);
impl Drop for LogSession {
fn drop(&mut self) {
use std::fs;
if cfg!(unix) {
parser.stop_printing_dot_graphs();
drop(self.2.take().unwrap());
let output = self.1.take().unwrap().wait_with_output().unwrap();
if output.status.success() {
if cfg!(target_os = "macos") && fs::metadata(&self.0).unwrap().len() > HTML_HEADER.len() as u64 {
Command::new("open").arg("log.html").output().unwrap();
}
} else {
eprintln!("Dot failed: {} {}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr));
}
}
session.0.wait()?;
if cfg!(target_os = "macos") {
Command::new("open").arg("log.html").output()?;
}
Ok(())
}