diff --git a/cli/src/main.rs b/cli/src/main.rs index 626a729c..8dbf345a 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -14,6 +14,7 @@ mod loader; mod logger; mod parse; mod test; +mod util; use self::loader::Loader; use clap::{App, Arg, SubCommand}; @@ -48,9 +49,22 @@ fn run() -> error::Result<()> { .subcommand( SubCommand::with_name("parse") .about("Parse a file") - .arg(Arg::with_name("path").index(1)), + .arg(Arg::with_name("path").index(1).required(true)) + .arg(Arg::with_name("debug").long("debug").short("d")) + .arg(Arg::with_name("debug-graph").long("debug-graph").short("D")), + ) + .subcommand( + SubCommand::with_name("test") + .about("Run a parser's tests") + .arg( + Arg::with_name("filter") + .long("filter") + .short("f") + .takes_value(true), + ) + .arg(Arg::with_name("debug").long("debug").short("d")) + .arg(Arg::with_name("debug-graph").long("debug-graph").short("D")), ) - .subcommand(SubCommand::with_name("test").about("Run a parser's tests")) .get_matches(); let home_dir = dirs::home_dir().unwrap(); @@ -74,20 +88,23 @@ fn run() -> error::Result<()> { generate::generate_parser_for_grammar(&grammar_path, minimize, state_ids_to_log)?; println!("{}", code); return Ok(()); - } else if let Some(_matches) = matches.subcommand_matches("test") { + } else if let Some(matches) = matches.subcommand_matches("test") { + let debug = matches.is_present("debug"); + let debug_graph = matches.is_present("debug-graph"); + let filter = matches.value_of("filter"); let corpus_path = current_dir.join("corpus"); - let home_dir = dirs::home_dir().unwrap(); - let mut loader = Loader::new(home_dir.join(".tree-sitter")); if let Some((language, _)) = loader.language_configuration_at_path(¤t_dir)? { - test::run_tests_at_path(language, &corpus_path)?; + test::run_tests_at_path(language, &corpus_path, debug, debug_graph, filter)?; } else { eprintln!("No language found"); } } else if let Some(matches) = matches.subcommand_matches("parse") { + let debug = matches.is_present("debug"); + let debug_graph = matches.is_present("debug-graph"); loader.find_parsers(&vec![home_dir.join("github")])?; let source_path = Path::new(matches.value_of("path").unwrap()); if let Some((language, _)) = loader.language_for_file_name(source_path)? { - parse::parse_file_at_path(language, source_path)?; + parse::parse_file_at_path(language, source_path, debug, debug_graph)?; } else { eprintln!("No language found"); } diff --git a/cli/src/parse.rs b/cli/src/parse.rs index 26fe9b9a..fde148b6 100644 --- a/cli/src/parse.rs +++ b/cli/src/parse.rs @@ -1,17 +1,41 @@ use super::error::Result; +use super::util; use std::fs; -use std::path::Path; -use tree_sitter::{Language, Parser}; use std::io::{self, Write}; +use std::path::Path; +use tree_sitter::{Language, LogType, Parser}; -pub fn parse_file_at_path(language: Language, path: &Path) -> Result<()> { +pub fn parse_file_at_path( + language: Language, + path: &Path, + debug: bool, + debug_graph: bool, +) -> Result<()> { let mut parser = Parser::new(); parser.set_language(language)?; let source_code = fs::read_to_string(path)?; + + let mut log_session = None; + + if debug_graph { + log_session = Some(util::start_logging_graphs(&mut parser, "log.html")?); + } else if debug { + parser.set_logger(Some(Box::new(|log_type, message| { + if log_type == LogType::Lex { + io::stderr().write(b" ").unwrap(); + } + write!(&mut io::stderr(), "{}\n", message).unwrap(); + }))); + } + let tree = parser .parse_str(&source_code, None) .expect("Incompatible language version"); + if let Some(log_session) = log_session { + util::stop_logging_graphs(&mut parser, log_session)?; + } + let stdout = io::stdout(); let mut stdout = stdout.lock(); let mut cursor = tree.walk(); diff --git a/cli/src/test.rs b/cli/src/test.rs index a693576d..790e9ec7 100644 --- a/cli/src/test.rs +++ b/cli/src/test.rs @@ -1,14 +1,15 @@ use super::error::Result; +use super::util; use ansi_term::Colour; use difference::{Changeset, Difference}; use regex::bytes::{Regex as ByteRegex, RegexBuilder as ByteRegexBuilder}; use regex::Regex; use std::char; use std::fs; -use std::io; +use std::io::{self, Write}; use std::path::Path; use std::str; -use tree_sitter::{Language, Parser}; +use tree_sitter::{Language, LogType, Parser}; lazy_static! { static ref HEADER_REGEX: ByteRegex = ByteRegexBuilder::new(r"^===+\r?\n([^=]*)\r?\n===+\r?\n") @@ -35,15 +36,34 @@ pub enum TestEntry { }, } -pub fn run_tests_at_path(language: Language, path: &Path) -> Result<()> { +pub fn run_tests_at_path( + language: Language, + path: &Path, + debug: bool, + debug_graph: bool, + filter: Option<&str>, +) -> Result<()> { let test_entry = parse_tests(path)?; let mut parser = Parser::new(); parser.set_language(language)?; + let mut log_session = None; + + if debug_graph { + log_session = Some(util::start_logging_graphs(&mut parser, "log.html")?); + } else if debug { + parser.set_logger(Some(Box::new(|log_type, message| { + if log_type == LogType::Lex { + io::stderr().write(b" ").unwrap(); + } + write!(&mut io::stderr(), "{}\n", message).unwrap(); + }))); + } + let mut failures = Vec::new(); if let TestEntry::Group { children, .. } = test_entry { for child in children { - run_tests(&mut parser, child, 0, &mut failures)?; + run_tests(&mut parser, child, filter, 0, &mut failures)?; } } @@ -83,28 +103,38 @@ pub fn run_tests_at_path(language: Language, path: &Path) -> Result<()> { } } + if let Some(log_session) = log_session { + util::stop_logging_graphs(&mut parser, log_session)?; + } + Ok(()) } fn run_tests( parser: &mut Parser, test_entry: TestEntry, + filter: Option<&str>, mut indent_level: i32, failures: &mut Vec<(String, String, String)>, ) -> Result<()> { - for _ in 0..indent_level { - print!(" "); - } match test_entry { TestEntry::Example { name, input, output, } => { + if let Some(filter) = filter { + if !name.contains(filter) { + return Ok(()); + } + } let tree = parser .parse_utf8(&mut |byte_offset, _| &input[byte_offset..], None) .unwrap(); let actual = tree.root_node().to_sexp(); + for _ in 0..indent_level { + print!(" "); + } if actual == output { println!("✓ {}", Colour::Green.paint(&name)); } else { @@ -113,10 +143,13 @@ fn run_tests( } } TestEntry::Group { name, children } => { + for _ in 0..indent_level { + print!(" "); + } println!("{}:", name); indent_level += 1; for child in children { - run_tests(parser, child, indent_level, failures)?; + run_tests(parser, child, filter, indent_level, failures)?; } } } diff --git a/cli/src/util.rs b/cli/src/util.rs new file mode 100644 index 00000000..d7d8572e --- /dev/null +++ b/cli/src/util.rs @@ -0,0 +1,36 @@ +use std::fs::File; +use std::io::{Result, Write}; +use std::process::{Child, ChildStdin, Command, Stdio}; +use std::str; +use tree_sitter::Parser; + +pub(crate) struct LogSession(Child, ChildStdin); + +pub(crate) fn start_logging_graphs(parser: &mut Parser, path: &str) -> Result { + let mut dot_file = File::create(path)?; + dot_file.write(b"\n\n\n")?; + let mut dot_process = Command::new("dot") + .arg("-Tsvg") + .stdin(Stdio::piped()) + .stdout(dot_file) + .spawn() + .expect("Failed to run Dot"); + let dot_stdin = dot_process + .stdin + .take() + .expect("Failed to open stdin for Dot"); + parser.print_dot_graphs(&dot_stdin); + Ok(LogSession(dot_process, dot_stdin)) +} + +pub(crate) fn stop_logging_graphs(parser: &mut Parser, mut session: LogSession) -> Result<()> { + drop(session.1); + parser.stop_printing_dot_graphs(); + session.0.wait()?; + + if cfg!(target_os = "macos") { + Command::new("open").arg("log.html").output()?; + } + + Ok(()) +} diff --git a/lib/binding/bindings.rs b/lib/binding/bindings.rs index 58d0e510..9d1f3490 100644 --- a/lib/binding/bindings.rs +++ b/lib/binding/bindings.rs @@ -1,5 +1,6 @@ /* automatically generated by rust-bindgen */ +pub type __darwin_size_t = ::std::os::raw::c_ulong; pub type FILE = [u64; 19usize]; pub type TSSymbol = u16; #[repr(C)] @@ -87,9 +88,9 @@ pub struct TSNode { #[repr(C)] #[derive(Debug, Copy, Clone)] pub struct TSTreeCursor { - pub context: [u32; 2usize], - pub id: *const ::std::os::raw::c_void, pub tree: *const ::std::os::raw::c_void, + pub id: *const ::std::os::raw::c_void, + pub context: [u32; 2usize], } extern "C" { pub fn ts_parser_new() -> *mut TSParser; @@ -110,7 +111,7 @@ extern "C" { pub fn ts_parser_set_logger(arg1: *mut TSParser, arg2: TSLogger); } extern "C" { - pub fn ts_parser_print_dot_graphs(arg1: *mut TSParser, arg2: *mut FILE); + pub fn ts_parser_print_dot_graphs(arg1: *mut TSParser, arg2: ::std::os::raw::c_int); } extern "C" { pub fn ts_parser_halt_on_error(arg1: *mut TSParser, arg2: bool); @@ -126,6 +127,15 @@ extern "C" { arg4: u32, ) -> *mut TSTree; } +extern "C" { + pub fn ts_parser_parse_string_encoding( + arg1: *mut TSParser, + arg2: *const TSTree, + arg3: *const ::std::os::raw::c_char, + arg4: u32, + arg5: TSInputEncoding, + ) -> *mut TSTree; +} extern "C" { pub fn ts_parser_enabled(arg1: *const TSParser) -> bool; } @@ -271,19 +281,22 @@ extern "C" { pub fn ts_tree_cursor_delete(arg1: *mut TSTreeCursor); } extern "C" { - pub fn ts_tree_cursor_goto_first_child(arg1: *mut TSTreeCursor) -> bool; + pub fn ts_tree_cursor_reset(arg1: *mut TSTreeCursor, arg2: TSNode); } extern "C" { - pub fn ts_tree_cursor_goto_first_child_for_byte(arg1: *mut TSTreeCursor, arg2: u32) -> i64; -} -extern "C" { - pub fn ts_tree_cursor_goto_next_sibling(arg1: *mut TSTreeCursor) -> bool; + pub fn ts_tree_cursor_current_node(arg1: *const TSTreeCursor) -> TSNode; } extern "C" { pub fn ts_tree_cursor_goto_parent(arg1: *mut TSTreeCursor) -> bool; } extern "C" { - pub fn ts_tree_cursor_current_node(arg1: *const TSTreeCursor) -> TSNode; + pub fn ts_tree_cursor_goto_next_sibling(arg1: *mut TSTreeCursor) -> bool; +} +extern "C" { + pub fn ts_tree_cursor_goto_first_child(arg1: *mut TSTreeCursor) -> bool; +} +extern "C" { + pub fn ts_tree_cursor_goto_first_child_for_byte(arg1: *mut TSTreeCursor, arg2: u32) -> i64; } extern "C" { pub fn ts_language_symbol_count(arg1: *const TSLanguage) -> u32; diff --git a/lib/binding/ffi.rs b/lib/binding/ffi.rs index 323609e0..685ed765 100644 --- a/lib/binding/ffi.rs +++ b/lib/binding/ffi.rs @@ -1,4 +1,9 @@ #![allow(dead_code)] #![allow(non_upper_case_globals)] +#![allow(non_camel_case_types)] include!("./bindings.rs"); + +extern "C" { + pub(crate) fn dup(fd: std::os::raw::c_int) -> std::os::raw::c_int; +} diff --git a/lib/binding/lib.rs b/lib/binding/lib.rs index 65a57d16..ae3c979c 100644 --- a/lib/binding/lib.rs +++ b/lib/binding/lib.rs @@ -6,6 +6,9 @@ extern crate regex; extern crate serde; extern crate serde_json; +#[cfg(unix)] +use std::os::unix::io::AsRawFd; + use regex::Regex; use serde::de::DeserializeOwned; use std::collections::HashMap; @@ -185,6 +188,17 @@ impl Parser { unsafe { ffi::ts_parser_set_logger(self.0, c_logger) }; } + #[cfg(unix)] + pub fn print_dot_graphs(&mut self, file: & impl AsRawFd) { + let fd = file.as_raw_fd(); + unsafe { ffi::ts_parser_print_dot_graphs(self.0, ffi::dup(fd)) } + } + + #[cfg(unix)] + pub fn stop_printing_dot_graphs(&mut self) { + unsafe { ffi::ts_parser_print_dot_graphs(self.0, -1) } + } + pub fn parse_str(&mut self, input: &str, old_tree: Option<&Tree>) -> Option { let bytes = input.as_bytes(); self.parse_utf8( diff --git a/lib/include/tree_sitter/runtime.h b/lib/include/tree_sitter/runtime.h index f0442477..ab69a0b5 100644 --- a/lib/include/tree_sitter/runtime.h +++ b/lib/include/tree_sitter/runtime.h @@ -83,7 +83,7 @@ const TSLanguage *ts_parser_language(const TSParser *); bool ts_parser_set_language(TSParser *, const TSLanguage *); TSLogger ts_parser_logger(const TSParser *); void ts_parser_set_logger(TSParser *, TSLogger); -void ts_parser_print_dot_graphs(TSParser *, FILE *); +void ts_parser_print_dot_graphs(TSParser *, int); void ts_parser_halt_on_error(TSParser *, bool); TSTree *ts_parser_parse(TSParser *, const TSTree *, TSInput); TSTree *ts_parser_parse_string(TSParser *, const TSTree *, const char *, uint32_t); diff --git a/lib/src/parser.c b/lib/src/parser.c index ef7f612d..a33dbc6f 100644 --- a/lib/src/parser.c +++ b/lib/src/parser.c @@ -1542,8 +1542,16 @@ void ts_parser_set_logger(TSParser *self, TSLogger logger) { self->lexer.logger = logger; } -void ts_parser_print_dot_graphs(TSParser *self, FILE *file) { - self->dot_graph_file = file; +void ts_parser_print_dot_graphs(TSParser *self, int fd) { + if (self->dot_graph_file) { + fclose(self->dot_graph_file); + } + + if (fd >= 0) { + self->dot_graph_file = fdopen(fd, "a"); + } else { + self->dot_graph_file = NULL; + } } void ts_parser_halt_on_error(TSParser *self, bool should_halt_on_error) { diff --git a/script/bindgen.sh b/script/bindgen.sh index 699f0339..0a536d20 100755 --- a/script/bindgen.sh +++ b/script/bindgen.sh @@ -1,7 +1,7 @@ #!/bin/bash -output_path=src/bindings.rs -header_path='vendor/tree-sitter/include/tree_sitter/runtime.h' +output_path=lib/binding/bindings.rs +header_path='lib/include/tree_sitter/runtime.h' bindgen \ --no-layout-tests \