Add debug and debug-graph flags to parse and test commands

cli/src/main.rs

@@ -14,6 +14,7 @@ mod loader;
 mod logger;
 mod parse;
 mod test;
+mod util;
 
 use self::loader::Loader;
 use clap::{App, Arg, SubCommand};
@@ -48,9 +49,22 @@ fn run() -> error::Result<()> {
         .subcommand(
             SubCommand::with_name("parse")
                 .about("Parse a file")
-                .arg(Arg::with_name("path").index(1)),
+                .arg(Arg::with_name("path").index(1).required(true))
+                .arg(Arg::with_name("debug").long("debug").short("d"))
+                .arg(Arg::with_name("debug-graph").long("debug-graph").short("D")),
+        )
+        .subcommand(
+            SubCommand::with_name("test")
+                .about("Run a parser's tests")
+                .arg(
+                    Arg::with_name("filter")
+                        .long("filter")
+                        .short("f")
+                        .takes_value(true),
+                )
+                .arg(Arg::with_name("debug").long("debug").short("d"))
+                .arg(Arg::with_name("debug-graph").long("debug-graph").short("D")),
         )
-        .subcommand(SubCommand::with_name("test").about("Run a parser's tests"))
         .get_matches();
 
     let home_dir = dirs::home_dir().unwrap();
@@ -74,20 +88,23 @@ fn run() -> error::Result<()> {
             generate::generate_parser_for_grammar(&grammar_path, minimize, state_ids_to_log)?;
         println!("{}", code);
         return Ok(());
-    } else if let Some(_matches) = matches.subcommand_matches("test") {
+    } else if let Some(matches) = matches.subcommand_matches("test") {
+        let debug = matches.is_present("debug");
+        let debug_graph = matches.is_present("debug-graph");
+        let filter = matches.value_of("filter");
         let corpus_path = current_dir.join("corpus");
-        let home_dir = dirs::home_dir().unwrap();
-        let mut loader = Loader::new(home_dir.join(".tree-sitter"));
         if let Some((language, _)) = loader.language_configuration_at_path(&current_dir)? {
-            test::run_tests_at_path(language, &corpus_path)?;
+            test::run_tests_at_path(language, &corpus_path, debug, debug_graph, filter)?;
         } else {
             eprintln!("No language found");
         }
     } else if let Some(matches) = matches.subcommand_matches("parse") {
+        let debug = matches.is_present("debug");
+        let debug_graph = matches.is_present("debug-graph");
         loader.find_parsers(&vec![home_dir.join("github")])?;
         let source_path = Path::new(matches.value_of("path").unwrap());
         if let Some((language, _)) = loader.language_for_file_name(source_path)? {
-            parse::parse_file_at_path(language, source_path)?;
+            parse::parse_file_at_path(language, source_path, debug, debug_graph)?;
         } else {
             eprintln!("No language found");
         }

cli/src/parse.rs

@@ -1,17 +1,41 @@
 use super::error::Result;
+use super::util;
 use std::fs;
-use std::path::Path;
-use tree_sitter::{Language, Parser};
 use std::io::{self, Write};
+use std::path::Path;
+use tree_sitter::{Language, LogType, Parser};
 
-pub fn parse_file_at_path(language: Language, path: &Path) -> Result<()> {
+pub fn parse_file_at_path(
+    language: Language,
+    path: &Path,
+    debug: bool,
+    debug_graph: bool,
+) -> Result<()> {
     let mut parser = Parser::new();
     parser.set_language(language)?;
     let source_code = fs::read_to_string(path)?;
+
+    let mut log_session = None;
+
+    if debug_graph {
+        log_session = Some(util::start_logging_graphs(&mut parser, "log.html")?);
+    } else if debug {
+        parser.set_logger(Some(Box::new(|log_type, message| {
+            if log_type == LogType::Lex {
+                io::stderr().write(b"  ").unwrap();
+            }
+            write!(&mut io::stderr(), "{}\n", message).unwrap();
+        })));
+    }
+
     let tree = parser
         .parse_str(&source_code, None)
         .expect("Incompatible language version");
 
+    if let Some(log_session) = log_session {
+        util::stop_logging_graphs(&mut parser, log_session)?;
+    }
+
     let stdout = io::stdout();
     let mut stdout = stdout.lock();
     let mut cursor = tree.walk();

cli/src/test.rs

@@ -1,14 +1,15 @@
 use super::error::Result;
+use super::util;
 use ansi_term::Colour;
 use difference::{Changeset, Difference};
 use regex::bytes::{Regex as ByteRegex, RegexBuilder as ByteRegexBuilder};
 use regex::Regex;
 use std::char;
 use std::fs;
-use std::io;
+use std::io::{self, Write};
 use std::path::Path;
 use std::str;
-use tree_sitter::{Language, Parser};
+use tree_sitter::{Language, LogType, Parser};
 
 lazy_static! {
     static ref HEADER_REGEX: ByteRegex = ByteRegexBuilder::new(r"^===+\r?\n([^=]*)\r?\n===+\r?\n")
@@ -35,15 +36,34 @@ pub enum TestEntry {
     },
 }
 
-pub fn run_tests_at_path(language: Language, path: &Path) -> Result<()> {
+pub fn run_tests_at_path(
+    language: Language,
+    path: &Path,
+    debug: bool,
+    debug_graph: bool,
+    filter: Option<&str>,
+) -> Result<()> {
     let test_entry = parse_tests(path)?;
     let mut parser = Parser::new();
     parser.set_language(language)?;
 
+    let mut log_session = None;
+
+    if debug_graph {
+        log_session = Some(util::start_logging_graphs(&mut parser, "log.html")?);
+    } else if debug {
+        parser.set_logger(Some(Box::new(|log_type, message| {
+            if log_type == LogType::Lex {
+                io::stderr().write(b"  ").unwrap();
+            }
+            write!(&mut io::stderr(), "{}\n", message).unwrap();
+        })));
+    }
+
     let mut failures = Vec::new();
     if let TestEntry::Group { children, .. } = test_entry {
         for child in children {
-            run_tests(&mut parser, child, 0, &mut failures)?;
+            run_tests(&mut parser, child, filter, 0, &mut failures)?;
         }
     }
 
@@ -83,28 +103,38 @@ pub fn run_tests_at_path(language: Language, path: &Path) -> Result<()> {
         }
     }
 
+    if let Some(log_session) = log_session {
+        util::stop_logging_graphs(&mut parser, log_session)?;
+    }
+
     Ok(())
 }
 
 fn run_tests(
     parser: &mut Parser,
     test_entry: TestEntry,
+    filter: Option<&str>,
     mut indent_level: i32,
     failures: &mut Vec<(String, String, String)>,
 ) -> Result<()> {
-    for _ in 0..indent_level {
-        print!("  ");
-    }
     match test_entry {
         TestEntry::Example {
             name,
             input,
             output,
         } => {
+            if let Some(filter) = filter {
+                if !name.contains(filter) {
+                    return Ok(());
+                }
+            }
             let tree = parser
                 .parse_utf8(&mut |byte_offset, _| &input[byte_offset..], None)
                 .unwrap();
             let actual = tree.root_node().to_sexp();
+            for _ in 0..indent_level {
+                print!("  ");
+            }
             if actual == output {
                 println!("✓ {}", Colour::Green.paint(&name));
             } else {
@@ -113,10 +143,13 @@ fn run_tests(
             }
         }
         TestEntry::Group { name, children } => {
+            for _ in 0..indent_level {
+                print!("  ");
+            }
             println!("{}:", name);
             indent_level += 1;
             for child in children {
-                run_tests(parser, child, indent_level, failures)?;
+                run_tests(parser, child, filter, indent_level, failures)?;
             }
         }
     }

cli/src/util.rs

@@ -0,0 +1,36 @@
+use std::fs::File;
+use std::io::{Result, Write};
+use std::process::{Child, ChildStdin, Command, Stdio};
+use std::str;
+use tree_sitter::Parser;
+
+pub(crate) struct LogSession(Child, ChildStdin);
+
+pub(crate) fn start_logging_graphs(parser: &mut Parser, path: &str) -> Result<LogSession> {
+    let mut dot_file = File::create(path)?;
+    dot_file.write(b"<!DOCTYPE html>\n<style>svg { width: 100%; }</style>\n\n")?;
+    let mut dot_process = Command::new("dot")
+        .arg("-Tsvg")
+        .stdin(Stdio::piped())
+        .stdout(dot_file)
+        .spawn()
+        .expect("Failed to run Dot");
+    let dot_stdin = dot_process
+        .stdin
+        .take()
+        .expect("Failed to open stdin for Dot");
+    parser.print_dot_graphs(&dot_stdin);
+    Ok(LogSession(dot_process, dot_stdin))
+}
+
+pub(crate) fn stop_logging_graphs(parser: &mut Parser, mut session: LogSession) -> Result<()> {
+    drop(session.1);
+    parser.stop_printing_dot_graphs();
+    session.0.wait()?;
+
+    if cfg!(target_os = "macos") {
+        Command::new("open").arg("log.html").output()?;
+    }
+
+    Ok(())
+}

lib/binding/bindings.rs

@@ -1,5 +1,6 @@
 /* automatically generated by rust-bindgen */
 
+pub type __darwin_size_t = ::std::os::raw::c_ulong;
 pub type FILE = [u64; 19usize];
 pub type TSSymbol = u16;
 #[repr(C)]
@@ -87,9 +88,9 @@ pub struct TSNode {
 #[repr(C)]
 #[derive(Debug, Copy, Clone)]
 pub struct TSTreeCursor {
-    pub context: [u32; 2usize],
-    pub id: *const ::std::os::raw::c_void,
     pub tree: *const ::std::os::raw::c_void,
+    pub id: *const ::std::os::raw::c_void,
+    pub context: [u32; 2usize],
 }
 extern "C" {
     pub fn ts_parser_new() -> *mut TSParser;
@@ -110,7 +111,7 @@ extern "C" {
     pub fn ts_parser_set_logger(arg1: *mut TSParser, arg2: TSLogger);
 }
 extern "C" {
-    pub fn ts_parser_print_dot_graphs(arg1: *mut TSParser, arg2: *mut FILE);
+    pub fn ts_parser_print_dot_graphs(arg1: *mut TSParser, arg2: ::std::os::raw::c_int);
 }
 extern "C" {
     pub fn ts_parser_halt_on_error(arg1: *mut TSParser, arg2: bool);
@@ -126,6 +127,15 @@ extern "C" {
         arg4: u32,
     ) -> *mut TSTree;
 }
+extern "C" {
+    pub fn ts_parser_parse_string_encoding(
+        arg1: *mut TSParser,
+        arg2: *const TSTree,
+        arg3: *const ::std::os::raw::c_char,
+        arg4: u32,
+        arg5: TSInputEncoding,
+    ) -> *mut TSTree;
+}
 extern "C" {
     pub fn ts_parser_enabled(arg1: *const TSParser) -> bool;
 }
@@ -271,19 +281,22 @@ extern "C" {
     pub fn ts_tree_cursor_delete(arg1: *mut TSTreeCursor);
 }
 extern "C" {
-    pub fn ts_tree_cursor_goto_first_child(arg1: *mut TSTreeCursor) -> bool;
+    pub fn ts_tree_cursor_reset(arg1: *mut TSTreeCursor, arg2: TSNode);
 }
 extern "C" {
-    pub fn ts_tree_cursor_goto_first_child_for_byte(arg1: *mut TSTreeCursor, arg2: u32) -> i64;
-}
-extern "C" {
-    pub fn ts_tree_cursor_goto_next_sibling(arg1: *mut TSTreeCursor) -> bool;
+    pub fn ts_tree_cursor_current_node(arg1: *const TSTreeCursor) -> TSNode;
 }
 extern "C" {
     pub fn ts_tree_cursor_goto_parent(arg1: *mut TSTreeCursor) -> bool;
 }
 extern "C" {
-    pub fn ts_tree_cursor_current_node(arg1: *const TSTreeCursor) -> TSNode;
+    pub fn ts_tree_cursor_goto_next_sibling(arg1: *mut TSTreeCursor) -> bool;
+}
+extern "C" {
+    pub fn ts_tree_cursor_goto_first_child(arg1: *mut TSTreeCursor) -> bool;
+}
+extern "C" {
+    pub fn ts_tree_cursor_goto_first_child_for_byte(arg1: *mut TSTreeCursor, arg2: u32) -> i64;
 }
 extern "C" {
     pub fn ts_language_symbol_count(arg1: *const TSLanguage) -> u32;

lib/binding/ffi.rs

@@ -1,4 +1,9 @@
 #![allow(dead_code)]
 #![allow(non_upper_case_globals)]
+#![allow(non_camel_case_types)]
 
 include!("./bindings.rs");
+
+extern "C" {
+    pub(crate) fn dup(fd: std::os::raw::c_int) -> std::os::raw::c_int;
+}

lib/binding/lib.rs

@@ -6,6 +6,9 @@ extern crate regex;
 extern crate serde;
 extern crate serde_json;
 
+#[cfg(unix)]
+use std::os::unix::io::AsRawFd;
+
 use regex::Regex;
 use serde::de::DeserializeOwned;
 use std::collections::HashMap;
@@ -185,6 +188,17 @@ impl Parser {
         unsafe { ffi::ts_parser_set_logger(self.0, c_logger) };
     }
 
+    #[cfg(unix)]
+    pub fn print_dot_graphs(&mut self, file: & impl AsRawFd) {
+        let fd = file.as_raw_fd();
+        unsafe { ffi::ts_parser_print_dot_graphs(self.0, ffi::dup(fd)) }
+    }
+
+    #[cfg(unix)]
+    pub fn stop_printing_dot_graphs(&mut self) {
+        unsafe { ffi::ts_parser_print_dot_graphs(self.0, -1) }
+    }
+
     pub fn parse_str(&mut self, input: &str, old_tree: Option<&Tree>) -> Option<Tree> {
         let bytes = input.as_bytes();
         self.parse_utf8(

lib/include/tree_sitter/runtime.h

@@ -83,7 +83,7 @@ const TSLanguage *ts_parser_language(const TSParser *);
 bool ts_parser_set_language(TSParser *, const TSLanguage *);
 TSLogger ts_parser_logger(const TSParser *);
 void ts_parser_set_logger(TSParser *, TSLogger);
-void ts_parser_print_dot_graphs(TSParser *, FILE *);
+void ts_parser_print_dot_graphs(TSParser *, int);
 void ts_parser_halt_on_error(TSParser *, bool);
 TSTree *ts_parser_parse(TSParser *, const TSTree *, TSInput);
 TSTree *ts_parser_parse_string(TSParser *, const TSTree *, const char *, uint32_t);

lib/src/parser.c

@@ -1542,8 +1542,16 @@ void ts_parser_set_logger(TSParser *self, TSLogger logger) {
   self->lexer.logger = logger;
 }
 
-void ts_parser_print_dot_graphs(TSParser *self, FILE *file) {
-  self->dot_graph_file = file;
+void ts_parser_print_dot_graphs(TSParser *self, int fd) {
+  if (self->dot_graph_file) {
+    fclose(self->dot_graph_file);
+  }
+
+  if (fd >= 0) {
+    self->dot_graph_file = fdopen(fd, "a");
+  } else {
+    self->dot_graph_file = NULL;
+  }
 }
 
 void ts_parser_halt_on_error(TSParser *self, bool should_halt_on_error) {

script/bindgen.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-output_path=src/bindings.rs
-header_path='vendor/tree-sitter/include/tree_sitter/runtime.h'
+output_path=lib/binding/bindings.rs
+header_path='lib/include/tree_sitter/runtime.h'
 
 bindgen                         \
   --no-layout-tests             \