Add benchmark script

* Structure `cli` crate as both a library and an executable, so that
benchmarks can import code from the crate.
* Import macros in the Rust 2018 style.

cli/src/generate/build_tables/build_lex_table.rs

@@ -5,6 +5,7 @@ use crate::generate::grammars::{LexicalGrammar, SyntaxGrammar};
 use crate::generate::nfa::{CharacterSet, NfaCursor};
 use crate::generate::rules::Symbol;
 use crate::generate::tables::{AdvanceAction, LexState, LexTable, ParseStateId, ParseTable};
+use log::info;
 use std::collections::hash_map::Entry;
 use std::collections::{BTreeMap, HashMap, VecDeque};
 

cli/src/generate/build_tables/item.rs

@@ -1,6 +1,7 @@
 use crate::generate::grammars::{LexicalGrammar, Production, ProductionStep, SyntaxGrammar};
 use crate::generate::rules::Associativity;
 use crate::generate::rules::{Symbol, SymbolType};
+use lazy_static::lazy_static;
 use smallbitvec::SmallBitVec;
 use std::cmp::Ordering;
 use std::fmt;

cli/src/generate/build_tables/minimize_parse_table.rs

@@ -4,6 +4,7 @@ use crate::generate::grammars::{LexicalGrammar, SyntaxGrammar, VariableType};
 use crate::generate::rules::{AliasMap, Symbol};
 use crate::generate::tables::{ParseAction, ParseState, ParseTable, ParseTableEntry};
 use hashbrown::{HashMap, HashSet};
+use log::info;
 
 pub(crate) fn minimize_parse_table(
     parse_table: &mut ParseTable,

cli/src/generate/build_tables/mod.rs

@@ -17,6 +17,7 @@ use crate::generate::grammars::{InlinedProductionMap, LexicalGrammar, SyntaxGram
 use crate::generate::nfa::{CharacterSet, NfaCursor};
 use crate::generate::rules::{AliasMap, Symbol, SymbolType};
 use crate::generate::tables::{LexTable, ParseAction, ParseTable, ParseTableEntry};
+use log::info;
 
 pub(crate) fn build_tables(
     syntax_grammar: &SyntaxGrammar,

cli/src/generate/mod.rs

@@ -3,6 +3,7 @@ use self::parse_grammar::parse_grammar;
 use self::prepare_grammar::prepare_grammar;
 use self::render::render_c_code;
 use crate::error::{Error, Result};
+use lazy_static::lazy_static;
 use regex::{Regex, RegexBuilder};
 use std::fs;
 use std::io::Write;
@@ -62,7 +63,6 @@ pub fn generate_parser_in_directory(
     Ok(())
 }
 
-#[cfg(test)]
 pub fn generate_parser_for_grammar(grammar_json: &String) -> Result<(String, String)> {
     let grammar_json = JSON_COMMENT_REGEX.replace_all(grammar_json, "\n");
     generate_parser_for_grammar_with_opts(&grammar_json, true, Vec::new())
@@ -141,6 +141,7 @@ fn ensure_file<T: AsRef<[u8]>>(path: &PathBuf, f: impl Fn() -> T) -> Result<()>
     if path.exists() {
         Ok(())
     } else {
-        fs::write(path, f().as_ref()).map_err(|e| Error(format!("Failed to write file {:?}: {}", path, e)))
+        fs::write(path, f().as_ref())
+            .map_err(|e| Error(format!("Failed to write file {:?}: {}", path, e)))
     }
 }

cli/src/generate/parse_grammar.rs

@@ -1,6 +1,7 @@
 use super::grammars::{InputGrammar, Variable, VariableType};
 use super::rules::Rule;
 use crate::error::Result;
+use serde_derive::Deserialize;
 use serde_json::{Map, Value};
 
 #[derive(Deserialize)]

cli/src/generate/prepare_grammar/expand_tokens.rs

@@ -3,6 +3,7 @@ use crate::error::{Error, Result};
 use crate::generate::grammars::{LexicalGrammar, LexicalVariable};
 use crate::generate::nfa::{CharacterSet, Nfa, NfaState};
 use crate::generate::rules::Rule;
+use lazy_static::lazy_static;
 use regex::Regex;
 use regex_syntax::ast::{
     parse, Ast, Class, ClassPerlKind, ClassSet, ClassSetItem, RepetitionKind, RepetitionRange,

cli/src/generate/properties.rs

@@ -1,6 +1,8 @@
 use crate::error::{Error, Result};
+use log::info;
 use rsass;
 use rsass::sass::Value;
+use serde_derive::Serialize;
 use std::cmp::Ordering;
 use std::collections::hash_map::Entry;
 use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};

cli/src/lib.rs

@@ -0,0 +1,10 @@
+pub mod error;
+pub mod generate;
+pub mod loader;
+pub mod logger;
+pub mod parse;
+pub mod test;
+pub mod util;
+
+#[cfg(test)]
+mod tests;

cli/src/loader.rs

@@ -1,5 +1,6 @@
 use libloading::{Library, Symbol};
 use regex::{Regex, RegexBuilder};
+use serde_derive::Deserialize;
 use std::collections::HashMap;
 use std::fs;
 use std::io;

cli/src/logger.rs

@@ -23,7 +23,7 @@ impl Log for Logger {
     fn flush(&self) {}
 }
 
-pub(crate) fn init() {
+pub fn init() {
     log::set_boxed_logger(Box::new(Logger { filter: None })).unwrap();
     log::set_max_level(LevelFilter::Info);
 }

cli/src/main.rs

@@ -1,31 +1,10 @@
-#[macro_use]
-extern crate lazy_static;
-#[macro_use]
-extern crate log;
-#[macro_use]
-extern crate serde_derive;
-extern crate hashbrown;
-extern crate regex;
-extern crate rsass;
-extern crate serde_json;
-
-mod error;
-mod generate;
-mod loader;
-mod logger;
-mod parse;
-mod test;
-mod util;
-
-#[cfg(test)]
-mod tests;
-
-use self::loader::Loader;
 use clap::{App, AppSettings, Arg, SubCommand};
 use std::env;
 use std::fs;
 use std::path::Path;
 use std::process::exit;
+use tree_sitter_cli::loader::Loader;
+use tree_sitter_cli::{error, generate, logger, parse, test};
 use std::usize;
 
 fn main() {

cli/src/test.rs

@@ -2,6 +2,7 @@ use super::error::Result;
 use super::util;
 use ansi_term::Colour;
 use difference::{Changeset, Difference};
+use lazy_static::lazy_static;
 use regex::bytes::{Regex as ByteRegex, RegexBuilder as ByteRegexBuilder};
 use regex::Regex;
 use std::char;
@@ -38,7 +39,10 @@ pub enum TestEntry {
 
 impl Default for TestEntry {
     fn default() -> Self {
-        TestEntry::Group { name: String::new(), children: Vec::new() }
+        TestEntry::Group {
+            name: String::new(),
+            children: Vec::new(),
+        }
     }
 }
 

cli/src/tests/corpus_test.rs

@@ -5,6 +5,7 @@ use super::helpers::scope_sequence::ScopeSequence;
 use crate::generate;
 use crate::test::{parse_tests, print_diff, print_diff_key, TestEntry};
 use crate::util;
+use lazy_static::lazy_static;
 use std::{env, fs, time, usize};
 use tree_sitter::{InputEdit, LogType, Node, Parser, Point, Tree};
 
@@ -373,7 +374,10 @@ fn check_consistent_sizes(tree: &Tree, input: &Vec<u8>) {
 
         assert!(start_byte <= end_byte);
         assert!(start_point <= end_point);
-        assert_eq!(start_byte, line_offsets[start_point.row] + start_point.column);
+        assert_eq!(
+            start_byte,
+            line_offsets[start_point.row] + start_point.column
+        );
         assert_eq!(end_byte, line_offsets[end_point.row] + end_point.column);
 
         let mut last_child_end_byte = start_byte;

cli/src/tests/helpers/allocations.rs

@@ -1,6 +1,7 @@
 #![cfg(test)]
 #![allow(dead_code)]
 
+use lazy_static::lazy_static;
 use spin::Mutex;
 use std::collections::HashMap;
 use std::os::raw::{c_ulong, c_void};
@@ -46,10 +47,7 @@ pub fn stop_recording() {
             .map(|e| e.1)
             .collect::<Vec<_>>();
         allocation_indices.sort_unstable();
-        panic!(
-            "Leaked allocation indices: {:?}",
-            allocation_indices
-        );
+        panic!("Leaked allocation indices: {:?}", allocation_indices);
     }
 }
 

cli/src/tests/helpers/dirs.rs

@@ -0,0 +1,11 @@
+lazy_static! {
+    static ref ROOT_DIR: PathBuf = PathBuf::from(env!("CARGO_MANIFEST_DIR")).parent().unwrap().to_owned();
+    static ref FIXTURES_DIR: PathBuf = ROOT_DIR.join("test").join("fixtures");
+    static ref HEADER_DIR: PathBuf = ROOT_DIR.join("lib").join("include");
+    static ref GRAMMARS_DIR: PathBuf = ROOT_DIR.join("test").join("fixtures").join("grammars");
+    static ref SCRATCH_DIR: PathBuf = {
+        let result = ROOT_DIR.join("target").join("scratch");
+        fs::create_dir_all(&result).unwrap();
+        result
+    };
+}

cli/src/tests/helpers/fixtures.rs

@@ -1,18 +1,12 @@
 use crate::loader::Loader;
+use lazy_static::lazy_static;
 use std::fs;
 use std::path::{Path, PathBuf};
 use tree_sitter::Language;
 
+include!("./dirs.rs");
+
 lazy_static! {
-    static ref ROOT_DIR: PathBuf = [env!("CARGO_MANIFEST_DIR"), ".."].iter().collect();
-    static ref FIXTURES_DIR: PathBuf = ROOT_DIR.join("test").join("fixtures");
-    static ref HEADER_DIR: PathBuf = ROOT_DIR.join("lib").join("include");
-    static ref GRAMMARS_DIR: PathBuf = ROOT_DIR.join("test").join("fixtures").join("grammars");
-    static ref SCRATCH_DIR: PathBuf = {
-        let result = ROOT_DIR.join("target").join("scratch");
-        fs::create_dir_all(&result).unwrap();
-        result
-    };
     static ref TEST_LOADER: Loader = Loader::new(SCRATCH_DIR.clone());
 }
 

cli/src/tests/parser_api_test.rs

@@ -1,4 +1,5 @@
 use super::helpers::fixtures::get_language;
+use serde_derive::Deserialize;
 use std::thread;
 use tree_sitter::{InputEdit, Language, LogType, Parser, Point, PropertySheet};
 

cli/src/util.rs

@@ -8,18 +8,18 @@ use tree_sitter::Parser;
 const HTML_HEADER: &[u8] = b"<!DOCTYPE html>\n<style>svg { width: 100%; }</style>\n\n";
 
 #[cfg(windows)]
-pub(crate) struct LogSession();
+pub struct LogSession();
 
 #[cfg(unix)]
-pub(crate) struct LogSession(PathBuf, Option<Child>, Option<ChildStdin>);
+pub struct LogSession(PathBuf, Option<Child>, Option<ChildStdin>);
 
 #[cfg(windows)]
-pub(crate) fn log_graphs(_parser: &mut Parser, _path: &str) -> std::io::Result<LogSession> {
+pub fn log_graphs(_parser: &mut Parser, _path: &str) -> std::io::Result<LogSession> {
     Ok(LogSession())
 }
 
 #[cfg(unix)]
-pub(crate) fn log_graphs(parser: &mut Parser, path: &str) -> std::io::Result<LogSession> {
+pub fn log_graphs(parser: &mut Parser, path: &str) -> std::io::Result<LogSession> {
     use std::io::Write;
 
     let mut dot_file = std::fs::File::create(path)?;