From 075bf2bd5cdfed6d463ce1f9c22ae7f60226b5d2 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 26 Feb 2021 12:45:02 -0800 Subject: [PATCH] In generate, create rust bindings Also, migrate node binding files into the same 'bindings' folder. --- cli/src/generate/binding_files.rs | 121 ++++++++++++++++++++++++ cli/src/generate/mod.rs | 20 +--- cli/src/generate/npm_files.rs | 18 ---- cli/src/generate/templates/binding.gyp | 3 +- cli/src/generate/templates/build.rs | 40 ++++++++ cli/src/generate/templates/cargo.toml | 25 +++++ cli/src/generate/templates/index.js | 22 +++-- cli/src/generate/templates/lib.rs | 52 ++++++++++ cli/src/generate/templates/package.json | 19 ++++ 9 files changed, 273 insertions(+), 47 deletions(-) create mode 100644 cli/src/generate/binding_files.rs delete mode 100644 cli/src/generate/npm_files.rs create mode 100644 cli/src/generate/templates/build.rs create mode 100644 cli/src/generate/templates/cargo.toml create mode 100644 cli/src/generate/templates/lib.rs create mode 100644 cli/src/generate/templates/package.json diff --git a/cli/src/generate/binding_files.rs b/cli/src/generate/binding_files.rs new file mode 100644 index 00000000..8ce7e4a4 --- /dev/null +++ b/cli/src/generate/binding_files.rs @@ -0,0 +1,121 @@ +use super::write_file; +use crate::error::{Error, Result}; +use std::path::Path; +use std::{fs, str}; + +const BINDING_CC_TEMPLATE: &'static str = include_str!("./templates/binding.cc"); +const BINDING_GYP_TEMPLATE: &'static str = include_str!("./templates/binding.gyp"); +const INDEX_JS_TEMPLATE: &'static str = include_str!("./templates/index.js"); +const LIB_RS_TEMPLATE: &'static str = include_str!("./templates/lib.rs"); +const BUILD_RS_TEMPLATE: &'static str = include_str!("./templates/build.rs"); +const CARGO_TOML_TEMPLATE: &'static str = include_str!("./templates/cargo.toml"); +const PACKAGE_JSON_TEMPLATE: &'static str = include_str!("./templates/package.json"); +const PARSER_NAME_PLACEHOLDER: &'static str = "PARSER_NAME"; + +pub fn generate_binding_files(repo_path: &Path, language_name: &str) -> Result<()> { + let bindings_dir = repo_path.join("bindings"); + + // Generate rust bindings if needed. + let rust_binding_dir = bindings_dir.join("rust"); + if !rust_binding_dir.exists() { + create_dir(&rust_binding_dir)?; + generate_file( + &rust_binding_dir.join("lib.rs"), + LIB_RS_TEMPLATE, + language_name, + )?; + generate_file( + &rust_binding_dir.join("build.rs"), + BUILD_RS_TEMPLATE, + language_name, + )?; + let cargo_toml_path = repo_path.join("Cargo.toml"); + if !cargo_toml_path.exists() { + generate_file(&cargo_toml_path, CARGO_TOML_TEMPLATE, language_name)?; + } + } + + // Generate node bindings + let node_binding_dir = bindings_dir.join("node"); + if !node_binding_dir.exists() { + create_dir(&node_binding_dir)?; + generate_file( + &node_binding_dir.join("index.js"), + INDEX_JS_TEMPLATE, + language_name, + )?; + generate_file( + &node_binding_dir.join("binding.cc"), + BINDING_CC_TEMPLATE, + language_name, + )?; + + // Create binding.gyp, or update it with new binding path. + let binding_gyp_path = repo_path.join("binding.gyp"); + if binding_gyp_path.exists() { + eprintln!("Updating binding.gyp with new binding path"); + + let binding_gyp = fs::read_to_string(&binding_gyp_path) + .map_err(Error::wrap(|| "Failed to read binding.gyp"))?; + let binding_gyp = binding_gyp.replace("src/binding.cc", "bindings/node/binding.cc"); + write_file(&binding_gyp_path, binding_gyp)?; + } else { + generate_file(&binding_gyp_path, BINDING_GYP_TEMPLATE, language_name)?; + } + + // Create package.json, or update it with new binding path. + let package_json_path = repo_path.join("package.json"); + if package_json_path.exists() { + let package_json_str = fs::read_to_string(&package_json_path) + .map_err(Error::wrap(|| "Failed to read package.json"))?; + let mut package_json = + serde_json::from_str::>( + &package_json_str, + ) + .map_err(Error::wrap(|| "Failed to parse package.json"))?; + let package_json_main = package_json.get("main"); + let package_json_needs_update = package_json_main.map_or(true, |v| { + let main_string = v.as_str(); + main_string == Some("index.js") || main_string == Some("./index.js") + }); + if package_json_needs_update { + eprintln!("Updating package.json with new binding path"); + + package_json.insert( + "main".to_string(), + serde_json::Value::String("bindings/node".to_string()), + ); + let mut package_json_str = serde_json::to_string_pretty(&package_json)?; + package_json_str.push('\n'); + write_file(&package_json_path, package_json_str)?; + } + } else { + generate_file(&package_json_path, PACKAGE_JSON_TEMPLATE, language_name)?; + } + + // Remove files from old node binding paths. + let old_index_js_path = repo_path.join("index.js"); + let old_binding_cc_path = repo_path.join("src").join("binding.cc"); + if old_index_js_path.exists() { + fs::remove_file(old_index_js_path).ok(); + } + if old_binding_cc_path.exists() { + fs::remove_file(old_binding_cc_path).ok(); + } + } + + Ok(()) +} + +fn generate_file(path: &Path, template: &str, language_name: &str) -> Result<()> { + write_file( + path, + template.replace(PARSER_NAME_PLACEHOLDER, language_name), + ) +} + +fn create_dir(path: &Path) -> Result<()> { + fs::create_dir_all(&path).map_err(Error::wrap(|| { + format!("Failed to create {:?}", path.to_string_lossy()) + })) +} diff --git a/cli/src/generate/mod.rs b/cli/src/generate/mod.rs index 9e9adb44..68b213ef 100644 --- a/cli/src/generate/mod.rs +++ b/cli/src/generate/mod.rs @@ -1,10 +1,10 @@ +mod binding_files; mod build_tables; mod char_tree; mod dedup; mod grammars; mod nfa; mod node_types; -mod npm_files; pub mod parse_grammar; mod prepare_grammar; mod render; @@ -105,15 +105,7 @@ pub fn generate_parser_in_directory( write_file(&header_path.join("parser.h"), header)?; } - ensure_file(&repo_path.join("index.js"), || { - npm_files::index_js(&language_name) - })?; - ensure_file(&src_path.join("binding.cc"), || { - npm_files::binding_cc(&language_name) - })?; - ensure_file(&repo_path.join("binding.gyp"), || { - npm_files::binding_gyp(&language_name) - })?; + binding_files::generate_binding_files(&repo_path, &language_name)?; Ok(()) } @@ -224,11 +216,3 @@ fn write_file(path: &Path, body: impl AsRef<[u8]>) -> Result<()> { format!("Failed to write {:?}", path.file_name().unwrap()) })) } - -fn ensure_file>(path: &PathBuf, f: impl Fn() -> T) -> Result<()> { - if path.exists() { - Ok(()) - } else { - write_file(path, f().as_ref()) - } -} diff --git a/cli/src/generate/npm_files.rs b/cli/src/generate/npm_files.rs deleted file mode 100644 index 5f813c88..00000000 --- a/cli/src/generate/npm_files.rs +++ /dev/null @@ -1,18 +0,0 @@ -use std::str; - -const BINDING_CC_TEMPLATE: &'static str = include_str!("./templates/binding.cc"); -const BINDING_GYP_TEMPLATE: &'static str = include_str!("./templates/binding.gyp"); -const INDEX_JS_TEMPLATE: &'static str = include_str!("./templates/index.js"); -const PARSER_NAME_PLACEHOLDER: &'static str = "PARSER_NAME"; - -pub fn binding_cc(parser_name: &str) -> String { - BINDING_CC_TEMPLATE.replace(PARSER_NAME_PLACEHOLDER, parser_name) -} - -pub fn binding_gyp(parser_name: &str) -> String { - BINDING_GYP_TEMPLATE.replace(PARSER_NAME_PLACEHOLDER, parser_name) -} - -pub fn index_js(parser_name: &str) -> String { - INDEX_JS_TEMPLATE.replace(PARSER_NAME_PLACEHOLDER, parser_name) -} diff --git a/cli/src/generate/templates/binding.gyp b/cli/src/generate/templates/binding.gyp index f273a007..ba86afb0 100644 --- a/cli/src/generate/templates/binding.gyp +++ b/cli/src/generate/templates/binding.gyp @@ -7,8 +7,9 @@ "src" ], "sources": [ + "bindings/node/binding.cc", "src/parser.c", - "src/binding.cc" + # If your language uses an external scanner, add it here. ], "cflags_c": [ "-std=c99", diff --git a/cli/src/generate/templates/build.rs b/cli/src/generate/templates/build.rs new file mode 100644 index 00000000..0a878ba6 --- /dev/null +++ b/cli/src/generate/templates/build.rs @@ -0,0 +1,40 @@ +fn main() { + let src_dir = std::path::Path::new("src"); + + let mut c_config = cc::Build::new(); + c_config.include(&src_dir); + c_config + .flag_if_supported("-Wno-unused-parameter") + .flag_if_supported("-Wno-unused-but-set-variable") + .flag_if_supported("-Wno-trigraphs"); + let parser_path = src_dir.join("parser.c"); + c_config.file(&parser_path); + + // If your language uses an external scanner written in C, + // then include this block of code: + + /* + let scanner_path = src_dir.join("scanner.c"); + c_config.file(&scanner_path); + println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap()); + */ + + println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap()); + c_config.compile("parser"); + + // If your language uses an external scanner written in C++, + // then include this block of code: + + /* + let mut cpp_config = cc::Build::new(); + cpp_config.cpp(true); + cpp_config.include(&src_dir); + cpp_config + .flag_if_supported("-Wno-unused-parameter") + .flag_if_supported("-Wno-unused-but-set-variable"); + let scanner_path = src_dir.join("scanner.cc"); + cpp_config.file(&scanner_path); + println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap()); + cpp_config.compile("scanner"); + */ +} diff --git a/cli/src/generate/templates/cargo.toml b/cli/src/generate/templates/cargo.toml new file mode 100644 index 00000000..9e912b62 --- /dev/null +++ b/cli/src/generate/templates/cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "tree-sitter-PARSER_NAME" +description = "PARSER_NAME grammar for the tree-sitter parsing library" +version = "0.0.1" +keywords = ["incremental", "parsing", "PARSER_NAME"] +categories = ["parsing", "text-editors"] +repository = "https://github.com/tree-sitter/tree-sitter-javascript" +edition = "2018" + +build = "bindings/rust/build.rs" +include = [ + "bindings/rust/*", + "grammar.js", + "queries/*", + "src/*", +] + +[lib] +path = "bindings/rust/lib.rs" + +[dependencies] +tree-sitter = "0.17" + +[build-dependencies] +cc = "1.0" diff --git a/cli/src/generate/templates/index.js b/cli/src/generate/templates/index.js index e6746235..bc5daf7c 100644 --- a/cli/src/generate/templates/index.js +++ b/cli/src/generate/templates/index.js @@ -1,17 +1,19 @@ try { - module.exports = require("./build/Release/tree_sitter_PARSER_NAME_binding"); -} catch (error) { - if (error.code !== 'MODULE_NOT_FOUND') - throw error - else try { - module.exports = require("./build/Debug/tree_sitter_PARSER_NAME_binding"); + module.exports = require("../../build/Release/tree_sitter_PARSER_NAME_binding"); +} catch (error1) { + if (error1.code !== 'MODULE_NOT_FOUND') { + throw error1; + } + try { + module.exports = require("../../build/Debug/tree_sitter_PARSER_NAME_binding"); } catch (error2) { - if (error2.code === 'MODULE_NOT_FOUND') - throw error - throw error2 + if (error2.code !== 'MODULE_NOT_FOUND') { + throw error2; + } + throw error1 } } try { - module.exports.nodeTypeInfo = require("./src/node-types.json"); + module.exports.nodeTypeInfo = require("../../src/node-types.json"); } catch (_) {} diff --git a/cli/src/generate/templates/lib.rs b/cli/src/generate/templates/lib.rs new file mode 100644 index 00000000..2811f3c5 --- /dev/null +++ b/cli/src/generate/templates/lib.rs @@ -0,0 +1,52 @@ +//! This crate provides PARSER_NAME language support for the [tree-sitter][] parsing library. +//! +//! Typically, you will use the [language][language func] function to add this language to a +//! tree-sitter [Parser][], and then use the parser to parse some code: +//! +//! ``` +//! let code = ""; +//! let mut parser = tree_sitter::Parser::new(); +//! parser.set_language(tree_sitter_javascript::language()).expect("Error loading PARSER_NAME grammar"); +//! let tree = parser.parse(code, None).unwrap(); +//! ``` +//! +//! [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html +//! [language func]: fn.language.html +//! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html +//! [tree-sitter]: https://tree-sitter.github.io/ + +use tree_sitter::Language; + +extern "C" { + fn tree_sitter_PARSER_NAME() -> Language; +} + +/// Get the tree-sitter [Language][] for this grammar. +/// +/// [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html +pub fn language() -> Language { + unsafe { tree_sitter_PARSER_NAME() } +} + +/// The content of the [`node-types.json`][] file for this grammar. +/// +/// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types +pub const NODE_TYPES: &'static str = include_str!("../../src/node-types.json"); + +// Uncomment these to include any queries that this grammar contains + +// pub const HIGHLIGHTS_QUERY: &'static str = include_str!("../../queries/highlights.scm"); +// pub const INJECTIONS_QUERY: &'static str = include_str!("../../queries/injections.scm"); +// pub const LOCALS_QUERY: &'static str = include_str!("../../queries/locals.scm"); +// pub const TAGS_QUERY: &'static str = include_str!("../../queries/tags.scm"); + +#[cfg(test)] +mod tests { + #[test] + fn test_can_load_grammar() { + let mut parser = tree_sitter::Parser::new(); + parser + .set_language(super::language()) + .expect("Error loading PARSER_NAME language"); + } +} diff --git a/cli/src/generate/templates/package.json b/cli/src/generate/templates/package.json new file mode 100644 index 00000000..bba22b82 --- /dev/null +++ b/cli/src/generate/templates/package.json @@ -0,0 +1,19 @@ +{ + "name": "tree-sitter-PARSER_NAME", + "version": "0.0.1", + "description": "PARSER_NAME grammar for tree-sitter", + "main": "bindings/node", + "keywords": [ + "parsing", + "incremental" + ], + "dependencies": { + "nan": "^2.12.1" + }, + "devDependencies": { + "tree-sitter-cli": "^0.17.3" + }, + "scripts": { + "test": "tree-sitter test" + } +}