diff --git a/Cargo.lock b/Cargo.lock index 547002ce..c7ae6744 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -16,9 +16,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a" dependencies = [ "memchr", ] @@ -133,7 +133,7 @@ dependencies = [ "rustc-hash", "shlex", "syn", - "which 4.4.2", + "which 4.4.0", ] [[package]] @@ -150,15 +150,15 @@ checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" [[package]] name = "bumpalo" -version = "3.15.3" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea184aa71bb362a1157c896979544cc23974e08fd265f29ea96b59f0b4a555b" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" [[package]] name = "bytes" -version = "1.5.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "cc" @@ -192,19 +192,19 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chunked_transfer" -version = "1.5.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901" +checksum = "cca491388666e04d7248af3f60f0c40cfb0991c72205595d7c396e3510207d1a" [[package]] name = "clang-sys" -version = "1.7.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" +checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f" dependencies = [ "glob", "libc", - "libloading", + "libloading 0.7.4", ] [[package]] @@ -265,9 +265,9 @@ dependencies = [ [[package]] name = "core-foundation" -version = "0.9.4" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" dependencies = [ "core-foundation-sys", "libc", @@ -275,9 +275,9 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cranelift-bforest" @@ -452,9 +452,9 @@ dependencies = [ [[package]] name = "either" -version = "1.10.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "equivalent" @@ -485,10 +485,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" [[package]] -name = "form_urlencoded" -version = "1.2.1" +name = "filetime" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.4.1", + "windows-sys 0.52.0", +] + +[[package]] +name = "form_urlencoded" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" dependencies = [ "percent-encoding", ] @@ -505,9 +517,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.12" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if", "libc", @@ -596,9 +608,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "idna" -version = "0.5.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -641,9 +653,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.10" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "jni" @@ -669,9 +681,9 @@ checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" [[package]] name = "js-sys" -version = "0.3.68" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "406cda4b368d531c842222cf9d2600a9a4acce8d29423695379c6868a143a9ee" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" dependencies = [ "wasm-bindgen", ] @@ -714,6 +726,16 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "libloading" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +dependencies = [ + "cfg-if", + "winapi", +] + [[package]] name = "libloading" version = "0.8.1" @@ -724,17 +746,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "libredox" -version = "0.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" -dependencies = [ - "bitflags 2.4.2", - "libc", - "redox_syscall", -] - [[package]] name = "libssh2-sys" version = "0.3.0" @@ -913,9 +924,9 @@ checksum = "1e91099d4268b0e11973f036e885d652fb0b21fedcf69738c627f94db6a44f42" [[package]] name = "percent-encoding" -version = "2.3.1" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pin-project-lite" @@ -947,9 +958,9 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.16" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" +checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62" dependencies = [ "proc-macro2", "syn", @@ -1018,6 +1029,15 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2ff9a1f06a88b01621b7ae906ef0211290d1c8a168a15542486a8f61c0833b9" +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -1029,12 +1049,12 @@ dependencies = [ [[package]] name = "redox_users" -version = "0.4.4" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ "getrandom", - "libredox", + "redox_syscall 0.2.16", "thiserror", ] @@ -1101,9 +1121,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.17" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "same-file" @@ -1163,9 +1183,9 @@ dependencies = [ [[package]] name = "shlex" -version = "1.3.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" [[package]] name = "slice-group-by" @@ -1367,7 +1387,9 @@ dependencies = [ "ctrlc", "difference", "dirs", + "filetime", "glob", + "heck", "html-escape", "indexmap", "indoc", @@ -1430,7 +1452,7 @@ dependencies = [ "dirs", "fs4", "indoc", - "libloading", + "libloading 0.8.1", "once_cell", "regex", "serde", @@ -1463,21 +1485,21 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.15" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" [[package]] name = "unicode-normalization" -version = "0.1.23" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" dependencies = [ "tinyvec", ] @@ -1490,9 +1512,9 @@ checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" [[package]] name = "url" -version = "2.5.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" dependencies = [ "form_urlencoded", "idna", @@ -1501,9 +1523,9 @@ dependencies = [ [[package]] name = "utf8-width" -version = "0.1.7" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3" +checksum = "5190c9442dcdaf0ddd50f37420417d219ae5261bbf5db120d0f9bab996c9cba1" [[package]] name = "utf8parse" @@ -1541,9 +1563,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.91" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1e124130aee3fb58c5bdd6b639a0509486b0338acaaae0c84a5124b0f588b7f" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -1551,9 +1573,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.91" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e7e1900c352b609c8488ad12639a311045f40a35491fb69ba8c12f758af70b" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" dependencies = [ "bumpalo", "log", @@ -1566,9 +1588,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.91" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b30af9e2d358182b5c7449424f017eba305ed32a7010509ede96cdc4696c46ed" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1576,9 +1598,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.91" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", @@ -1589,9 +1611,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.91" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" [[package]] name = "wasm-encoder" @@ -1819,9 +1841,9 @@ checksum = "acdf5b8da6ebf7549dad0cd32ca4a3a0461449ef4feec9d0d8450d8da9f51f9b" [[package]] name = "web-sys" -version = "0.3.68" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96565907687f7aceb35bc5fc03770a8a0471d82e479f25832f54a0e3f4b28446" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" dependencies = [ "js-sys", "wasm-bindgen", @@ -1846,14 +1868,13 @@ dependencies = [ [[package]] name = "which" -version = "4.4.2" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" dependencies = [ "either", - "home", + "libc", "once_cell", - "rustix", ] [[package]] @@ -1887,9 +1908,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.6" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ "winapi", ] diff --git a/Cargo.toml b/Cargo.toml index 8e3715b2..998d8311 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,9 +53,11 @@ ctor = "0.2.6" ctrlc = { version = "3.4.2", features = ["termination"] } difference = "2.0.0" dirs = "5.0.1" +filetime = "0.2.23" fs4 = "0.7.0" git2 = "0.18.2" glob = "0.3.1" +heck = "0.4.1" html-escape = "0.2.13" indexmap = "2.2.2" indoc = "2.0.4" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 21f95876..410f9f7c 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -32,7 +32,9 @@ clap.workspace = true ctrlc.workspace = true difference.workspace = true dirs.workspace = true +filetime.workspace = true glob.workspace = true +heck.workspace = true html-escape.workspace = true indexmap.workspace = true indoc.workspace = true diff --git a/cli/build.rs b/cli/build.rs index 94b474ea..bbd9e84c 100644 --- a/cli/build.rs +++ b/cli/build.rs @@ -1,6 +1,10 @@ -use std::ffi::OsStr; -use std::path::{Path, PathBuf}; -use std::{env, fs}; +use std::{ + env, + ffi::OsStr, + fs, + path::{Path, PathBuf}, + time::SystemTime, +}; fn main() { if let Some(git_sha) = read_git_sha() { @@ -10,6 +14,12 @@ fn main() { if web_playground_files_present() { println!("cargo:rustc-cfg=TREE_SITTER_EMBED_WASM_BINDING"); } + + let build_time = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs_f64(); + println!("cargo:rustc-env=BUILD_TIME={build_time}"); } fn web_playground_files_present() -> bool { @@ -30,7 +40,8 @@ fn read_git_sha() -> Option { git_path = repo_path.join(".git"); if git_path.exists() { break; - } else if !repo_path.pop() { + } + if !repo_path.pop() { return None; } } @@ -93,7 +104,7 @@ fn read_git_sha() -> Option { return fs::read_to_string(&ref_filename).ok(); } // If we're on a detached commit, then the `HEAD` file itself contains the sha. - else if head_content.len() == 40 { + if head_content.len() == 40 { return Some(head_content); } } diff --git a/cli/src/generate/grammar_files.rs b/cli/src/generate/grammar_files.rs new file mode 100644 index 00000000..0c052cc9 --- /dev/null +++ b/cli/src/generate/grammar_files.rs @@ -0,0 +1,460 @@ +use super::write_file; +use anyhow::{anyhow, Context, Result}; +use filetime::FileTime; +use heck::{ToKebabCase, ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase}; +use serde::Deserialize; +use serde_json::{Map, Value}; +use std::fs::File; +use std::io::BufReader; +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime}; +use std::{fs, str}; + +const BUILD_TIME: &str = env!("BUILD_TIME"); + +const CLI_VERSION: &str = env!("CARGO_PKG_VERSION"); +const CLI_VERSION_PLACEHOLDER: &str = "CLI_VERSION"; + +const PARSER_NAME_PLACEHOLDER: &str = "PARSER_NAME"; +const CAMEL_PARSER_NAME_PLACEHOLDER: &str = "CAMEL_PARSER_NAME"; +const UPPER_PARSER_NAME_PLACEHOLDER: &str = "UPPER_PARSER_NAME"; +const LOWER_PARSER_NAME_PLACEHOLDER: &str = "LOWER_PARSER_NAME"; + +const DSL_D_TS_FILE: &str = include_str!("../../npm/dsl.d.ts"); +const GRAMMAR_JS_TEMPLATE: &str = include_str!("./templates/grammar.js"); +const PACKAGE_JSON_TEMPLATE: &str = include_str!("./templates/package.json"); +const GITIGNORE_TEMPLATE: &str = include_str!("./templates/gitignore"); +const GITATTRIBUTES_TEMPLATE: &str = include_str!("./templates/gitattributes"); +const EDITORCONFIG_TEMPLATE: &str = include_str!("./templates/gitattributes"); + +const RUST_BINDING_VERSION: &str = env!("CARGO_PKG_VERSION"); +const RUST_BINDING_VERSION_PLACEHOLDER: &str = "RUST_BINDING_VERSION"; + +const LIB_RS_TEMPLATE: &str = include_str!("./templates/lib.rs"); +const BUILD_RS_TEMPLATE: &str = include_str!("./templates/build.rs"); +const CARGO_TOML_TEMPLATE: &str = include_str!("./templates/cargo.toml"); + +const INDEX_JS_TEMPLATE: &str = include_str!("./templates/index.js"); +const JS_BINDING_CC_TEMPLATE: &str = include_str!("./templates/js-binding.cc"); +const BINDING_GYP_TEMPLATE: &str = include_str!("./templates/binding.gyp"); + +const MAKEFILE_TEMPLATE: &str = include_str!("./templates/makefile"); +const PARSER_NAME_H_TEMPLATE: &str = include_str!("./templates/PARSER_NAME.h"); +const PARSER_NAME_PC_IN_TEMPLATE: &str = include_str!("./templates/PARSER_NAME.pc.in"); + +const GO_MOD_TEMPLATE: &str = include_str!("./templates/go.mod"); +const BINDING_GO_TEMPLATE: &str = include_str!("./templates/binding.go"); +const BINDING_GO_TEST_TEMPLATE: &str = include_str!("./templates/binding_test.go"); + +const SETUP_PY_TEMPLATE: &str = include_str!("./templates/setup.py"); +const INIT_PY_TEMPLATE: &str = include_str!("./templates/__init__.py"); +const INIT_PYI_TEMPLATE: &str = include_str!("./templates/__init__.pyi"); +const PYPROJECT_TOML_TEMPLATE: &str = include_str!("./templates/pyproject.toml"); +const PY_BINDING_C_TEMPLATE: &str = include_str!("./templates/py-binding.c"); + +const PACKAGE_SWIFT_TEMPLATE: &str = include_str!("./templates/Package.swift"); + +#[derive(Deserialize, Debug)] +struct LanguageConfiguration {} + +#[derive(Deserialize, Debug)] +struct PackageJSON { + #[serde(rename = "tree-sitter")] + tree_sitter: Option>, +} + +pub fn path_in_ignore(repo_path: &Path) -> bool { + [ + "bindings", + "build", + "examples", + "node_modules", + "queries", + "script", + "src", + "target", + "test", + "types", + ] + .iter() + .any(|dir| repo_path.ends_with(dir)) +} + +pub fn generate_grammar_files( + repo_path: &Path, + language_name: &str, + generate_bindings: bool, +) -> Result<()> { + let dashed_language_name = language_name.to_kebab_case(); + + // Create package.json, or update it with new binding path + let package_json_path_state = missing_path_else( + repo_path.join("package.json"), + |path| generate_file(path, PACKAGE_JSON_TEMPLATE, dashed_language_name.as_str()), + |path| { + let package_json_str = + fs::read_to_string(path).with_context(|| "Failed to read package.json")?; + let mut package_json = serde_json::from_str::>(&package_json_str) + .with_context(|| "Failed to parse package.json")?; + let package_json_main = package_json.get("main"); + let package_json_needs_update = package_json_main.map_or(true, |v| { + let main_string = v.as_str(); + main_string == Some("index.js") || main_string == Some("./index.js") + }); + if package_json_needs_update { + eprintln!("Updating package.json with new binding path"); + package_json.insert( + "main".to_string(), + Value::String("bindings/node".to_string()), + ); + let mut package_json_str = serde_json::to_string_pretty(&package_json)?; + package_json_str.push('\n'); + write_file(path, package_json_str)?; + } + + Ok(()) + }, + )?; + + let (_, package_json) = lookup_package_json_for_path(package_json_path_state.as_path())?; + + // Do not create a grammar.js file in a repo with multiple language configs + if !package_json.has_multiple_language_configs() { + missing_path(repo_path.join("grammar.js"), |path| { + generate_file(path, GRAMMAR_JS_TEMPLATE, language_name) + })?; + } + + // Rewrite dsl.d.ts only if its mtime differs from what was set on its creation + missing_path(repo_path.join("types"), create_dir)?.apply(|path| { + missing_path(path.join("dsl.d.ts"), |path| { + write_file(path, DSL_D_TS_FILE) + })? + .apply_state(|state| { + let build_time = + SystemTime::UNIX_EPOCH + Duration::from_secs_f64(BUILD_TIME.parse::()?); + + match state { + PathState::Exists(path) => { + let mtime = fs::metadata(path)?.modified()?; + if mtime != build_time { + write_file(path, DSL_D_TS_FILE)?; + filetime::set_file_mtime(path, FileTime::from_system_time(build_time))?; + } + } + PathState::Missing(path) => { + filetime::set_file_mtime(path, FileTime::from_system_time(build_time))?; + } + } + + Ok(()) + })?; + Ok(()) + })?; + + // Write .gitignore file + missing_path(repo_path.join(".gitignore"), |path| { + generate_file(path, GITIGNORE_TEMPLATE, language_name) + })?; + + // Write .gitattributes file + missing_path(repo_path.join(".gitattributes"), |path| { + generate_file(path, GITATTRIBUTES_TEMPLATE, language_name) + })?; + + // Write .editorconfig file + missing_path(repo_path.join(".editorconfig"), |path| { + generate_file(path, EDITORCONFIG_TEMPLATE, language_name) + })?; + + if generate_bindings { + let bindings_dir = repo_path.join("bindings"); + + // Generate Rust bindings + missing_path(bindings_dir.join("rust"), create_dir)?.apply(|path| { + missing_path(path.join("lib.rs"), |path| { + generate_file(path, LIB_RS_TEMPLATE, language_name) + })?; + + missing_path(path.join("build.rs"), |path| { + generate_file(path, BUILD_RS_TEMPLATE, language_name) + })?; + + missing_path(repo_path.join("Cargo.toml"), |path| { + generate_file(path, CARGO_TOML_TEMPLATE, dashed_language_name.as_str()) + })?; + + Ok(()) + })?; + + // Generate Node bindings + missing_path(bindings_dir.join("node"), create_dir)?.apply(|path| { + missing_path(path.join("index.js"), |path| { + generate_file(path, INDEX_JS_TEMPLATE, language_name) + })?; + + missing_path(path.join("binding.cc"), |path| { + generate_file(path, JS_BINDING_CC_TEMPLATE, language_name) + })?; + + // Create binding.gyp, or update it with new binding path. + missing_path_else( + repo_path.join("binding.gyp"), + |path| generate_file(path, BINDING_GYP_TEMPLATE, language_name), + |path| { + let binding_gyp = + fs::read_to_string(path).with_context(|| "Failed to read binding.gyp")?; + let old_path = "\"src/binding.cc\""; + if binding_gyp.contains(old_path) { + eprintln!("Updating binding.gyp with new binding path"); + let binding_gyp = + binding_gyp.replace(old_path, "\"bindings/node/binding.cc\""); + write_file(path, binding_gyp)?; + } + Ok(()) + }, + )?; + + // Remove files from old node binding paths. + existing_path(repo_path.join("index.js"), remove_file)?; + existing_path(repo_path.join("src").join("binding.cc"), remove_file)?; + + Ok(()) + })?; + + // Generate C bindings + missing_path(bindings_dir.join("c"), create_dir)?.apply(|path| { + missing_path( + path.join(format!("tree-sitter-{language_name}.h")), + |path| generate_file(path, PARSER_NAME_H_TEMPLATE, language_name), + )?; + + missing_path( + path.join(format!("tree-sitter-{language_name}.pc.in")), + |path| generate_file(path, PARSER_NAME_PC_IN_TEMPLATE, language_name), + )?; + + missing_path(repo_path.join("Makefile"), |path| { + generate_file(path, MAKEFILE_TEMPLATE, language_name) + })?; + + Ok(()) + })?; + + // Generate Go bindings + missing_path(bindings_dir.join("go"), create_dir)?.apply(|path| { + missing_path(path.join("binding.go"), |path| { + generate_file(path, BINDING_GO_TEMPLATE, language_name) + })?; + + missing_path(path.join("binding_test.go"), |path| { + generate_file(path, BINDING_GO_TEST_TEMPLATE, language_name) + })?; + + missing_path(path.join("go.mod"), |path| { + generate_file(path, GO_MOD_TEMPLATE, language_name) + })?; + + Ok(()) + })?; + + // Generate Python bindings + missing_path( + bindings_dir + .join("python") + .join(format!("tree_sitter_{}", language_name.to_snake_case())), + create_dir, + )? + .apply(|path| { + missing_path(path.join("binding.c"), |path| { + generate_file(path, PY_BINDING_C_TEMPLATE, language_name) + })?; + + missing_path(path.join("__init__.py"), |path| { + generate_file(path, INIT_PY_TEMPLATE, language_name) + })?; + + missing_path(path.join("__init__.pyi"), |path| { + generate_file(path, INIT_PYI_TEMPLATE, language_name) + })?; + + missing_path(path.join("py.typed"), |path| { + generate_file(path, "", language_name) // py.typed is empty + })?; + + missing_path(repo_path.join("setup.py"), |path| { + generate_file(path, SETUP_PY_TEMPLATE, language_name) + })?; + + missing_path(repo_path.join("pyproject.toml"), |path| { + generate_file(path, PYPROJECT_TOML_TEMPLATE, dashed_language_name.as_str()) + })?; + + Ok(()) + })?; + + // Generate Swift bindings + missing_path( + bindings_dir + .join("swift") + .join(format!("TreeSitter{}", language_name.to_upper_camel_case())), + create_dir, + )? + .apply(|path| { + missing_path(path.join(format!("{language_name}.h")), |path| { + generate_file(path, PARSER_NAME_H_TEMPLATE, language_name) + })?; + + missing_path(repo_path.join("Package.swift"), |path| { + generate_file(path, PACKAGE_SWIFT_TEMPLATE, language_name) + })?; + + Ok(()) + })?; + } + + Ok(()) +} + +fn lookup_package_json_for_path(path: &Path) -> Result<(PathBuf, PackageJSON)> { + let mut pathbuf = path.to_owned(); + loop { + let package_json = pathbuf + .exists() + .then(|| -> Result { + let file = + File::open(pathbuf.as_path()).with_context(|| "Failed to open package.json")?; + let package_json: PackageJSON = serde_json::from_reader(BufReader::new(file))?; + Ok(package_json) + }) + .transpose()?; + if let Some(package_json) = package_json { + if package_json.tree_sitter.is_some() { + return Ok((pathbuf, package_json)); + } + } + pathbuf.pop(); // package.json + if !pathbuf.pop() { + return Err(anyhow!(concat!( + "Failed to locate a package.json file that has a \"tree-sitter\" section,", + " please ensure you have one, and if you don't then consult the docs", + ))); + } + pathbuf.push("package.json"); + } +} + +fn generate_file(path: &Path, template: &str, language_name: &str) -> Result<()> { + write_file( + path, + template + .replace( + CAMEL_PARSER_NAME_PLACEHOLDER, + &language_name.to_upper_camel_case(), + ) + .replace( + UPPER_PARSER_NAME_PLACEHOLDER, + &language_name.to_shouty_snake_case(), + ) + .replace( + LOWER_PARSER_NAME_PLACEHOLDER, + &language_name.to_snake_case(), + ) + .replace(PARSER_NAME_PLACEHOLDER, language_name) + .replace(CLI_VERSION_PLACEHOLDER, CLI_VERSION) + .replace(RUST_BINDING_VERSION_PLACEHOLDER, RUST_BINDING_VERSION), + ) +} + +fn create_dir(path: &Path) -> Result<()> { + fs::create_dir_all(path) + .with_context(|| format!("Failed to create {:?}", path.to_string_lossy())) +} + +fn remove_file(path: &Path) -> Result<()> { + fs::remove_file(path).ok(); + Ok(()) +} + +#[derive(PartialEq, Eq, Debug)] +enum PathState { + Exists(PathBuf), + Missing(PathBuf), +} + +#[allow(dead_code)] +impl PathState { + fn exists(&self, mut action: impl FnMut(&Path) -> Result<()>) -> Result<&Self> { + if let Self::Exists(path) = self { + action(path.as_path())?; + } + Ok(self) + } + + fn missing(&self, mut action: impl FnMut(&Path) -> Result<()>) -> Result<&Self> { + if let Self::Missing(path) = self { + action(path.as_path())?; + } + Ok(self) + } + + fn apply(&self, mut action: impl FnMut(&Path) -> Result<()>) -> Result<&Self> { + action(self.as_path())?; + Ok(self) + } + + fn apply_state(&self, mut action: impl FnMut(&Self) -> Result<()>) -> Result<&Self> { + action(self)?; + Ok(self) + } + + fn as_path(&self) -> &Path { + match self { + Self::Exists(path) | Self::Missing(path) => path.as_path(), + } + } +} + +fn existing_path(path: PathBuf, mut action: F) -> Result +where + F: FnMut(&Path) -> Result<()>, +{ + if path.exists() { + action(path.as_path())?; + Ok(PathState::Exists(path)) + } else { + Ok(PathState::Missing(path)) + } +} + +fn missing_path(path: PathBuf, mut action: F) -> Result +where + F: FnMut(&Path) -> Result<()>, +{ + if !path.exists() { + action(path.as_path())?; + Ok(PathState::Missing(path)) + } else { + Ok(PathState::Exists(path)) + } +} + +fn missing_path_else(path: PathBuf, mut action: T, mut else_action: F) -> Result +where + T: FnMut(&Path) -> Result<()>, + F: FnMut(&Path) -> Result<()>, +{ + if !path.exists() { + action(path.as_path())?; + Ok(PathState::Missing(path)) + } else { + else_action(path.as_path())?; + Ok(PathState::Exists(path)) + } +} + +impl PackageJSON { + fn has_multiple_language_configs(&self) -> bool { + self.tree_sitter.as_ref().is_some_and(|c| c.len() > 1) + } +} diff --git a/cli/src/generate/mod.rs b/cli/src/generate/mod.rs index 44f8b6e5..8fb39a39 100644 --- a/cli/src/generate/mod.rs +++ b/cli/src/generate/mod.rs @@ -1,7 +1,25 @@ -mod binding_files; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::{env, fs}; + +use anyhow::{anyhow, Context, Result}; +use lazy_static::lazy_static; +use regex::{Regex, RegexBuilder}; +use semver::Version; + +use build_tables::build_tables; +use grammar_files::path_in_ignore; +use grammars::{InlinedProductionMap, LexicalGrammar, SyntaxGrammar}; +use parse_grammar::parse_grammar; +use prepare_grammar::prepare_grammar; +use render::render_c_code; +use rules::AliasMap; + mod build_tables; mod char_tree; mod dedup; +mod grammar_files; mod grammars; mod nfa; mod node_types; @@ -11,21 +29,6 @@ mod render; mod rules; mod tables; -use self::build_tables::build_tables; -use self::grammars::{InlinedProductionMap, LexicalGrammar, SyntaxGrammar}; -use self::parse_grammar::parse_grammar; -use self::prepare_grammar::prepare_grammar; -use self::render::render_c_code; -use self::rules::AliasMap; -use anyhow::{anyhow, Context, Result}; -use lazy_static::lazy_static; -use regex::{Regex, RegexBuilder}; -use semver::Version; -use std::io::Write; -use std::path::Path; -use std::process::{Command, Stdio}; -use std::{env, fs}; - lazy_static! { static ref JSON_COMMENT_REGEX: Regex = RegexBuilder::new("^\\s*//.*") .multi_line(true) @@ -46,8 +49,35 @@ pub fn generate_parser_in_directory( report_symbol_name: Option<&str>, js_runtime: Option<&str>, ) -> Result<()> { - let src_path = repo_path.join("src"); - let header_path = src_path.join("tree_sitter"); + let mut repo_path = repo_path.to_owned(); + let mut grammar_path = grammar_path; + + // Populate a new empty grammar directory. + if let Some(path) = grammar_path { + let path = PathBuf::from(path); + if !path + .try_exists() + .with_context(|| "Some error with specified path")? + { + fs::create_dir_all(&path)?; + grammar_path = None; + repo_path = path; + } + } + + if repo_path.is_dir() && !repo_path.join("grammar.js").exists() && !path_in_ignore(&repo_path) { + if let Some(dir_name) = repo_path + .file_name() + .map(|x| x.to_string_lossy().to_ascii_lowercase()) + { + if let Some(language_name) = dir_name + .strip_prefix("tree-sitter-") + .or_else(|| Some(dir_name.as_ref())) + { + grammar_files::generate_grammar_files(&repo_path, language_name, false)?; + } + } + } // Read the grammar.json. let grammar_json = if let Some(path) = grammar_path { @@ -58,6 +88,9 @@ pub fn generate_parser_in_directory( load_grammar_file(&grammar_js_path, js_runtime)? }; + let src_path = repo_path.join("src"); + let header_path = src_path.join("tree_sitter"); + // Ensure that the output directories exist. fs::create_dir_all(&src_path)?; fs::create_dir_all(&header_path)?; @@ -91,8 +124,8 @@ pub fn generate_parser_in_directory( write_file(&src_path.join("node-types.json"), node_types_json)?; write_file(&header_path.join("parser.h"), tree_sitter::PARSER_HEADER)?; - if generate_bindings { - binding_files::generate_binding_files(repo_path, &language_name)?; + if !path_in_ignore(&repo_path) { + grammar_files::generate_grammar_files(&repo_path, &language_name, generate_bindings)?; } Ok(()) diff --git a/cli/src/generate/templates/PARSER_NAME.h b/cli/src/generate/templates/PARSER_NAME.h new file mode 100644 index 00000000..7150f82e --- /dev/null +++ b/cli/src/generate/templates/PARSER_NAME.h @@ -0,0 +1,16 @@ +#ifndef TREE_SITTER_UPPER_PARSER_NAME_H_ +#define TREE_SITTER_UPPER_PARSER_NAME_H_ + +typedef struct TSLanguage TSLanguage; + +#ifdef __cplusplus +extern "C" { +#endif + +extern const TSLanguage *tree_sitter_PARSER_NAME(void); + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_UPPER_PARSER_NAME_H_ diff --git a/cli/src/generate/templates/PARSER_NAME.pc.in b/cli/src/generate/templates/PARSER_NAME.pc.in new file mode 100644 index 00000000..9eb63aa9 --- /dev/null +++ b/cli/src/generate/templates/PARSER_NAME.pc.in @@ -0,0 +1,11 @@ +prefix=@PREFIX@ +libdir=@LIBDIR@ +includedir=@INCLUDEDIR@ + +Name: tree-sitter-PARSER_NAME +Description: PARSER_NAME grammar for tree-sitter +URL: @URL@ +Version: @VERSION@ +Requires: @REQUIRES@ +Libs: -L${libdir} @ADDITIONAL_LIBS@ -ltree-sitter-PARSER_NAME +Cflags: -I${includedir} diff --git a/cli/src/generate/templates/Package.swift b/cli/src/generate/templates/Package.swift new file mode 100644 index 00000000..e914fbb9 --- /dev/null +++ b/cli/src/generate/templates/Package.swift @@ -0,0 +1,24 @@ +// swift-tools-version:5.3 +import PackageDescription + +let package = Package( + name: "TreeSitterCAMEL_PARSER_NAME", + platforms: [.macOS(.v10_13), .iOS(.v11)], + products: [ + .library(name: "TreeSitterCAMEL_PARSER_NAME", targets: ["TreeSitterCAMEL_PARSER_NAME"]), + ], + dependencies: [], + targets: [ + .target(name: "TreeSitterCAMEL_PARSER_NAME", + path: ".", + sources: [ + "src/parser.c", + // NOTE: if your language has an external scanner, add it here. + ], + resources: [ + .copy("queries") + ], + publicHeadersPath: "bindings/swift", + cSettings: [.headerSearchPath("src")]) + ] +) diff --git a/cli/src/generate/templates/__init__.py b/cli/src/generate/templates/__init__.py new file mode 100644 index 00000000..ee3cc398 --- /dev/null +++ b/cli/src/generate/templates/__init__.py @@ -0,0 +1,3 @@ +"CAMEL_PARSER_NAME grammar for tree-sitter" + +from ._binding import language diff --git a/cli/src/generate/templates/__init__.pyi b/cli/src/generate/templates/__init__.pyi new file mode 100644 index 00000000..5416666f --- /dev/null +++ b/cli/src/generate/templates/__init__.pyi @@ -0,0 +1 @@ +def language() -> int: ... diff --git a/cli/src/generate/templates/binding.cc b/cli/src/generate/templates/binding.cc deleted file mode 100644 index e4297974..00000000 --- a/cli/src/generate/templates/binding.cc +++ /dev/null @@ -1,28 +0,0 @@ -#include "tree_sitter/parser.h" -#include -#include "nan.h" - -using namespace v8; - -extern "C" TSLanguage * tree_sitter_PARSER_NAME(); - -namespace { - -NAN_METHOD(New) {} - -void Init(Local exports, Local module) { - Local tpl = Nan::New(New); - tpl->SetClassName(Nan::New("Language").ToLocalChecked()); - tpl->InstanceTemplate()->SetInternalFieldCount(1); - - Local constructor = Nan::GetFunction(tpl).ToLocalChecked(); - Local instance = constructor->NewInstance(Nan::GetCurrentContext()).ToLocalChecked(); - Nan::SetInternalFieldPointer(instance, 0, tree_sitter_PARSER_NAME()); - - Nan::Set(instance, Nan::New("name").ToLocalChecked(), Nan::New("PARSER_NAME").ToLocalChecked()); - Nan::Set(module, Nan::New("exports").ToLocalChecked(), instance); -} - -NODE_MODULE_CONTEXT_AWARE(tree_sitter_PARSER_NAME_binding, Init) - -} // namespace diff --git a/cli/src/generate/templates/binding.go b/cli/src/generate/templates/binding.go new file mode 100644 index 00000000..b41863c5 --- /dev/null +++ b/cli/src/generate/templates/binding.go @@ -0,0 +1,13 @@ +package tree_sitter_PARSER_NAME + +// #cgo CFLAGS: -std=c11 -fPIC +// #include "../../src/parser.c" +// // NOTE: if your language has an external scanner, add it here. +import "C" + +import "unsafe" + +// Get the tree-sitter Language for this grammar. +func Language() unsafe.Pointer { + return unsafe.Pointer(C.tree_sitter_LOWER_PARSER_NAME()) +} diff --git a/cli/src/generate/templates/binding.gyp b/cli/src/generate/templates/binding.gyp index ba86afb0..dd6ced2b 100644 --- a/cli/src/generate/templates/binding.gyp +++ b/cli/src/generate/templates/binding.gyp @@ -4,15 +4,18 @@ "target_name": "tree_sitter_PARSER_NAME_binding", "include_dirs": [ "=RUST_BINDING_VERSION" [build-dependencies] -cc = "1.0" +cc = "1.0.87" diff --git a/cli/src/generate/templates/editorconfig b/cli/src/generate/templates/editorconfig new file mode 100644 index 00000000..55e6298d --- /dev/null +++ b/cli/src/generate/templates/editorconfig @@ -0,0 +1,41 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +[*.{json,toml,yml}] +indent_style = space +indent_size = 2 + +[*.js] +quote_type = double +indent_style = space +indent_size = 2 + +[*.rs] +indent_style = space +indent_size = 4 + +[*.{c,cc,h}] +indent_style = space +indent_size = 4 + +[*.{py,pyi}] +quote_type = double +indent_style = space +indent_size = 4 + +[*.swift] +indent_style = space +indent_size = 4 + +[*.go] +indent_style = tab +indent_size = 8 + +[Makefile] +indent_style = tab +indent_size = 8 diff --git a/cli/src/generate/templates/gitattributes b/cli/src/generate/templates/gitattributes new file mode 100644 index 00000000..9f71c8f1 --- /dev/null +++ b/cli/src/generate/templates/gitattributes @@ -0,0 +1,10 @@ +* text eol=lf + +src/*.json linguist-generated +src/parser.c linguist-generated +src/tree_sitter/* linguist-generated + +bindings/** linguist-generated +binding.gyp linguist-generated +setup.py linguist-generated +Makefile linguist-generated diff --git a/cli/src/generate/templates/gitignore b/cli/src/generate/templates/gitignore new file mode 100644 index 00000000..c1ab2b25 --- /dev/null +++ b/cli/src/generate/templates/gitignore @@ -0,0 +1,36 @@ +# Rust artifacts +/Cargo.lock +/target/ + +# Node artifacts +/build/ +/node_modules/ + +# Swift artifacts +/.build/ + +# Python artifacts +/dist/ +*.egg-info +*.whl + +# Zig artifacts +/zig-cache/ +/zig-out/ + +# C artifacts +*.a +*.so +*.so.* +*.dylib +*.dll +*.pc + +# Example dirs +/examples/*/ + +# Grammar volatiles +dsl.d.ts +*.wasm +*.obj +*.o diff --git a/cli/src/generate/templates/go.mod b/cli/src/generate/templates/go.mod new file mode 100644 index 00000000..00e31a44 --- /dev/null +++ b/cli/src/generate/templates/go.mod @@ -0,0 +1,5 @@ +module github.com/tree-sitter/tree-sitter-PARSER_NAME + +go 1.22 + +require github.com/smacker/go-tree-sitter v0.0.0-20230720070738-0d0a9f78d8f8 diff --git a/cli/src/generate/templates/grammar.js b/cli/src/generate/templates/grammar.js new file mode 100644 index 00000000..b82d06e2 --- /dev/null +++ b/cli/src/generate/templates/grammar.js @@ -0,0 +1,11 @@ +/// +// @ts-check + +module.exports = grammar({ + name: "LOWER_PARSER_NAME", + + rules: { + // TODO: add the actual grammar rules + source_file: $ => "hello" + } +}); diff --git a/cli/src/generate/templates/index.js b/cli/src/generate/templates/index.js index bc5daf7c..ea822bfc 100644 --- a/cli/src/generate/templates/index.js +++ b/cli/src/generate/templates/index.js @@ -1,13 +1,13 @@ try { module.exports = require("../../build/Release/tree_sitter_PARSER_NAME_binding"); } catch (error1) { - if (error1.code !== 'MODULE_NOT_FOUND') { + if (error1.code !== "MODULE_NOT_FOUND") { throw error1; } try { module.exports = require("../../build/Debug/tree_sitter_PARSER_NAME_binding"); } catch (error2) { - if (error2.code !== 'MODULE_NOT_FOUND') { + if (error2.code !== "MODULE_NOT_FOUND") { throw error2; } throw error1 diff --git a/cli/src/generate/templates/js-binding.cc b/cli/src/generate/templates/js-binding.cc new file mode 100644 index 00000000..39be7054 --- /dev/null +++ b/cli/src/generate/templates/js-binding.cc @@ -0,0 +1,29 @@ +#include "nan.h" +#include + +using namespace v8; + +typedef struct TSLanguage TSLanguage; + +extern "C" const TSLanguage *tree_sitter_PARSER_NAME(void); + +namespace { + +NAN_METHOD(New) {} + +void Init(Local exports, Local module) { + Local tpl = Nan::New(New); + tpl->SetClassName(Nan::New("Language").ToLocalChecked()); + tpl->InstanceTemplate()->SetInternalFieldCount(1); + + Local constructor = Nan::GetFunction(tpl).ToLocalChecked(); + Local instance = constructor->NewInstance(Nan::GetCurrentContext()).ToLocalChecked(); + Nan::SetInternalFieldPointer(instance, 0, (void *)tree_sitter_PARSER_NAME()); + + Nan::Set(instance, Nan::New("name").ToLocalChecked(), Nan::New("PARSER_NAME").ToLocalChecked()); + Nan::Set(module, Nan::New("exports").ToLocalChecked(), instance); +} + +NODE_MODULE_CONTEXT_AWARE(tree_sitter_PARSER_NAME_binding, Init) + +} // namespace diff --git a/cli/src/generate/templates/lib.rs b/cli/src/generate/templates/lib.rs index dab87e4f..a8101945 100644 --- a/cli/src/generate/templates/lib.rs +++ b/cli/src/generate/templates/lib.rs @@ -1,13 +1,15 @@ -//! This crate provides PARSER_NAME language support for the [tree-sitter][] parsing library. +//! This crate provides CAMEL_PARSER_NAME language support for the [tree-sitter][] parsing library. //! //! Typically, you will use the [language][language func] function to add this language to a //! tree-sitter [Parser][], and then use the parser to parse some code: //! //! ``` -//! let code = ""; +//! let code = r#" +//! "#; //! let mut parser = tree_sitter::Parser::new(); -//! parser.set_language(tree_sitter_PARSER_NAME::language()).expect("Error loading PARSER_NAME grammar"); +//! parser.set_language(&tree_sitter_PARSER_NAME::language()).expect("Error loading CAMEL_PARSER_NAME grammar"); //! let tree = parser.parse(code, None).unwrap(); +//! assert!(!tree.root_node().has_error()); //! ``` //! //! [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html @@ -31,14 +33,14 @@ pub fn language() -> Language { /// The content of the [`node-types.json`][] file for this grammar. /// /// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types -pub const NODE_TYPES: &'static str = include_str!("../../src/node-types.json"); +pub const NODE_TYPES: &str = include_str!("../../src/node-types.json"); // Uncomment these to include any queries that this grammar contains -// pub const HIGHLIGHTS_QUERY: &'static str = include_str!("../../queries/highlights.scm"); -// pub const INJECTIONS_QUERY: &'static str = include_str!("../../queries/injections.scm"); -// pub const LOCALS_QUERY: &'static str = include_str!("../../queries/locals.scm"); -// pub const TAGS_QUERY: &'static str = include_str!("../../queries/tags.scm"); +// pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm"); +// pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm"); +// pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm"); +// pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm"); #[cfg(test)] mod tests { @@ -46,7 +48,7 @@ mod tests { fn test_can_load_grammar() { let mut parser = tree_sitter::Parser::new(); parser - .set_language(super::language()) - .expect("Error loading PARSER_NAME language"); + .set_language(&super::language()) + .expect("Error loading CAMEL_PARSER_NAME language"); } } diff --git a/cli/src/generate/templates/makefile b/cli/src/generate/templates/makefile new file mode 100644 index 00000000..aec4f8e9 --- /dev/null +++ b/cli/src/generate/templates/makefile @@ -0,0 +1,94 @@ +VERSION := 0.0.1 + +LANGUAGE_NAME := tree-sitter-PARSER_NAME + +# repository +SRC_DIR := src + +PARSER_REPO_URL := $(shell git -C $(SRC_DIR) remote get-url origin 2>/dev/null) + +ifeq ($(PARSER_URL),) + PARSER_URL := $(subst .git,,$(PARSER_REPO_URL)) +ifeq ($(shell echo $(PARSER_URL) | grep '^[a-z][-+.0-9a-z]*://'),) + PARSER_URL := $(subst :,/,$(PARSER_URL)) + PARSER_URL := $(subst git@,https://,$(PARSER_URL)) +endif +endif + +# ABI versioning +SONAME_MAJOR := $(word 1,$(subst ., ,$(VERSION))) +SONAME_MINOR := $(word 2,$(subst ., ,$(VERSION))) + +# install directory layout +PREFIX ?= /usr/local +INCLUDEDIR ?= $(PREFIX)/include +LIBDIR ?= $(PREFIX)/lib +PCLIBDIR ?= $(LIBDIR)/pkgconfig + +# object files +OBJS := $(patsubst %.c,%.o,$(wildcard $(SRC_DIR)/*.c)) + +# flags +ARFLAGS := rcs +override CFLAGS += -I$(SRC_DIR) -std=c11 + +# OS-specific bits +ifeq ($(shell uname),Darwin) + SOEXT = dylib + SOEXTVER_MAJOR = $(SONAME_MAJOR).dylib + SOEXTVER = $(SONAME_MAJOR).$(SONAME_MINOR).dylib + LINKSHARED := $(LINKSHARED)-dynamiclib -Wl, + ifneq ($(ADDITIONAL_LIBS),) + LINKSHARED := $(LINKSHARED)$(ADDITIONAL_LIBS), + endif + LINKSHARED := $(LINKSHARED)-install_name,$(LIBDIR)/lib$(LANGUAGE_NAME).$(SONAME_MAJOR).dylib,-rpath,@executable_path/../Frameworks +else ifneq ($(filter $(shell uname),Linux FreeBSD NetBSD DragonFly),) + SOEXT = so + SOEXTVER_MAJOR = so.$(SONAME_MAJOR) + SOEXTVER = so.$(SONAME_MAJOR).$(SONAME_MINOR) + LINKSHARED := $(LINKSHARED)-shared -Wl, + ifneq ($(ADDITIONAL_LIBS),) + LINKSHARED := $(LINKSHARED)$(ADDITIONAL_LIBS) + endif + LINKSHARED := $(LINKSHARED)-soname,lib$(LANGUAGE_NAME).so.$(SONAME_MAJOR) +else ifeq ($(OS),Windows_NT) + $(error "Windows is not supported") +endif +ifneq ($(filter $(shell uname),FreeBSD NetBSD DragonFly),) + PCLIBDIR := $(PREFIX)/libdata/pkgconfig +endif + +all: lib$(LANGUAGE_NAME).a lib$(LANGUAGE_NAME).$(SOEXT) $(LANGUAGE_NAME).pc + +$(SRC_DIR)/%.o: $(SRC_DIR)/%.c + $(CC) -c $^ -o $@ + +lib$(LANGUAGE_NAME).a: $(OBJS) + $(AR) $(ARFLAGS) $@ $^ + +lib$(LANGUAGE_NAME).$(SOEXT): $(OBJS) + $(CC) -fPIC $(LDFLAGS) $(LINKSHARED) $^ $(LDLIBS) -o $@ + +$(LANGUAGE_NAME).pc: + sed > $@ bindings/c/$(LANGUAGE_NAME).pc.in \ + -e 's|@URL@|$(PARSER_URL)|' \ + -e 's|@VERSION@|$(VERSION)|' \ + -e 's|@LIBDIR@|$(LIBDIR)|;' \ + -e 's|@INCLUDEDIR@|$(INCLUDEDIR)|;' \ + -e 's|=$(PREFIX)|=$${prefix}|' \ + -e 's|@PREFIX@|$(PREFIX)|' \ + -e 's|@REQUIRES@|$(REQUIRES)|' \ + -e 's|@ADDITIONAL_LIBS@|$(ADDITIONAL_LIBS)|' + +install: all + install -Dm644 bindings/c/$(LANGUAGE_NAME).h '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/$(LANGUAGE_NAME).h + install -Dm644 $(LANGUAGE_NAME).pc '$(DESTDIR)$(PCLIBDIR)'/$(LANGUAGE_NAME).pc + install -Dm755 lib$(LANGUAGE_NAME).a '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).a + install -Dm755 lib$(LANGUAGE_NAME).$(SOEXT) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER) + ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) + ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXT) + +clean: + $(RM) $(OBJS) $(LANGUAGE_NAME).pc lib$(LANGUAGE_NAME).a lib$(LANGUAGE_NAME).$(SOEXT) + +.PHONY: all install clean diff --git a/cli/src/generate/templates/package.json b/cli/src/generate/templates/package.json index 18598797..dacdaa61 100644 --- a/cli/src/generate/templates/package.json +++ b/cli/src/generate/templates/package.json @@ -1,19 +1,31 @@ { "name": "tree-sitter-PARSER_NAME", "version": "0.0.1", - "description": "PARSER_NAME grammar for tree-sitter", + "description": "CAMEL_PARSER_NAME grammar for tree-sitter", + "repository": "github:tree-sitter/tree-sitter-PARSER_NAME", + "license": "MIT", "main": "bindings/node", "keywords": [ "parsing", - "incremental" + "incremental", + "LOWER_PARSER_NAME" ], "dependencies": { - "nan": "^2.12.1" + "nan": "^2.18.0" }, "devDependencies": { "tree-sitter-cli": "^CLI_VERSION" }, "scripts": { - "test": "tree-sitter test" - } + "build": "tree-sitter generate --no-bindings", + "build-wasm": "tree-sitter build-wasm", + "test": "tree-sitter test", + "parse": "tree-sitter parse" + }, + "tree-sitter": [ + { + "scope": "source.LOWER_PARSER_NAME", + "injection-regex": "LOWER_PARSER_NAME" + } + ] } diff --git a/cli/src/generate/templates/py-binding.c b/cli/src/generate/templates/py-binding.c new file mode 100644 index 00000000..5edbea71 --- /dev/null +++ b/cli/src/generate/templates/py-binding.c @@ -0,0 +1,27 @@ +#include + +typedef struct TSLanguage TSLanguage; + +extern const TSLanguage *tree_sitter_LOWER_PARSER_NAME(void); + +static PyObject* _binding_language(PyObject *self, PyObject *args) { + return PyLong_FromVoidPtr((void *)tree_sitter_LOWER_PARSER_NAME()); +} + +static PyMethodDef methods[] = { + {"language", _binding_language, METH_NOARGS, + "Get the tree-sitter language for this grammar."}, + {NULL, NULL, 0, NULL} +}; + +static struct PyModuleDef module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "_binding", + .m_doc = NULL, + .m_size = -1, + .m_methods = methods +}; + +PyMODINIT_FUNC PyInit__binding(void) { + return PyModule_Create(&module); +} diff --git a/cli/src/generate/templates/pyproject.toml b/cli/src/generate/templates/pyproject.toml new file mode 100644 index 00000000..ae454c85 --- /dev/null +++ b/cli/src/generate/templates/pyproject.toml @@ -0,0 +1,26 @@ +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "tree-sitter-PARSER_NAME" +description = "CAMEL_PARSER_NAME grammar for tree-sitter" +version = "0.0.1" +keywords = ["parsing", "incremental", "PARSER_NAME"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Topic :: Software Development :: Compilers", + "Topic :: Text Processing :: Linguistic", +] +requires-python = ">=3.8" +license.file = "LICENSE" +readme = "README.md" + +[project.optional-dependencies] +core = ["tree-sitter~=0.21"] + +[tool.cibuildwheel] +build = "cp38-*" +build-frontend = "build" diff --git a/cli/src/generate/templates/setup.py b/cli/src/generate/templates/setup.py new file mode 100644 index 00000000..037f8074 --- /dev/null +++ b/cli/src/generate/templates/setup.py @@ -0,0 +1,49 @@ +from os.path import join +from setuptools import Extension, find_packages, setup +from setuptools.command.build import build +from wheel.bdist_wheel import bdist_wheel + + +class Build(build): + def run(self): + dest = join(self.build_lib, "tree_sitter_PARSER_NAME", "queries") + try: + self.copy_tree("queries", dest) + except: + pass + super().run() + + +class BdistWheel(bdist_wheel): + def get_tag(self): + python, abi, platform = super().get_tag() + if python.startswith("cp"): + python, abi = "cp38", "abi3" + return python, abi, platform + + +setup( + packages=find_packages("bindings/python"), + package_dir={"": "bindings/python"}, + package_data={ + "tree_sitter_LOWER_PARSER_NAME": ["*.pyi", "py.typed"], + "tree_sitter_LOWER_PARSER_NAME.queries": ["*.scm"], + }, + ext_package="tree_sitter_LOWER_PARSER_NAME", + ext_modules=[ + Extension( + name="_binding", + sources=[ + "bindings/python/tree_sitter_LOWER_PARSER_NAME/binding.c", + "src/parser.c", + # NOTE: if your language uses an external scanner, add it here. + ], + extra_compile_args=["-std=c11"], + define_macros=[("Py_LIMITED_API", "0x03080000"), ("PY_SSIZE_T_CLEAN", None)], + include_dirs=["src"], + py_limited_api=True, + ) + ], + cmdclass={"build": Build, "bdist_wheel": BdistWheel}, + zip_safe=False, +) diff --git a/docs/section-2-using-parsers.md b/docs/section-2-using-parsers.md index 893b7143..51c2da3f 100644 --- a/docs/section-2-using-parsers.md +++ b/docs/section-2-using-parsers.md @@ -51,7 +51,7 @@ Here's an example of a simple C program that uses the Tree-sitter [JSON parser]( // Declare the `tree_sitter_json` function, which is // implemented by the `tree-sitter-json` library. -TSLanguage *tree_sitter_json(); +extern const TSLanguage *tree_sitter_json(); int main() { // Create a parser. @@ -326,9 +326,9 @@ Conceptually, it can be represented by three syntax trees with overlapping range #include // These functions are each implemented in their own repo. -const TSLanguage *tree_sitter_embedded_template(); -const TSLanguage *tree_sitter_html(); -const TSLanguage *tree_sitter_ruby(); +extern const TSLanguage *tree_sitter_embedded_template(); +extern const TSLanguage *tree_sitter_html(); +extern const TSLanguage *tree_sitter_ruby(); int main(int argc, const char **argv) { const char *text = argv[1]; diff --git a/docs/section-3-creating-parsers.md b/docs/section-3-creating-parsers.md index 2642f23c..714864e4 100644 --- a/docs/section-3-creating-parsers.md +++ b/docs/section-3-creating-parsers.md @@ -104,14 +104,46 @@ Let's go over all of the functionality of the `tree-sitter` command line tool. The most important command you'll use is `tree-sitter generate`. This command reads the `grammar.js` file in your current working directory and creates a file called `src/parser.c`, which implements the parser. After making changes to your grammar, just run `tree-sitter generate` again. -The first time you run `tree-sitter generate`, it will also generate a few other files: +The first time you run `tree-sitter generate`, it will also generate a few other files for bindings for the following languages: + +#### C/C++ + +* `Makefile` - This file tells `make` how to compile your language. +* `bindings/c/tree-sitter-language.h` - This file provides the C interface of your language. +* `bindings/c/tree-sitter-language.pc` - This file provides pkg-config metadata about your language's C library. +* `src/tree_sitter/parser.h` - This file provides some basic C definitions that are used in your generated `parser.c` file. + +#### Go + +* `bindings/go/binding.go` - This file wraps your language in a Go module. +* `bindings/go/binding_test.go` - This file contains a test for the Go package. + +#### Node * `binding.gyp` - This file tells Node.js how to compile your language. * `bindings/node/index.js` - This is the file that Node.js initially loads when using your language. -* `bindings/node/binding.cc` - This file wraps your language in a JavaScript object when used in Node.js. +* `bindings/node/binding.cc` - This file wraps your language in a JavaScript module for Node.js. + +#### Python + +* `pyproject.toml` - This file is the manifest of the Python package. +* `setup.py` - This file tells Python how to compile your language. +* `bindings/python/binding.c` - This file wraps your language in a Python module. +* `bindings/python/tree_sitter_language/__init__.py` - This file tells Python how to load your language. +* `bindings/python/tree_sitter_language/__init__.pyi` - This file provides type hints for your parser when used in Python. +* `bindings/python/tree_sitter_language/py.typed` - This file provides type hints for your parser when used in Python. + +#### Rust + +* `Cargo.toml` - This file is the manifest of the Rust package. * `bindings/rust/lib.rs` - This file wraps your language in a Rust crate when used in Rust. * `bindings/rust/build.rs` - This file wraps the building process for the Rust crate. -* `src/tree_sitter/parser.h` - This file provides some basic C definitions that are used in your generated `parser.c` file. + +#### Swift + +* `Package.swift` - This file tells Swift how to compile your language. +* `bindings/swift/TreeSitterLanguage/language.h` - This file wraps your language in a Swift module when used in Swift. + If there is an ambiguity or *local ambiguity* in your grammar, Tree-sitter will detect it during parser generation, and it will exit with a `Unresolved conflict` error message. See below for more information on these errors.