Merge pull request #2540 from ahlinc/build-bindgen

feat: optionally generate Rust bindings during build process

Cargo.lock

@@ -43,6 +43,29 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "bindgen"
+version = "0.66.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2b84e06fc203107bfbad243f4aba2af864eb7db3b1cf46ea0a023b0b433d2a7"
+dependencies = [
+ "bitflags 2.4.0",
+ "cexpr",
+ "clang-sys",
+ "lazy_static",
+ "lazycell",
+ "log",
+ "peeking_take_while",
+ "prettyplease",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "rustc-hash",
+ "shlex",
+ "syn 2.0.29",
+ "which",
+]
+
 [[package]]
 name = "bitflags"
 version = "1.3.2"
@@ -82,6 +105,15 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c"
 
+[[package]]
+name = "cexpr"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
+dependencies = [
+ "nom",
+]
+
 [[package]]
 name = "cfg-if"
 version = "1.0.0"
@@ -94,6 +126,17 @@ version = "1.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cca491388666e04d7248af3f60f0c40cfb0991c72205595d7c396e3510207d1a"
 
+[[package]]
+name = "clang-sys"
+version = "1.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f"
+dependencies = [
+ "glob",
+ "libc",
+ "libloading",
+]
+
 [[package]]
 name = "clap"
 version = "2.34.0"
@@ -381,6 +424,12 @@ version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
+[[package]]
+name = "lazycell"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
+
 [[package]]
 name = "libc"
 version = "0.2.147"
@@ -424,6 +473,12 @@ version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
 
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
 [[package]]
 name = "ndk-context"
 version = "0.1.1"
@@ -442,6 +497,16 @@ dependencies = [
  "static_assertions",
 ]
 
+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
 [[package]]
 name = "objc"
 version = "0.2.7"
@@ -469,6 +534,12 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1e91099d4268b0e11973f036e885d652fb0b21fedcf69738c627f94db6a44f42"
 
+[[package]]
+name = "peeking_take_while"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
+
 [[package]]
 name = "percent-encoding"
 version = "2.3.0"
@@ -491,6 +562,16 @@ dependencies = [
  "yansi",
 ]
 
+[[package]]
+name = "prettyplease"
+version = "0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62"
+dependencies = [
+ "proc-macro2",
+ "syn 2.0.29",
+]
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.66"
@@ -684,6 +765,12 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "shlex"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3"
+
 [[package]]
 name = "smallbitvec"
 version = "2.5.1"
@@ -831,6 +918,7 @@ dependencies = [
 name = "tree-sitter"
 version = "0.20.10"
 dependencies = [
+ "bindgen",
  "cc",
  "regex",
 ]

lib/Cargo.toml

@@ -26,6 +26,7 @@ include = [
 regex = "1.9.1"
 
 [build-dependencies]
+bindgen = { version = "^0.66.1", optional = true }
 cc = "^1.0.79"
 
 [lib]

lib/binding_rust/bindings.rs

@@ -1,5 +1,7 @@
 /* automatically generated by rust-bindgen 0.66.1 */
 
+pub const TREE_SITTER_LANGUAGE_VERSION: u32 = 14;
+pub const TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION: u32 = 13;
 pub type TSStateId = u16;
 pub type TSSymbol = u16;
 pub type TSFieldId = u16;
@@ -783,6 +785,3 @@ extern "C" {
         new_free: ::std::option::Option<unsafe extern "C" fn(arg1: *mut ::std::os::raw::c_void)>,
     );
 }
-
-pub const TREE_SITTER_LANGUAGE_VERSION: usize = 14;
-pub const TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION: usize = 13;

lib/binding_rust/build.rs

@@ -17,6 +17,9 @@ fn main() {
         }
     }
 
+    #[cfg(feature = "bindgen")]
+    generate_bindings();
+
     let src_path = Path::new("src");
     for entry in fs::read_dir(&src_path).unwrap() {
         let entry = entry.unwrap();
@@ -34,6 +37,44 @@ fn main() {
         .compile("tree-sitter");
 }
 
+#[cfg(feature = "bindgen")]
+fn generate_bindings() {
+    const HEADER_PATH: &str = "include/tree_sitter/api.h";
+
+    println!("cargo:rerun-if-changed={}", HEADER_PATH);
+
+    let no_copy = [
+        "TSInput",
+        "TSLanguage",
+        "TSLogger",
+        "TSLookaheadIterator",
+        "TSParser",
+        "TSTree",
+        "TSQuery",
+        "TSQueryCursor",
+        "TSQueryCapture",
+        "TSQueryMatch",
+        "TSQueryPredicateStep",
+    ];
+
+    let bindings = bindgen::Builder::default()
+        .header(HEADER_PATH)
+        .layout_tests(false)
+        .allowlist_type("^TS.*")
+        .allowlist_function("^ts_.*")
+        .allowlist_var("^TREE_SITTER.*")
+        .no_copy(no_copy.join("|"))
+        .generate()
+        .expect("Failed to generate bindings");
+
+    let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
+    let bindings_rs = out_dir.join("bindings.rs");
+
+    bindings.write_to_file(&bindings_rs).expect(&*format!(
+        "Failed to write bindings into path: {bindings_rs:?}"
+    ));
+}
+
 fn which(exe_name: impl AsRef<Path>) -> Option<PathBuf> {
     env::var_os("PATH").and_then(|paths| {
         env::split_paths(&paths).find_map(|dir| {

lib/binding_rust/ffi.rs

@@ -2,6 +2,10 @@
 #![allow(non_upper_case_globals)]
 #![allow(non_camel_case_types)]
 
+#[cfg(feature = "bindgen")]
+include!(concat!(env!("OUT_DIR"), "/bindings.rs"));
+
+#[cfg(not(feature = "bindgen"))]
 include!("./bindings.rs");
 
 extern "C" {

lib/binding_rust/lib.rs

@@ -27,12 +27,13 @@ use std::{
 /// The Tree-sitter library is generally backwards-compatible with languages
 /// generated using older CLI versions, but is not forwards-compatible.
 #[doc(alias = "TREE_SITTER_LANGUAGE_VERSION")]
-pub const LANGUAGE_VERSION: usize = ffi::TREE_SITTER_LANGUAGE_VERSION;
+pub const LANGUAGE_VERSION: usize = ffi::TREE_SITTER_LANGUAGE_VERSION as usize;
 
 /// The earliest ABI version that is supported by the current version of the
 /// library.
 #[doc(alias = "TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION")]
-pub const MIN_COMPATIBLE_LANGUAGE_VERSION: usize = ffi::TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION;
+pub const MIN_COMPATIBLE_LANGUAGE_VERSION: usize =
+    ffi::TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION as usize;
 
 pub const PARSER_HEADER: &'static str = include_str!("../include/tree_sitter/parser.h");
 

script/generate-bindings

@@ -33,18 +33,7 @@ bindgen                                        \
   --no-layout-tests                            \
   --allowlist-type '^TS.*'                     \
   --allowlist-function '^ts_.*'                \
+  --allowlist-var "^TREE_SITTER.*"             \
   --blocklist-type '^__.*'                     \
   --no-copy "$no_copy"                         \
   $header_path > $output_path
-
-echo "" >> $output_path
-
-defines=(
-  TREE_SITTER_LANGUAGE_VERSION
-  TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION
-)
-
-for define in "${defines[@]}"; do
-  define_value=$(grep -E "#define $define (.*)" $header_path | cut -d' ' -f3)
-  echo "pub const $define: usize = $define_value;" >> $output_path
-done