feat!: introduce tree-sitter-language crate for grammar crates to depend on

Co-authored-by: Conrad <conrad@zed.dev>
Co-authored-by: Marshall <marshall@zed.dev>
Co-authored-by: Amaan Qureshi <amaanq12@gmail.com>

Cargo.lock

@@ -1389,6 +1389,7 @@ dependencies = [
  "bindgen",
  "cc",
  "regex",
+ "tree-sitter-language",
  "wasmtime-c-api-impl",
 ]
 
@@ -1458,6 +1459,10 @@ dependencies = [
  "tree-sitter",
 ]
 
+[[package]]
+name = "tree-sitter-language"
+version = "0.1.0"
+
 [[package]]
 name = "tree-sitter-loader"
 version = "0.22.6"

Cargo.toml

@@ -5,6 +5,7 @@ members = [
   "cli/config",
   "cli/loader",
   "lib",
+  "lib/language",
   "tags",
   "highlight",
   "xtask",

cli/src/generate/grammar_files.rs

@@ -274,9 +274,19 @@ pub fn generate_grammar_files(
 
     // Generate Rust bindings
     missing_path(bindings_dir.join("rust"), create_dir)?.apply(|path| {
-        missing_path(path.join("lib.rs"), |path| {
-            generate_file(path, LIB_RS_TEMPLATE, language_name)
-        })?;
+        missing_path_else(
+            path.join("lib.rs"),
+            |path| generate_file(path, LIB_RS_TEMPLATE, language_name),
+            |path| {
+                let lib_rs =
+                    fs::read_to_string(path).with_context(|| "Failed to read lib.rs")?;
+                if !lib_rs.contains("tree_sitter_language") {
+                    generate_file(path, LIB_RS_TEMPLATE, language_name)?;
+                    eprintln!("Updated lib.rs with `tree_sitter_language` dependency");
+                }
+                Ok(())
+            },
+        )?;
 
         missing_path_else(
             path.join("build.rs"),
@@ -306,9 +316,36 @@ pub fn generate_grammar_files(
             },
         )?;
 
-        missing_path(repo_path.join("Cargo.toml"), |path| {
-            generate_file(path, CARGO_TOML_TEMPLATE, dashed_language_name.as_str())
-        })?;
+        missing_path_else(
+            repo_path.join("Cargo.toml"),
+            |path| generate_file(path, CARGO_TOML_TEMPLATE, dashed_language_name.as_str()),
+            |path| {
+                let cargo_toml =
+                    fs::read_to_string(path).with_context(|| "Failed to read Cargo.toml")?;
+                if !cargo_toml.contains("tree-sitter-language") {
+                    let start_index = cargo_toml
+                        .find("tree-sitter = \"")
+                        .ok_or_else(|| anyhow!("Failed to find the `tree-sitter` dependency in Cargo.toml"))?;
+
+                    let version_start_index = start_index + "tree-sitter = \"".len();
+                    let version_end_index = cargo_toml[version_start_index..]
+                        .find('\"')
+                        .map(|i| i + version_start_index)
+                        .ok_or_else(|| anyhow!("Failed to find the end of the `tree-sitter` version in Cargo.toml"))?;
+
+                    let cargo_toml = format!(
+                        "{}{}{}",
+                        &cargo_toml[..start_index],
+                        "tree-sitter-language = \"0.1.0\"",
+                        &cargo_toml[version_end_index + 1..],
+                    );
+
+                    write_file(path, cargo_toml)?;
+                    eprintln!("Updated Cargo.toml with the `tree-sitter-language` dependency");
+                }
+                Ok(())
+            },
+        )?;
 
         Ok(())
     })?;

cli/src/generate/templates/cargo.toml

@@ -17,7 +17,10 @@ include = ["bindings/rust/*", "grammar.js", "queries/*", "src/*"]
 path = "bindings/rust/lib.rs"
 
 [dependencies]
-tree-sitter = ">=RUST_BINDING_VERSION"
+tree-sitter-language = "0.1"
+
+[dev-dependencies]
+tree-sitter = { version = "0.22" }
 
 [build-dependencies]
 cc = "1.0.87"

cli/src/generate/templates/lib.rs

@@ -7,7 +7,10 @@
 //! let code = r#"
 //! "#;
 //! let mut parser = tree_sitter::Parser::new();
-//! parser.set_language(&tree_sitter_PARSER_NAME::language()).expect("Error loading CAMEL_PARSER_NAME grammar");
+//! let language = tree_sitter_PARSER_NAME::LANGUAGE;
+//! parser
+//!     .set_language(&language.into())
+//!     .expect("Error loading CAMEL_PARSER_NAME parser");
 //! let tree = parser.parse(code, None).unwrap();
 //! assert!(!tree.root_node().has_error());
 //! ```
@@ -17,18 +20,14 @@
 //! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
 //! [tree-sitter]: https://tree-sitter.github.io/
 
-use tree_sitter::Language;
+use tree_sitter_language::LanguageFn;
 
 extern "C" {
-    fn tree_sitter_PARSER_NAME() -> Language;
+    fn tree_sitter_PARSER_NAME() -> *const ();
 }
 
-/// Get the tree-sitter [Language][] for this grammar.
-///
-/// [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
-pub fn language() -> Language {
-    unsafe { tree_sitter_PARSER_NAME() }
-}
+/// The tree-sitter [`LanguageFn`] for this grammar.
+pub const LANGUAGE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_PARSER_NAME) };
 
 /// The content of the [`node-types.json`][] file for this grammar.
 ///
@@ -48,7 +47,7 @@ mod tests {
     fn test_can_load_grammar() {
         let mut parser = tree_sitter::Parser::new();
         parser
-            .set_language(&super::language())
-            .expect("Error loading CAMEL_PARSER_NAME grammar");
+            .set_language(&super::LANGUAGE.into())
+            .expect("Error loading CAMEL_PARSER_NAME parser");
     }
 }

lib/Cargo.toml

@@ -30,6 +30,7 @@ wasm = ["wasmtime-c-api"]
 
 [dependencies]
 regex.workspace = true
+tree-sitter-language = { version = "0.1", path = "language" }
 
 [dependencies.wasmtime-c-api]
 version = "19"

lib/binding_rust/lib.rs

@@ -22,6 +22,8 @@ use std::{
     sync::atomic::AtomicUsize,
 };
 
+use tree_sitter_language::LanguageFn;
+
 #[cfg(feature = "wasm")]
 mod wasm_language;
 #[cfg(feature = "wasm")]
@@ -284,6 +286,10 @@ pub struct LossyUtf8<'a> {
 }
 
 impl Language {
+    pub fn new(builder: LanguageFn) -> Self {
+        Self(unsafe { (builder.into_raw())() as _ })
+    }
+
     /// Get the ABI version number that indicates which version of the
     /// Tree-sitter CLI that was used to generate this [`Language`].
     #[doc(alias = "ts_language_version")]
@@ -406,6 +412,12 @@ impl Language {
     }
 }
 
+impl From<LanguageFn> for Language {
+    fn from(value: LanguageFn) -> Self {
+        Self::new(value)
+    }
+}
+
 impl Clone for Language {
     fn clone(&self) -> Self {
         unsafe { Self(ffi::ts_language_copy(self.0)) }

lib/language/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "tree-sitter-language"
+description = "The tree-sitter Language type, used by the library and by language implementations"
+version = "0.1.0"
+authors.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+homepage.workspace = true
+repository.workspace = true
+license.workspace = true
+keywords.workspace = true
+
+[lib]
+path = "language.rs"

lib/language/language.rs

@@ -0,0 +1,20 @@
+/// LanguageFn wraps a C function that returns a pointer to a tree-sitter grammer.
+#[repr(transparent)]
+pub struct LanguageFn(unsafe extern "C" fn() -> *const ());
+
+impl LanguageFn {
+    /// Creates a `LanguageFn`.
+    ///
+    /// # Safety
+    ///
+    /// Only call this with language functions generated from grammars
+    /// by the Tree-sitter CLI.
+    pub const unsafe fn from_raw(f: unsafe extern "C" fn() -> *const ()) -> Self {
+        Self(f)
+    }
+
+    /// Gets the function wrapped by this `LanguageFn`.
+    pub const fn into_raw(self) -> unsafe extern "C" fn() -> *const () {
+        self.0
+    }
+}