feat: expose the allocator and array header files for external scanners

cli/build.rs

@@ -20,6 +20,19 @@ fn main() {
         .unwrap()
         .as_secs_f64();
     println!("cargo:rustc-env=BUILD_TIME={build_time}");
+
+    #[cfg(any(
+        target_os = "linux",
+        target_os = "android",
+        target_os = "freebsd",
+        target_os = "openbsd",
+        target_os = "netbsd",
+        target_os = "dragonfly",
+    ))]
+    println!("cargo:rustc-link-arg=-Wl,--dynamic-list=cli/dynamic-symbols.txt");
+
+    #[cfg(any(target_os = "macos", target_os = "ios"))]
+    println!("cargo:rustc-link-arg=-Wl,-exported_symbols_list,cli/dynamic-symbols-darwin.txt");
 }
 
 fn web_playground_files_present() -> bool {

cli/dynamic-symbols-darwin.txt

@@ -0,0 +1,4 @@
+_ts_current_malloc
+_ts_current_calloc
+_ts_current_realloc
+_ts_current_free

cli/dynamic-symbols.txt

@@ -0,0 +1,6 @@
+{
+  ts_current_malloc;
+  ts_current_calloc;
+  ts_current_realloc;
+  ts_current_free;
+};

cli/loader/src/lib.rs

@@ -529,6 +529,7 @@ impl Loader {
             .cpp(true)
             .opt_level(2)
             .cargo_metadata(false)
+            .cargo_warnings(false)
             .target(BUILD_TARGET)
             .host(BUILD_TARGET)
             .flag_if_supported("-Werror=implicit-function-declaration");
@@ -584,10 +585,6 @@ impl Loader {
                 command.arg("-O2");
             }
 
-            // For conditional compilation of external scanner code when
-            // used internally by `tree-siteer parse` and other sub commands.
-            command.arg("-DTREE_SITTER_INTERNAL_BUILD");
-
             if let Some(scanner_path) = scanner_path.as_ref() {
                 if scanner_path.extension() == Some("c".as_ref()) {
                     command.arg("-xc").arg("-std=c99").arg(scanner_path);
@@ -599,6 +596,14 @@ impl Loader {
             command.arg("-xc").arg(parser_path);
         }
 
+        // For conditional compilation of external scanner code when
+        // used internally by `tree-sitter parse` and other sub commands.
+        command.arg("-DTREE_SITTER_INTERNAL_BUILD");
+
+        // Always use the same allocator in the CLI as any scanner, useful for debugging and
+        // tracking memory leaks in tests.
+        command.arg("-DTS_REUSE_ALLOCATOR");
+
         let output = command.output().with_context(|| {
             format!("Failed to execute the C compiler with the following command:\n{command:?}")
         })?;

cli/src/generate/mod.rs

@@ -122,6 +122,8 @@ pub fn generate_parser_in_directory(
 
     write_file(&src_path.join("parser.c"), c_code)?;
     write_file(&src_path.join("node-types.json"), node_types_json)?;
+    write_file(&header_path.join("alloc.h"), tree_sitter::ALLOC_HEADER)?;
+    write_file(&header_path.join("array.h"), tree_sitter::ARRAY_HEADER)?;
     write_file(&header_path.join("parser.h"), tree_sitter::PARSER_HEADER)?;
 
     if !path_in_ignore(&repo_path) {

cli/src/generate/templates/alloc.h

@@ -0,0 +1,60 @@
+#ifndef TREE_SITTER_ALLOC_H_
+#define TREE_SITTER_ALLOC_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#ifdef _WIN32
+#define TS_PUBLIC __declspec(dllexport)
+#else
+#define TS_PUBLIC __attribute__((visibility("default")))
+#endif
+
+TS_PUBLIC extern void *(*ts_current_malloc)(size_t);
+TS_PUBLIC extern void *(*ts_current_calloc)(size_t, size_t);
+TS_PUBLIC extern void *(*ts_current_realloc)(void *, size_t);
+TS_PUBLIC extern void (*ts_current_free)(void *);
+
+// Allow clients to override allocation functions
+#ifdef TS_REUSE_ALLOCATOR
+
+#ifndef ts_malloc
+#define ts_malloc  ts_current_malloc
+#endif
+#ifndef ts_calloc
+#define ts_calloc  ts_current_calloc
+#endif
+#ifndef ts_realloc
+#define ts_realloc ts_current_realloc
+#endif
+#ifndef ts_free
+#define ts_free    ts_current_free
+#endif
+
+#else
+
+#ifndef ts_malloc
+#define ts_malloc  malloc
+#endif
+#ifndef ts_calloc
+#define ts_calloc  calloc
+#endif
+#ifndef ts_realloc
+#define ts_realloc realloc
+#endif
+#ifndef ts_free
+#define ts_free    free
+#endif
+
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // TREE_SITTER_ALLOC_H_

cli/src/tests/helpers/fixtures.rs

@@ -107,6 +107,25 @@ pub fn get_test_language(name: &str, parser_code: &str, path: Option<&Path>) ->
 
     let header_path = src_dir.join("tree_sitter");
     fs::create_dir_all(&header_path).unwrap();
+
+    fs::write(header_path.join("alloc.h"), tree_sitter::PARSER_HEADER)
+        .with_context(|| {
+            format!(
+                "Failed to write {:?}",
+                header_path.join("alloc.h").file_name().unwrap()
+            )
+        })
+        .unwrap();
+
+    fs::write(header_path.join("array.h"), tree_sitter::PARSER_HEADER)
+        .with_context(|| {
+            format!(
+                "Failed to write {:?}",
+                header_path.join("array.h").file_name().unwrap()
+            )
+        })
+        .unwrap();
+
     fs::write(header_path.join("parser.h"), tree_sitter::PARSER_HEADER)
         .with_context(|| {
             format!(