Move allocation tracking into lib crate

We have several test cases defined in the `cli` crate that depend on the
`lib` crate's `allocation-tracking` feature.  The implementation of the
actual allocation tracker used to live in the `cli` crate, close to the
test cases that use it.  The `allocation-tracking` feature in the `lib`
crate was just used to tell the tree-sitter implementation to expect
that the allocation tracker exists, and to use it.

That pattern meant that we had a circular dependency: `cli` depends on
`lib`, but `lib` required some code that was implemented in `cli`.
That, in turn, caused linker errors — but only when compiling in certain
configurations! [1]

This patch moves all of the allocation tracking implementation into the
`lib` crate, gated on the existing `allocation-tracking` feature, which
fixes the circular dependency.

Note that this patch does **not** fix the fact that feature unification
causes the `lib` crate to be built with the `allocation-tracking`
feature enabled, even though it's not a default.  Fixing that depends on
the forthcoming version 2 feature resolver [2], or using the `dev_dep`
workaround [3] in the meantime.

[1] https://github.com/tree-sitter/tree-sitter/issues/919
[2] https://doc.rust-lang.org/nightly/cargo/reference/features.html#feature-resolver-version-2
[3] https://github.com/tree-sitter/tree-sitter/issues/919#issuecomment-777107086

Cargo.lock

@@ -836,7 +836,9 @@ name = "tree-sitter"
 version = "0.17.1"
 dependencies = [
  "cc",
+ "lazy_static",
  "regex",
+ "spin",
 ]
 
 [[package]]
@@ -862,7 +864,6 @@ dependencies = [
  "serde_derive",
  "serde_json",
  "smallbitvec",
- "spin",
  "tempfile",
  "tiny_http",
  "tree-sitter",

cli/Cargo.toml

@@ -65,5 +65,4 @@ features = ["std"]
 
 [dev-dependencies]
 rand = "0.7.0"
-spin = "0.5"
 tempfile = "3"

cli/src/tests/corpus_test.rs

@@ -1,4 +1,3 @@
-use super::helpers::allocations;
 use super::helpers::edits::{get_random_edit, invert_edit};
 use super::helpers::fixtures::{fixtures_dir, get_language, get_test_language};
 use super::helpers::random::Rand;
@@ -9,7 +8,7 @@ use crate::test::{parse_tests, print_diff, print_diff_key, strip_sexp_fields, Te
 use crate::util;
 use lazy_static::lazy_static;
 use std::{env, fs, time, usize};
-use tree_sitter::{LogType, Node, Parser, Tree};
+use tree_sitter::{allocations, LogType, Node, Parser, Tree};
 
 const EDIT_COUNT: usize = 3;
 const TRIAL_COUNT: usize = 10;
@@ -390,7 +389,9 @@ fn flatten_tests(test: TestEntry) -> Vec<(String, Vec<u8>, String, bool)> {
                 }
                 result.push((name, input, output, has_fields));
             }
-            TestEntry::Group { mut name, children, .. } => {
+            TestEntry::Group {
+                mut name, children, ..
+            } => {
                 if !prefix.is_empty() {
                     name.insert_str(0, " - ");
                     name.insert_str(0, prefix);

cli/src/tests/helpers/mod.rs

@@ -1,4 +1,3 @@
-pub(super) mod allocations;
 pub(super) mod edits;
 pub(super) mod fixtures;
 pub(super) mod random;

cli/src/tests/parser_test.rs

@@ -1,11 +1,10 @@
-use super::helpers::allocations;
 use super::helpers::edits::ReadRecorder;
 use super::helpers::fixtures::{get_language, get_test_language};
 use crate::generate::generate_parser_for_grammar;
 use crate::parse::{perform_edit, Edit};
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::{thread, time};
-use tree_sitter::{IncludedRangesError, InputEdit, LogType, Parser, Point, Range};
+use tree_sitter::{allocations, IncludedRangesError, InputEdit, LogType, Parser, Point, Range};
 
 #[test]
 fn test_parsing_simple_string() {

cli/src/tests/pathological_test.rs

@@ -1,6 +1,5 @@
-use super::helpers::allocations;
 use super::helpers::fixtures::get_language;
-use tree_sitter::Parser;
+use tree_sitter::{allocations, Parser};
 
 #[test]
 fn test_pathological_example_1() {

cli/src/tests/query_test.rs

@@ -1,11 +1,10 @@
-use super::helpers::allocations;
 use super::helpers::fixtures::get_language;
 use lazy_static::lazy_static;
 use std::env;
 use std::fmt::Write;
 use tree_sitter::{
-    Language, Node, Parser, Query, QueryCapture, QueryCursor, QueryError, QueryErrorKind,
-    QueryMatch, QueryPredicate, QueryPredicateArg, QueryProperty,
+    allocations, Language, Node, Parser, Query, QueryCapture, QueryCursor, QueryError,
+    QueryErrorKind, QueryMatch, QueryPredicate, QueryPredicateArg, QueryProperty,
 };
 
 lazy_static! {

cli/src/tests/tags_test.rs

@@ -1,9 +1,8 @@
-use super::helpers::allocations;
 use super::helpers::fixtures::{get_language, get_language_queries_path};
 use std::ffi::CStr;
 use std::ffi::CString;
 use std::{fs, ptr, slice, str};
-use tree_sitter::Point;
+use tree_sitter::{allocations, Point};
 use tree_sitter_tags::c_lib as c;
 use tree_sitter_tags::{Error, TagsConfiguration, TagsContext};
 

lib/Cargo.toml

@@ -3,6 +3,7 @@ name = "tree-sitter"
 description = "Rust bindings to the Tree-sitter parsing library"
 version = "0.17.1"
 authors = ["Max Brunsfeld <maxbrunsfeld@gmail.com>"]
+edition = "2018"
 license = "MIT"
 readme = "binding_rust/README.md"
 keywords = ["incremental", "parsing"]
@@ -21,7 +22,9 @@ include = [
 ]
 
 [dependencies]
+lazy_static = { version="1.2.0", optional=true }
 regex = "1"
+spin = { version="0.5", optional=true }
 
 [build-dependencies]
 cc = "^1.0.58"
@@ -32,4 +35,4 @@ path = "binding_rust/lib.rs"
 # This feature is only useful for testing the Tree-sitter library itself.
 # It is exposed because all of Tree-sitter's tests live in the Tree-sitter CLI crate.
 [features]
-allocation-tracking = []
+allocation-tracking = ["lazy_static", "spin"]

lib/binding_rust/allocations.rs

@@ -1,6 +1,3 @@
-#![cfg(test)]
-#![allow(dead_code)]
-
 use lazy_static::lazy_static;
 use spin::Mutex;
 use std::collections::HashMap;
@@ -82,35 +79,38 @@ fn record_dealloc(ptr: *mut c_void) {
 }
 
 #[no_mangle]
-extern "C" fn ts_record_malloc(size: c_ulong) -> *const c_void {
+pub extern "C" fn ts_record_malloc(size: c_ulong) -> *const c_void {
     let result = unsafe { malloc(size) };
     record_alloc(result);
     result
 }
 
 #[no_mangle]
-extern "C" fn ts_record_calloc(count: c_ulong, size: c_ulong) -> *const c_void {
+pub extern "C" fn ts_record_calloc(count: c_ulong, size: c_ulong) -> *const c_void {
     let result = unsafe { calloc(count, size) };
     record_alloc(result);
     result
 }
 
 #[no_mangle]
-extern "C" fn ts_record_realloc(ptr: *mut c_void, size: c_ulong) -> *const c_void {
+pub extern "C" fn ts_record_realloc(ptr: *mut c_void, size: c_ulong) -> *const c_void {
     record_dealloc(ptr);
     let result = unsafe { realloc(ptr, size) };
     record_alloc(result);
     result
 }
 
+// This needs to be unsafe because it's reexported as crate::util::free_ptr, which is mapped to
+// libc's `free` function when the allocation-tracking feature is disabled.  Since `free` is
+// unsafe, this function needs to be too.
 #[no_mangle]
-extern "C" fn ts_record_free(ptr: *mut c_void) {
+pub unsafe extern "C" fn ts_record_free(ptr: *mut c_void) {
     record_dealloc(ptr);
-    unsafe { free(ptr) };
+    free(ptr);
 }
 
 #[no_mangle]
-extern "C" fn ts_toggle_allocation_recording(enabled: bool) -> bool {
+pub extern "C" fn ts_toggle_allocation_recording(enabled: bool) -> bool {
     let mut recorder = RECORDER.lock();
     let was_enabled = recorder.enabled;
     recorder.enabled = enabled;

lib/binding_rust/lib.rs

@@ -1,6 +1,9 @@
 mod ffi;
 mod util;
 
+#[cfg(feature = "allocation-tracking")]
+pub mod allocations;
+
 #[cfg(unix)]
 use std::os::unix::io::AsRawFd;
 

lib/binding_rust/util.rs

@@ -1,21 +1,19 @@
 use std::os::raw::c_void;
 
+#[cfg(not(feature = "allocation-tracking"))]
 extern "C" {
     /// Normally, use `free(1)` to free memory allocated from C.
-    #[cfg(not(feature = "allocation-tracking"))]
     #[link_name = "free"]
     pub fn free_ptr(ptr: *mut c_void);
-
-    /// When the `allocation-tracking` feature is enabled, the C library is compiled with
-    /// the `TREE_SITTER_TEST` macro, so all calls to `malloc`, `free`, etc are linked
-    /// against wrapper functions called `ts_record_malloc`, `ts_record_free`, etc.
-    /// When freeing buffers allocated from C, use the wrapper `free` function.
-    #[cfg(feature = "allocation-tracking")]
-    #[link_name = "ts_record_free"]
-    pub fn free_ptr(ptr: *mut c_void);
-
 }
 
+/// When the `allocation-tracking` feature is enabled, the C library is compiled with
+/// the `TREE_SITTER_TEST` macro, so all calls to `malloc`, `free`, etc are linked
+/// against wrapper functions called `ts_record_malloc`, `ts_record_free`, etc.
+/// When freeing buffers allocated from C, use the wrapper `free` function.
+#[cfg(feature = "allocation-tracking")]
+pub use crate::allocations::ts_record_free as free_ptr;
+
 /// A raw pointer and a length, exposed as an iterator.
 pub struct CBufferIter<T> {
     ptr: *mut T,