diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..3fe315ba --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,97 @@ +name: CI + +on: + push: + branches: + - master + pull_request: + branches: + - "**" + +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: 0 + EMSCRIPTEN_VERSION: 2.0.11 + +jobs: + tests: + name: Unix tests + runs-on: ${{ matrix.os }} + strategy: + fail-fast: true + matrix: + os: [macos-latest, ubuntu-latest] + toolchain: [stable] + steps: + - name: Checkout repo + uses: actions/checkout@v2 + + # Work around https://github.com/actions/cache/issues/403. + - name: Use GNU tar + if: matrix.os == 'macos-latest' + run: | + echo PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH" >> $GITHUB_ENV + + - name: Cache artifacts + id: cache + uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }}-emscripten-${{ env.EMSCRIPTEN_VERSION }} + + - name: Install rust + if: steps.cache.outputs.cache-hit != 'true' + uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ matrix.toolchain }} + profile: minimal + + - name: Install emscripten + uses: mymindstorm/setup-emsdk@v7 + with: + version: ${{ env.EMSCRIPTEN_VERSION }} + + - name: Build C library + run: make + + - name: Build wasm library + run: script/build-wasm + + - name: Build CLI + run: cargo build --release + + - name: Set up fixture parsers + run: | + script/fetch-fixtures + script/generate-fixtures + script/generate-fixtures-wasm + + - name: Run main tests + run: script/test + + - name: Run wasm tests + run: script/test-wasm + + - name: Run benchmarks + run: script/benchmark + + - name: Compress CLI binary + if: startsWith(github.ref, 'refs/tags/v') + run: | + cp target/release/tree-sitter . + gzip --keep --suffix "-${{ runner.os }}-x64.gz" tree-sitter + + - name: Release + uses: softprops/action-gh-release@v1 + if: startsWith(github.ref, 'refs/tags/v') + with: + draft: true + files: | + tree-sitter-*.gz + lib/binding_web/tree-sitter.js + lib/binding_web/tree-sitter.wasm + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 282ba02d..00000000 --- a/.travis.yml +++ /dev/null @@ -1,70 +0,0 @@ -language: rust -rust: - - stable - -env: - CFLAGS="-Wall -Wextra -Werror -Wstrict-prototypes" - -matrix: - include: - - os: osx - env: USE_EMSCRIPTEN=1 - - os: linux - services: docker - -before_install: - # Install node - - nvm install 12 - - nvm use 12 - - # Download emscripten and create a shorthand for adding it to the PATH. - # Don't add it to the path globally because it overrides the default - # clang and node. - - if [ -n "$USE_EMSCRIPTEN" ]; then export WASM_ENV="$(script/fetch-emscripten)"; fi - -script: - # Build the WASM binding - - (eval "$WASM_ENV" && script/build-wasm) - - # build the shared/static libraries - - make - - # Build the CLI - - cargo build --release - - # Fetch and regenerate the fixture parsers - - script/fetch-fixtures - - script/generate-fixtures - - (eval "$WASM_ENV" && script/generate-fixtures-wasm) - - # Run the tests - - script/test - - script/test-wasm - - script/benchmark - -branches: - only: - - master - - /\d+\.\d+\.\d+/ - -before_deploy: - - cp target/release/tree-sitter . - - gzip --suffix "-${TRAVIS_OS_NAME}-x64.gz" tree-sitter - -deploy: - provider: releases - api_key: - secure: "cAd2mQP+Q55v3zedo5ZyOVc3hq3XKMW93lp5LuXV6CYKYbIhkyfym4qfs+C9GJQiIP27cnePYM7B3+OMIFwSPIgXHWWSsuloMtDgYSc/PAwb2dZnJqAyog3BohW/QiGTSnvbVlxPF6P9RMQU6+JP0HJzEJy6QBTa4Und/j0jm24=" - file_glob: true - file: - - "tree-sitter-*.gz" - draft: true - overwrite: true - skip_cleanup: true - on: - tags: true - -cache: - cargo: true - directories: - - target/emsdk diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 47becc3d..b3029c6a 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -42,6 +42,11 @@ html-escape = "0.2.6" version = ">= 0.17.0" path = "../lib" +[dev-dependencies.tree-sitter] +version = ">= 0.17.0" +path = "../lib" +features = ["allocation-tracking"] + [dependencies.tree-sitter-highlight] version = ">= 0.3.0" path = "../highlight" diff --git a/cli/src/allocations_stubs.rs b/cli/src/allocations_stubs.rs new file mode 100644 index 00000000..b4b6dde1 --- /dev/null +++ b/cli/src/allocations_stubs.rs @@ -0,0 +1,40 @@ +// In all dev builds, the tree-sitter library is built with the `allocation-tracking` +// feature enabled. This causes the library to link against a set of externally +// defined C functions like `ts_record_malloc` and `ts_record_free`. In tests, these +// are defined to actually keep track of outstanding allocations. But when not running +// tests, the symbols still need to be defined. This file provides pass-through +// implementations of all of these functions. + +use std::os::raw::c_void; + +extern "C" { + fn malloc(size: usize) -> *mut c_void; + fn calloc(count: usize, size: usize) -> *mut c_void; + fn realloc(ptr: *mut c_void, size: usize) -> *mut c_void; + fn free(ptr: *mut c_void); +} + +#[no_mangle] +unsafe extern "C" fn ts_record_malloc(size: usize) -> *const c_void { + malloc(size) +} + +#[no_mangle] +unsafe extern "C" fn ts_record_calloc(count: usize, size: usize) -> *const c_void { + calloc(count, size) +} + +#[no_mangle] +unsafe extern "C" fn ts_record_realloc(ptr: *mut c_void, size: usize) -> *const c_void { + realloc(ptr, size) +} + +#[no_mangle] +unsafe extern "C" fn ts_record_free(ptr: *mut c_void) { + free(ptr) +} + +#[no_mangle] +extern "C" fn ts_toggle_allocation_recording(_: bool) -> bool { + false +} diff --git a/cli/src/lib.rs b/cli/src/lib.rs index e00323b7..5b491574 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -16,3 +16,6 @@ pub mod web_ui; #[cfg(test)] mod tests; + +#[cfg(not(test))] +mod allocations_stubs; diff --git a/cli/src/tests/helpers/allocations.rs b/cli/src/tests/helpers/allocations.rs index 2f89c173..0e1cef22 100644 --- a/cli/src/tests/helpers/allocations.rs +++ b/cli/src/tests/helpers/allocations.rs @@ -4,6 +4,7 @@ use lazy_static::lazy_static; use spin::Mutex; use std::collections::HashMap; +use std::env; use std::os::raw::{c_ulong, c_void}; #[derive(Debug, PartialEq, Eq, Hash)] @@ -31,9 +32,14 @@ extern "C" { pub fn start_recording() { let mut recorder = RECORDER.lock(); - recorder.enabled = true; recorder.allocation_count = 0; recorder.outstanding_allocations.clear(); + + if env::var("RUST_TEST_THREADS").map_or(false, |s| s == "1") { + recorder.enabled = true; + } else { + panic!("This test must be run with RUST_TEST_THREADS=1. Use script/test."); + } } pub fn stop_recording() { diff --git a/lib/Cargo.toml b/lib/Cargo.toml index e8305c0e..41349738 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -28,3 +28,8 @@ cc = "^1.0.58" [lib] path = "binding_rust/lib.rs" + +# This feature is only useful for testing the Tree-sitter library itself. +# It is exposed because all of Tree-sitter's tests live in the Tree-sitter CLI crate. +[features] +allocation-tracking = [] diff --git a/lib/binding_rust/build.rs b/lib/binding_rust/build.rs index 0ec7a4ad..f1fd6bfa 100644 --- a/lib/binding_rust/build.rs +++ b/lib/binding_rust/build.rs @@ -21,9 +21,9 @@ fn main() { let mut config = cc::Build::new(); - println!("cargo:rerun-if-env-changed=PROFILE"); - if env::var("PROFILE").map_or(false, |s| s == "debug") { - config.define("TREE_SITTER_TEST", ""); + println!("cargo:rerun-if-env-changed=CARGO_FEATURE_ALLOCATION_TRACKING"); + if env::var("CARGO_FEATURE_ALLOCATION_TRACKING").is_ok() { + config.define("TREE_SITTER_ALLOCATION_TRACKING", ""); } let src_path = Path::new("src"); diff --git a/lib/binding_rust/util.rs b/lib/binding_rust/util.rs index 1a4ac1b7..4e5efb7e 100644 --- a/lib/binding_rust/util.rs +++ b/lib/binding_rust/util.rs @@ -1,71 +1,22 @@ use std::os::raw::c_void; extern "C" { - /// In *Release* builds, the C library links directly against `malloc` and `free`. - /// - /// When freeing memory that was allocated by C code, use `free` directly. - #[cfg(not(debug_assertions))] + /// Normally, use `free(1)` to free memory allocated from C. + #[cfg(not(feature = "allocation-tracking"))] #[link_name = "free"] pub fn free_ptr(ptr: *mut c_void); - /// In *Test* builds, the C library is compiled with the `TREE_SITTER_TEST` macro, - /// so all calls to `malloc`, `free`, etc are linked against wrapper functions - /// called `ts_record_malloc`, `ts_record_free`, etc. These symbols are defined - /// in the `tree_sitter_cli::tests::helpers::allocations` module. - /// - /// When freeing memory that was allocated by C code, use the `free` function - /// from that module. - #[cfg(debug_assertions)] + /// When the `allocation-tracking` feature is enabled, the C library is compiled with + /// the `TREE_SITTER_TEST` macro, so all calls to `malloc`, `free`, etc are linked + /// against wrapper functions called `ts_record_malloc`, `ts_record_free`, etc. + /// When freeing buffers allocated from C, use the wrapper `free` function. + #[cfg(feature = "allocation-tracking")] #[link_name = "ts_record_free"] pub fn free_ptr(ptr: *mut c_void); - /// In *Debug* builds, the C library is compiled the same as in test builds: using - /// the wrapper functions. This prevents the C library from having to be recompiled - /// constantly when switching between running tests and compiling with RLS. - /// - /// But we don't want to actually record allocations when running the library in - /// debug mode, so we define symbols like `ts_record_malloc` to just delegate to - /// the normal `malloc` functions. - #[cfg(all(debug_assertions, not(test)))] - fn malloc(size: usize) -> *mut c_void; - #[cfg(all(debug_assertions, not(test)))] - fn calloc(count: usize, size: usize) -> *mut c_void; - #[cfg(all(debug_assertions, not(test)))] - fn realloc(ptr: *mut c_void, size: usize) -> *mut c_void; - #[cfg(all(debug_assertions, not(test)))] - fn free(ptr: *mut c_void); -} - -#[cfg(all(debug_assertions, not(test)))] -#[no_mangle] -unsafe extern "C" fn ts_record_malloc(size: usize) -> *const c_void { - malloc(size) -} - -#[cfg(all(debug_assertions, not(test)))] -#[no_mangle] -unsafe extern "C" fn ts_record_calloc(count: usize, size: usize) -> *const c_void { - calloc(count, size) -} - -#[cfg(all(debug_assertions, not(test)))] -#[no_mangle] -unsafe extern "C" fn ts_record_realloc(ptr: *mut c_void, size: usize) -> *const c_void { - realloc(ptr, size) -} - -#[cfg(all(debug_assertions, not(test)))] -#[no_mangle] -unsafe extern "C" fn ts_record_free(ptr: *mut c_void) { - free(ptr) -} - -#[cfg(all(debug_assertions, not(test)))] -#[no_mangle] -extern "C" fn ts_toggle_allocation_recording(_: bool) -> bool { - false } +/// A raw pointer and a length, exposed as an iterator. pub struct CBufferIter { ptr: *mut T, count: usize, diff --git a/lib/src/alloc.h b/lib/src/alloc.h index 6e22a0ab..dd487ca2 100644 --- a/lib/src/alloc.h +++ b/lib/src/alloc.h @@ -9,7 +9,7 @@ extern "C" { #include #include -#if defined(TREE_SITTER_TEST) +#if defined(TREE_SITTER_ALLOCATION_TRACKING) void *ts_record_malloc(size_t); void *ts_record_calloc(size_t, size_t); diff --git a/script/benchmark b/script/benchmark index 7599e989..4f480868 100755 --- a/script/benchmark +++ b/script/benchmark @@ -48,11 +48,11 @@ done if [[ "${mode}" == "debug" ]]; then test_binary=$( - cargo bench benchmark --no-run --message-format=json 2> /dev/null |\ + cargo bench benchmark -p tree-sitter-cli --no-run --message-format=json 2> /dev/null |\ jq -rs 'map(select(.target.name == "benchmark" and .executable))[0].executable' ) env | grep TREE_SITTER echo $test_binary else - exec cargo bench benchmark + exec cargo bench benchmark -p tree-sitter-cli fi diff --git a/script/benchmark.cmd b/script/benchmark.cmd index f5608d9d..bbb2df5c 100644 --- a/script/benchmark.cmd +++ b/script/benchmark.cmd @@ -1,3 +1,3 @@ @echo off -cargo bench +cargo bench benchmark -p tree-sitter-cli diff --git a/script/test b/script/test index 9b578dcf..dfb29da7 100755 --- a/script/test +++ b/script/test @@ -31,22 +31,12 @@ OPTIONS EOF } -export TREE_SITTER_TEST=1 export RUST_TEST_THREADS=1 export RUST_BACKTRACE=full mode=normal +test_flags="-p tree-sitter-cli" -# Specify a `--target` explicitly. For some reason, this is required for -# address sanitizer support. -toolchain=$(rustup show active-toolchain) -toolchain_regex='(stable|beta|nightly)-([_a-z0-9-]+).*' -if [[ $toolchain =~ $toolchain_regex ]]; then - release=${BASH_REMATCH[1]} - current_target=${BASH_REMATCH[2]} -else - echo "Failed to parse toolchain '${toolchain}'" -fi while getopts "adDghl:e:s:t:" option; do case ${option} in @@ -56,6 +46,17 @@ while getopts "adDghl:e:s:t:" option; do ;; a) export RUSTFLAGS="-Z sanitizer=address" + # Specify a `--target` explicitly. For some reason, this is required for + # address sanitizer support. + toolchain=$(rustup show active-toolchain) + toolchain_regex='(stable|beta|nightly)-([_a-z0-9-]+).*' + if [[ $toolchain =~ $toolchain_regex ]]; then + release=${BASH_REMATCH[1]} + current_target=${BASH_REMATCH[2]} + else + echo "Failed to parse toolchain '${toolchain}'" + fi + test_flags="${test_flags} --target ${current_target}" ;; l) export TREE_SITTER_TEST_LANGUAGE_FILTER=${OPTARG} @@ -95,10 +96,10 @@ fi if [[ "${mode}" == "debug" ]]; then test_binary=$( - cargo test -p tree-sitter-cli --no-run --message-format=json 2> /dev/null |\ + cargo test $test_flags --no-run --message-format=json 2> /dev/null |\ jq -rs 'map(select(.target.name == "tree-sitter-cli" and .executable))[0].executable' ) lldb "${test_binary}" -- $top_level_filter else - cargo test --target=${current_target} -p tree-sitter-cli --jobs 1 $top_level_filter -- --nocapture + cargo test $test_flags --jobs 1 $top_level_filter -- --nocapture fi diff --git a/script/test.cmd b/script/test.cmd index de1f8500..7b988c43 100644 --- a/script/test.cmd +++ b/script/test.cmd @@ -1,7 +1,6 @@ @echo off setlocal -set TREE_SITTER_TEST=1 set RUST_TEST_THREADS=1 set RUST_BACKTRACE=full cargo test -p tree-sitter-cli "%~1" -- --nocapture