Merge pull request #801 from tree-sitter/actions-ci

Switch to GitHub Actions CI.

.github/workflows/ci.yml

@@ -0,0 +1,97 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - "**"
+
+env:
+  CARGO_TERM_COLOR: always
+  CARGO_INCREMENTAL: 0
+  EMSCRIPTEN_VERSION: 2.0.11
+
+jobs:
+  tests:
+    name: Unix tests
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: true
+      matrix:
+        os: [macos-latest, ubuntu-latest]
+        toolchain: [stable]
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v2
+
+      # Work around https://github.com/actions/cache/issues/403.
+      - name: Use GNU tar
+        if: matrix.os == 'macos-latest'
+        run: |
+          echo PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH" >> $GITHUB_ENV
+
+      - name: Cache artifacts
+        id: cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }}-emscripten-${{ env.EMSCRIPTEN_VERSION }}
+
+      - name: Install rust
+        if: steps.cache.outputs.cache-hit != 'true'
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ matrix.toolchain }}
+          profile: minimal
+
+      - name: Install emscripten
+        uses: mymindstorm/setup-emsdk@v7
+        with:
+          version: ${{ env.EMSCRIPTEN_VERSION }}
+
+      - name: Build C library
+        run: make
+
+      - name: Build wasm library
+        run: script/build-wasm
+
+      - name: Build CLI
+        run: cargo build --release
+
+      - name: Set up fixture parsers
+        run: |
+          script/fetch-fixtures
+          script/generate-fixtures
+          script/generate-fixtures-wasm
+
+      - name: Run main tests
+        run: script/test
+
+      - name: Run wasm tests
+        run: script/test-wasm
+
+      - name: Run benchmarks
+        run: script/benchmark
+
+      - name: Compress CLI binary
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: |
+          cp target/release/tree-sitter .
+          gzip --keep --suffix "-${{ runner.os }}-x64.gz" tree-sitter
+
+      - name: Release
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/v')
+        with:
+          draft: true
+          files: |
+            tree-sitter-*.gz
+            lib/binding_web/tree-sitter.js
+            lib/binding_web/tree-sitter.wasm
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.travis.yml

@@ -1,70 +0,0 @@
-language: rust
-rust:
-  - stable
-
-env:
-  CFLAGS="-Wall -Wextra -Werror -Wstrict-prototypes"
-
-matrix:
-  include:
-    - os: osx
-      env: USE_EMSCRIPTEN=1
-    - os: linux
-      services: docker
-
-before_install:
-  # Install node
-  - nvm install 12
-  - nvm use 12
-
-  # Download emscripten and create a shorthand for adding it to the PATH.
-  # Don't add it to the path globally because it overrides the default
-  # clang and node.
-  - if [ -n "$USE_EMSCRIPTEN" ]; then export WASM_ENV="$(script/fetch-emscripten)"; fi
-
-script:
-  # Build the WASM binding
-  - (eval "$WASM_ENV" && script/build-wasm)
-
-  # build the shared/static libraries
-  - make
-
-  # Build the CLI
-  - cargo build --release
-
-  # Fetch and regenerate the fixture parsers
-  - script/fetch-fixtures
-  - script/generate-fixtures
-  - (eval "$WASM_ENV" && script/generate-fixtures-wasm)
-
-  # Run the tests
-  - script/test
-  - script/test-wasm
-  - script/benchmark
-
-branches:
-  only:
-  - master
-  - /\d+\.\d+\.\d+/
-
-before_deploy:
-  - cp target/release/tree-sitter .
-  - gzip --suffix "-${TRAVIS_OS_NAME}-x64.gz" tree-sitter
-
-deploy:
-  provider: releases
-  api_key:
-    secure: "cAd2mQP+Q55v3zedo5ZyOVc3hq3XKMW93lp5LuXV6CYKYbIhkyfym4qfs+C9GJQiIP27cnePYM7B3+OMIFwSPIgXHWWSsuloMtDgYSc/PAwb2dZnJqAyog3BohW/QiGTSnvbVlxPF6P9RMQU6+JP0HJzEJy6QBTa4Und/j0jm24="
-  file_glob: true
-  file:
-    - "tree-sitter-*.gz"
-  draft: true
-  overwrite: true
-  skip_cleanup: true
-  on:
-    tags: true
-
-cache:
-  cargo: true
-  directories:
-    - target/emsdk

cli/Cargo.toml

@@ -42,6 +42,11 @@ html-escape = "0.2.6"
 version = ">= 0.17.0"
 path = "../lib"
 
+[dev-dependencies.tree-sitter]
+version = ">= 0.17.0"
+path = "../lib"
+features = ["allocation-tracking"]
+
 [dependencies.tree-sitter-highlight]
 version = ">= 0.3.0"
 path = "../highlight"

cli/src/allocations_stubs.rs

@@ -0,0 +1,40 @@
+// In all dev builds, the tree-sitter library is built with the `allocation-tracking`
+// feature enabled. This causes the library to link against a set of externally
+// defined C functions like `ts_record_malloc` and `ts_record_free`. In tests, these
+// are defined to actually keep track of outstanding allocations. But when not running
+// tests, the symbols still need to be defined. This file provides pass-through
+// implementations of all of these functions.
+
+use std::os::raw::c_void;
+
+extern "C" {
+    fn malloc(size: usize) -> *mut c_void;
+    fn calloc(count: usize, size: usize) -> *mut c_void;
+    fn realloc(ptr: *mut c_void, size: usize) -> *mut c_void;
+    fn free(ptr: *mut c_void);
+}
+
+#[no_mangle]
+unsafe extern "C" fn ts_record_malloc(size: usize) -> *const c_void {
+    malloc(size)
+}
+
+#[no_mangle]
+unsafe extern "C" fn ts_record_calloc(count: usize, size: usize) -> *const c_void {
+    calloc(count, size)
+}
+
+#[no_mangle]
+unsafe extern "C" fn ts_record_realloc(ptr: *mut c_void, size: usize) -> *const c_void {
+    realloc(ptr, size)
+}
+
+#[no_mangle]
+unsafe extern "C" fn ts_record_free(ptr: *mut c_void) {
+    free(ptr)
+}
+
+#[no_mangle]
+extern "C" fn ts_toggle_allocation_recording(_: bool) -> bool {
+    false
+}

cli/src/lib.rs

@@ -16,3 +16,6 @@ pub mod web_ui;
 
 #[cfg(test)]
 mod tests;
+
+#[cfg(not(test))]
+mod allocations_stubs;

cli/src/tests/helpers/allocations.rs

@@ -4,6 +4,7 @@
 use lazy_static::lazy_static;
 use spin::Mutex;
 use std::collections::HashMap;
+use std::env;
 use std::os::raw::{c_ulong, c_void};
 
 #[derive(Debug, PartialEq, Eq, Hash)]
@@ -31,9 +32,14 @@ extern "C" {
 
 pub fn start_recording() {
     let mut recorder = RECORDER.lock();
-    recorder.enabled = true;
     recorder.allocation_count = 0;
     recorder.outstanding_allocations.clear();
+
+    if env::var("RUST_TEST_THREADS").map_or(false, |s| s == "1") {
+        recorder.enabled = true;
+    } else {
+        panic!("This test must be run with RUST_TEST_THREADS=1. Use script/test.");
+    }
 }
 
 pub fn stop_recording() {

lib/Cargo.toml

@@ -28,3 +28,8 @@ cc = "^1.0.58"
 
 [lib]
 path = "binding_rust/lib.rs"
+
+# This feature is only useful for testing the Tree-sitter library itself.
+# It is exposed because all of Tree-sitter's tests live in the Tree-sitter CLI crate.
+[features]
+allocation-tracking = []

lib/binding_rust/build.rs

@@ -21,9 +21,9 @@ fn main() {
 
     let mut config = cc::Build::new();
 
-    println!("cargo:rerun-if-env-changed=PROFILE");
-    if env::var("PROFILE").map_or(false, |s| s == "debug") {
-        config.define("TREE_SITTER_TEST", "");
+    println!("cargo:rerun-if-env-changed=CARGO_FEATURE_ALLOCATION_TRACKING");
+    if env::var("CARGO_FEATURE_ALLOCATION_TRACKING").is_ok() {
+        config.define("TREE_SITTER_ALLOCATION_TRACKING", "");
     }
 
     let src_path = Path::new("src");

lib/binding_rust/util.rs

@@ -1,71 +1,22 @@
 use std::os::raw::c_void;
 
 extern "C" {
-    /// In *Release* builds, the C library links directly against `malloc` and `free`.
-    ///
-    /// When freeing memory that was allocated by C code, use `free` directly.
-    #[cfg(not(debug_assertions))]
+    /// Normally, use `free(1)` to free memory allocated from C.
+    #[cfg(not(feature = "allocation-tracking"))]
     #[link_name = "free"]
     pub fn free_ptr(ptr: *mut c_void);
 
-    /// In *Test* builds, the C library is compiled with the `TREE_SITTER_TEST` macro,
-    /// so all calls to `malloc`, `free`, etc are linked against wrapper functions
-    /// called `ts_record_malloc`, `ts_record_free`, etc. These symbols are defined
-    /// in the `tree_sitter_cli::tests::helpers::allocations` module.
-    ///
-    /// When freeing memory that was allocated by C code, use the `free` function
-    /// from that module.
-    #[cfg(debug_assertions)]
+    /// When the `allocation-tracking` feature is enabled, the C library is compiled with
+    /// the `TREE_SITTER_TEST` macro, so all calls to `malloc`, `free`, etc are linked
+    /// against wrapper functions called `ts_record_malloc`, `ts_record_free`, etc.
+    /// When freeing buffers allocated from C, use the wrapper `free` function.
+    #[cfg(feature = "allocation-tracking")]
     #[link_name = "ts_record_free"]
     pub fn free_ptr(ptr: *mut c_void);
 
-    /// In *Debug* builds, the C library is compiled the same as in test builds: using
-    /// the wrapper functions. This prevents the C library from having to be recompiled
-    /// constantly when switching between running tests and compiling with RLS.
-    ///
-    /// But we don't want to actually record allocations when running the library in
-    /// debug mode, so we define symbols like `ts_record_malloc` to just delegate to
-    /// the normal `malloc` functions.
-    #[cfg(all(debug_assertions, not(test)))]
-    fn malloc(size: usize) -> *mut c_void;
-    #[cfg(all(debug_assertions, not(test)))]
-    fn calloc(count: usize, size: usize) -> *mut c_void;
-    #[cfg(all(debug_assertions, not(test)))]
-    fn realloc(ptr: *mut c_void, size: usize) -> *mut c_void;
-    #[cfg(all(debug_assertions, not(test)))]
-    fn free(ptr: *mut c_void);
-}
-
-#[cfg(all(debug_assertions, not(test)))]
-#[no_mangle]
-unsafe extern "C" fn ts_record_malloc(size: usize) -> *const c_void {
-    malloc(size)
-}
-
-#[cfg(all(debug_assertions, not(test)))]
-#[no_mangle]
-unsafe extern "C" fn ts_record_calloc(count: usize, size: usize) -> *const c_void {
-    calloc(count, size)
-}
-
-#[cfg(all(debug_assertions, not(test)))]
-#[no_mangle]
-unsafe extern "C" fn ts_record_realloc(ptr: *mut c_void, size: usize) -> *const c_void {
-    realloc(ptr, size)
-}
-
-#[cfg(all(debug_assertions, not(test)))]
-#[no_mangle]
-unsafe extern "C" fn ts_record_free(ptr: *mut c_void) {
-    free(ptr)
-}
-
-#[cfg(all(debug_assertions, not(test)))]
-#[no_mangle]
-extern "C" fn ts_toggle_allocation_recording(_: bool) -> bool {
-    false
 }
 
+/// A raw pointer and a length, exposed as an iterator.
 pub struct CBufferIter<T> {
     ptr: *mut T,
     count: usize,

lib/src/alloc.h

@@ -9,7 +9,7 @@ extern "C" {
 #include <stdbool.h>
 #include <stdio.h>
 
-#if defined(TREE_SITTER_TEST)
+#if defined(TREE_SITTER_ALLOCATION_TRACKING)
 
 void *ts_record_malloc(size_t);
 void *ts_record_calloc(size_t, size_t);

script/benchmark

@@ -48,11 +48,11 @@ done
 
 if [[ "${mode}" == "debug" ]]; then
   test_binary=$(
-    cargo bench benchmark --no-run --message-format=json 2> /dev/null |\
+    cargo bench benchmark -p tree-sitter-cli --no-run --message-format=json 2> /dev/null |\
     jq -rs 'map(select(.target.name == "benchmark" and .executable))[0].executable'
   )
   env | grep TREE_SITTER
   echo $test_binary
 else
-  exec cargo bench benchmark
+  exec cargo bench benchmark -p tree-sitter-cli
 fi

script/benchmark.cmd

@@ -1,3 +1,3 @@
 @echo off
 
-cargo bench
+cargo bench benchmark -p tree-sitter-cli

script/test

@@ -31,22 +31,12 @@ OPTIONS
 EOF
 }
 
-export TREE_SITTER_TEST=1
 export RUST_TEST_THREADS=1
 export RUST_BACKTRACE=full
 
 mode=normal
+test_flags="-p tree-sitter-cli"
 
-# Specify a `--target` explicitly. For some reason, this is required for
-# address sanitizer support.
-toolchain=$(rustup show active-toolchain)
-toolchain_regex='(stable|beta|nightly)-([_a-z0-9-]+).*'
-if [[ $toolchain =~ $toolchain_regex ]]; then
-  release=${BASH_REMATCH[1]}
-  current_target=${BASH_REMATCH[2]}
-else
-  echo "Failed to parse toolchain '${toolchain}'"
-fi
 
 while getopts "adDghl:e:s:t:" option; do
   case ${option} in
@@ -56,6 +46,17 @@ while getopts "adDghl:e:s:t:" option; do
       ;;
     a)
       export RUSTFLAGS="-Z sanitizer=address"
+      # Specify a `--target` explicitly. For some reason, this is required for
+      # address sanitizer support.
+      toolchain=$(rustup show active-toolchain)
+      toolchain_regex='(stable|beta|nightly)-([_a-z0-9-]+).*'
+      if [[ $toolchain =~ $toolchain_regex ]]; then
+        release=${BASH_REMATCH[1]}
+        current_target=${BASH_REMATCH[2]}
+      else
+        echo "Failed to parse toolchain '${toolchain}'"
+      fi
+      test_flags="${test_flags} --target ${current_target}"
       ;;
     l)
       export TREE_SITTER_TEST_LANGUAGE_FILTER=${OPTARG}
@@ -95,10 +96,10 @@ fi
 
 if [[ "${mode}" == "debug" ]]; then
   test_binary=$(
-    cargo test -p tree-sitter-cli --no-run --message-format=json 2> /dev/null |\
+    cargo test $test_flags --no-run --message-format=json 2> /dev/null |\
     jq -rs 'map(select(.target.name == "tree-sitter-cli" and .executable))[0].executable'
   )
   lldb "${test_binary}" -- $top_level_filter
 else
-  cargo test --target=${current_target} -p tree-sitter-cli --jobs 1 $top_level_filter -- --nocapture
+  cargo test $test_flags --jobs 1 $top_level_filter -- --nocapture
 fi

script/test.cmd

@@ -1,7 +1,6 @@
 @echo off
 
 setlocal
-set TREE_SITTER_TEST=1
 set RUST_TEST_THREADS=1
 set RUST_BACKTRACE=full
 cargo test -p tree-sitter-cli "%~1" -- --nocapture