From fb6590b35e6a120da710657df14ba34b0ced330a Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Wed, 11 Nov 2020 08:42:52 -0500 Subject: [PATCH 01/32] Remove Travis and Appveyor. --- .appveyor.yml | 50 ----------------------------------- .travis.yml | 72 --------------------------------------------------- 2 files changed, 122 deletions(-) delete mode 100644 .appveyor.yml delete mode 100644 .travis.yml diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index d463b7a2..00000000 --- a/.appveyor.yml +++ /dev/null @@ -1,50 +0,0 @@ -build: false -install: - # Terminate early unless building either a tag or a PR. - - if "%APPVEYOR_REPO_TAG%" == "false" if not "%APPVEYOR_REPO_BRANCH%" == "master" appveyor exit - - # Install rust - - appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe - - IF "%PLATFORM%" == "x86" rustup-init -y --default-toolchain stable --default-host i686-pc-windows-msvc - - IF "%PLATFORM%" == "x64" rustup-init -y --default-toolchain stable --default-host x86_64-pc-windows-msvc - - set PATH=%PATH%;C:\Users\appveyor\.cargo\bin - - rustc -vV - - cargo -vV - - # Install dependencies - - git submodule update --init - -platform: - - x64 - - x86 - -test_script: - # Fetch and regenerate the fixture parsers - - script\fetch-fixtures.cmd - - cargo build --release - - script\generate-fixtures.cmd - - # Run tests - - script\test.cmd - - script\benchmark.cmd - -before_deploy: - - move target\release\tree-sitter.exe tree-sitter.exe - - 7z a -tgzip tree-sitter-windows-%PLATFORM%.gz tree-sitter.exe - - appveyor PushArtifact tree-sitter-windows-%PLATFORM%.gz - -deploy: - description: '' - provider: GitHub - auth_token: - secure: VC9ntV5+inKoNteZyLQksKzWMKXF46P+Jx3JHKVSfF+o1rWtZn2iIHAVsQv5LaUi - artifact: /tree-sitter-windows-.*/ - draft: true - force_update: true - on: - APPVEYOR_REPO_TAG: true - -cache: - - target - - test\fixtures\grammars - - C:\Users\appveyor\.cargo diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 7205ae03..00000000 --- a/.travis.yml +++ /dev/null @@ -1,72 +0,0 @@ -language: rust -rust: - - stable - -env: - CFLAGS="-Wall -Wextra -Werror -Wstrict-prototypes" - -matrix: - include: - - os: osx - env: USE_EMSCRIPTEN=1 - - os: linux - services: docker - -before_install: - # Install node - - nvm install 10 - - nvm use 10 - - # Download emscripten and create a shorthand for adding it to the PATH. - # Don't add it to the path globally because it overrides the default - # clang and node. - - if [ -n "$USE_EMSCRIPTEN" ]; then export WASM_ENV="$(script/fetch-emscripten)"; fi - -script: - # Build the WASM binding - - (eval "$WASM_ENV" && script/build-wasm) - - # build the shared/static libraries - - make - - # Build the CLI - - cargo build --release - - # Fetch and regenerate the fixture parsers - - script/fetch-fixtures - - script/generate-fixtures - - (eval "$WASM_ENV" && script/generate-fixtures-wasm) - - # Run the tests - - script/test - - script/test-wasm - - script/benchmark - -branches: - only: - - master - - /\d+\.\d+\.\d+/ - -before_deploy: - - cp target/release/tree-sitter . - - gzip --suffix "-${TRAVIS_OS_NAME}-x64.gz" tree-sitter - -deploy: - provider: releases - api_key: - secure: "cAd2mQP+Q55v3zedo5ZyOVc3hq3XKMW93lp5LuXV6CYKYbIhkyfym4qfs+C9GJQiIP27cnePYM7B3+OMIFwSPIgXHWWSsuloMtDgYSc/PAwb2dZnJqAyog3BohW/QiGTSnvbVlxPF6P9RMQU6+JP0HJzEJy6QBTa4Und/j0jm24=" - file_glob: true - file: - - "tree-sitter-*.gz" - - "target/release/tree-sitter.js" - - "target/release/tree-sitter.wasm" - draft: true - overwrite: true - skip_cleanup: true - on: - tags: true - -cache: - cargo: true - directories: - - target/emsdk From 89046608046f78ab4a9292d0a8db3b5ef84eb57d Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Wed, 11 Nov 2020 08:44:14 -0500 Subject: [PATCH 02/32] Institute Actions .yml --- .github/workflows/ci.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..e69de29b From 8b64c12517c5ded1234a2b633e74b47545ab9138 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Wed, 11 Nov 2020 08:49:33 -0500 Subject: [PATCH 03/32] Simple workflow derived from Amethyst's; let's see if it works --- .github/workflows/ci.yml | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e69de29b..a4761e86 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -0,0 +1,38 @@ +name: CI + +on: + push: + branches: + - master + pull_request: + branches: + - "**" + +env: + RUSTFLAGS: -Cdebuginfo=0 + + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: 0 + +jobs: + tests: + name: Tests + runs-on: ${{ matrix.os }} + strategy: + fail-fast: true + matrix: + os: [macos-latest, windows-latest, ubuntu-latest] + toolchain: [nightly] + steps: + - uses: actions/checkout@v2 + + - name: install ${{ matrix.toolchain }} toolchain + id: install_toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ matrix.toolchain }} + profile: minimal + # Don't use a 'components:' entry--we don't need them with beta/nightly, plus nightly often doesn't have them + override: true + + - run: cargo test From 711d4fdace537413c4178904af46ca7e445437e3 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Wed, 11 Nov 2020 09:00:26 -0500 Subject: [PATCH 04/32] make instead of cargo --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4761e86..5f667890 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,4 +35,4 @@ jobs: # Don't use a 'components:' entry--we don't need them with beta/nightly, plus nightly often doesn't have them override: true - - run: cargo test + - run: make test From 07948dfa42022a4a06eef6421abc2d4c78c7d80e Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Wed, 11 Nov 2020 09:05:17 -0500 Subject: [PATCH 05/32] regular make? --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f667890..9cc09e6b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,4 +35,4 @@ jobs: # Don't use a 'components:' entry--we don't need them with beta/nightly, plus nightly often doesn't have them override: true - - run: make test + - run: make From 9ea5e491a9869145369b96cbf2977e5352311261 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Tue, 24 Nov 2020 13:42:59 -0500 Subject: [PATCH 06/32] Actually run all the tests, not just `make`. --- .github/workflows/ci.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9cc09e6b..28fb1230 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,12 +16,12 @@ env: jobs: tests: - name: Tests + name: Unix tests runs-on: ${{ matrix.os }} strategy: fail-fast: true matrix: - os: [macos-latest, windows-latest, ubuntu-latest] + os: [macos-latest, ubuntu-latest] toolchain: [nightly] steps: - uses: actions/checkout@v2 @@ -35,4 +35,13 @@ jobs: # Don't use a 'components:' entry--we don't need them with beta/nightly, plus nightly often doesn't have them override: true - - run: make + - run: | + (eval "$WASM_ENV" && script/build-wasm) + make + cargo build --release + script/fetch-features + script/generate-fixtures + (eval "$WASM_ENV" && script/generate-fixtures-wasm) + script/test + script/test-wasm + script/benchmark From 25c58214006382b719e94aecc2b1a59ba63b392d Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Tue, 24 Nov 2020 13:45:45 -0500 Subject: [PATCH 07/32] Try just Ubuntu. Justbuntu --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28fb1230..a923c0ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: strategy: fail-fast: true matrix: - os: [macos-latest, ubuntu-latest] + os: [ubuntu-latest] toolchain: [nightly] steps: - uses: actions/checkout@v2 From dd78781403f759507090e739c368b9da8fbc0d87 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Tue, 24 Nov 2020 13:49:43 -0500 Subject: [PATCH 08/32] whoops --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a923c0ea..e009c854 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,7 @@ jobs: (eval "$WASM_ENV" && script/build-wasm) make cargo build --release - script/fetch-features + script/fetch-fixtures script/generate-fixtures (eval "$WASM_ENV" && script/generate-fixtures-wasm) script/test From cdb108b7815a10ec6a322a8ab2d108f55467ca49 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Tue, 24 Nov 2020 14:24:25 -0500 Subject: [PATCH 09/32] Try not building wasm because it's making actions very sad --- .github/workflows/ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e009c854..184fda02 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,12 +36,9 @@ jobs: override: true - run: | - (eval "$WASM_ENV" && script/build-wasm) make cargo build --release script/fetch-fixtures script/generate-fixtures - (eval "$WASM_ENV" && script/generate-fixtures-wasm) script/test - script/test-wasm script/benchmark From 3b121894c276a60dfd10634a04315111920001f4 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Tue, 24 Nov 2020 14:38:29 -0500 Subject: [PATCH 10/32] Is it make that's whining? --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 184fda02..3c984f12 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,6 @@ jobs: override: true - run: | - make cargo build --release script/fetch-fixtures script/generate-fixtures From abef4a74177c4ffaa3124b1dca413aa4014a858d Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 1 Dec 2020 17:38:22 -0800 Subject: [PATCH 11/32] Use stable rust, remove custom rustc flags --- .github/workflows/ci.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c984f12..b052ebb2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,8 +9,6 @@ on: - "**" env: - RUSTFLAGS: -Cdebuginfo=0 - CARGO_TERM_COLOR: always CARGO_INCREMENTAL: 0 @@ -22,17 +20,15 @@ jobs: fail-fast: true matrix: os: [ubuntu-latest] - toolchain: [nightly] + toolchain: [stable] steps: - uses: actions/checkout@v2 - - name: install ${{ matrix.toolchain }} toolchain - id: install_toolchain + - name: Install Rust uses: actions-rs/toolchain@v1 with: toolchain: ${{ matrix.toolchain }} profile: minimal - # Don't use a 'components:' entry--we don't need them with beta/nightly, plus nightly often doesn't have them override: true - run: | From 00d25e82985877c8472e094e686d4157fdd5ee6f Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 1 Dec 2020 17:55:56 -0800 Subject: [PATCH 12/32] Use caching in GH actions --- .github/workflows/ci.yml | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b052ebb2..6b3fc567 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,18 +22,38 @@ jobs: os: [ubuntu-latest] toolchain: [stable] steps: - - uses: actions/checkout@v2 + - name: Checkout repo + uses: actions/checkout@v2 - - name: Install Rust + - name: Setup cache + uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }} + + - name: Install rust uses: actions-rs/toolchain@v1 with: toolchain: ${{ matrix.toolchain }} profile: minimal override: true - - run: | - cargo build --release + - name: Build C library + run: make + + - name: Build CLI + run: cargo build --release + + - name: Setup fixture parsers + run: | script/fetch-fixtures script/generate-fixtures - script/test - script/benchmark + + - name: Run tests + run: script/test + + - name: Run benchmarks + run: script/benchmark From b661050a61c1c318acca2309003fc4e5f0905b58 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 2 Dec 2020 13:17:13 -0800 Subject: [PATCH 13/32] Simplify setup for enabling/disabling allocation recording in the C lib --- cli/Cargo.toml | 5 +++ cli/src/allocations_stubs.rs | 40 +++++++++++++++++ cli/src/lib.rs | 3 ++ cli/src/tests/helpers/allocations.rs | 8 +++- lib/Cargo.toml | 5 +++ lib/binding_rust/build.rs | 6 +-- lib/binding_rust/util.rs | 65 ++++------------------------ lib/src/alloc.h | 2 +- script/test | 1 - script/test.cmd | 1 - 10 files changed, 72 insertions(+), 64 deletions(-) create mode 100644 cli/src/allocations_stubs.rs diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 48dbbff7..1748516d 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -41,6 +41,11 @@ webbrowser = "0.5.1" version = ">= 0.17.0" path = "../lib" +[dev-dependencies.tree-sitter] +version = ">= 0.17.0" +path = "../lib" +features = ["allocation-tracking"] + [dependencies.tree-sitter-highlight] version = ">= 0.3.0" path = "../highlight" diff --git a/cli/src/allocations_stubs.rs b/cli/src/allocations_stubs.rs new file mode 100644 index 00000000..b4b6dde1 --- /dev/null +++ b/cli/src/allocations_stubs.rs @@ -0,0 +1,40 @@ +// In all dev builds, the tree-sitter library is built with the `allocation-tracking` +// feature enabled. This causes the library to link against a set of externally +// defined C functions like `ts_record_malloc` and `ts_record_free`. In tests, these +// are defined to actually keep track of outstanding allocations. But when not running +// tests, the symbols still need to be defined. This file provides pass-through +// implementations of all of these functions. + +use std::os::raw::c_void; + +extern "C" { + fn malloc(size: usize) -> *mut c_void; + fn calloc(count: usize, size: usize) -> *mut c_void; + fn realloc(ptr: *mut c_void, size: usize) -> *mut c_void; + fn free(ptr: *mut c_void); +} + +#[no_mangle] +unsafe extern "C" fn ts_record_malloc(size: usize) -> *const c_void { + malloc(size) +} + +#[no_mangle] +unsafe extern "C" fn ts_record_calloc(count: usize, size: usize) -> *const c_void { + calloc(count, size) +} + +#[no_mangle] +unsafe extern "C" fn ts_record_realloc(ptr: *mut c_void, size: usize) -> *const c_void { + realloc(ptr, size) +} + +#[no_mangle] +unsafe extern "C" fn ts_record_free(ptr: *mut c_void) { + free(ptr) +} + +#[no_mangle] +extern "C" fn ts_toggle_allocation_recording(_: bool) -> bool { + false +} diff --git a/cli/src/lib.rs b/cli/src/lib.rs index e00323b7..5b491574 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -16,3 +16,6 @@ pub mod web_ui; #[cfg(test)] mod tests; + +#[cfg(not(test))] +mod allocations_stubs; diff --git a/cli/src/tests/helpers/allocations.rs b/cli/src/tests/helpers/allocations.rs index 2f89c173..0e1cef22 100644 --- a/cli/src/tests/helpers/allocations.rs +++ b/cli/src/tests/helpers/allocations.rs @@ -4,6 +4,7 @@ use lazy_static::lazy_static; use spin::Mutex; use std::collections::HashMap; +use std::env; use std::os::raw::{c_ulong, c_void}; #[derive(Debug, PartialEq, Eq, Hash)] @@ -31,9 +32,14 @@ extern "C" { pub fn start_recording() { let mut recorder = RECORDER.lock(); - recorder.enabled = true; recorder.allocation_count = 0; recorder.outstanding_allocations.clear(); + + if env::var("RUST_TEST_THREADS").map_or(false, |s| s == "1") { + recorder.enabled = true; + } else { + panic!("This test must be run with RUST_TEST_THREADS=1. Use script/test."); + } } pub fn stop_recording() { diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 8f88966f..071846a4 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -28,3 +28,8 @@ cc = "1.0" [lib] path = "binding_rust/lib.rs" + +# This feature is only useful for testing the Tree-sitter library itself. +# It is exposed because all of Tree-sitter's tests live in the Tree-sitter CLI crate. +[features] +allocation-tracking = [] diff --git a/lib/binding_rust/build.rs b/lib/binding_rust/build.rs index 0ec7a4ad..f1fd6bfa 100644 --- a/lib/binding_rust/build.rs +++ b/lib/binding_rust/build.rs @@ -21,9 +21,9 @@ fn main() { let mut config = cc::Build::new(); - println!("cargo:rerun-if-env-changed=PROFILE"); - if env::var("PROFILE").map_or(false, |s| s == "debug") { - config.define("TREE_SITTER_TEST", ""); + println!("cargo:rerun-if-env-changed=CARGO_FEATURE_ALLOCATION_TRACKING"); + if env::var("CARGO_FEATURE_ALLOCATION_TRACKING").is_ok() { + config.define("TREE_SITTER_ALLOCATION_TRACKING", ""); } let src_path = Path::new("src"); diff --git a/lib/binding_rust/util.rs b/lib/binding_rust/util.rs index 1a4ac1b7..4e5efb7e 100644 --- a/lib/binding_rust/util.rs +++ b/lib/binding_rust/util.rs @@ -1,71 +1,22 @@ use std::os::raw::c_void; extern "C" { - /// In *Release* builds, the C library links directly against `malloc` and `free`. - /// - /// When freeing memory that was allocated by C code, use `free` directly. - #[cfg(not(debug_assertions))] + /// Normally, use `free(1)` to free memory allocated from C. + #[cfg(not(feature = "allocation-tracking"))] #[link_name = "free"] pub fn free_ptr(ptr: *mut c_void); - /// In *Test* builds, the C library is compiled with the `TREE_SITTER_TEST` macro, - /// so all calls to `malloc`, `free`, etc are linked against wrapper functions - /// called `ts_record_malloc`, `ts_record_free`, etc. These symbols are defined - /// in the `tree_sitter_cli::tests::helpers::allocations` module. - /// - /// When freeing memory that was allocated by C code, use the `free` function - /// from that module. - #[cfg(debug_assertions)] + /// When the `allocation-tracking` feature is enabled, the C library is compiled with + /// the `TREE_SITTER_TEST` macro, so all calls to `malloc`, `free`, etc are linked + /// against wrapper functions called `ts_record_malloc`, `ts_record_free`, etc. + /// When freeing buffers allocated from C, use the wrapper `free` function. + #[cfg(feature = "allocation-tracking")] #[link_name = "ts_record_free"] pub fn free_ptr(ptr: *mut c_void); - /// In *Debug* builds, the C library is compiled the same as in test builds: using - /// the wrapper functions. This prevents the C library from having to be recompiled - /// constantly when switching between running tests and compiling with RLS. - /// - /// But we don't want to actually record allocations when running the library in - /// debug mode, so we define symbols like `ts_record_malloc` to just delegate to - /// the normal `malloc` functions. - #[cfg(all(debug_assertions, not(test)))] - fn malloc(size: usize) -> *mut c_void; - #[cfg(all(debug_assertions, not(test)))] - fn calloc(count: usize, size: usize) -> *mut c_void; - #[cfg(all(debug_assertions, not(test)))] - fn realloc(ptr: *mut c_void, size: usize) -> *mut c_void; - #[cfg(all(debug_assertions, not(test)))] - fn free(ptr: *mut c_void); -} - -#[cfg(all(debug_assertions, not(test)))] -#[no_mangle] -unsafe extern "C" fn ts_record_malloc(size: usize) -> *const c_void { - malloc(size) -} - -#[cfg(all(debug_assertions, not(test)))] -#[no_mangle] -unsafe extern "C" fn ts_record_calloc(count: usize, size: usize) -> *const c_void { - calloc(count, size) -} - -#[cfg(all(debug_assertions, not(test)))] -#[no_mangle] -unsafe extern "C" fn ts_record_realloc(ptr: *mut c_void, size: usize) -> *const c_void { - realloc(ptr, size) -} - -#[cfg(all(debug_assertions, not(test)))] -#[no_mangle] -unsafe extern "C" fn ts_record_free(ptr: *mut c_void) { - free(ptr) -} - -#[cfg(all(debug_assertions, not(test)))] -#[no_mangle] -extern "C" fn ts_toggle_allocation_recording(_: bool) -> bool { - false } +/// A raw pointer and a length, exposed as an iterator. pub struct CBufferIter { ptr: *mut T, count: usize, diff --git a/lib/src/alloc.h b/lib/src/alloc.h index 6e22a0ab..dd487ca2 100644 --- a/lib/src/alloc.h +++ b/lib/src/alloc.h @@ -9,7 +9,7 @@ extern "C" { #include #include -#if defined(TREE_SITTER_TEST) +#if defined(TREE_SITTER_ALLOCATION_TRACKING) void *ts_record_malloc(size_t); void *ts_record_calloc(size_t, size_t); diff --git a/script/test b/script/test index 9b578dcf..b7f1a5b4 100755 --- a/script/test +++ b/script/test @@ -31,7 +31,6 @@ OPTIONS EOF } -export TREE_SITTER_TEST=1 export RUST_TEST_THREADS=1 export RUST_BACKTRACE=full diff --git a/script/test.cmd b/script/test.cmd index de1f8500..7b988c43 100644 --- a/script/test.cmd +++ b/script/test.cmd @@ -1,7 +1,6 @@ @echo off setlocal -set TREE_SITTER_TEST=1 set RUST_TEST_THREADS=1 set RUST_BACKTRACE=full cargo test -p tree-sitter-cli "%~1" -- --nocapture From 6c8a928253768488d247ac65c0179783d69a0172 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 2 Dec 2020 15:35:28 -0800 Subject: [PATCH 14/32] Avoid always specifying a --target flag in test script --- script/test | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/script/test b/script/test index b7f1a5b4..dfb29da7 100755 --- a/script/test +++ b/script/test @@ -35,17 +35,8 @@ export RUST_TEST_THREADS=1 export RUST_BACKTRACE=full mode=normal +test_flags="-p tree-sitter-cli" -# Specify a `--target` explicitly. For some reason, this is required for -# address sanitizer support. -toolchain=$(rustup show active-toolchain) -toolchain_regex='(stable|beta|nightly)-([_a-z0-9-]+).*' -if [[ $toolchain =~ $toolchain_regex ]]; then - release=${BASH_REMATCH[1]} - current_target=${BASH_REMATCH[2]} -else - echo "Failed to parse toolchain '${toolchain}'" -fi while getopts "adDghl:e:s:t:" option; do case ${option} in @@ -55,6 +46,17 @@ while getopts "adDghl:e:s:t:" option; do ;; a) export RUSTFLAGS="-Z sanitizer=address" + # Specify a `--target` explicitly. For some reason, this is required for + # address sanitizer support. + toolchain=$(rustup show active-toolchain) + toolchain_regex='(stable|beta|nightly)-([_a-z0-9-]+).*' + if [[ $toolchain =~ $toolchain_regex ]]; then + release=${BASH_REMATCH[1]} + current_target=${BASH_REMATCH[2]} + else + echo "Failed to parse toolchain '${toolchain}'" + fi + test_flags="${test_flags} --target ${current_target}" ;; l) export TREE_SITTER_TEST_LANGUAGE_FILTER=${OPTARG} @@ -94,10 +96,10 @@ fi if [[ "${mode}" == "debug" ]]; then test_binary=$( - cargo test -p tree-sitter-cli --no-run --message-format=json 2> /dev/null |\ + cargo test $test_flags --no-run --message-format=json 2> /dev/null |\ jq -rs 'map(select(.target.name == "tree-sitter-cli" and .executable))[0].executable' ) lldb "${test_binary}" -- $top_level_filter else - cargo test --target=${current_target} -p tree-sitter-cli --jobs 1 $top_level_filter -- --nocapture + cargo test $test_flags --jobs 1 $top_level_filter -- --nocapture fi From 36cb83aa054991ae2187d677e6f7efafb779cc53 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 2 Dec 2020 16:12:01 -0800 Subject: [PATCH 15/32] Update JS error recovery test to reflect grammar change --- test/fixtures/error_corpus/javascript_errors.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/fixtures/error_corpus/javascript_errors.txt b/test/fixtures/error_corpus/javascript_errors.txt index ad71037c..4359ae68 100644 --- a/test/fixtures/error_corpus/javascript_errors.txt +++ b/test/fixtures/error_corpus/javascript_errors.txt @@ -148,7 +148,8 @@ const h = `i ${j(k} l` (lexical_declaration (variable_declarator (identifier) - (template_string (template_substitution (identifier) (ERROR))))) + (template_string (template_substitution + (augmented_assignment_expression (identifier) (MISSING identifier)))))) (lexical_declaration (variable_declarator (identifier) From 9dd35e7853e498c9681188374ad13c257a4e5ca1 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 2 Dec 2020 16:27:37 -0800 Subject: [PATCH 16/32] Build wasm binding, run wasm tests on GH actions --- .github/workflows/ci.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6b3fc567..5cd1eb46 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: - name: Checkout repo uses: actions/checkout@v2 - - name: Setup cache + - name: Cache artifacts uses: actions/cache@v2 with: path: | @@ -44,16 +44,23 @@ jobs: - name: Build C library run: make + - name: Build wasm library + run: script/build-wasm + - name: Build CLI run: cargo build --release - - name: Setup fixture parsers + - name: Set up fixture parsers run: | script/fetch-fixtures script/generate-fixtures + script/generate-fixtures-wasm - - name: Run tests + - name: Run main tests run: script/test + - name: Run wasm tests + run: script/test-wasm + - name: Run benchmarks run: script/benchmark From ea8a2dcb8ee7a2f40c29e0b42a78532ee88871df Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 5 Jan 2021 12:21:51 -0800 Subject: [PATCH 17/32] Add macOS build to GH actions CI --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5cd1eb46..02336bfe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: true matrix: - os: [ubuntu-latest] + os: [macos-latest, ubuntu-latest] toolchain: [stable] steps: - name: Checkout repo @@ -41,6 +41,10 @@ jobs: profile: minimal override: true + - name: Install emscripten + run: eval $(script/fetch-emscripten) + if: matrix.os == 'macos-latest' + - name: Build C library run: make From f94d67b42a8c2f9b7191b98653a6039290630ed3 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 11 Jan 2021 14:21:52 -0800 Subject: [PATCH 18/32] Use a publisehd action for setting up emscripten on CI --- .github/workflows/ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02336bfe..2baee151 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,7 @@ jobs: path: | ~/.cargo/registry ~/.cargo/git + emsdk-cache target key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }} @@ -42,8 +43,10 @@ jobs: override: true - name: Install emscripten - run: eval $(script/fetch-emscripten) - if: matrix.os == 'macos-latest' + uses: mymindstorm/setup-emsdk@v7 + with: + version: 2.0.11 + actions-cache-folder: emsdk-cache - name: Build C library run: make From c1660d201d2547ae18f9a4512412879d238a86c3 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 11 Jan 2021 14:37:42 -0800 Subject: [PATCH 19/32] Don't cache emscripten stuff for now --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2baee151..66d90968 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,6 @@ jobs: path: | ~/.cargo/registry ~/.cargo/git - emsdk-cache target key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }} @@ -46,7 +45,6 @@ jobs: uses: mymindstorm/setup-emsdk@v7 with: version: 2.0.11 - actions-cache-folder: emsdk-cache - name: Build C library run: make From 2f5a6598faf7e6c4d454b2763b015261c86a6d58 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 11 Jan 2021 14:59:03 -0800 Subject: [PATCH 20/32] Try again with caching for emscripten --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 66d90968..f5af6f7a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,7 @@ jobs: path: | ~/.cargo/registry ~/.cargo/git + emsdk-cache-${{ runner.os }} target key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }} @@ -45,6 +46,7 @@ jobs: uses: mymindstorm/setup-emsdk@v7 with: version: 2.0.11 + actions-cache-folder: emsdk-cache-${{ runner.os }} - name: Build C library run: make From ffef8f53f6e01b728b536ca4c87e7124f4d252e6 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 11 Jan 2021 15:46:17 -0800 Subject: [PATCH 21/32] Publish assets to GH releases from CI on tagged builds --- .github/workflows/ci.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5af6f7a..ddab927b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,3 +71,21 @@ jobs: - name: Run benchmarks run: script/benchmark + + - name: Compress CLI binary + if: startsWith(github.ref, 'refs/tags/v') + run: | + cp target/release/tree-sitter . + gzip --keep --suffix "-${{ runner.os }}-x64.gz" tree-sitter + + - name: Release + uses: softprops/action-gh-release@v1 + if: startsWith(github.ref, 'refs/tags/v') + with: + draft: true + files: | + tree-sitter-*.gz + lib/binding_web/tree-sitter.js + lib/binding_web/tree-sitter.wasm + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 4bedff9c7513f32cda98542cc9e562188f4c4a74 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 11 Jan 2021 16:03:54 -0800 Subject: [PATCH 22/32] Include emscripten version in cache key --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ddab927b..54c6d43d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,7 @@ on: env: CARGO_TERM_COLOR: always CARGO_INCREMENTAL: 0 + EMSCRIPTEN_VERSION: 2.0.11 jobs: tests: @@ -33,9 +34,10 @@ jobs: ~/.cargo/git emsdk-cache-${{ runner.os }} target - key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }} + key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }}-emscripten-${{ env.EMSCRIPTEN_VERSION }} - name: Install rust + if: steps.cache.outputs.cache-hit != 'true' uses: actions-rs/toolchain@v1 with: toolchain: ${{ matrix.toolchain }} @@ -45,7 +47,7 @@ jobs: - name: Install emscripten uses: mymindstorm/setup-emsdk@v7 with: - version: 2.0.11 + version: ${{ env.EMSCRIPTEN_VERSION }} actions-cache-folder: emsdk-cache-${{ runner.os }} - name: Build C library From 94eeb0662fe17f9c32a5e1709f8f9397cde8475f Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 11 Jan 2021 16:13:43 -0800 Subject: [PATCH 23/32] Remove emscripten caching again --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54c6d43d..5f43d8aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,7 +32,6 @@ jobs: path: | ~/.cargo/registry ~/.cargo/git - emsdk-cache-${{ runner.os }} target key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }}-emscripten-${{ env.EMSCRIPTEN_VERSION }} @@ -48,7 +47,6 @@ jobs: uses: mymindstorm/setup-emsdk@v7 with: version: ${{ env.EMSCRIPTEN_VERSION }} - actions-cache-folder: emsdk-cache-${{ runner.os }} - name: Build C library run: make From 03ff52a31a072cab7bfb0ff34d3096baa82c1498 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 11 Jan 2021 16:38:19 -0800 Subject: [PATCH 24/32] Noop to run CI again and test caching From d6ad7484a6cca09f4cba9fbb6a830a379220275c Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 11 Jan 2021 16:59:05 -0800 Subject: [PATCH 25/32] Fix install-rust step in GH CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f43d8aa..a6ffd2fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,7 @@ jobs: uses: actions/checkout@v2 - name: Cache artifacts + id: cache uses: actions/cache@v2 with: path: | @@ -41,7 +42,6 @@ jobs: with: toolchain: ${{ matrix.toolchain }} profile: minimal - override: true - name: Install emscripten uses: mymindstorm/setup-emsdk@v7 From 8050c4026ce121b05878a90489e411fa6cc7adae Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 5 Feb 2021 09:44:29 -0800 Subject: [PATCH 26/32] ci: Use gnu tar on macOS --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a6ffd2fb..3fe315ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,6 +26,12 @@ jobs: - name: Checkout repo uses: actions/checkout@v2 + # Work around https://github.com/actions/cache/issues/403. + - name: Use GNU tar + if: matrix.os == 'macos-latest' + run: | + echo PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH" >> $GITHUB_ENV + - name: Cache artifacts id: cache uses: actions/cache@v2 From 82dad76b98519e56685e73b1dc3d2cdc86a074d1 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 5 Feb 2021 09:44:32 -0800 Subject: [PATCH 27/32] Revert "Remove emscripten caching again" This reverts commit 94eeb0662fe17f9c32a5e1709f8f9397cde8475f. --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3fe315ba..e884f816 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,7 @@ jobs: path: | ~/.cargo/registry ~/.cargo/git + emsdk-cache-${{ runner.os }} target key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }}-emscripten-${{ env.EMSCRIPTEN_VERSION }} @@ -53,6 +54,7 @@ jobs: uses: mymindstorm/setup-emsdk@v7 with: version: ${{ env.EMSCRIPTEN_VERSION }} + actions-cache-folder: emsdk-cache-${{ runner.os }} - name: Build C library run: make From 4c931784880d07748e17631d7bb76ac6d9edc26a Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 5 Feb 2021 10:27:18 -0800 Subject: [PATCH 28/32] Revert "Revert "Remove emscripten caching again"" This reverts commit 82dad76b98519e56685e73b1dc3d2cdc86a074d1. --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e884f816..3fe315ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,6 @@ jobs: path: | ~/.cargo/registry ~/.cargo/git - emsdk-cache-${{ runner.os }} target key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }}-emscripten-${{ env.EMSCRIPTEN_VERSION }} @@ -54,7 +53,6 @@ jobs: uses: mymindstorm/setup-emsdk@v7 with: version: ${{ env.EMSCRIPTEN_VERSION }} - actions-cache-folder: emsdk-cache-${{ runner.os }} - name: Build C library run: make From 9c480bee2369138d31cfa1032d28e505ebd2bafc Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 5 Feb 2021 10:28:46 -0800 Subject: [PATCH 29/32] Print out tar version in CI --- .github/workflows/ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3fe315ba..1281dcf7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,6 +32,14 @@ jobs: run: | echo PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH" >> $GITHUB_ENV + - name: Check GNU tar + if: matrix.os == 'macos-latest' + run: | + echo "gtar:" + gtar --version + echo "tar:" + tar --version + - name: Cache artifacts id: cache uses: actions/cache@v2 From cd2c571f7290c8d3068bb8b51a09273ff51c7198 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 5 Feb 2021 11:09:33 -0800 Subject: [PATCH 30/32] Restore the appveyor config file --- .appveyor.yml | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .appveyor.yml diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 00000000..d463b7a2 --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,50 @@ +build: false +install: + # Terminate early unless building either a tag or a PR. + - if "%APPVEYOR_REPO_TAG%" == "false" if not "%APPVEYOR_REPO_BRANCH%" == "master" appveyor exit + + # Install rust + - appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe + - IF "%PLATFORM%" == "x86" rustup-init -y --default-toolchain stable --default-host i686-pc-windows-msvc + - IF "%PLATFORM%" == "x64" rustup-init -y --default-toolchain stable --default-host x86_64-pc-windows-msvc + - set PATH=%PATH%;C:\Users\appveyor\.cargo\bin + - rustc -vV + - cargo -vV + + # Install dependencies + - git submodule update --init + +platform: + - x64 + - x86 + +test_script: + # Fetch and regenerate the fixture parsers + - script\fetch-fixtures.cmd + - cargo build --release + - script\generate-fixtures.cmd + + # Run tests + - script\test.cmd + - script\benchmark.cmd + +before_deploy: + - move target\release\tree-sitter.exe tree-sitter.exe + - 7z a -tgzip tree-sitter-windows-%PLATFORM%.gz tree-sitter.exe + - appveyor PushArtifact tree-sitter-windows-%PLATFORM%.gz + +deploy: + description: '' + provider: GitHub + auth_token: + secure: VC9ntV5+inKoNteZyLQksKzWMKXF46P+Jx3JHKVSfF+o1rWtZn2iIHAVsQv5LaUi + artifact: /tree-sitter-windows-.*/ + draft: true + force_update: true + on: + APPVEYOR_REPO_TAG: true + +cache: + - target + - test\fixtures\grammars + - C:\Users\appveyor\.cargo From 9857438d9b91cb694f7f9b8f3a245824b693a4f5 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 5 Feb 2021 11:26:40 -0800 Subject: [PATCH 31/32] Revert "Print out tar version in CI" This reverts commit 9c480bee2369138d31cfa1032d28e505ebd2bafc. --- .github/workflows/ci.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1281dcf7..3fe315ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,14 +32,6 @@ jobs: run: | echo PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH" >> $GITHUB_ENV - - name: Check GNU tar - if: matrix.os == 'macos-latest' - run: | - echo "gtar:" - gtar --version - echo "tar:" - tar --version - - name: Cache artifacts id: cache uses: actions/cache@v2 From 242ad90770568646e853fd85c997671f5bebb8fb Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 5 Feb 2021 11:55:54 -0800 Subject: [PATCH 32/32] Only build the CLI crate when running benchmarks --- script/benchmark | 4 ++-- script/benchmark.cmd | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/script/benchmark b/script/benchmark index 7599e989..4f480868 100755 --- a/script/benchmark +++ b/script/benchmark @@ -48,11 +48,11 @@ done if [[ "${mode}" == "debug" ]]; then test_binary=$( - cargo bench benchmark --no-run --message-format=json 2> /dev/null |\ + cargo bench benchmark -p tree-sitter-cli --no-run --message-format=json 2> /dev/null |\ jq -rs 'map(select(.target.name == "benchmark" and .executable))[0].executable' ) env | grep TREE_SITTER echo $test_binary else - exec cargo bench benchmark + exec cargo bench benchmark -p tree-sitter-cli fi diff --git a/script/benchmark.cmd b/script/benchmark.cmd index f5608d9d..bbb2df5c 100644 --- a/script/benchmark.cmd +++ b/script/benchmark.cmd @@ -1,3 +1,3 @@ @echo off -cargo bench +cargo bench benchmark -p tree-sitter-cli