ci: major overhaul

- Simplify some workflow steps and auxiliary scripts
- Build library using cmake when not cross-compiling
- Try to fetch fixtures from cache first
- Use `actions-rust-lang/setup-rust-toolchain`
This commit is contained in:
ObserverOfTime 2024-10-11 20:10:24 +03:00 committed by Amaan Qureshi
parent 7715001692
commit e8e56255bd
11 changed files with 268 additions and 255 deletions

View file

@ -1,26 +1,29 @@
name: backport
name: Backport Pull Request
on:
pull_request_target:
types: [closed, labeled]
permissions:
contents: write
pull-requests: write
jobs:
backport:
permissions:
contents: write
pull-requests: write
name: Backport Pull Request
if: github.event.pull_request.merged
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Checkout repository
uses: actions/checkout@v4
- uses: actions/create-github-app-token@v1
- name: Create app token
uses: actions/create-github-app-token@v1
id: app-token
with:
app-id: ${{ vars.BACKPORT_APP }}
private-key: ${{ secrets.BACKPORT_KEY }}
- name: Create backport PR
id: backport
uses: korthout/backport-action@v3
with:
pull_title: "${pull_title}"

View file

@ -8,7 +8,7 @@ env:
on:
workflow_call:
inputs:
run_test:
run-test:
default: true
type: boolean
@ -21,99 +21,93 @@ jobs:
fail-fast: false
matrix:
platform:
- linux-arm64 #
- linux-arm #
- linux-x64 #
- linux-x86 #
- linux-powerpc64 #
- windows-arm64 #
- windows-x64 # <-- No C library build - requires an additional adapted Makefile for `cl.exe` compiler
- windows-x86 # -- // --
- macos-arm64 #
- macos-x64 #
- linux-arm64
- linux-arm
- linux-x64
- linux-x86
- linux-powerpc64
- windows-arm64
- windows-x64
- windows-x86
- macos-arm64
- macos-x64
include:
# When adding a new `target`:
# 1. Define a new platform alias above
# 2. Add a new record to a matrix map in `cli/npm/install.js`
- { platform: linux-arm64 , target: aarch64-unknown-linux-gnu , os: ubuntu-latest , use-cross: true }
- { platform: linux-arm , target: arm-unknown-linux-gnueabi , os: ubuntu-latest , use-cross: true }
- { platform: linux-x64 , target: x86_64-unknown-linux-gnu , os: ubuntu-20.04 , cli_features: wasm } #2272
- { platform: linux-x86 , target: i686-unknown-linux-gnu , os: ubuntu-latest , use-cross: true }
- { platform: linux-powerpc64 , target: powerpc64-unknown-linux-gnu , os: ubuntu-latest , use-cross: true }
- { platform: windows-arm64 , target: aarch64-pc-windows-msvc , os: windows-latest }
- { platform: windows-x64 , target: x86_64-pc-windows-msvc , os: windows-latest , cli_features: wasm }
- { platform: windows-x86 , target: i686-pc-windows-msvc , os: windows-latest }
- { platform: macos-arm64 , target: aarch64-apple-darwin , os: macos-14 , cli_features: wasm }
- { platform: macos-x64 , target: x86_64-apple-darwin , os: macos-12 , cli_features: wasm }
# When adding a new `target`:
# 1. Define a new platform alias above
# 2. Add a new record to the matrix map in `cli/npm/install.js`
- { platform: linux-arm64 , target: aarch64-unknown-linux-gnu , os: ubuntu-latest , use-cross: true }
- { platform: linux-arm , target: arm-unknown-linux-gnueabi , os: ubuntu-latest , use-cross: true }
- { platform: linux-x64 , target: x86_64-unknown-linux-gnu , os: ubuntu-20.04 , features: wasm } # See #2272
- { platform: linux-x86 , target: i686-unknown-linux-gnu , os: ubuntu-latest , use-cross: true }
- { platform: linux-powerpc64 , target: powerpc64-unknown-linux-gnu , os: ubuntu-latest , use-cross: true }
- { platform: windows-arm64 , target: aarch64-pc-windows-msvc , os: windows-latest }
- { platform: windows-x64 , target: x86_64-pc-windows-msvc , os: windows-latest , features: wasm }
- { platform: windows-x86 , target: i686-pc-windows-msvc , os: windows-latest }
- { platform: macos-arm64 , target: aarch64-apple-darwin , os: macos-14 , features: wasm }
- { platform: macos-x64 , target: x86_64-apple-darwin , os: macos-12 , features: wasm }
# Cross compilers for C library
- { platform: linux-arm64 , cc: aarch64-linux-gnu-gcc , ar: aarch64-linux-gnu-ar }
- { platform: linux-arm , cc: arm-linux-gnueabi-gcc , ar: arm-linux-gnueabi-ar }
- { platform: linux-x86 , cc: i686-linux-gnu-gcc , ar: i686-linux-gnu-ar }
- { platform: linux-powerpc64 , cc: powerpc64-linux-gnu-gcc , ar: powerpc64-linux-gnu-ar }
# Cross compilers for C library
- { platform: linux-arm64 , cc: aarch64-linux-gnu-gcc , ar: aarch64-linux-gnu-ar }
- { platform: linux-arm , cc: arm-linux-gnueabi-gcc , ar: arm-linux-gnueabi-ar }
- { platform: linux-x86 , cc: i686-linux-gnu-gcc , ar: i686-linux-gnu-ar }
- { platform: linux-powerpc64 , cc: powerpc64-linux-gnu-gcc , ar: powerpc64-linux-gnu-ar }
# See #2041 tree-sitter issue
- { platform: windows-x64 , rust-test-threads: 1 }
- { platform: windows-x86 , rust-test-threads: 1 }
# Prevent race condition (see #2041)
- { platform: windows-x64 , rust-test-threads: 1 }
- { platform: windows-x86 , rust-test-threads: 1 }
# CLI only build
- { platform: windows-arm64 , cli-only: true }
# Can't natively run CLI on Github runner's host
- { platform: windows-arm64 , no-run: true }
env:
BUILD_CMD: cargo
EXE: ${{ contains(matrix.target, 'windows') && '.exe' || '' }}
SUFFIX: ${{ contains(matrix.target, 'windows') && '.exe' || '' }}
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v4
- name: Checkout repository
uses: actions/checkout@v4
- name: Read Emscripten version
run: echo "EMSCRIPTEN_VERSION=$(cat cli/loader/emscripten-version)" >> $GITHUB_ENV
run: printf 'EMSCRIPTEN_VERSION=%s\n' "$(<cli/loader/emscripten-version)" >> $GITHUB_ENV
- name: Install Emscripten
if: ${{ !matrix.cli-only && !matrix.use-cross }}
if: ${{ !matrix.no-run && !matrix.use-cross }}
uses: mymindstorm/setup-emsdk@v14
with:
version: ${{ env.EMSCRIPTEN_VERSION }}
- run: rustup toolchain install stable --profile minimal
- run: rustup target add ${{ matrix.target }}
- uses: Swatinem/rust-cache@v2
- name: Set up Rust
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
target: ${{ matrix.target }}
- name: Install cross
if: ${{ matrix.use-cross }}
run: cargo install cross --git https://github.com/cross-rs/cross
- name: Build custom cross image
- name: Configure cross
if: ${{ matrix.use-cross }}
run: |
target="${{ matrix.target }}"
image=ghcr.io/cross-rs/$target:custom
printf '%s\n' > Cross.toml \
'[target.${{ matrix.target }}]' \
'image = "ghcr.io/cross-rs/${{ matrix.target }}:edge"' \
'[build]' \
'pre-build = [' \
' "dpkg --add-architecture $CROSS_DEB_ARCH",' \
' "curl -fsSL https://deb.nodesource.com/setup_22.x | bash -",' \
' "apt-get update && apt-get -y install libssl-dev nodejs"' \
']'
cat - Cross.toml <<< 'Cross.toml:'
printf '%s\n' >> $GITHUB_ENV \
"CROSS_CONFIG=$PWD/Cross.toml" \
"CROSS_IMAGE=ghcr.io/cross-rs/${{ matrix.target }}:edge"
echo "[target.$target]" >> Cross.toml
echo "image = \"$image\"" >> Cross.toml
echo "[build]" >> Cross.toml
echo "pre-build = [" >> Cross.toml
echo " \"dpkg --add-architecture \$CROSS_DEB_ARCH\"," >> Cross.toml
echo " \"apt-get update && apt-get -y install libssl-dev\"" >> Cross.toml
echo "]" >> Cross.toml
echo "Cross.toml:"
cat Cross.toml
echo "CROSS_IMAGE=$image" >> $GITHUB_ENV
echo "CROSS_CONFIG=$PWD/Cross.toml" >> $GITHUB_ENV
echo "FROM ghcr.io/cross-rs/$target:edge" >> Dockerfile
echo "RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash -" >> Dockerfile
echo "RUN apt-get update && apt-get -y install nodejs" >> Dockerfile
docker build -t $image .
- name: Setup env extras
- name: Set up environment
env:
RUST_TEST_THREADS: ${{ matrix.rust-test-threads }}
USE_CROSS: ${{ matrix.use-cross }}
@ -122,68 +116,93 @@ jobs:
AR: ${{ matrix.ar }}
run: |
PATH="$PWD/.github/scripts:$PATH"
echo "$PWD/.github/scripts" >> $GITHUB_PATH
printf '%s/.github/scripts\n' "$PWD" >> $GITHUB_PATH
echo "TREE_SITTER=tree-sitter.sh" >> $GITHUB_ENV
echo "TARGET=$TARGET" >> $GITHUB_ENV
echo "ROOT=$PWD" >> $GITHUB_ENV
printf '%s\n' >> $GITHUB_ENV \
'TREE_SITTER=tree-sitter.sh' \
"TARGET=$TARGET" \
"ROOT=$PWD"
[ -n "$RUST_TEST_THREADS" ] && \
echo "RUST_TEST_THREADS=$RUST_TEST_THREADS" >> $GITHUB_ENV
[[ -n $RUST_TEST_THREADS ]] && \
printf 'RUST_TEST_THREADS=%s\n' "$RUST_TEST_THREADS" >> $GITHUB_ENV
[ -n "$CC" ] && echo "CC=$CC" >> $GITHUB_ENV
[ -n "$AR" ] && echo "AR=$AR" >> $GITHUB_ENV
[[ -n $CC ]] && printf 'CC=%s\n' "$CC" >> $GITHUB_ENV
[[ -n $AR ]] && printf 'AR=%s\n' "$AR" >> $GITHUB_ENV
if [ "$USE_CROSS" == "true" ]; then
echo "BUILD_CMD=cross" >> $GITHUB_ENV
runner=$(BUILD_CMD=cross cross.sh bash -c "env | sed -nr '/^CARGO_TARGET_.*_RUNNER=/s///p'")
[ -n "$runner" ] && echo "CROSS_RUNNER=$runner" >> $GITHUB_ENV
if [[ $USE_CROSS == true ]]; then
printf 'BUILD_CMD=cross\n' >> $GITHUB_ENV
runner=$(cross.sh bash -c "env | sed -n 's/^CARGO_TARGET_.*_RUNNER=//p'")
[[ -n $runner ]] && printf 'CROSS_RUNNER=%s\n' "$runner" >> $GITHUB_ENV
fi
- name: Build C library
if: ${{ !contains(matrix.os, 'windows') }} # Requires an additional adapted Makefile for `cl.exe` compiler
run: make.sh -j CFLAGS="-Werror"
- name: Build C library (make)
if: ${{ runner.os != 'Windows' }}
run: make.sh -j CFLAGS="$CFLAGS"
env:
CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wno-unused-parameter -pedantic
- name: Build C library (CMake)
if: ${{ !matrix.use-cross }}
run: |-
cmake -S lib -B build \
-DBUILD_SHARED_LIBS=OFF \
-DCMAKE_BUILD_TYPE=Debug \
-DCMAKE_COMPILE_WARNING_AS_ERROR=ON
cmake --build build --verbose
cmake -S lib -B build \
-DBUILD_SHARED_LIBS=ON \
-DCMAKE_BUILD_TYPE=Debug \
-DCMAKE_COMPILE_WARNING_AS_ERROR=ON
cmake --build build --verbose
- name: Build wasm library
if: ${{ !matrix.cli-only && !matrix.use-cross }} # No sense to build on the same Github runner hosts many times
run: $BUILD_CMD run --package xtask -- build-wasm
# No reason to build on the same Github runner hosts many times
if: ${{ !matrix.no-run && !matrix.use-cross }}
run: $BUILD_CMD run -p xtask -- build-wasm
- run: $BUILD_CMD build --release --target=${{ matrix.target }} --features=${{ matrix.cli_features }}
- name: Build target
run: $BUILD_CMD build --release --target=${{ matrix.target }} --features=${{ matrix.features }}
- run: $BUILD_CMD run --package xtask -- fetch-fixtures
- uses: ./.github/actions/cache
- name: Cache fixtures
id: cache
if: ${{ !matrix.no-run && inputs.run-test }}
uses: ./.github/actions/cache
- name: Fetch fixtures
if: ${{ !matrix.no-run && inputs.run-test }}
run: $BUILD_CMD run -p xtask -- fetch-fixtures
- name: Generate fixtures
if: ${{ !matrix.cli-only && inputs.run_test && steps.cache.outputs.cache-hit != 'true' }} # Can't natively run CLI on Github runner's host
run: $BUILD_CMD run --package xtask -- generate-fixtures
if: ${{ !matrix.no-run && inputs.run-test && steps.cache.outputs.cache-hit != 'true' }}
run: $BUILD_CMD run -p xtask -- generate-fixtures
- name: Generate WASM fixtures
if: ${{ !matrix.cli-only && !matrix.use-cross && inputs.run_test && steps.cache.outputs.cache-hit != 'true' }} # See comment for the "Build wasm library" step
run: $BUILD_CMD run --package xtask -- generate-fixtures --wasm
- name: Generate Wasm fixtures
if: ${{ !matrix.no-run && !matrix.use-cross && inputs.run-test && steps.cache.outputs.cache-hit != 'true' }}
run: $BUILD_CMD run -p xtask -- generate-fixtures --wasm
- name: Run main tests
if: ${{ !matrix.cli-only && inputs.run_test }} # Can't natively run CLI on Github runner's host
run: $BUILD_CMD test --target=${{ matrix.target }} --features=${{ matrix.cli_features }}
if: ${{ !matrix.no-run && inputs.run-test }}
run: $BUILD_CMD test --target=${{ matrix.target }} --features=${{ matrix.features }}
- name: Run wasm tests
if: ${{ !matrix.cli-only && !matrix.use-cross && inputs.run_test }} # See comment for the "Build wasm library" step
run: $BUILD_CMD run --package xtask -- test-wasm
if: ${{ !matrix.no-run && !matrix.use-cross && inputs.run-test }}
run: $BUILD_CMD run -p xtask -- test-wasm
- name: Run benchmarks
if: ${{ !matrix.cli-only && !matrix.use-cross && inputs.run_test }} # Cross-compiled benchmarks make no sense
# Cross-compiled benchmarks are pointless
if: ${{ !matrix.no-run && !matrix.use-cross && inputs.run-test }}
run: $BUILD_CMD bench benchmark -p tree-sitter-cli --target=${{ matrix.target }}
- name: Upload CLI artifact
uses: actions/upload-artifact@v4
with:
name: tree-sitter.${{ matrix.platform }}
path: target/${{ matrix.target }}/release/tree-sitter${{ env.EXE }}
path: target/${{ matrix.target }}/release/tree-sitter${{ env.SUFFIX }}
if-no-files-found: error
retention-days: 7
- name: Upload WASM artifacts
- name: Upload Wasm artifacts
if: ${{ matrix.platform == 'linux-x64' }}
uses: actions/upload-artifact@v4
with:

View file

@ -1,9 +1,9 @@
name: CI
on:
pull_request:
push:
branches:
- 'master'
branches: [master]
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@ -13,13 +13,22 @@ jobs:
checks:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- run: rustup toolchain install stable --profile minimal
- run: rustup toolchain install nightly --profile minimal
- run: rustup component add --toolchain nightly rustfmt
- run: rustup component add --toolchain nightly clippy
- uses: Swatinem/rust-cache@v2
- run: make lint
- name: Checkout repository
uses: actions/checkout@v4
- name: Set up stable Rust toolchain
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
toolchain: stable
- name: Set up nightly Rust toolchain
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
toolchain: nightly
components: clippy, rustfmt
- name: Lint files
run: make lint
sanitize:
uses: ./.github/workflows/sanitize.yml

View file

@ -1,4 +1,5 @@
name: Release
on:
workflow_dispatch:
push:
@ -9,16 +10,17 @@ jobs:
build:
uses: ./.github/workflows/build.yml
with:
run_test: false
run-test: false
release:
name: Release
name: Release on GitHub
runs-on: ubuntu-latest
needs: build
permissions:
contents: write
steps:
- uses: actions/checkout@v4
- name: Checkout repository
uses: actions/checkout@v4
- name: Download build artifacts
uses: actions/download-artifact@v4
@ -42,29 +44,24 @@ jobs:
ls -l target/
- name: Create release
uses: softprops/action-gh-release@v2
with:
name: ${{ github.ref_name }}
tag_name: ${{ github.ref_name }}
fail_on_unmatched_files: true
files: |
target/tree-sitter-*.gz
target/tree-sitter.wasm
run: |-
gh release create \
target/tree-sitter-*.gz \
target/tree-sitter.wasm \
target/tree-sitter.js
env:
GH_TOKEN: ${{ github.token }}
crates_io:
name: Publish CLI to Crates.io
name: Publish packages to Crates.io
runs-on: ubuntu-latest
needs: release
steps:
- uses: actions/checkout@v4
- name: Checkout repository
uses: actions/checkout@v4
- name: Setup Rust
uses: actions-rs/toolchain@v1
with:
profile: minimal
toolchain: stable
override: true
- name: Set up Rust
uses: actions-rust-lang/setup-rust-toolchain@v1
- name: Publish crates to Crates.io
uses: katyo/publish-crates@v2
@ -72,29 +69,32 @@ jobs:
registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
npm:
name: Publish lib to npmjs.com
name: Publish packages to npmjs.com
runs-on: ubuntu-latest
needs: release
strategy:
fail-fast: false
matrix:
directory: ["cli/npm", "lib/binding_web"]
directory: [cli/npm, lib/binding_web]
steps:
- uses: actions/checkout@v4
- name: CHeckout repository
uses: actions/checkout@v4
- name: Set up Node
uses: actions/setup-node@v4
with:
node-version: 20
registry-url: https://registry.npmjs.org
- name: Set up Rust
uses: actions-rust-lang/setup-rust-toolchain@v1
- name: Build wasm
if: matrix.directory == 'lib/binding_web'
run: cargo xtask build-wasm
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: 20
registry-url: "https://registry.npmjs.org"
- name: Publish lib to npmjs.com
- name: Publish to npmjs.com
working-directory: ${{ matrix.directory }}
run: npm publish
env:
NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}
run: |
cd ${{ matrix.directory }}
npm publish
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

View file

@ -1,34 +1,47 @@
name: no_response
name: No response
on:
schedule:
- cron: '30 1 * * *' # Run every day at 01:30
- cron: "30 1 * * *" # Run every day at 01:30
workflow_dispatch:
issue_comment:
permissions:
issues: write
pull-requests: write
jobs:
close:
name: Close issues with no response
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
permissions:
issues: write
pull-requests: write
steps:
- uses: actions/checkout@v4
- uses: actions/github-script@v7
- name: Checkout script
uses: actions/checkout@v4
with:
sparse-checkout: .github/scripts/close_unresponsive.js
sparse-checkout-cone-mode: false
- name: Run script
uses: actions/github-script@v7
with:
script: |
const script = require('./.github/scripts/close_unresponsive.js')
await script({github, context})
remove_label:
name: Remove response label
if: github.event_name == 'issue_comment'
runs-on: ubuntu-latest
permissions:
issues: write
pull-requests: write
steps:
- uses: actions/checkout@v4
- uses: actions/github-script@v7
- name: Checkout script
uses: actions/checkout@v4
with:
sparse-checkout: .github/scripts/remove_response_label.js
sparse-checkout-cone-mode: false
- name: Run script
uses: actions/github-script@v7
with:
script: |
const script = require('./.github/scripts/remove_response_label.js')

View file

@ -1,15 +1,23 @@
name: "reviewers: remove"
name: Remove Reviewers
on:
pull_request_target:
types: [converted_to_draft, closed]
permissions:
pull-requests: write
jobs:
remove-reviewers:
runs-on: ubuntu-latest
permissions:
pull-requests: write
steps:
- uses: actions/checkout@v4
- name: 'Remove reviewers'
- name: Checkout script
uses: actions/checkout@v4
with:
sparse-checkout: .github/scripts/reviewers_remove.js
sparse-checkout-cone-mode: false
- name: Run script
uses: actions/github-script@v7
with:
script: |

View file

@ -8,39 +8,44 @@ on:
workflow_call:
jobs:
check_undefined_behaviour:
name: Sanitizer checks
check-undefined-behaviour:
runs-on: ubuntu-latest
timeout-minutes: 20
env:
TREE_SITTER: ${{ github.workspace }}/target/release/tree-sitter
steps:
- name: Checkout source code
uses: actions/checkout@v4
- name: Checkout repository
uses: actions/checkout@v4
- name: Install UBSAN library
run: sudo apt-get update -y && sudo apt-get install -y libubsan1
- name: Install UBSAN library
run: sudo apt-get update -y && sudo apt-get install -y libubsan1
- run: rustup toolchain install stable --profile minimal
- uses: Swatinem/rust-cache@v2
- run: cargo build --release
- run: cargo xtask fetch-fixtures
- name: Set up Rust
uses: actions-rust-lang/setup-rust-toolchain@v1
- uses: ./.github/actions/cache
id: cache
- name: Build project
run: cargo build --release
- if: ${{ steps.cache.outputs.cache-hit != 'true' }}
run: cargo xtask generate-fixtures
- name: Cache fixtures
uses: ./.github/actions/cache
id: cache
- name: Run main tests with undefined behaviour sanitizer (UBSAN)
env:
CFLAGS: -fsanitize=undefined
RUSTFLAGS: ${{ env.RUSTFLAGS }} -lubsan
run: cargo test -- --test-threads 1
- name: Fetch fixtures
run: cargo xtask fetch-fixtures
- name: Run main tests with address sanitizer (ASAN)
env:
ASAN_OPTIONS: verify_asan_link_order=0
CFLAGS: -fsanitize=address
RUSTFLAGS: ${{ env.RUSTFLAGS }} -lasan --cfg sanitizing
run: cargo test -- --test-threads 1
- name: Generate fixtures
if: ${{ steps.cache.outputs.cache-hit != 'true' }}
run: cargo xtask generate-fixtures
- name: Run main tests with undefined behaviour sanitizer (UBSAN)
run: cargo test -- --test-threads 1
env:
CFLAGS: -fsanitize=undefined
RUSTFLAGS: ${{ env.RUSTFLAGS }} -lubsan
- name: Run main tests with address sanitizer (ASAN)
run: cargo test -- --test-threads 1
env:
ASAN_OPTIONS: verify_asan_link_order=0
CFLAGS: -fsanitize=address
RUSTFLAGS: ${{ env.RUSTFLAGS }} -lasan --cfg sanitizing