From bb122d6d4788205f72ad3acaa95157dc63d15f4b Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Sat, 8 Apr 2023 01:47:57 +0300 Subject: [PATCH 1/4] cicd: separate sanitize workflow + manual trigger for build workflow --- .github/workflows/CICD.yml | 7 +++++- .github/workflows/build.yml | 32 +-------------------------- .github/workflows/sanitize.yml | 40 ++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 32 deletions(-) create mode 100644 .github/workflows/sanitize.yml diff --git a/.github/workflows/CICD.yml b/.github/workflows/CICD.yml index 7c2351a8..e675781f 100644 --- a/.github/workflows/CICD.yml +++ b/.github/workflows/CICD.yml @@ -46,6 +46,11 @@ jobs: with: package: tree-sitter-cli + sanitize: + name: Sanitize + needs: [init, fast_checks] + uses: ./.github/workflows/sanitize.yml + build: name: Build & Test needs: [init, fast_checks] @@ -55,7 +60,7 @@ jobs: release: name: Release - needs: [init, fast_checks, full_checks, min_version, build] + needs: [init, fast_checks, full_checks, min_version, build, sanitize] if: > github.event.pull_request.head.repo.full_name == github.repository && startsWith(github.head_ref, 'release/v') diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 61543552..6cfa0d48 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,6 +6,7 @@ env: CROSS_DEBUG: 1 on: + workflow_dispatch: workflow_call: inputs: ref: @@ -174,34 +175,3 @@ jobs: lib/binding_web/tree-sitter.wasm if-no-files-found: error retention-days: 7 - - check_undefined_behaviour: - name: Undefined behaviour checks - runs-on: ubuntu-latest - env: - TREE_SITTER: ${{ github.workspace }}/target/release/tree-sitter - steps: - - name: Checkout source code - uses: actions/checkout@v3 - - - name: Install UBSAN library - run: sudo apt-get install -y libubsan1 - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - - - name: Build CLI - run: cargo build --release - - - name: Fetch fixtures - run: script/fetch-fixtures - - - name: Generate fixtures - run: script/generate-fixtures - - - name: Run main tests with undefined behaviour sanitizer (UBSAN) - env: - UBSAN_OPTIONS: halt_on_error=1 - CFLAGS: -fsanitize=undefined - RUSTFLAGS: -lubsan - run: cargo test -- --test-threads 1 diff --git a/.github/workflows/sanitize.yml b/.github/workflows/sanitize.yml new file mode 100644 index 00000000..ebfb477c --- /dev/null +++ b/.github/workflows/sanitize.yml @@ -0,0 +1,40 @@ +name: Sunitize + +env: + CARGO_TERM_COLOR: always + RUSTFLAGS: "-D warnings" + +on: + workflow_call: + +jobs: + check_undefined_behaviour: + name: Undefined behaviour checks + runs-on: ubuntu-latest + env: + TREE_SITTER: ${{ github.workspace }}/target/release/tree-sitter + steps: + - name: Checkout source code + uses: actions/checkout@v3 + + - name: Install UBSAN library + run: sudo apt-get install -y libubsan1 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Build CLI + run: cargo build --release + + - name: Fetch fixtures + run: script/fetch-fixtures + + - name: Generate fixtures + run: script/generate-fixtures + + - name: Run main tests with undefined behaviour sanitizer (UBSAN) + env: + UBSAN_OPTIONS: halt_on_error=1 + CFLAGS: -fsanitize=undefined + RUSTFLAGS: -lubsan + run: cargo test -- --test-threads 1 From 96086806d56404e65e82fc17a144506e98d30e43 Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Sat, 8 Apr 2023 05:14:12 +0300 Subject: [PATCH 2/4] cicd: change author configuring for a tag --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 87a06761..979d95a3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -78,8 +78,8 @@ jobs: tag: ${{ steps.tag_name.outputs.tag }} message: "Release ${{ steps.tag_name.outputs.tag }}" run: | - git config user.name "${GITHUB_ACTOR}" - git config user.email "${GITHUB_ACTOR}@users.noreply.github.com" + git config user.name "$(git log -1 --pretty='%cn')" + git config user.email "$(git log -1 --pretty='%ce')" git tag -a "$tag" HEAD -m "$message" git push origin "$tag" From f03f024ec45c092f4a314565d65f641b5f13741c Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Sat, 8 Apr 2023 07:35:34 +0300 Subject: [PATCH 3/4] cicd: change approach for matrix shaping with an anchor name plus extras --- .github/workflows/build.yml | 112 ++++++++++++++++++++---------------- 1 file changed, 61 insertions(+), 51 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6cfa0d48..2df9bc27 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,20 +15,39 @@ on: jobs: build: - name: ${{ matrix.job.name }} (${{ matrix.job.target }}) (${{ matrix.job.os }}) - runs-on: ${{ matrix.job.os }} + name: ${{ matrix.name }} (${{ matrix.target }}) (${{ matrix.os }}) + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - job: - - { name: linux-aarch64 , target: aarch64-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } - - { name: linux-arm , target: arm-unknown-linux-gnueabihf , os: ubuntu-latest , use-cross: true } - - { name: linux-x64 , target: x86_64-unknown-linux-gnu , os: ubuntu-latest } - - { name: linux-x86 , target: i686-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } - - { name: windows-x64 , target: x86_64-pc-windows-msvc , os: windows-latest } - - { name: windows-x86 , target: i686-pc-windows-msvc , os: windows-latest } - - { name: macos-arm64 , target: aarch64-apple-darwin , os: macos-latest } - - { name: macos-x64 , target: x86_64-apple-darwin , os: macos-latest } + name: + - linux-aarch64 # + - linux-arm # + - linux-x64 # + - linux-x86 # + - windows-x64 # <-- No C library build - requires an additional adapted Makefile for `cl.exe` compiler + - windows-x86 # -- // -- + - macos-arm64 # <-- MacOS M1/M2 - no tests, only CLI build to be published on release artifacts + - macos-x64 # + + include: + - { name: linux-aarch64 , target: aarch64-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } + - { name: linux-arm , target: arm-unknown-linux-gnueabihf , os: ubuntu-latest , use-cross: true } + - { name: linux-x64 , target: x86_64-unknown-linux-gnu , os: ubuntu-latest } + - { name: linux-x86 , target: i686-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } + - { name: windows-x64 , target: x86_64-pc-windows-msvc , os: windows-latest } + - { name: windows-x86 , target: i686-pc-windows-msvc , os: windows-latest } + - { name: macos-arm64 , target: aarch64-apple-darwin , os: macos-latest } + - { name: macos-x64 , target: x86_64-apple-darwin , os: macos-latest } + + # Cross compilers for C library + - { name: linux-aarch64 , cc: aarch64-linux-gnu-gcc , ar: aarch64-linux-gnu-ar } + - { name: linux-arm , cc: arm-unknown-linux-gnueabihf-gcc , ar: arm-unknown-linux-gnueabihf-gcc-ar } + - { name: linux-x86 , cc: i686-linux-gnu-gcc , ar: i686-linux-gnu-ar } + + # See #2041 tree-sitter issue + - { name: windows-x64 , rust_test_threads: 1 } + - { name: windows-x86 , rust_test_threads: 1 } env: BUILD_CMD: cargo @@ -55,20 +74,20 @@ jobs: - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable with: - targets: ${{ matrix.job.target }} + targets: ${{ matrix.target }} - name: Install cross - if: matrix.job.use-cross + if: matrix.use-cross uses: taiki-e/install-action@v2 with: tool: cross - name: Build custom cross image - if: ${{ matrix.job.use-cross && matrix.job.os == 'ubuntu-latest' }} + if: ${{ matrix.use-cross && matrix.os == 'ubuntu-latest' }} run: | cd .. - target="${{ matrix.job.target }}" + target="${{ matrix.target }}" image=ghcr.io/cross-rs/$target:custom echo "CROSS_IMAGE=$image" >> $GITHUB_ENV @@ -85,88 +104,79 @@ jobs: cd - - - name: Setup extra env + - name: Setup env extras + env: + RUST_TEST_THREADS: ${{ matrix.rust_test_threads }} + USE_CROSS: ${{ matrix.use-cross }} + CC: ${{ matrix.cc }} + AR: ${{ matrix.ar }} run: | PATH="$PWD/.github/scripts:$PATH" echo "PATH=$PATH" >> $GITHUB_ENV echo "ROOT=$PWD" >> $GITHUB_ENV echo "TREE_SITTER=tree-sitter.sh" >> $GITHUB_ENV - export TARGET=${{ matrix.job.target }} + export TARGET=${{ matrix.target }} echo "TARGET=$TARGET" >> $GITHUB_ENV - USE_CROSS="${{ matrix.job.use-cross }}" + [ -n "$RUST_TEST_THREADS" ] && \ + echo "RUST_TEST_THREADS=$RUST_TEST_THREADS" >> $GITHUB_ENV + + [ -n "$CC" ] && echo "CC=$CC" >> $GITHUB_ENV + [ -n "$AR" ] && echo "AR=$AR" >> $GITHUB_ENV if [ "$USE_CROSS" == "true" ]; then echo "BUILD_CMD=cross" >> $GITHUB_ENV - - export CROSS=1; echo "CROSS=$CROSS" >> $GITHUB_ENV - + export CROSS=1; echo "CROSS=1" >> $GITHUB_ENV runner=$(cross.sh bash -c "env | sed -nr '/^CARGO_TARGET_.*_RUNNER=/s///p'") [ -n "$runner" ] && echo "CROSS_RUNNER=$runner" >> $GITHUB_ENV - echo "runner: $runner" - - case "$TARGET" in - i686-unknown-linux-gnu) CC=i686-linux-gnu-gcc AR=i686-linux-gnu-ar ;; - aarch64-unknown-linux-gnu) CC=aarch64-linux-gnu-gcc AR=aarch64-linux-gnu-ar ;; - arm-unknown-linux-gnueabihf) CC=arm-unknown-linux-gnueabihf-gcc AR=arm-unknown-linux-gnueabihf-gcc-ar ;; - esac - - [ -n "$CC" ] && echo "CC=$CC" >> $GITHUB_ENV - [ -n "$AR" ] && echo "AR=$AR" >> $GITHUB_ENV fi - case "$TARGET" in - *-windows-*) - echo "RUST_TEST_THREADS=1" >> $GITHUB_ENV # See #2041 tree-sitter issue - ;; - esac - - name: Build C library - if: ${{ !contains(matrix.job.os, 'windows') }} # Requires an additional adapted Makefile for `cl.exe` compiler + if: ${{ !contains(matrix.os, 'windows') }} # Requires an additional adapted Makefile for `cl.exe` compiler run: make.sh CFLAGS="-Werror" -j - name: Build wasm library - if: ${{ !matrix.job.use-cross && matrix.job.name != 'macos-arm64' }} # Not used + if: ${{ !matrix.use-cross && matrix.name != 'macos-arm64' }} # Not used run: script/build-wasm - name: Build CLI - run: $BUILD_CMD build --release --target=${{ matrix.job.target }} + run: $BUILD_CMD build --release --target=${{ matrix.target }} - name: Fetch fixtures - if: ${{ matrix.job.name != 'macos-arm64' }} # Not used + if: ${{ matrix.name != 'macos-arm64' }} # Not used run: script/fetch-fixtures - name: Generate fixtures - if: ${{ matrix.job.name != 'macos-arm64' }} # Can't natively run CLI on runner's host + if: ${{ matrix.name != 'macos-arm64' }} # Can't natively run CLI on runner's host run: script/generate-fixtures - name: Generate WASM fixtures - if: ${{ !matrix.job.use-cross && matrix.job.name != 'macos-arm64' }} # Not used + if: ${{ !matrix.use-cross && matrix.name != 'macos-arm64' }} # Not used run: script/generate-fixtures-wasm - name: Run main tests - if: ${{ matrix.job.name != 'macos-arm64' }} # Can't natively run CLI on runner's host - run: $BUILD_CMD test --target=${{ matrix.job.target }} + if: ${{ matrix.name != 'macos-arm64' }} # Can't natively run CLI on runner's host + run: $BUILD_CMD test --target=${{ matrix.target }} - name: Run wasm tests - if: ${{ !matrix.job.use-cross && matrix.job.name != 'macos-arm64' }} # Not used + if: ${{ !matrix.use-cross && matrix.name != 'macos-arm64' }} # Not used run: script/test-wasm - name: Run benchmarks - if: ${{ !matrix.job.use-cross && matrix.job.name != 'macos-arm64' }} # Cross-compiled benchmarks make no sense - run: $BUILD_CMD bench benchmark -p tree-sitter-cli --target=${{ matrix.job.target }} + if: ${{ !matrix.use-cross && matrix.name != 'macos-arm64' }} # Cross-compiled benchmarks make no sense + run: $BUILD_CMD bench benchmark -p tree-sitter-cli --target=${{ matrix.target }} - name: Upload CLI artifact uses: actions/upload-artifact@v3 with: - name: tree-sitter.${{ matrix.job.name }} - path: target/${{ matrix.job.target }}/release/tree-sitter${{ contains(matrix.job.target, 'windows') && '.exe' || '' }} + name: tree-sitter.${{ matrix.name }} + path: target/${{ matrix.target }}/release/tree-sitter${{ contains(matrix.target, 'windows') && '.exe' || '' }} if-no-files-found: error retention-days: 7 - name: Upload WASM artifacts - if: ${{ matrix.job.name == 'linux-x64' }} + if: ${{ matrix.name == 'linux-x64' }} uses: actions/upload-artifact@v3 with: name: tree-sitter.wasm From d8caf6f8c530bef60f5ee419e0f4e96cb8452c63 Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Sat, 8 Apr 2023 08:42:17 +0300 Subject: [PATCH 4/4] chore(cicd): change formatting --- .github/workflows/build.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2df9bc27..b5c7ab60 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -89,15 +89,15 @@ jobs: target="${{ matrix.target }}" image=ghcr.io/cross-rs/$target:custom - echo "CROSS_IMAGE=$image" >> $GITHUB_ENV + echo "CROSS_IMAGE=$image" >> $GITHUB_ENV - echo "[target.$target]" >> Cross.toml - echo "image = \"$image\"" >> Cross.toml - echo "CROSS_CONFIG=$PWD/Cross.toml" >> $GITHUB_ENV + echo "[target.$target]" >> Cross.toml + echo "image = \"$image\"" >> Cross.toml + echo "CROSS_CONFIG=$PWD/Cross.toml" >> $GITHUB_ENV - echo "FROM ghcr.io/cross-rs/$target:edge" >> Dockerfile - echo "ENV DEBIAN_FRONTEND=noninteractive" >> Dockerfile - echo "RUN apt-get update && apt-get install -y nodejs" >> Dockerfile + echo "FROM ghcr.io/cross-rs/$target:edge" >> Dockerfile + echo "ENV DEBIAN_FRONTEND=noninteractive" >> Dockerfile + echo "RUN apt-get update && apt-get install -y nodejs" >> Dockerfile docker build -t $image . docker images docker run --rm $image env