Merge pull request #2187 from tree-sitter/cicd-tweaks

cicd: better matrix construction, a separate sanitize workflow, manual trigger for the build workflow

.github/workflows/CICD.yml

@@ -46,6 +46,11 @@ jobs:
     with:
       package: tree-sitter-cli
 
+  sanitize:
+    name: Sanitize
+    needs: [init, fast_checks]
+    uses: ./.github/workflows/sanitize.yml
+
   build:
     name: Build & Test
     needs: [init, fast_checks]
@@ -55,7 +60,7 @@ jobs:
 
   release:
     name: Release
-    needs: [init, fast_checks, full_checks, min_version, build]
+    needs: [init, fast_checks, full_checks, min_version, build, sanitize]
     if: >
       github.event.pull_request.head.repo.full_name == github.repository &&
       startsWith(github.head_ref, 'release/v')

.github/workflows/build.yml

@@ -6,6 +6,7 @@ env:
   CROSS_DEBUG: 1
 
 on:
+  workflow_dispatch:
   workflow_call:
     inputs:
       ref:
@@ -14,20 +15,39 @@ on:
 
 jobs:
   build:
-    name: ${{ matrix.job.name }} (${{ matrix.job.target }}) (${{ matrix.job.os }})
-    runs-on: ${{ matrix.job.os }}
+    name: ${{ matrix.name }} (${{ matrix.target }}) (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        job:
-          - { name: linux-aarch64 , target: aarch64-unknown-linux-gnu   , os: ubuntu-latest  , use-cross: true }
-          - { name: linux-arm     , target: arm-unknown-linux-gnueabihf , os: ubuntu-latest  , use-cross: true }
-          - { name: linux-x64     , target: x86_64-unknown-linux-gnu    , os: ubuntu-latest                    }
-          - { name: linux-x86     , target: i686-unknown-linux-gnu      , os: ubuntu-latest  , use-cross: true }
-          - { name: windows-x64   , target: x86_64-pc-windows-msvc      , os: windows-latest                   }
-          - { name: windows-x86   , target: i686-pc-windows-msvc        , os: windows-latest                   }
-          - { name: macos-arm64   , target: aarch64-apple-darwin        , os: macos-latest                     }
-          - { name: macos-x64     , target: x86_64-apple-darwin         , os: macos-latest                     }
+        name:
+        - linux-aarch64 #
+        - linux-arm     #
+        - linux-x64     #
+        - linux-x86     #
+        - windows-x64   # <-- No C library build - requires an additional adapted Makefile for `cl.exe` compiler
+        - windows-x86   #     -- // --
+        - macos-arm64   # <-- MacOS M1/M2 - no tests, only CLI build to be published on release artifacts
+        - macos-x64     #
+
+        include:
+        - { name: linux-aarch64 , target: aarch64-unknown-linux-gnu   , os: ubuntu-latest  , use-cross: true }
+        - { name: linux-arm     , target: arm-unknown-linux-gnueabihf , os: ubuntu-latest  , use-cross: true }
+        - { name: linux-x64     , target: x86_64-unknown-linux-gnu    , os: ubuntu-latest                    }
+        - { name: linux-x86     , target: i686-unknown-linux-gnu      , os: ubuntu-latest  , use-cross: true }
+        - { name: windows-x64   , target: x86_64-pc-windows-msvc      , os: windows-latest                   }
+        - { name: windows-x86   , target: i686-pc-windows-msvc        , os: windows-latest                   }
+        - { name: macos-arm64   , target: aarch64-apple-darwin        , os: macos-latest                     }
+        - { name: macos-x64     , target: x86_64-apple-darwin         , os: macos-latest                     }
+
+        # Cross compilers for C library
+        - { name: linux-aarch64 , cc: aarch64-linux-gnu-gcc           , ar: aarch64-linux-gnu-ar               }
+        - { name: linux-arm     , cc: arm-unknown-linux-gnueabihf-gcc , ar: arm-unknown-linux-gnueabihf-gcc-ar }
+        - { name: linux-x86     , cc: i686-linux-gnu-gcc              , ar: i686-linux-gnu-ar                  }
+
+        # See #2041 tree-sitter issue
+        - { name: windows-x64   , rust_test_threads: 1 }
+        - { name: windows-x86   , rust_test_threads: 1 }
 
     env:
       BUILD_CMD: cargo
@@ -54,118 +74,109 @@ jobs:
     - name: Install Rust toolchain
       uses: dtolnay/rust-toolchain@stable
       with:
-        targets: ${{ matrix.job.target }}
+        targets: ${{ matrix.target }}
 
     - name: Install cross
-      if: matrix.job.use-cross
+      if: matrix.use-cross
       uses: taiki-e/install-action@v2
       with:
         tool: cross
 
     - name: Build custom cross image
-      if: ${{ matrix.job.use-cross && matrix.job.os == 'ubuntu-latest' }}
+      if: ${{ matrix.use-cross && matrix.os == 'ubuntu-latest' }}
       run: |
         cd ..
 
-        target="${{ matrix.job.target }}"
+        target="${{ matrix.target }}"
         image=ghcr.io/cross-rs/$target:custom
-        echo "CROSS_IMAGE=$image"                                  >> $GITHUB_ENV
+        echo "CROSS_IMAGE=$image"                              >> $GITHUB_ENV
 
-        echo "[target.$target]"                                    >> Cross.toml
-        echo "image = \"$image\""                                  >> Cross.toml
-        echo "CROSS_CONFIG=$PWD/Cross.toml"                        >> $GITHUB_ENV
+        echo "[target.$target]"                                >> Cross.toml
+        echo "image = \"$image\""                              >> Cross.toml
+        echo "CROSS_CONFIG=$PWD/Cross.toml"                    >> $GITHUB_ENV
 
-        echo "FROM ghcr.io/cross-rs/$target:edge"                  >> Dockerfile
-        echo "ENV DEBIAN_FRONTEND=noninteractive"                  >> Dockerfile
-        echo "RUN apt-get update && apt-get install -y nodejs"     >> Dockerfile
+        echo "FROM ghcr.io/cross-rs/$target:edge"              >> Dockerfile
+        echo "ENV DEBIAN_FRONTEND=noninteractive"              >> Dockerfile
+        echo "RUN apt-get update && apt-get install -y nodejs" >> Dockerfile
         docker build -t $image .
         docker images
         docker run --rm $image env
 
         cd -
 
-    - name: Setup extra env
+    - name: Setup env extras
+      env:
+        RUST_TEST_THREADS: ${{ matrix.rust_test_threads }}
+        USE_CROSS: ${{ matrix.use-cross }}
+        CC: ${{ matrix.cc }}
+        AR: ${{ matrix.ar }}
       run: |
         PATH="$PWD/.github/scripts:$PATH"
         echo "PATH=$PATH" >> $GITHUB_ENV
         echo "ROOT=$PWD" >> $GITHUB_ENV
         echo "TREE_SITTER=tree-sitter.sh" >> $GITHUB_ENV
 
-        export TARGET=${{ matrix.job.target }}
+        export TARGET=${{ matrix.target }}
         echo "TARGET=$TARGET" >> $GITHUB_ENV
 
-        USE_CROSS="${{ matrix.job.use-cross }}"
+        [ -n "$RUST_TEST_THREADS" ] && \
+        echo "RUST_TEST_THREADS=$RUST_TEST_THREADS" >> $GITHUB_ENV
+
+        [ -n "$CC" ] && echo "CC=$CC" >> $GITHUB_ENV
+        [ -n "$AR" ] && echo "AR=$AR" >> $GITHUB_ENV
 
         if [ "$USE_CROSS" == "true" ]; then
           echo "BUILD_CMD=cross" >> $GITHUB_ENV
-
-          export CROSS=1; echo "CROSS=$CROSS" >> $GITHUB_ENV
-
+          export CROSS=1; echo "CROSS=1" >> $GITHUB_ENV
           runner=$(cross.sh bash -c "env | sed -nr '/^CARGO_TARGET_.*_RUNNER=/s///p'")
           [ -n "$runner" ] && echo "CROSS_RUNNER=$runner" >> $GITHUB_ENV
-          echo "runner: $runner"
-
-          case "$TARGET" in
-            i686-unknown-linux-gnu)      CC=i686-linux-gnu-gcc              AR=i686-linux-gnu-ar                  ;;
-            aarch64-unknown-linux-gnu)   CC=aarch64-linux-gnu-gcc           AR=aarch64-linux-gnu-ar               ;;
-            arm-unknown-linux-gnueabihf) CC=arm-unknown-linux-gnueabihf-gcc AR=arm-unknown-linux-gnueabihf-gcc-ar ;;
-          esac
-
-          [ -n "$CC" ] && echo "CC=$CC" >> $GITHUB_ENV
-          [ -n "$AR" ] && echo "AR=$AR" >> $GITHUB_ENV
         fi
 
-        case "$TARGET" in
-          *-windows-*)
-            echo "RUST_TEST_THREADS=1" >> $GITHUB_ENV # See #2041 tree-sitter issue
-            ;;
-        esac
-
     - name: Build C library
-      if: ${{ !contains(matrix.job.os, 'windows') }} # Requires an additional adapted Makefile for `cl.exe` compiler
+      if: ${{ !contains(matrix.os, 'windows') }} # Requires an additional adapted Makefile for `cl.exe` compiler
       run: make.sh CFLAGS="-Werror" -j
 
     - name: Build wasm library
-      if: ${{ !matrix.job.use-cross && matrix.job.name != 'macos-arm64' }} # Not used
+      if: ${{ !matrix.use-cross && matrix.name != 'macos-arm64' }} # Not used
       run: script/build-wasm
 
     - name: Build CLI
-      run: $BUILD_CMD build --release --target=${{ matrix.job.target }}
+      run: $BUILD_CMD build --release --target=${{ matrix.target }}
 
     - name: Fetch fixtures
-      if: ${{ matrix.job.name != 'macos-arm64' }} # Not used
+      if: ${{ matrix.name != 'macos-arm64' }} # Not used
       run: script/fetch-fixtures
 
     - name: Generate fixtures
-      if: ${{ matrix.job.name != 'macos-arm64' }} # Can't natively run CLI on runner's host
+      if: ${{ matrix.name != 'macos-arm64' }} # Can't natively run CLI on runner's host
       run: script/generate-fixtures
 
     - name: Generate WASM fixtures
-      if: ${{ !matrix.job.use-cross && matrix.job.name != 'macos-arm64' }} # Not used
+      if: ${{ !matrix.use-cross && matrix.name != 'macos-arm64' }} # Not used
       run: script/generate-fixtures-wasm
 
     - name: Run main tests
-      if: ${{ matrix.job.name != 'macos-arm64' }} # Can't natively run CLI on runner's host
-      run: $BUILD_CMD test --target=${{ matrix.job.target }}
+      if: ${{ matrix.name != 'macos-arm64' }} # Can't natively run CLI on runner's host
+      run: $BUILD_CMD test --target=${{ matrix.target }}
 
     - name: Run wasm tests
-      if: ${{ !matrix.job.use-cross && matrix.job.name != 'macos-arm64' }} # Not used
+      if: ${{ !matrix.use-cross && matrix.name != 'macos-arm64' }} # Not used
       run: script/test-wasm
 
     - name: Run benchmarks
-      if: ${{ !matrix.job.use-cross && matrix.job.name != 'macos-arm64' }} # Cross-compiled benchmarks make no sense
-      run: $BUILD_CMD bench benchmark -p tree-sitter-cli --target=${{ matrix.job.target }}
+      if: ${{ !matrix.use-cross && matrix.name != 'macos-arm64' }} # Cross-compiled benchmarks make no sense
+      run: $BUILD_CMD bench benchmark -p tree-sitter-cli --target=${{ matrix.target }}
 
     - name: Upload CLI artifact
       uses: actions/upload-artifact@v3
       with:
-        name: tree-sitter.${{ matrix.job.name }}
-        path: target/${{ matrix.job.target }}/release/tree-sitter${{ contains(matrix.job.target, 'windows') && '.exe' || '' }}
+        name: tree-sitter.${{ matrix.name }}
+        path: target/${{ matrix.target }}/release/tree-sitter${{ contains(matrix.target, 'windows') && '.exe' || '' }}
         if-no-files-found: error
         retention-days: 7
 
     - name: Upload WASM artifacts
-      if: ${{ matrix.job.name == 'linux-x64' }}
+      if: ${{ matrix.name == 'linux-x64' }}
       uses: actions/upload-artifact@v3
       with:
         name: tree-sitter.wasm
@@ -174,34 +185,3 @@ jobs:
           lib/binding_web/tree-sitter.wasm
         if-no-files-found: error
         retention-days: 7
-
-  check_undefined_behaviour:
-    name: Undefined behaviour checks
-    runs-on: ubuntu-latest
-    env:
-      TREE_SITTER: ${{ github.workspace }}/target/release/tree-sitter
-    steps:
-    - name: Checkout source code
-      uses: actions/checkout@v3
-
-    - name: Install UBSAN library
-      run: sudo apt-get install -y libubsan1
-
-    - name: Install Rust toolchain
-      uses: dtolnay/rust-toolchain@stable
-
-    - name: Build CLI
-      run: cargo build --release
-
-    - name: Fetch fixtures
-      run: script/fetch-fixtures
-
-    - name: Generate fixtures
-      run: script/generate-fixtures
-
-    - name: Run main tests with undefined behaviour sanitizer (UBSAN)
-      env:
-        UBSAN_OPTIONS: halt_on_error=1
-        CFLAGS: -fsanitize=undefined
-        RUSTFLAGS: -lubsan
-      run: cargo test -- --test-threads 1

.github/workflows/release.yml

@@ -78,8 +78,8 @@ jobs:
           tag: ${{ steps.tag_name.outputs.tag }}
           message: "Release ${{ steps.tag_name.outputs.tag }}"
         run: |
-          git config user.name "${GITHUB_ACTOR}"
-          git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
+          git config user.name "$(git log -1 --pretty='%cn')"
+          git config user.email "$(git log -1 --pretty='%ce')"
           git tag -a "$tag" HEAD -m "$message"
           git push origin "$tag"
 

.github/workflows/sanitize.yml

@@ -0,0 +1,40 @@
+name: Sunitize
+
+env:
+  CARGO_TERM_COLOR: always
+  RUSTFLAGS: "-D warnings"
+
+on:
+  workflow_call:
+
+jobs:
+  check_undefined_behaviour:
+    name: Undefined behaviour checks
+    runs-on: ubuntu-latest
+    env:
+      TREE_SITTER: ${{ github.workspace }}/target/release/tree-sitter
+    steps:
+    - name: Checkout source code
+      uses: actions/checkout@v3
+
+    - name: Install UBSAN library
+      run: sudo apt-get install -y libubsan1
+
+    - name: Install Rust toolchain
+      uses: dtolnay/rust-toolchain@stable
+
+    - name: Build CLI
+      run: cargo build --release
+
+    - name: Fetch fixtures
+      run: script/fetch-fixtures
+
+    - name: Generate fixtures
+      run: script/generate-fixtures
+
+    - name: Run main tests with undefined behaviour sanitizer (UBSAN)
+      env:
+        UBSAN_OPTIONS: halt_on_error=1
+        CFLAGS: -fsanitize=undefined
+        RUSTFLAGS: -lubsan
+      run: cargo test -- --test-threads 1