cicd: new workflow

.github/workflows/CICD.yml

@@ -0,0 +1,69 @@
+name: CICD
+
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - master
+      - check/*
+
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
+  cancel-in-progress: true
+
+jobs:
+  init:
+    name: Init
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get PR head ref
+        if: ${{ github.event_name == 'pull_request' }}
+        id: ref
+        run: |
+          echo "ref=refs/pull/${{ github.event.pull_request.number }}/head" >> $GITHUB_OUTPUT
+    outputs:
+      ref: >-
+        ${{
+          (github.event_name == 'pull_request' && startsWith(github.head_ref, 'release/v'))
+          && steps.ref.outputs.ref
+          || github.ref
+        }}
+
+  fast_checks:
+    name: Fast checks
+    uses: ./.github/workflows/fast_checks.yml
+
+  full_checks:
+    name: Full Rust checks
+    needs: fast_checks
+    uses: ./.github/workflows/full_rust_checks.yml
+
+  min_version:
+    name: Minimum supported rust version
+    needs: fast_checks
+    uses: ./.github/workflows/msrv.yml
+    with:
+      package: tree-sitter-cli
+
+  build:
+    name: Build & Test
+    needs: [init, fast_checks]
+    uses: ./.github/workflows/build.yml
+    with:
+      ref: ${{ needs.init.outputs.ref }}
+
+  release:
+    name: Release
+    needs: [init, fast_checks, full_checks, min_version, build]
+    if: >
+      github.event.pull_request.head.repo.full_name == github.repository &&
+      startsWith(github.head_ref, 'release/v')
+    uses: ./.github/workflows/release.yml
+    with:
+      ref: ${{ needs.init.outputs.ref }}
+
+  publish:
+    name: Publish
+    needs: release
+    uses: ./.github/workflows/publish.yml

.github/workflows/build.yml

@@ -0,0 +1,171 @@
+name: Build & Test
+
+env:
+  CARGO_TERM_COLOR: always
+  RUSTFLAGS: "-D warnings"
+  CROSS_DEBUG: 1
+
+on:
+  workflow_call:
+    inputs:
+      ref:
+        default: ${{ github.ref }}
+        type: string
+
+jobs:
+  build:
+    name: ${{ matrix.job.name }} (${{ matrix.job.target }}) (${{ matrix.job.os }})
+    runs-on: ${{ matrix.job.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        job:
+          - { name: linux-aarch64 , target: aarch64-unknown-linux-gnu   , os: ubuntu-latest  , use-cross: true }
+          - { name: linux-arm     , target: arm-unknown-linux-gnueabihf , os: ubuntu-latest  , use-cross: true }
+          - { name: linux-x64     , target: x86_64-unknown-linux-gnu    , os: ubuntu-latest                    }
+          - { name: linux-x86     , target: i686-unknown-linux-gnu      , os: ubuntu-latest  , use-cross: true }
+          - { name: windows-x64   , target: x86_64-pc-windows-msvc      , os: windows-latest                   }
+          - { name: windows-x86   , target: i686-pc-windows-msvc        , os: windows-latest                   }
+          - { name: macos-x64     , target: x86_64-apple-darwin         , os: macos-latest                     }
+
+    env:
+      BUILD_CMD: cargo
+
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+    - name: Checkout source code
+      uses: actions/checkout@v3
+      with:
+        ref: ${{ inputs.ref }}
+
+    - name: Read Emscripten version
+      run: |
+        echo "EMSCRIPTEN_VERSION=$(cat cli/emscripten-version)" >> $GITHUB_ENV
+
+    - name: Install Emscripten
+      uses: mymindstorm/setup-emsdk@v12
+      with:
+        version: ${{ env.EMSCRIPTEN_VERSION }}
+
+    - name: Install Rust toolchain
+      uses: dtolnay/rust-toolchain@stable
+      with:
+        targets: ${{ matrix.job.target }}
+
+    - name: Install cross
+      if: matrix.job.use-cross
+      uses: taiki-e/install-action@v2
+      with:
+        tool: cross
+
+    - name: Build custom cross image
+      if: ${{ matrix.job.use-cross && matrix.job.os == 'ubuntu-latest' }}
+      run: |
+        cd ..
+
+        target="${{ matrix.job.target }}"
+        image=ghcr.io/cross-rs/$target:custom
+        echo "CROSS_IMAGE=$image"                                  >> $GITHUB_ENV
+
+        echo "[target.$target]"                                    >> Cross.toml
+        echo "image = \"$image\""                                  >> Cross.toml
+        echo "CROSS_CONFIG=$PWD/Cross.toml"                        >> $GITHUB_ENV
+
+        echo "FROM ghcr.io/cross-rs/$target:edge"                  >> Dockerfile
+        echo "ENV DEBIAN_FRONTEND=noninteractive"                  >> Dockerfile
+        echo "RUN apt-get update && apt-get install -y nodejs"     >> Dockerfile
+        docker build -t $image .
+        docker images
+        docker run --rm $image env
+
+        cd -
+
+    - name: Setup extra env
+      run: |
+        PATH="$PWD/.github/scripts:$PATH"
+        echo "PATH=$PATH" >> $GITHUB_ENV
+        echo "ROOT=$PWD" >> $GITHUB_ENV
+        echo "TREE_SITTER=tree-sitter.sh" >> $GITHUB_ENV
+
+        export TARGET=${{ matrix.job.target }}
+        echo "TARGET=$TARGET" >> $GITHUB_ENV
+
+        USE_CROSS="${{ matrix.job.use-cross }}"
+
+        if [ "$USE_CROSS" == "true" ]; then
+          echo "BUILD_CMD=cross" >> $GITHUB_ENV
+
+          export CROSS=1; echo "CROSS=$CROSS" >> $GITHUB_ENV
+
+          runner=$(cross.sh bash -c "env | sed -nr '/^CARGO_TARGET_.*_RUNNER=/s///p'")
+          [ -n "$runner" ] && echo "CROSS_RUNNER=$runner" >> $GITHUB_ENV
+          echo "runner: $runner"
+
+          case "$TARGET" in
+            i686-unknown-linux-gnu)      CC=i686-linux-gnu-gcc              AR=i686-linux-gnu-ar                  ;;
+            aarch64-unknown-linux-gnu)   CC=aarch64-linux-gnu-gcc           AR=aarch64-linux-gnu-ar               ;;
+            arm-unknown-linux-gnueabihf) CC=arm-unknown-linux-gnueabihf-gcc AR=arm-unknown-linux-gnueabihf-gcc-ar ;;
+          esac
+
+          [ -n "$CC" ] && echo "CC=$CC" >> $GITHUB_ENV
+          [ -n "$AR" ] && echo "AR=$AR" >> $GITHUB_ENV
+        fi
+
+        case "$TARGET" in
+          *-windows-*)
+            echo "RUST_TEST_THREADS=1" >> $GITHUB_ENV # See #2041 tree-sitter issue
+            ;;
+        esac
+
+    - name: Build C library
+      if: "!contains(matrix.job.os, 'windows')" # Requires an additional adapted Makefile for `cl.exe` compiler
+      run: make.sh CFLAGS="-Werror" -j
+
+    - name: Build wasm library
+      run: script/build-wasm
+
+    - name: Build CLI
+      run: $BUILD_CMD build --release --target=${{ matrix.job.target }}
+
+    - name: Fetch fixtures
+      run: script/fetch-fixtures
+
+    - name: Generate fixtures
+      run: script/generate-fixtures
+
+    - name: Generate WASM fixtures
+      if: "!matrix.job.use-cross"
+      run: script/generate-fixtures-wasm
+
+    - name: Run main tests
+      run: $BUILD_CMD test --target=${{ matrix.job.target }}
+
+    - name: Run wasm tests
+      if: "!matrix.job.use-cross" # TODO: Install Emscripten into custom cross images
+      run: script/test-wasm
+
+    - name: Run benchmarks
+      if: "!matrix.job.use-cross" # It doesn't make sense to benchmark something in an emulator
+      run: $BUILD_CMD bench benchmark -p tree-sitter-cli --target=${{ matrix.job.target }}
+
+    - name: Upload CLI artifact
+      uses: actions/upload-artifact@v3
+      with:
+        name: tree-sitter.${{ matrix.job.name }}
+        path: target/${{ matrix.job.target }}/release/tree-sitter${{ contains(matrix.job.target, 'windows') && '.exe' || '' }}
+        if-no-files-found: error
+        retention-days: 7
+
+    - name: Upload WASM artifacts
+      if: ${{ matrix.job.name == 'linux-x64' }}
+      uses: actions/upload-artifact@v3
+      with:
+        name: tree-sitter.wasm
+        path: |
+          lib/binding_web/tree-sitter.js
+          lib/binding_web/tree-sitter.wasm
+        if-no-files-found: error
+        retention-days: 7

.github/workflows/ci.yml

@@ -1,150 +0,0 @@
-name: CI
-
-on:
-  push:
-    branches:
-      - master
-    tags:
-      - v*
-  pull_request:
-    branches:
-      - "**"
-
-env:
-  CARGO_TERM_COLOR: always
-  CARGO_INCREMENTAL: 0
-
-jobs:
-  unix-tests:
-    name: Unix tests
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: true
-      matrix:
-        os:
-          - macos-latest
-          - ubuntu-latest
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v3
-
-      # Work around https://github.com/actions/cache/issues/403.
-      - name: Use GNU tar
-        if: matrix.os == 'macos-latest'
-        run: |
-          echo PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH" >> $GITHUB_ENV
-
-      - name: Read Emscripten version
-        run: |
-          printf 'EMSCRIPTEN_VERSION=%s\n' "$(cat cli/emscripten-version)" >> $GITHUB_ENV
-
-      - name: Cache artifacts
-        id: cache
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            target
-          key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }}-emscripten-${{ env.EMSCRIPTEN_VERSION }}
-
-      - name: Install rust
-        if: steps.cache.outputs.cache-hit != 'true'
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: minimal
-          components: rustfmt, clippy
-
-      - name: Check Rust code formatting
-        run: cargo fmt -- --check
-
-      - name: Install emscripten
-        uses: mymindstorm/setup-emsdk@v12
-        with:
-          version: ${{ env.EMSCRIPTEN_VERSION }}
-
-      - name: Build C library
-        run: make
-
-      - name: Build wasm library
-        run: script/build-wasm
-
-      - name: Build CLI
-        run: |
-          RUSTFLAGS="-D warnings"
-          cargo build --release
-
-      - name: Set up fixture parsers
-        run: |
-          script/fetch-fixtures
-          script/generate-fixtures
-          script/generate-fixtures-wasm
-
-      - name: Run main tests
-        run: cargo test
-
-      - name: Run wasm tests
-        run: script/test-wasm
-
-      - name: Run benchmarks
-        run: script/benchmark
-
-      - name: Compress CLI binary
-        if: startsWith(github.ref, 'refs/tags/v')
-        run: |
-          cp target/release/tree-sitter .
-          export platform=$(echo ${{ runner.os }} | awk '{print tolower($0)}')
-          gzip --suffix "-${platform}-x64.gz" tree-sitter
-
-      - name: Release
-        uses: softprops/action-gh-release@v1
-        if: startsWith(github.ref, 'refs/tags/v')
-        with:
-          draft: true
-          files: |
-            tree-sitter-*.gz
-            lib/binding_web/tree-sitter.js
-            lib/binding_web/tree-sitter.wasm
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-  windows-tests:
-    name: Windows tests
-    runs-on: windows-latest
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v3
-
-      - name: Cache artifacts
-        id: cache
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            target
-          key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }}
-
-      - name: Install rust
-        if: steps.cache.outputs.cache-hit != 'true'
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: minimal
-
-      - name: Check Rust code formatting
-        run: cargo fmt -- --check
-
-      - name: Build CLI
-        run: |
-          $env:RUSTFLAGS="-D warnings"
-          cargo build --release
-
-      - name: Set up fixture parsers
-        run: |
-          script/fetch-fixtures.cmd
-          script/generate-fixtures.cmd
-
-      - name: Run main tests
-        run: script/test

.github/workflows/fast_checks.yml

@@ -0,0 +1,31 @@
+name: Fast checks to fail fast on any simple code issues
+
+env:
+  CARGO_TERM_COLOR: always
+  RUSTFLAGS: "-D warnings"
+
+on:
+  workflow_call:
+
+jobs:
+  check_rust_formatting:
+    name: Check Rust formating
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Checkout source code
+      uses: actions/checkout@v3
+
+    - name: Run cargo fmt
+      run: cargo fmt -- --check
+
+  check_c_warnings:
+    name: Check C warnings
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Checkout source code
+      uses: actions/checkout@v3
+
+    - name: Make C library to check that it's able to compile without warnings
+      run: make -j CFLAGS="-Werror"

.github/workflows/full_rust_checks.yml

@@ -0,0 +1,32 @@
+name: Full Rust codebase checks
+
+env:
+  CARGO_TERM_COLOR: always
+  RUSTFLAGS: "-D warnings"
+
+on:
+  workflow_call:
+
+jobs:
+  run:
+    name: Run checks
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Checkout source code
+      uses: actions/checkout@v3
+
+    - name: Install rust toolchain
+      uses: dtolnay/rust-toolchain@master
+      with:
+        toolchain: stable
+        components: clippy, rustfmt
+
+    - name: Run cargo fmt
+      run: cargo fmt -- --check
+
+    # - name: Run clippy
+    #   run: cargo clippy --all-targets
+
+    - name: Run cargo check
+      run: cargo check --workspace --examples --tests --benches --bins

.github/workflows/msrv.yml

@@ -0,0 +1,42 @@
+name: Minimum supported rust version
+
+env:
+  CARGO_TERM_COLOR: always
+  RUSTFLAGS: "-D warnings"
+
+on:
+  workflow_call:
+    inputs:
+      package:
+        description: Target cargo package name
+        required: true
+        type: string
+
+
+jobs:
+  run:
+    name: Run checks
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Checkout source code
+      uses: actions/checkout@v3
+
+    - name: Get the MSRV from the package metadata
+      id: msrv
+      run: cargo metadata --no-deps --format-version 1 | jq -r '"version=" + (.packages[] | select(.name == "${{ inputs.package }}").rust_version)' >> $GITHUB_OUTPUT
+
+    - name: Install rust toolchain (v${{ steps.msrv.outputs.version }})
+      uses: dtolnay/rust-toolchain@master
+      with:
+        toolchain: ${{ steps.msrv.outputs.version }}
+        components: clippy, rustfmt
+
+    - name: Run cargo fmt
+      run: cargo fmt -- --check
+
+    # - name: Run clippy (on minimum supported rust version to prevent warnings we can't fix)
+    #   run: cargo clippy --all-targets
+
+    # - name: Run main tests
+    #   run: cargo test

.github/workflows/publish.yml

@@ -0,0 +1,21 @@
+name: Publish to registries
+
+on:
+  workflow_call:
+
+jobs:
+  crates_io:
+    name: Publish to Crates.io
+    runs-on: ubuntu-latest
+    steps:
+      - name: Publish packages
+        run: |
+          echo "::warning::TODO: add a Crates.io publish logic"
+
+  npm:
+    name: Publish to npmjs.com
+    runs-on: ubuntu-latest
+    steps:
+      - name: Publish packages
+        run: |
+          echo "::warning::TODO: add a npmjs.com publish logic"

.github/workflows/release.yml

@@ -0,0 +1,101 @@
+name: Release
+
+on:
+  workflow_call:
+    inputs:
+      ref:
+        default: ${{ github.ref }}
+        type: string
+
+jobs:
+  permissions:
+    name: Check permissions
+    runs-on: ubuntu-latest
+    outputs:
+      release_allowed: ${{ steps.maintainer.outputs.is_maintainer == 'true' }}
+    steps:
+
+    - name: Is maintainer
+      id: maintainer
+      env:
+        GH_TOKEN: ${{ github.token }}
+        repo: ${{ github.repository }}
+        actor: ${{ github.actor }}
+      run: |
+        maintainer=$(
+          gh api "/repos/${repo}/collaborators" |
+          jq ".[] | {login, maintainer: .permissions | .maintain} | select(.login == \"${actor}\") | .maintainer"
+        );
+        if [ "$maintainer" == "true" ]; then
+          echo "@${actor} has maintainer level permissions :rocket:" >> $GITHUB_STEP_SUMMARY;
+          echo "is_maintainer=true" >> $GITHUB_OUTPUT
+        fi
+
+  release:
+    name: Release
+    needs: permissions
+    if: needs.permissions.outputs.release_allowed
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+
+      - name: Checkout source code
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.ref }}
+
+      - name: Download build artifacts
+        uses: actions/download-artifact@v3
+        with:
+          path: artifacts
+
+      - name: Display structure of downloaded files
+        run: ls -lR
+        working-directory: artifacts
+
+      - name: Prepare release artifacts
+        run: |
+          mkdir -p target
+          mv artifacts/tree-sitter.wasm/* target/
+          rm -r artifacts/tree-sitter.wasm
+          for platform in $(cd artifacts; ls); do
+            exe=$(ls artifacts/$platform/tree-sitter*)
+            gzip --stdout --name $exe > target/tree-sitter-$platform.gz
+          done
+          rm -rf artifacts
+          ls -l target/
+
+      - name: Get tag name from a release/v* branch name
+        id: tag_name
+        env:
+          tag: ${{ github.head_ref }}
+        run: echo "tag=${tag#release/}" >> $GITHUB_OUTPUT
+
+      - name: Add a release tag
+        env:
+          ref: ${{ inputs.ref }}
+          tag: ${{ steps.tag_name.outputs.tag }}
+          message: "Release ${{ steps.tag_name.outputs.tag }}"
+        run: |
+          git config user.name "${GITHUB_ACTOR}"
+          git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
+          git tag -a "$tag" HEAD -m "$message"
+          git push origin "$tag"
+
+      - name: Create release
+        uses: softprops/action-gh-release@v1
+        with:
+          name: ${{ steps.tag_name.outputs.tag }}
+          tag_name: ${{ steps.tag_name.outputs.tag }}
+          fail_on_unmatched_files: true
+          files: |
+            tree-sitter-*.gz
+            tree-sitter.wasm
+            tree-sitter.js
+
+      - name: Merge release PR
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh pr merge ${{ github.event.pull_request.html_url }} --match-head-commit $(git rev-parse HEAD) --merge --delete-branch