diff --git a/.github/actions/cache/action.yml b/.github/actions/cache/action.yml index cc816682..a32cc294 100644 --- a/.github/actions/cache/action.yml +++ b/.github/actions/cache/action.yml @@ -1,24 +1,23 @@ -name: 'Cache' -description: "This action caches fixtures" +name: Cache + +description: This action caches fixtures + outputs: cache-hit: - description: 'Cache hit' - value: ${{ steps.cache_output.outputs.cache-hit }} + description: Cache hit + value: ${{ steps.cache.outputs.cache-hit }} + runs: - using: "composite" + using: composite steps: - uses: actions/cache@v4 - id: cache_fixtures + id: cache with: path: | test/fixtures/grammars target/release/tree-sitter-*.wasm key: fixtures-${{ join(matrix.*, '_') }}-${{ hashFiles( - 'cli/src/generate/**', - 'script/generate-fixtures*', + 'cli/generate/src/**', + 'xtask/src/*', 'test/fixtures/grammars/*/**/src/*.c', '.github/actions/cache/action.yml') }} - - - run: echo "cache-hit=${{ steps.cache_fixtures.outputs.cache-hit }}" >> $GITHUB_OUTPUT - shell: bash - id: cache_output diff --git a/.github/scripts/cross.sh b/.github/scripts/cross.sh index a52f0873..de1d4e94 100755 --- a/.github/scripts/cross.sh +++ b/.github/scripts/cross.sh @@ -1,17 +1,3 @@ -#!/bin/bash +#!/bin/bash -eu -# set -x -set -e - -if [ "$BUILD_CMD" != "cross" ]; then - echo "cross.sh - is a helper to assist only in cross compiling environments" >&2 - echo "To use this tool set the BUILD_CMD env var to the \"cross\" value" >&2 - exit 111 -fi - -if [ -z "$CROSS_IMAGE" ]; then - echo "The CROSS_IMAGE env var should be provided" >&2 - exit 111 -fi - -docker run --rm -v /home/runner:/home/runner -w "$PWD" "$CROSS_IMAGE" "$@" +exec docker run --rm -v /home/runner:/home/runner -w "$PWD" "$CROSS_IMAGE" "$@" diff --git a/.github/scripts/make.sh b/.github/scripts/make.sh index 79192541..175074f9 100755 --- a/.github/scripts/make.sh +++ b/.github/scripts/make.sh @@ -1,19 +1,9 @@ -#!/bin/bash +#!/bin/bash -eu -# set -x -set -e +tree_sitter="$ROOT"/target/"$TARGET"/release/tree-sitter -if [ "$BUILD_CMD" == "cross" ]; then - if [ -z "$CC" ]; then - echo "make.sh: CC is not set" >&2 - exit 111 - fi - if [ -z "$AR" ]; then - echo "make.sh: AR is not set" >&2 - exit 111 - fi - - cross.sh make CC=$CC AR=$AR "$@" +if [[ $BUILD_CMD == cross ]]; then + cross.sh make CC="$CC" AR="$AR" "$@" else - make "$@" + exec make "$@" fi diff --git a/.github/scripts/tree-sitter.sh b/.github/scripts/tree-sitter.sh index 0cac9153..125a2d92 100755 --- a/.github/scripts/tree-sitter.sh +++ b/.github/scripts/tree-sitter.sh @@ -1,28 +1,9 @@ -#!/bin/bash - -# set -x -set -e - -if [ -z "$ROOT" ]; then - echo "The ROOT env var should be set to absolute path of a repo root folder" >&2 - exit 111 -fi - -if [ -z "$TARGET" ]; then - echo "The TARGET env var should be equal to a \`cargo build --target \` command value" >&2 - exit 111 -fi +#!/bin/bash -eu tree_sitter="$ROOT"/target/"$TARGET"/release/tree-sitter -if [ "$BUILD_CMD" == "cross" ]; then - if [ -z "$CROSS_RUNNER" ]; then - echo "The CROSS_RUNNER env var should be set to a CARGO_TARGET_*_RUNNER env var value" >&2 - echo "that is available in a docker image used by the cross tool under the hood" >&2 - exit 111 - fi - - cross.sh $CROSS_RUNNER "$tree_sitter" "$@" +if [[ $BUILD_CMD == cross ]]; then + cross.sh "$CROSS_RUNNER" "$tree_sitter" "$@" else - "$tree_sitter" "$@" + exec "$tree_sitter" "$@" fi diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 0c3ba6be..a0c15e01 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -1,26 +1,29 @@ -name: backport +name: Backport Pull Request + on: pull_request_target: types: [closed, labeled] + +permissions: + contents: write + pull-requests: write + jobs: backport: - permissions: - contents: write - pull-requests: write - name: Backport Pull Request if: github.event.pull_request.merged runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v4 - - uses: actions/create-github-app-token@v1 + - name: Create app token + uses: actions/create-github-app-token@v1 id: app-token with: app-id: ${{ vars.BACKPORT_APP }} private-key: ${{ secrets.BACKPORT_KEY }} - name: Create backport PR - id: backport uses: korthout/backport-action@v3 with: pull_title: "${pull_title}" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 85624cbf..74abe69d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,7 +8,7 @@ env: on: workflow_call: inputs: - run_test: + run-test: default: true type: boolean @@ -21,92 +21,93 @@ jobs: fail-fast: false matrix: platform: - - linux-arm64 # - - linux-arm # - - linux-x64 # - - linux-x86 # - - linux-powerpc64 # - - windows-arm64 # - - windows-x64 # <-- No C library build - requires an additional adapted Makefile for `cl.exe` compiler - - windows-x86 # -- // -- - - macos-arm64 # - - macos-x64 # + - linux-arm64 + - linux-arm + - linux-x64 + - linux-x86 + - linux-powerpc64 + - windows-arm64 + - windows-x64 + - windows-x86 + - macos-arm64 + - macos-x64 include: - # When adding a new `target`: - # 1. Define a new platform alias above - # 2. Add a new record to a matrix map in `cli/npm/install.js` - - { platform: linux-arm64 , target: aarch64-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } - - { platform: linux-arm , target: arm-unknown-linux-gnueabi , os: ubuntu-latest , use-cross: true } - - { platform: linux-x64 , target: x86_64-unknown-linux-gnu , os: ubuntu-20.04 , cli_features: wasm } #2272 - - { platform: linux-x86 , target: i686-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } - - { platform: linux-powerpc64 , target: powerpc64-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } - - { platform: windows-arm64 , target: aarch64-pc-windows-msvc , os: windows-latest } - - { platform: windows-x64 , target: x86_64-pc-windows-msvc , os: windows-latest , cli_features: wasm } - - { platform: windows-x86 , target: i686-pc-windows-msvc , os: windows-latest } - - { platform: macos-arm64 , target: aarch64-apple-darwin , os: macos-14 , cli_features: wasm } - - { platform: macos-x64 , target: x86_64-apple-darwin , os: macos-12 , cli_features: wasm } + # When adding a new `target`: + # 1. Define a new platform alias above + # 2. Add a new record to the matrix map in `cli/npm/install.js` + - { platform: linux-arm64 , target: aarch64-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } + - { platform: linux-arm , target: arm-unknown-linux-gnueabi , os: ubuntu-latest , use-cross: true } + - { platform: linux-x64 , target: x86_64-unknown-linux-gnu , os: ubuntu-20.04 , features: wasm } # See #2272 + - { platform: linux-x86 , target: i686-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } + - { platform: linux-powerpc64 , target: powerpc64-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } + - { platform: windows-arm64 , target: aarch64-pc-windows-msvc , os: windows-latest } + - { platform: windows-x64 , target: x86_64-pc-windows-msvc , os: windows-latest , features: wasm } + - { platform: windows-x86 , target: i686-pc-windows-msvc , os: windows-latest } + - { platform: macos-arm64 , target: aarch64-apple-darwin , os: macos-14 , features: wasm } + - { platform: macos-x64 , target: x86_64-apple-darwin , os: macos-13 , features: wasm } - # Cross compilers for C library - - { platform: linux-arm64 , cc: aarch64-linux-gnu-gcc , ar: aarch64-linux-gnu-ar } - - { platform: linux-arm , cc: arm-linux-gnueabi-gcc , ar: arm-linux-gnueabi-ar } - - { platform: linux-x86 , cc: i686-linux-gnu-gcc , ar: i686-linux-gnu-ar } - - { platform: linux-powerpc64 , cc: powerpc64-linux-gnu-gcc , ar: powerpc64-linux-gnu-ar } + # Cross compilers for C library + - { platform: linux-arm64 , cc: aarch64-linux-gnu-gcc , ar: aarch64-linux-gnu-ar } + - { platform: linux-arm , cc: arm-linux-gnueabi-gcc , ar: arm-linux-gnueabi-ar } + - { platform: linux-x86 , cc: i686-linux-gnu-gcc , ar: i686-linux-gnu-ar } + - { platform: linux-powerpc64 , cc: powerpc64-linux-gnu-gcc , ar: powerpc64-linux-gnu-ar } - # See #2041 tree-sitter issue - - { platform: windows-x64 , rust-test-threads: 1 } - - { platform: windows-x86 , rust-test-threads: 1 } + # Prevent race condition (see #2041) + - { platform: windows-x64 , rust-test-threads: 1 } + - { platform: windows-x86 , rust-test-threads: 1 } - # CLI only build - - { platform: windows-arm64 , cli-only: true } + # Can't natively run CLI on Github runner's host + - { platform: windows-arm64 , no-run: true } env: BUILD_CMD: cargo - EXE: ${{ contains(matrix.target, 'windows') && '.exe' || '' }} + SUFFIX: ${{ contains(matrix.target, 'windows') && '.exe' || '' }} defaults: run: shell: bash steps: - - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v4 - name: Read Emscripten version - run: echo "EMSCRIPTEN_VERSION=$(cat cli/loader/emscripten-version)" >> $GITHUB_ENV + run: printf 'EMSCRIPTEN_VERSION=%s\n' "$(> $GITHUB_ENV - name: Install Emscripten - if: ${{ !matrix.cli-only && !matrix.use-cross }} + if: ${{ !matrix.no-run && !matrix.use-cross }} uses: mymindstorm/setup-emsdk@v14 with: version: ${{ env.EMSCRIPTEN_VERSION }} - - run: rustup toolchain install stable --profile minimal - - run: rustup target add ${{ matrix.targetĀ }} - - uses: Swatinem/rust-cache@v2 + - name: Set up Rust + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + target: ${{ matrix.target }} - name: Install cross if: ${{ matrix.use-cross }} - uses: taiki-e/install-action@v2 - with: - tool: cross + run: RUSTFLAGS="" cargo install cross --git https://github.com/cross-rs/cross - - name: Build custom cross image - if: ${{ matrix.use-cross && matrix.os == 'ubuntu-latest' }} + - name: Configure cross + if: ${{ matrix.use-cross }} run: | - target="${{ matrix.target }}" - image=ghcr.io/cross-rs/$target:custom - echo "CROSS_IMAGE=$image" >> $GITHUB_ENV + printf '%s\n' > Cross.toml \ + '[target.${{ matrix.target }}]' \ + 'image = "ghcr.io/cross-rs/${{ matrix.target }}:edge"' \ + '[build]' \ + 'pre-build = [' \ + ' "dpkg --add-architecture $CROSS_DEB_ARCH",' \ + ' "curl -fsSL https://deb.nodesource.com/setup_22.x | bash -",' \ + ' "apt-get update && apt-get -y install libssl-dev nodejs"' \ + ']' + cat - Cross.toml <<< 'Cross.toml:' + printf '%s\n' >> $GITHUB_ENV \ + "CROSS_CONFIG=$PWD/Cross.toml" \ + "CROSS_IMAGE=ghcr.io/cross-rs/${{ matrix.target }}:edge" - echo "[target.$target]" >> Cross.toml - echo "image = \"$image\"" >> Cross.toml - echo "CROSS_CONFIG=$PWD/Cross.toml" >> $GITHUB_ENV - - echo "FROM ghcr.io/cross-rs/$target:edge" >> Dockerfile - echo "RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash -" >> Dockerfile - echo "RUN apt-get update && apt-get -y install nodejs" >> Dockerfile - docker build -t $image . - - - name: Setup env extras + - name: Set up environment env: RUST_TEST_THREADS: ${{ matrix.rust-test-threads }} USE_CROSS: ${{ matrix.use-cross }} @@ -115,68 +116,117 @@ jobs: AR: ${{ matrix.ar }} run: | PATH="$PWD/.github/scripts:$PATH" - echo "$PWD/.github/scripts" >> $GITHUB_PATH + printf '%s/.github/scripts\n' "$PWD" >> $GITHUB_PATH - echo "TREE_SITTER=tree-sitter.sh" >> $GITHUB_ENV - echo "TARGET=$TARGET" >> $GITHUB_ENV - echo "ROOT=$PWD" >> $GITHUB_ENV + printf '%s\n' >> $GITHUB_ENV \ + 'TREE_SITTER=tree-sitter.sh' \ + "TARGET=$TARGET" \ + "ROOT=$PWD" - [ -n "$RUST_TEST_THREADS" ] && \ - echo "RUST_TEST_THREADS=$RUST_TEST_THREADS" >> $GITHUB_ENV + [[ -n $RUST_TEST_THREADS ]] && \ + printf 'RUST_TEST_THREADS=%s\n' "$RUST_TEST_THREADS" >> $GITHUB_ENV - [ -n "$CC" ] && echo "CC=$CC" >> $GITHUB_ENV - [ -n "$AR" ] && echo "AR=$AR" >> $GITHUB_ENV + [[ -n $CC ]] && printf 'CC=%s\n' "$CC" >> $GITHUB_ENV + [[ -n $AR ]] && printf 'AR=%s\n' "$AR" >> $GITHUB_ENV - if [ "$USE_CROSS" == "true" ]; then - echo "BUILD_CMD=cross" >> $GITHUB_ENV - runner=$(BUILD_CMD=cross cross.sh bash -c "env | sed -nr '/^CARGO_TARGET_.*_RUNNER=/s///p'") - [ -n "$runner" ] && echo "CROSS_RUNNER=$runner" >> $GITHUB_ENV + if [[ $USE_CROSS == true ]]; then + printf 'BUILD_CMD=cross\n' >> $GITHUB_ENV + runner=$(cross.sh bash -c "env | sed -n 's/^CARGO_TARGET_.*_RUNNER=//p'") + [[ -n $runner ]] && printf 'CROSS_RUNNER=%s\n' "$runner" >> $GITHUB_ENV fi - - name: Build C library - if: ${{ !contains(matrix.os, 'windows') }} # Requires an additional adapted Makefile for `cl.exe` compiler - run: make.sh -j CFLAGS="-Werror" + - name: Build wasmtime library + if: ${{ !matrix.use-cross && contains(matrix.features, 'wasm') }} + run: | + mkdir -p target + WASMTIME_VERSION=$(cargo metadata --format-version=1 --locked --features wasm | \ + jq -r '.packages[] | select(.name == "wasmtime-c-api-impl") | .version') + curl -LSs "$WASMTIME_REPO/archive/refs/tags/v${WASMTIME_VERSION}.tar.gz" | tar xzf - -C target + cd target/wasmtime-${WASMTIME_VERSION} + cmake -S crates/c-api -B target/c-api \ + -DCMAKE_INSTALL_PREFIX="$PWD/artifacts" \ + -DWASMTIME_DISABLE_ALL_FEATURES=ON \ + -DWASMTIME_FEATURE_CRANELIFT=ON \ + -DWASMTIME_TARGET='${{ matrix.target }}' + cmake --build target/c-api && cmake --install target/c-api + printf 'CMAKE_PREFIX_PATH=%s\n' "$PWD/artifacts" >> $GITHUB_ENV + env: + WASMTIME_REPO: https://github.com/bytecodealliance/wasmtime + RUSTFLAGS: "" + + - name: Build C library (make) + if: ${{ runner.os != 'Windows' }} + run: make.sh -j CFLAGS="$CFLAGS" + env: + CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types + + - name: Build C library (CMake) + if: ${{ !matrix.use-cross }} + run: | + cmake -S lib -B build/static \ + -DBUILD_SHARED_LIBS=OFF \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \ + -DTREE_SITTER_FEATURE_WASM=$WASM + cmake --build build/static --verbose + + cmake -S lib -B build/shared \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \ + -DTREE_SITTER_FEATURE_WASM=$WASM + cmake --build build/shared --verbose + env: + CC: ${{ contains(matrix.target, 'linux') && 'clang' || '' }} + WASM: ${{ contains(matrix.features, 'wasm') && 'ON' || 'OFF' }} - name: Build wasm library - if: ${{ !matrix.cli-only && !matrix.use-cross }} # No sense to build on the same Github runner hosts many times - run: script/build-wasm + # No reason to build on the same Github runner hosts many times + if: ${{ !matrix.no-run && !matrix.use-cross }} + run: $BUILD_CMD run -p xtask -- build-wasm - - run: $BUILD_CMD build --release --target=${{ matrix.target }} --features=${{ matrix.cli_features }} + - name: Build target + run: $BUILD_CMD build --release --target=${{ matrix.target }} --features=${{ matrix.features }} - - run: script/fetch-fixtures - - - uses: ./.github/actions/cache + - name: Cache fixtures id: cache + if: ${{ !matrix.no-run && inputs.run-test }} + uses: ./.github/actions/cache + + - name: Fetch fixtures + if: ${{ !matrix.no-run && inputs.run-test }} + run: $BUILD_CMD run -p xtask -- fetch-fixtures - name: Generate fixtures - if: ${{ !matrix.cli-only && inputs.run_test && steps.cache.outputs.cache-hit != 'true' }} # Can't natively run CLI on Github runner's host - run: script/generate-fixtures + if: ${{ !matrix.no-run && inputs.run-test && steps.cache.outputs.cache-hit != 'true' }} + run: $BUILD_CMD run -p xtask -- generate-fixtures - - name: Generate WASM fixtures - if: ${{ !matrix.cli-only && !matrix.use-cross && inputs.run_test && steps.cache.outputs.cache-hit != 'true' }} # See comment for the "Build wasm library" step - run: script/generate-fixtures-wasm + - name: Generate Wasm fixtures + if: ${{ !matrix.no-run && !matrix.use-cross && inputs.run-test && steps.cache.outputs.cache-hit != 'true' }} + run: $BUILD_CMD run -p xtask -- generate-fixtures --wasm - name: Run main tests - if: ${{ !matrix.cli-only && inputs.run_test }} # Can't natively run CLI on Github runner's host - run: $BUILD_CMD test --target=${{ matrix.target }} --features=${{ matrix.cli_features }} + if: ${{ !matrix.no-run && inputs.run-test }} + run: $BUILD_CMD test --target=${{ matrix.target }} --features=${{ matrix.features }} - name: Run wasm tests - if: ${{ !matrix.cli-only && !matrix.use-cross && inputs.run_test }} # See comment for the "Build wasm library" step - run: script/test-wasm + if: ${{ !matrix.no-run && !matrix.use-cross && inputs.run-test }} + run: $BUILD_CMD run -p xtask -- test-wasm - name: Run benchmarks - if: ${{ !matrix.cli-only && !matrix.use-cross && inputs.run_test }} # Cross-compiled benchmarks make no sense + # Cross-compiled benchmarks are pointless + if: ${{ !matrix.no-run && !matrix.use-cross && inputs.run-test }} run: $BUILD_CMD bench benchmark -p tree-sitter-cli --target=${{ matrix.target }} - name: Upload CLI artifact uses: actions/upload-artifact@v4 with: name: tree-sitter.${{ matrix.platform }} - path: target/${{ matrix.target }}/release/tree-sitter${{ env.EXE }} + path: target/${{ matrix.target }}/release/tree-sitter${{ env.SUFFIX }} if-no-files-found: error retention-days: 7 - - name: Upload WASM artifacts + - name: Upload Wasm artifacts if: ${{ matrix.platform == 'linux-x64' }} uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de51faf5..dac4cdbf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,9 +1,9 @@ name: CI + on: pull_request: push: - branches: - - 'master' + branches: [master] concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -13,12 +13,22 @@ jobs: checks: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - run: rustup toolchain install stable --profile minimal - - run: rustup toolchain install nightly --profile minimal - - run: rustup component add --toolchain nightly rustfmt - - uses: Swatinem/rust-cache@v2 - - run: make lint + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up stable Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: stable + + - name: Set up nightly Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: nightly + components: clippy, rustfmt + + - name: Lint files + run: make lint sanitize: uses: ./.github/workflows/sanitize.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cd25fd71..7c0152ff 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,5 @@ name: Release + on: workflow_dispatch: push: @@ -9,16 +10,17 @@ jobs: build: uses: ./.github/workflows/build.yml with: - run_test: false + run-test: false release: - name: Release + name: Release on GitHub runs-on: ubuntu-latest needs: build permissions: contents: write steps: - - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v4 - name: Download build artifacts uses: actions/download-artifact@v4 @@ -42,29 +44,24 @@ jobs: ls -l target/ - name: Create release - uses: softprops/action-gh-release@v2 - with: - name: ${{ github.ref_name }} - tag_name: ${{ github.ref_name }} - fail_on_unmatched_files: true - files: | - target/tree-sitter-*.gz - target/tree-sitter.wasm + run: |- + gh release create \ + target/tree-sitter-*.gz \ + target/tree-sitter.wasm \ target/tree-sitter.js + env: + GH_TOKEN: ${{ github.token }} crates_io: - name: Publish CLI to Crates.io + name: Publish packages to Crates.io runs-on: ubuntu-latest needs: release steps: - - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v4 - - name: Setup Rust - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true + - name: Set up Rust + uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Publish crates to Crates.io uses: katyo/publish-crates@v2 @@ -72,29 +69,32 @@ jobs: registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }} npm: - name: Publish lib to npmjs.com + name: Publish packages to npmjs.com runs-on: ubuntu-latest needs: release strategy: fail-fast: false matrix: - directory: ["cli/npm", "lib/binding_web"] + directory: [cli/npm, lib/binding_web] steps: - - uses: actions/checkout@v4 + - name: CHeckout repository + uses: actions/checkout@v4 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: 20 + registry-url: https://registry.npmjs.org + + - name: Set up Rust + uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Build wasm if: matrix.directory == 'lib/binding_web' - run: ./script/build-wasm + run: cargo xtask build-wasm - - name: Setup Node - uses: actions/setup-node@v4 - with: - node-version: 18 - registry-url: "https://registry.npmjs.org" - - - name: Publish lib to npmjs.com + - name: Publish to npmjs.com + working-directory: ${{ matrix.directory }} + run: npm publish env: - NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} - run: | - cd ${{ matrix.directory }} - npm publish + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.github/workflows/response.yml b/.github/workflows/response.yml index 663ae6ad..576b9474 100644 --- a/.github/workflows/response.yml +++ b/.github/workflows/response.yml @@ -1,34 +1,47 @@ -name: no_response +name: No response + on: schedule: - - cron: '30 1 * * *' # Run every day at 01:30 + - cron: "30 1 * * *" # Run every day at 01:30 workflow_dispatch: issue_comment: +permissions: + issues: write + pull-requests: write + jobs: close: + name: Close issues with no response if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest - permissions: - issues: write - pull-requests: write steps: - - uses: actions/checkout@v4 - - uses: actions/github-script@v7 + - name: Checkout script + uses: actions/checkout@v4 + with: + sparse-checkout: .github/scripts/close_unresponsive.js + sparse-checkout-cone-mode: false + + - name: Run script + uses: actions/github-script@v7 with: script: | const script = require('./.github/scripts/close_unresponsive.js') await script({github, context}) remove_label: + name: Remove response label if: github.event_name == 'issue_comment' runs-on: ubuntu-latest - permissions: - issues: write - pull-requests: write steps: - - uses: actions/checkout@v4 - - uses: actions/github-script@v7 + - name: Checkout script + uses: actions/checkout@v4 + with: + sparse-checkout: .github/scripts/remove_response_label.js + sparse-checkout-cone-mode: false + + - name: Run script + uses: actions/github-script@v7 with: script: | const script = require('./.github/scripts/remove_response_label.js') diff --git a/.github/workflows/reviewers_remove.yml b/.github/workflows/reviewers_remove.yml index b10d8c3d..b99f0caa 100644 --- a/.github/workflows/reviewers_remove.yml +++ b/.github/workflows/reviewers_remove.yml @@ -1,15 +1,23 @@ -name: "reviewers: remove" +name: Remove Reviewers + on: pull_request_target: types: [converted_to_draft, closed] + +permissions: + pull-requests: write + jobs: remove-reviewers: runs-on: ubuntu-latest - permissions: - pull-requests: write steps: - - uses: actions/checkout@v4 - - name: 'Remove reviewers' + - name: Checkout script + uses: actions/checkout@v4 + with: + sparse-checkout: .github/scripts/reviewers_remove.js + sparse-checkout-cone-mode: false + + - name: Run script uses: actions/github-script@v7 with: script: | diff --git a/.github/workflows/sanitize.yml b/.github/workflows/sanitize.yml index 72bdd079..875b8278 100644 --- a/.github/workflows/sanitize.yml +++ b/.github/workflows/sanitize.yml @@ -8,39 +8,44 @@ on: workflow_call: jobs: - check_undefined_behaviour: - name: Sanitizer checks + check-undefined-behaviour: runs-on: ubuntu-latest timeout-minutes: 20 env: TREE_SITTER: ${{ github.workspace }}/target/release/tree-sitter steps: - - name: Checkout source code - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v4 - - name: Install UBSAN library - run: sudo apt-get update -y && sudo apt-get install -y libubsan1 + - name: Install UBSAN library + run: sudo apt-get update -y && sudo apt-get install -y libubsan1 - - run: rustup toolchain install stable --profile minimal - - uses: Swatinem/rust-cache@v2 - - run: cargo build --release - - run: script/fetch-fixtures + - name: Set up Rust + uses: actions-rust-lang/setup-rust-toolchain@v1 - - uses: ./.github/actions/cache - id: cache + - name: Build project + run: cargo build --release - - if: ${{ steps.cache.outputs.cache-hit != 'true' }} - run: script/generate-fixtures + - name: Cache fixtures + uses: ./.github/actions/cache + id: cache - - name: Run main tests with undefined behaviour sanitizer (UBSAN) - env: - CFLAGS: -fsanitize=undefined - RUSTFLAGS: ${{ env.RUSTFLAGS }} -lubsan - run: cargo test -- --test-threads 1 + - name: Fetch fixtures + run: cargo xtask fetch-fixtures - - name: Run main tests with address sanitizer (ASAN) - env: - ASAN_OPTIONS: verify_asan_link_order=0 - CFLAGS: -fsanitize=address - RUSTFLAGS: ${{ env.RUSTFLAGS }} -lasan --cfg sanitizing - run: cargo test -- --test-threads 1 + - name: Generate fixtures + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + run: cargo xtask generate-fixtures + + - name: Run main tests with undefined behaviour sanitizer (UBSAN) + run: cargo test -- --test-threads 1 + env: + CFLAGS: -fsanitize=undefined + RUSTFLAGS: ${{ env.RUSTFLAGS }} -lubsan + + - name: Run main tests with address sanitizer (ASAN) + run: cargo test -- --test-threads 1 + env: + ASAN_OPTIONS: verify_asan_link_order=0 + CFLAGS: -fsanitize=address + RUSTFLAGS: ${{ env.RUSTFLAGS }} -lasan --cfg sanitizing diff --git a/Cargo.lock b/Cargo.lock index e65d81dd..88860f32 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -156,9 +156,9 @@ checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" [[package]] name = "cc" -version = "1.1.24" +version = "1.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812acba72f0a070b003d3697490d2b55b837230ae7c6c6497f05cc2ddbb8d938" +checksum = "2e80e3b6a3ab07840e1cae9b0666a63970dc28e8ed5ffbcdacbfc760c281bfc1" dependencies = [ "jobserver", "libc", @@ -311,18 +311,18 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cranelift-bforest" -version = "0.112.1" +version = "0.112.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6e376bd92bddd03dcfc443b14382611cae5d10012aa0b1628bbf18bb73f12f7" +checksum = "7b765ed4349e66bedd9b88c7691da42e24c7f62067a6be17ddffa949367b6e17" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-bitset" -version = "0.112.1" +version = "0.112.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45ecbe07f25a8100e5077933516200e97808f1d7196b5a073edb85fa08fde32e" +checksum = "9eaa2aece6237198afd32bff57699e08d4dccb8d3902c214fc1e6ba907247ca4" dependencies = [ "serde", "serde_derive", @@ -330,9 +330,9 @@ dependencies = [ [[package]] name = "cranelift-codegen" -version = "0.112.1" +version = "0.112.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc60913f32c1de18538c28bef74b8c87cf16de7841a1b0956fcf01b23237853a" +checksum = "351824439e59d42f0e4fa5aac1d13deded155120043565769e55cd4ad3ca8ed9" dependencies = [ "bumpalo", "cranelift-bforest", @@ -353,33 +353,33 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.112.1" +version = "0.112.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bae009e7822f47aa55e7dcef846ccf3aa4eb102ca6b4bcb8a44b36f3f49aa85c" +checksum = "5a0ce0273d7a493ef8f31f606849a4e931c19187a4923f5f87fc1f2b13109981" dependencies = [ "cranelift-codegen-shared", ] [[package]] name = "cranelift-codegen-shared" -version = "0.112.1" +version = "0.112.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c78f01a852536c68e34444450f845ed6e0782a1f047f85397fe460b8fbce8f1" +checksum = "0f72016ac35579051913f4f07f6b36c509ed69412d852fd44c8e1d7b7fa6d92a" [[package]] name = "cranelift-control" -version = "0.112.1" +version = "0.112.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a061b22e00a9e36b31f2660dfb05a9617b7775bd54b79754d3bb75a990dac06" +checksum = "db28951d21512c4fd0554ef179bfb11e4eb6815062957a9173824eee5de0c46c" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.112.1" +version = "0.112.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95e2b261a3e74ae42f4e606906d5ffa44ee2684e8b1ae23bdf75d21908dc9233" +checksum = "14ebe592a2f81af9237cf9be29dd3854ecb72108cfffa59e85ef12389bf939e3" dependencies = [ "cranelift-bitset", "serde", @@ -388,9 +388,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.112.1" +version = "0.112.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe14abba0e6bab42aca0f9ce757f96880f9187e88bc6cb975ed6acd8a42f7770" +checksum = "4437db9d60c7053ac91ded0802740c2ccf123ee6d6898dd906c34f8c530cd119" dependencies = [ "cranelift-codegen", "log", @@ -400,15 +400,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.112.1" +version = "0.112.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "311d91ae72b37d4262b51217baf8c9e01f1afd5148931468da1fdb7e9d011347" +checksum = "230cb33572b9926e210f2ca28145f2bc87f389e1456560932168e2591feb65c1" [[package]] name = "cranelift-native" -version = "0.112.1" +version = "0.112.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a3f84c75e578189ff7a716c24ad83740b553bf583f2510b323bfe4c1a74bb93" +checksum = "364524ac7aef7070b1141478724abebeec297d4ea1e87ad8b8986465e91146d9" dependencies = [ "cranelift-codegen", "libc", @@ -417,9 +417,9 @@ dependencies = [ [[package]] name = "cranelift-wasm" -version = "0.112.1" +version = "0.112.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f56b7b2476c47b2091eee5a20bc54a80fbb29ca5313ae2bd0dea52621abcfca1" +checksum = "0572cbd9d136a62c0f39837b6bce3b0978b96b8586794042bec0c214668fd6f5" dependencies = [ "cranelift-codegen", "cranelift-entity", @@ -442,9 +442,9 @@ dependencies = [ [[package]] name = "ctor" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f" +checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501" dependencies = [ "quote", "syn", @@ -565,6 +565,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "foldhash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -651,6 +657,9 @@ name = "hashbrown" version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +dependencies = [ + "foldhash", +] [[package]] name = "heck" @@ -982,24 +991,21 @@ dependencies = [ [[package]] name = "object" -version = "0.36.4" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "crc32fast", - "hashbrown 0.14.5", + "hashbrown 0.15.0", "indexmap", "memchr", ] [[package]] name = "once_cell" -version = "1.20.1" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1" -dependencies = [ - "portable-atomic", -] +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "openssl-probe" @@ -1055,12 +1061,6 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" -[[package]] -name = "portable-atomic" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" - [[package]] name = "postcard" version = "1.0.10" @@ -1104,9 +1104,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "b3e4daa0dcf6feba26f985457cdf104d4b4256fc5a09547140f3631bb076b19a" dependencies = [ "unicode-ident", ] @@ -1538,7 +1538,7 @@ dependencies = [ [[package]] name = "tree-sitter" -version = "0.24.1" +version = "0.24.7" dependencies = [ "bindgen", "cc", @@ -1551,7 +1551,7 @@ dependencies = [ [[package]] name = "tree-sitter-cli" -version = "0.24.1" +version = "0.24.7" dependencies = [ "anstyle", "anyhow", @@ -1601,7 +1601,7 @@ dependencies = [ [[package]] name = "tree-sitter-config" -version = "0.24.1" +version = "0.24.7" dependencies = [ "anyhow", "dirs", @@ -1611,7 +1611,7 @@ dependencies = [ [[package]] name = "tree-sitter-generate" -version = "0.24.1" +version = "0.24.7" dependencies = [ "anyhow", "heck 0.5.0", @@ -1632,7 +1632,7 @@ dependencies = [ [[package]] name = "tree-sitter-highlight" -version = "0.24.1" +version = "0.24.7" dependencies = [ "lazy_static", "regex", @@ -1643,17 +1643,18 @@ dependencies = [ [[package]] name = "tree-sitter-language" -version = "0.1.2" +version = "0.1.3" [[package]] name = "tree-sitter-loader" -version = "0.24.1" +version = "0.24.7" dependencies = [ "anyhow", "cc", "dirs", "fs4", "indoc", + "lazy_static", "libloading", "once_cell", "path-slash", @@ -1670,7 +1671,7 @@ dependencies = [ [[package]] name = "tree-sitter-tags" -version = "0.24.1" +version = "0.24.7" dependencies = [ "memchr", "regex", @@ -1871,9 +1872,9 @@ dependencies = [ [[package]] name = "wasmtime" -version = "25.0.1" +version = "25.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03601559991d459a228236a49135364eac85ac00dc07b65fb95ae61a957793af" +checksum = "ef01f9cb9636ed42a7ec5a09d785c0643590199dc7372dc22c7e2ba7a31a97d4" dependencies = [ "anyhow", "bitflags", @@ -1911,18 +1912,18 @@ dependencies = [ [[package]] name = "wasmtime-asm-macros" -version = "25.0.1" +version = "25.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e453b3bde07312874c0c6703e2de9281daab46646172c1b71fa59a97226f858e" +checksum = "ba5b20797419d6baf2296db2354f864e8bb3447cacca9d151ce7700ae08b4460" dependencies = [ "cfg-if", ] [[package]] name = "wasmtime-c-api-impl" -version = "25.0.1" +version = "25.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4def1c38f8981c88d92e10acc7efb01da5b5775897fca2ab81caad76e930bd6d" +checksum = "2852f09a087c740683a32a33b8f34268d1d33c1298b4f707d25f4bee158ccd75" dependencies = [ "anyhow", "log", @@ -1934,9 +1935,9 @@ dependencies = [ [[package]] name = "wasmtime-c-api-macros" -version = "25.0.1" +version = "25.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c3feb5a461c52a376e80ef7ce7cee37a3a8395cb1794ac8eb340c0cd0b5d715" +checksum = "fa52cecfad085e7a9725bcbf3c2b15a900e5dc14f5ddcc305c9779c19936618b" dependencies = [ "proc-macro2", "quote", @@ -1944,9 +1945,9 @@ dependencies = [ [[package]] name = "wasmtime-component-macro" -version = "25.0.1" +version = "25.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a6faeabbdbfd27e24e8d5204207ba9c247a13cf84181ea721b5f209f281fe01" +checksum = "26593c4b18c76ca3c3fbdd813d6692256537b639b851d8a6fe827e3d6966fc01" dependencies = [ "anyhow", "proc-macro2", @@ -1959,15 +1960,15 @@ dependencies = [ [[package]] name = "wasmtime-component-util" -version = "25.0.1" +version = "25.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b1b24db4aa3dc7c0d3181d1833b4fe9ec0cd3f08780b746415c84c0a9ec9011" +checksum = "a2ed562fbb0cbed20a56c369c8de146c1de06a48c19e26ed9aa45f073514ee60" [[package]] name = "wasmtime-cranelift" -version = "25.0.1" +version = "25.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c737bef9ea94aab874e29ac6a8688b89ceb43c7b51f047079c43387972c07ee3" +checksum = "f389b789cbcb53a8499131182135dea21d7d97ad77e7fb66830f69479ef0e68c" dependencies = [ "anyhow", "cfg-if", @@ -1990,9 +1991,9 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "25.0.1" +version = "25.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "817bfa9ea878ec37aa24f85fd6912844e8d87d321662824cf920d561b698cdfd" +checksum = "84b72debe8899f19bedf66f7071310f06ef62de943a1369ba9b373613e77dd3d" dependencies = [ "anyhow", "cranelift-bitset", @@ -2013,9 +2014,9 @@ dependencies = [ [[package]] name = "wasmtime-jit-icache-coherence" -version = "25.0.1" +version = "25.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48011232c0da424f89c3752a378d0b7f512fae321ea414a43e1e7a302a6a1f7e" +checksum = "1d930bc1325bc0448be6a11754156d770f56f6c3a61f440e9567f36cd2ea3065" dependencies = [ "anyhow", "cfg-if", @@ -2025,15 +2026,15 @@ dependencies = [ [[package]] name = "wasmtime-slab" -version = "25.0.1" +version = "25.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9858a22e656ae8574631221b474b8bebf63f1367fcac3f179873833eabc2ced" +checksum = "055a181b8d03998511294faea14798df436503f14d7fd20edcf7370ec583e80a" [[package]] name = "wasmtime-types" -version = "25.0.1" +version = "25.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d14b8a9206fe94485a03edb1654cd530dbd2a859a85a43502cb4e99653a568c" +checksum = "c8340d976673ac3fdacac781f2afdc4933920c1adc738c3409e825dab3955399" dependencies = [ "anyhow", "cranelift-entity", @@ -2045,9 +2046,9 @@ dependencies = [ [[package]] name = "wasmtime-versioned-export-macros" -version = "25.0.1" +version = "25.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9bb1f01efb8b542eadfda511e8ea1cc54309451aba97b69969e5b1a59cb7ded" +checksum = "a4b0c1f76891f778db9602ee3fbb4eb7e9a3f511847d1fb1b69eddbcea28303c" dependencies = [ "proc-macro2", "quote", @@ -2056,9 +2057,9 @@ dependencies = [ [[package]] name = "wasmtime-wit-bindgen" -version = "25.0.1" +version = "25.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1596caa67b31ac675fd3da61685c4260f8b10832021db42c85d227b7ba8133" +checksum = "b2fca2cbb5bb390f65d4434c19bf8d9873dfc60f10802918ebcd6f819a38d703" dependencies = [ "anyhow", "heck 0.4.1", @@ -2348,8 +2349,14 @@ dependencies = [ name = "xtask" version = "0.1.0" dependencies = [ + "anstyle", + "anyhow", + "bindgen", + "cc", + "clap", "git2", "indoc", + "regex", "semver", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 3be44012..955ddb0e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.24.1" +version = "0.24.7" authors = ["Max Brunsfeld "] edition = "2021" rust-version = "1.74.1" @@ -23,6 +23,55 @@ license = "MIT" keywords = ["incremental", "parsing"] categories = ["command-line-utilities", "parsing"] +[workspace.lints.clippy] +dbg_macro = "deny" +todo = "deny" +pedantic = { level = "warn", priority = -1 } +nursery = { level = "warn", priority = -1 } +cargo = { level = "warn", priority = -1 } + +# The lints below are a specific subset of the pedantic+nursery lints +# that we explicitly allow in the tree-sitter codebase because they either: +# +# 1. Contain false positives, +# 2. Are unnecessary, or +# 3. Worsen the code + +branches_sharing_code = "allow" +cast_lossless = "allow" +cast_possible_truncation = "allow" +cast_possible_wrap = "allow" +cast_precision_loss = "allow" +cast_sign_loss = "allow" +checked_conversions = "allow" +cognitive_complexity = "allow" +collection_is_never_read = "allow" +fallible_impl_from = "allow" +fn_params_excessive_bools = "allow" +inline_always = "allow" +if_not_else = "allow" +items_after_statements = "allow" +match_wildcard_for_single_variants = "allow" +missing_errors_doc = "allow" +missing_panics_doc = "allow" +module_name_repetitions = "allow" +multiple_crate_versions = "allow" +option_if_let_else = "allow" +or_fun_call = "allow" +range_plus_one = "allow" +redundant_clone = "allow" +redundant_closure_for_method_calls = "allow" +ref_option = "allow" +similar_names = "allow" +string_lit_as_bytes = "allow" +struct_excessive_bools = "allow" +struct_field_names = "allow" +transmute_undefined_repr = "allow" +too_many_lines = "allow" +unnecessary_wraps = "allow" +unused_self = "allow" +used_underscore_items = "allow" + [profile.optimize] inherits = "release" strip = true # Automatically strip symbols from the binary. @@ -56,7 +105,7 @@ clap = { version = "4.5.18", features = [ "unstable-styles", ] } clap_complete = "4.5.29" -ctor = "0.2.8" +ctor = "0.2.9" ctrlc = { version = "3.4.5", features = ["termination"] } dialoguer = { version = "0.11.0", features = ["fuzzy-select"] } dirs = "5.0.1" @@ -96,9 +145,9 @@ walkdir = "2.5.0" wasmparser = "0.217.0" webbrowser = "1.0.2" -tree-sitter = { version = "0.24.0", path = "./lib" } -tree-sitter-generate = { version = "0.24.0", path = "./cli/generate" } -tree-sitter-loader = { version = "0.24.0", path = "./cli/loader" } -tree-sitter-config = { version = "0.24.0", path = "./cli/config" } -tree-sitter-highlight = { version = "0.24.0", path = "./highlight" } -tree-sitter-tags = { version = "0.24.0", path = "./tags" } +tree-sitter = { version = "0.24.5", path = "./lib" } +tree-sitter-generate = { version = "0.24.5", path = "./cli/generate" } +tree-sitter-loader = { version = "0.24.5", path = "./cli/loader" } +tree-sitter-config = { version = "0.24.5", path = "./cli/config" } +tree-sitter-highlight = { version = "0.24.5", path = "./highlight" } +tree-sitter-tags = { version = "0.24.5", path = "./tags" } diff --git a/Makefile b/Makefile index aafbf20b..b93e5be9 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ ifeq ($(OS),Windows_NT) $(error Windows is not supported) endif -VERSION := 0.24.1 +VERSION := 0.24.7 DESCRIPTION := An incremental parsing system for programming tools HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/ @@ -25,7 +25,7 @@ OBJ := $(SRC:.c=.o) # define default flags, and override to append mandatory flags ARFLAGS := rcs -CFLAGS ?= -O3 -Wall -Wextra -Wshadow -pedantic +CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types override CFLAGS += -std=c11 -fPIC -fvisibility=hidden override CFLAGS += -Ilib/src -Ilib/src/wasm -Ilib/include @@ -62,8 +62,8 @@ endif tree-sitter.pc: lib/tree-sitter.pc.in sed -e 's|@PROJECT_VERSION@|$(VERSION)|' \ - -e 's|@CMAKE_INSTALL_LIBDIR@|$(LIBDIR)|' \ - -e 's|@CMAKE_INSTALL_INCLUDEDIR@|$(INCLUDEDIR)|' \ + -e 's|@CMAKE_INSTALL_LIBDIR@|$(LIBDIR:$(PREFIX)/%=%)|' \ + -e 's|@CMAKE_INSTALL_INCLUDEDIR@|$(INCLUDEDIR:$(PREFIX)/%=%)|' \ -e 's|@PROJECT_DESCRIPTION@|$(DESCRIPTION)|' \ -e 's|@PROJECT_HOMEPAGE_URL@|$(HOMEPAGE_URL)|' \ -e 's|@CMAKE_INSTALL_PREFIX@|$(PREFIX)|' $< > $@ @@ -94,13 +94,13 @@ uninstall: ##### Dev targets ##### test: - script/fetch-fixtures - script/generate-fixtures - script/test + cargo xtask fetch-fixtures + cargo xtask generate-fixtures + cargo xtask test test_wasm: - script/generate-fixtures-wasm - script/test-wasm + cargo xtask generate-fixtures-wasm + cargo xtask test-wasm lint: cargo update --workspace --locked --quiet diff --git a/build.zig.zon b/build.zig.zon index dd188771..e582103d 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,6 +1,6 @@ .{ .name = "tree-sitter", - .version = "0.24.1", + .version = "0.24.7", .paths = .{ "build.zig", "build.zig.zon", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 37192824..c96dcb78 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -12,6 +12,9 @@ license.workspace = true keywords.workspace = true categories.workspace = true +[lints] +workspace = true + [[bin]] name = "tree-sitter" path = "src/main.rs" diff --git a/cli/config/Cargo.toml b/cli/config/Cargo.toml index 8379a546..5ad7b88f 100644 --- a/cli/config/Cargo.toml +++ b/cli/config/Cargo.toml @@ -12,6 +12,9 @@ license.workspace = true keywords.workspace = true categories.workspace = true +[lints] +workspace = true + [dependencies] anyhow.workspace = true dirs.workspace = true diff --git a/cli/generate/Cargo.toml b/cli/generate/Cargo.toml index b4600343..8f374ae1 100644 --- a/cli/generate/Cargo.toml +++ b/cli/generate/Cargo.toml @@ -12,6 +12,9 @@ license.workspace = true keywords.workspace = true categories.workspace = true +[lints] +workspace = true + [dependencies] anyhow.workspace = true heck.workspace = true diff --git a/cli/generate/src/build_tables/minimize_parse_table.rs b/cli/generate/src/build_tables/minimize_parse_table.rs index 74f70869..70950dff 100644 --- a/cli/generate/src/build_tables/minimize_parse_table.rs +++ b/cli/generate/src/build_tables/minimize_parse_table.rs @@ -70,18 +70,17 @@ impl<'a> Minimizer<'a> { production_id: 0, symbol, .. - } => { - if !self.simple_aliases.contains_key(symbol) - && !self.syntax_grammar.supertype_symbols.contains(symbol) - && !aliased_symbols.contains(symbol) - && self.syntax_grammar.variables[symbol.index].kind - != VariableType::Named - && (unit_reduction_symbol.is_none() - || unit_reduction_symbol == Some(symbol)) - { - unit_reduction_symbol = Some(symbol); - continue; - } + } if !self.simple_aliases.contains_key(symbol) + && !self.syntax_grammar.supertype_symbols.contains(symbol) + && !self.syntax_grammar.extra_symbols.contains(symbol) + && !aliased_symbols.contains(symbol) + && self.syntax_grammar.variables[symbol.index].kind + != VariableType::Named + && (unit_reduction_symbol.is_none() + || unit_reduction_symbol == Some(symbol)) => + { + unit_reduction_symbol = Some(symbol); + continue; } _ => {} } diff --git a/cli/generate/src/prepare_grammar/intern_symbols.rs b/cli/generate/src/prepare_grammar/intern_symbols.rs index 0941676d..02845f35 100644 --- a/cli/generate/src/prepare_grammar/intern_symbols.rs +++ b/cli/generate/src/prepare_grammar/intern_symbols.rs @@ -149,7 +149,7 @@ impl<'a> Interner<'a> { fn check_single(&self, elements: &[Rule], name: Option<&str>) { if elements.len() == 1 && matches!(elements[0], Rule::String(_) | Rule::Pattern(_, _)) { eprintln!( - "Warning: rule {} is just a `seq` or `choice` rule with a single element. This is unnecessary.", + "Warning: rule {} contains a `seq` or `choice` rule with a single element. This is unnecessary.", name.unwrap_or_default() ); } diff --git a/cli/generate/src/templates/array.h b/cli/generate/src/templates/array.h index 15a3b233..a17a574f 100644 --- a/cli/generate/src/templates/array.h +++ b/cli/generate/src/templates/array.h @@ -14,6 +14,7 @@ extern "C" { #include #ifdef _MSC_VER +#pragma warning(push) #pragma warning(disable : 4101) #elif defined(__GNUC__) || defined(__clang__) #pragma GCC diagnostic push @@ -278,7 +279,7 @@ static inline void _array__splice(Array *self, size_t element_size, #define _compare_int(a, b) ((int)*(a) - (int)(b)) #ifdef _MSC_VER -#pragma warning(default : 4101) +#pragma warning(pop) #elif defined(__GNUC__) || defined(__clang__) #pragma GCC diagnostic pop #endif diff --git a/cli/loader/Cargo.toml b/cli/loader/Cargo.toml index ec2b35d1..8c5b160b 100644 --- a/cli/loader/Cargo.toml +++ b/cli/loader/Cargo.toml @@ -12,6 +12,9 @@ license.workspace = true keywords.workspace = true categories.workspace = true +[lints] +workspace = true + [features] wasm = ["tree-sitter/wasm"] # TODO: For backward compatibility these must be enabled by default, @@ -24,6 +27,7 @@ cc.workspace = true dirs.workspace = true fs4.workspace = true indoc.workspace = true +lazy_static.workspace = true libloading.workspace = true once_cell.workspace = true path-slash.workspace = true diff --git a/cli/loader/emscripten-version b/cli/loader/emscripten-version index 331ba4bc..96cef7dd 100644 --- a/cli/loader/emscripten-version +++ b/cli/loader/emscripten-version @@ -1 +1 @@ -3.1.64 +3.1.64 \ No newline at end of file diff --git a/cli/loader/src/lib.rs b/cli/loader/src/lib.rs index 1b8083c0..31ef448c 100644 --- a/cli/loader/src/lib.rs +++ b/cli/loader/src/lib.rs @@ -21,6 +21,7 @@ use anyhow::Error; use anyhow::{anyhow, Context, Result}; use fs4::fs_std::FileExt; use indoc::indoc; +use lazy_static::lazy_static; use libloading::{Library, Symbol}; use once_cell::unsync::OnceCell; use path_slash::PathBufExt as _; @@ -38,6 +39,10 @@ use tree_sitter_highlight::HighlightConfiguration; use tree_sitter_tags::{Error as TagsError, TagsConfiguration}; use url::Url; +lazy_static! { + static ref GRAMMAR_NAME_REGEX: Regex = Regex::new(r#""name":\s*"(.*?)""#).unwrap(); +} + pub const EMSCRIPTEN_TAG: &str = concat!("docker.io/emscripten/emsdk:", env!("EMSCRIPTEN_VERSION")); #[derive(Default, Deserialize, Serialize)] @@ -141,12 +146,10 @@ pub struct TreeSitterJSON { } impl TreeSitterJSON { - pub fn from_file(path: &Path) -> Option { - if let Ok(file) = fs::File::open(path.join("tree-sitter.json")) { - Some(serde_json::from_reader(file).ok()?) - } else { - None - } + pub fn from_file(path: &Path) -> Result { + Ok(serde_json::from_str(&fs::read_to_string( + path.join("tree-sitter.json"), + )?)?) } pub fn has_multiple_language_configs(&self) -> bool { @@ -161,7 +164,8 @@ pub struct Grammar { #[serde(skip_serializing_if = "Option::is_none")] pub camelcase: Option, pub scope: String, - pub path: PathBuf, + #[serde(skip_serializing_if = "Option::is_none")] + pub path: Option, #[serde(default, skip_serializing_if = "PathsJSON::is_empty")] pub external_files: PathsJSON, pub file_types: Option>, @@ -192,7 +196,6 @@ pub struct Metadata { pub authors: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub links: Option, - // #[serde(skip_serializing_if = "Option::is_none")] #[serde(skip)] pub namespace: Option, } @@ -600,6 +603,13 @@ impl Loader { } } + pub fn language_for_configuration( + &self, + configuration: &LanguageConfiguration, + ) -> Result { + self.language_for_id(configuration.language_id) + } + fn language_for_id(&self, id: usize) -> Result { let (path, language, externals) = &self.languages_by_id[id]; language @@ -628,27 +638,7 @@ impl Loader { pub fn load_language_at_path(&self, mut config: CompileConfig) -> Result { let grammar_path = config.src_path.join("grammar.json"); - - #[derive(Deserialize)] - struct GrammarJSON { - name: String, - } - let mut grammar_file = fs::File::open(&grammar_path).with_context(|| { - format!( - "Failed to read grammar.json file at the following path:\n{:?}", - &grammar_path - ) - })?; - let grammar_json: GrammarJSON = serde_json::from_reader(BufReader::new(&mut grammar_file)) - .with_context(|| { - format!( - "Failed to parse grammar.json file at the following path:\n{:?}", - &grammar_path - ) - })?; - - config.name = grammar_json.name; - + config.name = Self::grammar_json_name(&grammar_path)?; self.load_language_at_path_with_name(config) } @@ -856,7 +846,7 @@ impl Loader { format!("Failed to execute the C compiler with the following command:\n{command:?}") })?; - lock_file.unlock()?; + FileExt::unlock(lock_file)?; fs::remove_file(lock_path)?; if output.status.success() { @@ -1125,27 +1115,16 @@ impl Loader { parser_path: &Path, set_current_path_config: bool, ) -> Result<&[LanguageConfiguration]> { - #[derive(Deserialize)] - struct GrammarJSON { - name: String, - } - let initial_language_configuration_count = self.language_configurations.len(); - if let Some(config) = TreeSitterJSON::from_file(parser_path) { + let ts_json = TreeSitterJSON::from_file(parser_path); + if let Ok(config) = ts_json { let language_count = self.languages_by_id.len(); for grammar in config.grammars { // Determine the path to the parser directory. This can be specified in - // the package.json, but defaults to the directory containing the - // package.json. - let language_path = parser_path.join(grammar.path); - - let grammar_path = language_path.join("src").join("grammar.json"); - let mut grammar_file = - fs::File::open(grammar_path).with_context(|| "Failed to read grammar.json")?; - let grammar_json: GrammarJSON = - serde_json::from_reader(BufReader::new(&mut grammar_file)) - .with_context(|| "Failed to parse grammar.json")?; + // the tree-sitter.json, but defaults to the directory containing the + // tree-sitter.json. + let language_path = parser_path.join(grammar.path.unwrap_or(PathBuf::from("."))); // Determine if a previous language configuration in this package.json file // already uses the same language. @@ -1184,7 +1163,7 @@ impl Loader { let configuration = LanguageConfiguration { root_path: parser_path.to_path_buf(), - language_name: grammar_json.name, + language_name: grammar.name, scope: Some(grammar.scope), language_id, file_types: grammar.file_types.unwrap_or_default(), @@ -1230,20 +1209,30 @@ impl Loader { Some(self.language_configurations.len() - 1); } } + } else if let Err(e) = ts_json { + match e.downcast_ref::() { + // This is noisy, and not really an issue. + Some(e) if e.kind() == std::io::ErrorKind::NotFound => {} + _ => { + eprintln!( + "Warning: Failed to parse {} -- {e}", + parser_path.join("tree-sitter.json").display() + ); + } + } } + // If we didn't find any language configurations in the tree-sitter.json file, + // but there is a grammar.json file, then use the grammar file to form a simple + // language configuration. if self.language_configurations.len() == initial_language_configuration_count && parser_path.join("src").join("grammar.json").exists() { let grammar_path = parser_path.join("src").join("grammar.json"); - let mut grammar_file = - fs::File::open(grammar_path).with_context(|| "Failed to read grammar.json")?; - let grammar_json: GrammarJSON = - serde_json::from_reader(BufReader::new(&mut grammar_file)) - .with_context(|| "Failed to parse grammar.json")?; + let language_name = Self::grammar_json_name(&grammar_path)?; let configuration = LanguageConfiguration { root_path: parser_path.to_owned(), - language_name: grammar_json.name, + language_name, language_id: self.languages_by_id.len(), file_types: Vec::new(), scope: None, @@ -1279,6 +1268,36 @@ impl Loader { pattern.and_then(|r| RegexBuilder::new(r).multi_line(true).build().ok()) } + fn grammar_json_name(grammar_path: &Path) -> Result { + let file = fs::File::open(grammar_path).with_context(|| { + format!("Failed to open grammar.json at {}", grammar_path.display()) + })?; + + let first_three_lines = BufReader::new(file) + .lines() + .take(3) + .collect::, _>>() + .with_context(|| { + format!( + "Failed to read the first three lines of grammar.json at {}", + grammar_path.display() + ) + })? + .join("\n"); + + let name = GRAMMAR_NAME_REGEX + .captures(&first_three_lines) + .and_then(|c| c.get(1)) + .ok_or_else(|| { + anyhow!( + "Failed to parse the language name from grammar.json at {}", + grammar_path.display() + ) + })?; + + Ok(name.as_str().to_string()) + } + pub fn select_language( &mut self, path: &Path, diff --git a/cli/npm/package.json b/cli/npm/package.json index 52788514..c24f8bfa 100644 --- a/cli/npm/package.json +++ b/cli/npm/package.json @@ -1,6 +1,6 @@ { "name": "tree-sitter-cli", - "version": "0.24.1", + "version": "0.24.7", "author": "Max Brunsfeld", "license": "MIT", "repository": { diff --git a/cli/src/init.rs b/cli/src/init.rs index 4cab1aa8..d9d0fb3e 100644 --- a/cli/src/init.rs +++ b/cli/src/init.rs @@ -1,17 +1,15 @@ use std::{ - fs::{self, File}, - io::BufReader, + fs, path::{Path, PathBuf}, str::{self, FromStr}, }; use anyhow::{anyhow, Context, Result}; use heck::{ToKebabCase, ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase}; -use indoc::indoc; use regex::Regex; use semver::Version; use serde::{Deserialize, Serialize}; -use serde_json::{json, Map, Value}; +use serde_json::{Map, Value}; use tree_sitter_generate::write_file; use tree_sitter_loader::{ Author, Bindings, Grammar, Links, Metadata, PackageJSON, PackageJSONAuthor, @@ -77,7 +75,7 @@ const BINDING_GYP_TEMPLATE: &str = include_str!("./templates/binding.gyp"); const BINDING_TEST_JS_TEMPLATE: &str = include_str!("./templates/binding_test.js"); const MAKEFILE_TEMPLATE: &str = include_str!("./templates/makefile"); -const CMAKELISTS_TXT_TEMPLATE: &str = include_str!("./templates/cmakelists.txt"); +const CMAKELISTS_TXT_TEMPLATE: &str = include_str!("./templates/cmakelists.cmake"); const PARSER_NAME_H_TEMPLATE: &str = include_str!("./templates/PARSER_NAME.h"); const PARSER_NAME_PC_IN_TEMPLATE: &str = include_str!("./templates/PARSER_NAME.pc.in"); @@ -112,22 +110,6 @@ pub fn path_in_ignore(repo_path: &Path) -> bool { .any(|dir| repo_path.ends_with(dir)) } -fn insert_after( - map: Map, - after: &str, - key: &str, - value: Value, -) -> Map { - let mut entries = map.into_iter().collect::>(); - let after_index = entries - .iter() - .position(|(k, _)| k == after) - .unwrap_or(entries.len() - 1) - + 1; - entries.insert(after_index, (key.to_string(), value)); - entries.into_iter().collect() -} - #[derive(Serialize, Deserialize, Clone)] pub struct JsonConfigOpts { pub name: String, @@ -153,9 +135,9 @@ impl JsonConfigOpts { name: self.name.clone(), camelcase: Some(self.camelcase), scope: self.scope, - path: PathBuf::from("."), + path: None, external_files: PathsJSON::Empty, - file_types: None, + file_types: Some(self.file_types), highlights: PathsJSON::Empty, injections: PathsJSON::Empty, locals: PathsJSON::Empty, @@ -171,7 +153,7 @@ impl JsonConfigOpts { authors: Some(vec![Author { name: self.author, email: self.email, - url: None, + url: self.url.map(|url| url.to_string()), }]), links: Some(Links { repository: self.repository.unwrap_or_else(|| { @@ -216,6 +198,7 @@ struct GenerateOpts<'a> { description: Option<&'a str>, repository: Option<&'a str>, version: &'a Version, + camel_parser_name: &'a str, } // TODO: remove in 0.25 @@ -228,9 +211,9 @@ pub fn migrate_package_json(repo_path: &Path) -> Result { root_path.join("tree-sitter.json"), ); - let old_config = serde_json::from_reader::<_, PackageJSON>( - File::open(&package_json_path) - .with_context(|| format!("Failed to open package.json in {}", root_path.display()))?, + let old_config = serde_json::from_str::( + &fs::read_to_string(&package_json_path) + .with_context(|| format!("Failed to read package.json in {}", root_path.display()))?, )?; if old_config.tree_sitter.is_none() { @@ -249,7 +232,7 @@ pub fn migrate_package_json(repo_path: &Path) -> Result { name: name.clone(), camelcase: Some(name.to_upper_camel_case()), scope: l.scope.unwrap_or_else(|| format!("source.{name}")), - path: l.path, + path: Some(l.path), external_files: l.external_files, file_types: l.file_types, highlights: l.highlights, @@ -352,19 +335,19 @@ pub fn migrate_package_json(repo_path: &Path) -> Result { write_file( &tree_sitter_json_path, - serde_json::to_string_pretty(&new_config)?, + serde_json::to_string_pretty(&new_config)? + "\n", )?; // Remove the `tree-sitter` field in-place - let mut package_json = serde_json::from_reader::<_, Map>( - File::open(&package_json_path) - .with_context(|| format!("Failed to open package.json in {}", root_path.display()))?, + let mut package_json = serde_json::from_str::>( + &fs::read_to_string(&package_json_path) + .with_context(|| format!("Failed to read package.json in {}", root_path.display()))?, ) .unwrap(); package_json.remove("tree-sitter"); write_file( &root_path.join("package.json"), - serde_json::to_string_pretty(&package_json)?, + serde_json::to_string_pretty(&package_json)? + "\n", )?; println!("Warning: your package.json's `tree-sitter` field has been automatically migrated to the new `tree-sitter.json` config file"); @@ -383,8 +366,6 @@ pub fn generate_grammar_files( ) -> Result<()> { let dashed_language_name = language_name.to_kebab_case(); - // TODO: remove legacy code updates in v0.24.0 - let tree_sitter_config = missing_path_else( repo_path.join("tree-sitter.json"), true, @@ -407,12 +388,16 @@ pub fn generate_grammar_files( }, )?; - let tree_sitter_config = serde_json::from_reader::<_, TreeSitterJSON>( - File::open(tree_sitter_config.as_path()) - .with_context(|| "Failed to open tree-sitter.json")?, + let tree_sitter_config = serde_json::from_str::( + &fs::read_to_string(tree_sitter_config.as_path()) + .with_context(|| "Failed to read tree-sitter.json")?, )?; let authors = tree_sitter_config.metadata.authors.as_ref(); + let camel_name = tree_sitter_config.grammars[0] + .camelcase + .clone() + .unwrap_or_else(|| language_name.to_upper_camel_case()); let generate_opts = GenerateOpts { author_name: authors @@ -432,126 +417,18 @@ pub fn generate_grammar_files( .as_ref() .map(|l| l.repository.as_str()), version: &tree_sitter_config.metadata.version, + camel_parser_name: &camel_name, }; - // Create or update package.json - missing_path_else( - repo_path.join("package.json"), - allow_update, - |path| { - generate_file( - path, - PACKAGE_JSON_TEMPLATE, - dashed_language_name.as_str(), - &generate_opts, - ) - }, - |path| { - let package_json_str = - fs::read_to_string(path).with_context(|| "Failed to read package.json")?; - let mut package_json = serde_json::from_str::>(&package_json_str) - .with_context(|| "Failed to parse package.json")?; - let mut updated = false; - - let dependencies = package_json - .entry("dependencies".to_string()) - .or_insert_with(|| Value::Object(Map::new())) - .as_object_mut() - .unwrap(); - if dependencies.remove("nan").is_some() { - eprintln!("Replacing nan dependency with node-addon-api in package.json"); - dependencies.insert("node-addon-api".to_string(), "^8.0.0".into()); - updated = true; - } - if !dependencies.contains_key("node-gyp-build") { - eprintln!("Adding node-gyp-build dependency to package.json"); - dependencies.insert("node-gyp-build".to_string(), "^4.8.1".into()); - updated = true; - } - - let dev_dependencies = package_json - .entry("devDependencies".to_string()) - .or_insert_with(|| Value::Object(Map::new())) - .as_object_mut() - .unwrap(); - if !dev_dependencies.contains_key("prebuildify") { - eprintln!("Adding prebuildify devDependency to package.json"); - dev_dependencies.insert("prebuildify".to_string(), "^6.0.1".into()); - updated = true; - } - - let node_test = "node --test bindings/node/*_test.js"; - let scripts = package_json - .entry("scripts".to_string()) - .or_insert_with(|| Value::Object(Map::new())) - .as_object_mut() - .unwrap(); - if !scripts.get("test").is_some_and(|v| v == node_test) { - eprintln!("Updating package.json scripts"); - *scripts = Map::from_iter([ - ("install".to_string(), "node-gyp-build".into()), - ("prestart".to_string(), "tree-sitter build --wasm".into()), - ("start".to_string(), "tree-sitter playground".into()), - ("test".to_string(), node_test.into()), - ]); - updated = true; - } - - // insert `peerDependencies` after `dependencies` - if !package_json.contains_key("peerDependencies") { - eprintln!("Adding peerDependencies to package.json"); - package_json = insert_after( - package_json, - "dependencies", - "peerDependencies", - json!({"tree-sitter": "^0.21.1"}), - ); - - package_json = insert_after( - package_json, - "peerDependencies", - "peerDependenciesMeta", - json!({"tree_sitter": {"optional": true}}), - ); - updated = true; - } - - // insert `types` right after `main` - if !package_json.contains_key("types") { - eprintln!("Adding types to package.json"); - package_json = insert_after(package_json, "main", "types", "bindings/node".into()); - updated = true; - } - - // insert `files` right after `keywords` - if !package_json.contains_key("files") { - eprintln!("Adding files to package.json"); - package_json = insert_after( - package_json, - "keywords", - "files", - json!([ - "grammar.js", - "binding.gyp", - "prebuilds/**", - "bindings/node/*", - "queries/*", - "src/**", - "*.wasm" - ]), - ); - updated = true; - } - - if updated { - let mut package_json_str = serde_json::to_string_pretty(&package_json)?; - package_json_str.push('\n'); - write_file(path, package_json_str)?; - } - - Ok(()) - }, - )?; + // Create package.json + missing_path(repo_path.join("package.json"), |path| { + generate_file( + path, + PACKAGE_JSON_TEMPLATE, + dashed_language_name.as_str(), + &generate_opts, + ) + })?; // Do not create a grammar.js file in a repo with multiple language configs if !tree_sitter_config.has_multiple_language_configs() { @@ -580,83 +457,22 @@ pub fn generate_grammar_files( // Generate Rust bindings if tree_sitter_config.bindings.rust { missing_path(bindings_dir.join("rust"), create_dir)?.apply(|path| { - missing_path_else( - path.join("lib.rs"), - allow_update, - |path| generate_file(path, LIB_RS_TEMPLATE, language_name, &generate_opts), - |path| { - let lib_rs = - fs::read_to_string(path).with_context(|| "Failed to read lib.rs")?; - if !lib_rs.contains("tree_sitter_language") { - generate_file(path, LIB_RS_TEMPLATE, language_name, &generate_opts)?; - eprintln!("Updated lib.rs with `tree_sitter_language` dependency"); - } - Ok(()) - }, - )?; + missing_path(path.join("lib.rs"), |path| { + generate_file(path, LIB_RS_TEMPLATE, language_name, &generate_opts) + })?; - missing_path_else( - path.join("build.rs"), - allow_update, - |path| generate_file(path, BUILD_RS_TEMPLATE, language_name, &generate_opts), - |path| { - let build_rs = - fs::read_to_string(path).with_context(|| "Failed to read build.rs")?; - if !build_rs.contains("-utf-8") { - let index = build_rs - .find(" let parser_path = src_dir.join(\"parser.c\")") - .ok_or_else(|| anyhow!(indoc!{ - "Failed to auto-update build.rs with the `/utf-8` flag for windows. - To fix this, remove `bindings/rust/build.rs` and re-run `tree-sitter generate`"}))?; + missing_path(path.join("build.rs"), |path| { + generate_file(path, BUILD_RS_TEMPLATE, language_name, &generate_opts) + })?; - let build_rs = format!( - "{}{}{}\n{}", - &build_rs[..index], - " #[cfg(target_env = \"msvc\")]\n", - " c_config.flag(\"-utf-8\");\n", - &build_rs[index..] - ); - - write_file(path, build_rs)?; - eprintln!("Updated build.rs with the /utf-8 flag for Windows compilation"); - } - Ok(()) - }, - )?; - - missing_path_else( - repo_path.join("Cargo.toml"), - allow_update, - |path| generate_file(path, CARGO_TOML_TEMPLATE, dashed_language_name.as_str(), &generate_opts), - |path| { - let cargo_toml = - fs::read_to_string(path).with_context(|| "Failed to read Cargo.toml")?; - if !cargo_toml.contains("tree-sitter-language") { - let start_index = cargo_toml - .find("tree-sitter = \"") - .ok_or_else(|| anyhow!("Failed to find the `tree-sitter` dependency in Cargo.toml"))?; - - let version_start_index = start_index + "tree-sitter = \"".len(); - let version_end_index = cargo_toml[version_start_index..] - .find('\"') - .map(|i| i + version_start_index) - .ok_or_else(|| anyhow!("Failed to find the end of the `tree-sitter` version in Cargo.toml"))?; - - let cargo_toml = format!( - "{}{}{}\n{}\n{}", - &cargo_toml[..start_index], - "tree-sitter-language = \"0.1.0\"", - &cargo_toml[version_end_index + 1..], - "[dev-dependencies]", - "tree-sitter = \"0.23\"", - ); - - write_file(path, cargo_toml)?; - eprintln!("Updated Cargo.toml with the `tree-sitter-language` dependency"); - } - Ok(()) - }, - )?; + missing_path(repo_path.join("Cargo.toml"), |path| { + generate_file( + path, + CARGO_TOML_TEMPLATE, + dashed_language_name.as_str(), + &generate_opts, + ) + })?; Ok(()) })?; @@ -670,10 +486,8 @@ pub fn generate_grammar_files( allow_update, |path| generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts), |path| { - let index_js = - fs::read_to_string(path).with_context(|| "Failed to read index.js")?; - if index_js.contains("../../build/Release") { - eprintln!("Replacing index.js with new binding API"); + let contents = fs::read_to_string(path)?; + if !contents.contains("bun") { generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts)?; } Ok(()) @@ -693,36 +507,13 @@ pub fn generate_grammar_files( ) })?; - missing_path_else( - path.join("binding.cc"), - allow_update, - |path| generate_file(path, JS_BINDING_CC_TEMPLATE, language_name, &generate_opts), - |path| { - let binding_cc = - fs::read_to_string(path).with_context(|| "Failed to read binding.cc")?; - if binding_cc.contains("NAN_METHOD(New) {}") { - eprintln!("Replacing binding.cc with new binding API"); - generate_file(path, JS_BINDING_CC_TEMPLATE, language_name, &generate_opts)?; - } - Ok(()) - }, - )?; + missing_path(path.join("binding.cc"), |path| { + generate_file(path, JS_BINDING_CC_TEMPLATE, language_name, &generate_opts) + })?; - // Create binding.gyp, or update it with new binding API. - missing_path_else( - repo_path.join("binding.gyp"), - allow_update, - |path| generate_file(path, BINDING_GYP_TEMPLATE, language_name, &generate_opts), - |path| { - let binding_gyp = - fs::read_to_string(path).with_context(|| "Failed to read binding.gyp")?; - if binding_gyp.contains("require('nan')") { - eprintln!("Replacing binding.gyp with new binding API"); - generate_file(path, BINDING_GYP_TEMPLATE, language_name, &generate_opts)?; - } - Ok(()) - }, - )?; + missing_path(repo_path.join("binding.gyp"), |path| { + generate_file(path, BINDING_GYP_TEMPLATE, language_name, &generate_opts) + })?; Ok(()) })?; @@ -752,9 +543,20 @@ pub fn generate_grammar_files( generate_file(path, MAKEFILE_TEMPLATE, language_name, &generate_opts) })?; - missing_path(repo_path.join("CMakeLists.txt"), |path| { - generate_file(path, CMAKELISTS_TXT_TEMPLATE, language_name, &generate_opts) - })?; + missing_path_else( + repo_path.join("CMakeLists.txt"), + allow_update, + |path| generate_file(path, CMAKELISTS_TXT_TEMPLATE, language_name, &generate_opts), + |path| { + let contents = fs::read_to_string(path)?; + let old = "add_custom_target(test"; + if contents.contains(old) { + write_file(path, contents.replace(old, "add_custom_target(ts-test")) + } else { + Ok(()) + } + }, + )?; Ok(()) })?; @@ -767,39 +569,14 @@ pub fn generate_grammar_files( generate_file(path, BINDING_GO_TEMPLATE, language_name, &generate_opts) })?; - missing_path_else( - path.join("binding_test.go"), - allow_update, - |path| { - generate_file( - path, - BINDING_TEST_GO_TEMPLATE, - language_name, - &generate_opts, - ) - }, - |path| { - let binding_test_go = fs::read_to_string(path) - .with_context(|| "Failed to read binding_test.go")?; - if binding_test_go.contains("smacker") { - eprintln!("Replacing binding_test.go with new binding API"); - generate_file( - path, - BINDING_TEST_GO_TEMPLATE, - language_name, - &generate_opts, - )?; - } - Ok(()) - }, - )?; - - // Delete the old go.mod file that lives inside bindings/go, it now lives in the root - // dir - let go_mod_path = path.join("go.mod"); - if allow_update && go_mod_path.exists() { - fs::remove_file(go_mod_path).with_context(|| "Failed to remove old go.mod file")?; - } + missing_path(path.join("binding_test.go"), |path| { + generate_file( + path, + BINDING_TEST_GO_TEMPLATE, + language_name, + &generate_opts, + ) + })?; missing_path(repo_path.join("go.mod"), |path| { generate_file(path, GO_MOD_TEMPLATE, language_name, &generate_opts) @@ -815,20 +592,9 @@ pub fn generate_grammar_files( let lang_path = path.join(format!("tree_sitter_{}", language_name.to_snake_case())); missing_path(&lang_path, create_dir)?; - missing_path_else( - lang_path.join("binding.c"), - allow_update, - |path| generate_file(path, PY_BINDING_C_TEMPLATE, language_name, &generate_opts), - |path| { - let binding_c = fs::read_to_string(path) - .with_context(|| "Failed to read bindings/python/binding.c")?; - if !binding_c.contains("PyCapsule_New") { - eprintln!("Replacing bindings/python/binding.c with new binding API"); - generate_file(path, PY_BINDING_C_TEMPLATE, language_name, &generate_opts)?; - } - Ok(()) - }, - )?; + missing_path(lang_path.join("binding.c"), |path| { + generate_file(path, PY_BINDING_C_TEMPLATE, language_name, &generate_opts) + })?; missing_path(lang_path.join("__init__.py"), |path| { generate_file(path, INIT_PY_TEMPLATE, language_name, &generate_opts) @@ -874,7 +640,7 @@ pub fn generate_grammar_files( // Generate Swift bindings if tree_sitter_config.bindings.swift { missing_path(bindings_dir.join("swift"), create_dir)?.apply(|path| { - let lang_path = path.join(format!("TreeSitter{}", language_name.to_upper_camel_case())); + let lang_path = path.join(format!("TreeSitter{camel_name}",)); missing_path(&lang_path, create_dir)?; missing_path(lang_path.join(format!("{language_name}.h")), |path| { @@ -882,18 +648,12 @@ pub fn generate_grammar_files( })?; missing_path( - path.join(format!( - "TreeSitter{}Tests", - language_name.to_upper_camel_case() - )), + path.join(format!("TreeSitter{camel_name}Tests",)), create_dir, )? .apply(|path| { missing_path( - path.join(format!( - "TreeSitter{}Tests.swift", - language_name.to_upper_camel_case() - )), + path.join(format!("TreeSitter{camel_name}Tests.swift")), |path| generate_file(path, TESTS_SWIFT_TEMPLATE, language_name, &generate_opts), )?; @@ -919,15 +679,14 @@ pub fn get_root_path(path: &Path) -> Result { let json = pathbuf .exists() .then(|| { - let file = File::open(pathbuf.as_path()) - .with_context(|| format!("Failed to open {filename}"))?; - let reader = BufReader::new(file); + let contents = fs::read_to_string(pathbuf.as_path()) + .with_context(|| format!("Failed to read {filename}"))?; if is_package_json { - serde_json::from_reader::<_, Map>(reader) + serde_json::from_str::>(&contents) .context(format!("Failed to parse {filename}")) .map(|v| v.contains_key("tree-sitter")) } else { - serde_json::from_reader::<_, TreeSitterJSON>(reader) + serde_json::from_str::(&contents) .context(format!("Failed to parse {filename}")) .map(|_| true) } @@ -961,7 +720,7 @@ fn generate_file( let mut replacement = template .replace( CAMEL_PARSER_NAME_PLACEHOLDER, - &language_name.to_upper_camel_case(), + generate_opts.camel_parser_name, ) .replace( UPPER_PARSER_NAME_PLACEHOLDER, @@ -1001,7 +760,12 @@ fn generate_file( } if let Some(email) = generate_opts.author_email { - replacement = replacement.replace(AUTHOR_EMAIL_PLACEHOLDER, email); + replacement = match filename { + "Cargo.toml" | "grammar.js" => { + replacement.replace(AUTHOR_EMAIL_PLACEHOLDER, &format!("<{email}>")) + } + _ => replacement.replace(AUTHOR_EMAIL_PLACEHOLDER, email), + } } else { match filename { "package.json" => { @@ -1099,7 +863,7 @@ fn generate_file( PARSER_DESCRIPTION_PLACEHOLDER, &format!( "{} grammar for tree-sitter", - language_name.to_upper_camel_case() + generate_opts.camel_parser_name, ), ) } diff --git a/cli/src/main.rs b/cli/src/main.rs index 1758fada..4926f256 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -653,11 +653,11 @@ impl Init { (opts.name.clone(), Some(opts)) } else { - let json = serde_json::from_reader::<_, TreeSitterJSON>( - fs::File::open(current_dir.join("tree-sitter.json")) - .with_context(|| "Failed to open tree-sitter.json")?, + let mut json = serde_json::from_str::( + &fs::read_to_string(current_dir.join("tree-sitter.json")) + .with_context(|| "Failed to read tree-sitter.json")?, )?; - (json.grammars[0].name.clone(), None) + (json.grammars.swap_remove(0).name, None) }; generate_grammar_files(current_dir, &language_name, self.update, json_config_opts)?; diff --git a/cli/src/playground.html b/cli/src/playground.html index 420cd28d..10a52488 100644 --- a/cli/src/playground.html +++ b/cli/src/playground.html @@ -19,6 +19,11 @@ +
+ + +
+
@@ -67,6 +72,12 @@ + +