diff --git a/.envrc b/.envrc deleted file mode 100644 index 3550a30f..00000000 --- a/.envrc +++ /dev/null @@ -1 +0,0 @@ -use flake diff --git a/.gitattributes b/.gitattributes index acd15fd1..1d9b8cb4 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,4 +3,5 @@ /lib/src/unicode/*.h linguist-vendored /lib/src/unicode/LICENSE linguist-vendored +/cli/src/generate/prepare_grammar/*.json -diff Cargo.lock -diff diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 88437c52..6c8ad05d 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -1,6 +1,6 @@ name: Bug Report description: Report a problem -type: Bug +labels: [bug] body: - type: textarea attributes: diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 05b37e6f..388f3675 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -1,6 +1,6 @@ name: Feature request description: Request an enhancement -type: Feature +labels: [enhancement] body: - type: markdown attributes: diff --git a/.github/actions/cache/action.yml b/.github/actions/cache/action.yml index f04dda31..12d3d7b3 100644 --- a/.github/actions/cache/action.yml +++ b/.github/actions/cache/action.yml @@ -17,9 +17,10 @@ runs: test/fixtures/grammars target/release/tree-sitter-*.wasm key: fixtures-${{ join(matrix.*, '_') }}-${{ hashFiles( - 'crates/generate/src/**', + 'cli/generate/src/**', 'lib/src/parser.h', 'lib/src/array.h', 'lib/src/alloc.h', + 'xtask/src/*', 'test/fixtures/grammars/*/**/src/*.c', '.github/actions/cache/action.yml') }} diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c75a67e6..ccba319b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,8 +4,6 @@ updates: directory: "/" schedule: interval: "weekly" - cooldown: - default-days: 3 commit-message: prefix: "build(deps)" labels: @@ -14,16 +12,10 @@ updates: groups: cargo: patterns: ["*"] - ignore: - - dependency-name: "*" - update-types: ["version-update:semver-major", "version-update:semver-minor"] - - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" - cooldown: - default-days: 3 commit-message: prefix: "ci" labels: @@ -32,22 +24,3 @@ updates: groups: actions: patterns: ["*"] - - - package-ecosystem: "npm" - versioning-strategy: increase - directories: - - "/crates/npm" - - "/crates/eslint" - - "/lib/binding_web" - schedule: - interval: "weekly" - cooldown: - default-days: 3 - commit-message: - prefix: "build(deps)" - labels: - - "dependencies" - - "npm" - groups: - npm: - patterns: ["*"] diff --git a/.github/scripts/close_spam.js b/.github/scripts/close_spam.js deleted file mode 100644 index 41046964..00000000 --- a/.github/scripts/close_spam.js +++ /dev/null @@ -1,29 +0,0 @@ -module.exports = async ({ github, context }) => { - let target = context.payload.issue; - if (target) { - await github.rest.issues.update({ - ...context.repo, - issue_number: target.number, - state: "closed", - state_reason: "not_planned", - title: "[spam]", - body: "", - type: null, - }); - } else { - target = context.payload.pull_request; - await github.rest.pulls.update({ - ...context.repo, - pull_number: target.number, - state: "closed", - title: "[spam]", - body: "", - }); - } - - await github.rest.issues.lock({ - ...context.repo, - issue_number: target.number, - lock_reason: "spam", - }); -}; diff --git a/.github/scripts/cross.sh b/.github/scripts/cross.sh new file mode 100755 index 00000000..de1d4e94 --- /dev/null +++ b/.github/scripts/cross.sh @@ -0,0 +1,3 @@ +#!/bin/bash -eu + +exec docker run --rm -v /home/runner:/home/runner -w "$PWD" "$CROSS_IMAGE" "$@" diff --git a/.github/scripts/make.sh b/.github/scripts/make.sh new file mode 100755 index 00000000..175074f9 --- /dev/null +++ b/.github/scripts/make.sh @@ -0,0 +1,9 @@ +#!/bin/bash -eu + +tree_sitter="$ROOT"/target/"$TARGET"/release/tree-sitter + +if [[ $BUILD_CMD == cross ]]; then + cross.sh make CC="$CC" AR="$AR" "$@" +else + exec make "$@" +fi diff --git a/.github/scripts/tree-sitter.sh b/.github/scripts/tree-sitter.sh new file mode 100755 index 00000000..125a2d92 --- /dev/null +++ b/.github/scripts/tree-sitter.sh @@ -0,0 +1,9 @@ +#!/bin/bash -eu + +tree_sitter="$ROOT"/target/"$TARGET"/release/tree-sitter + +if [[ $BUILD_CMD == cross ]]; then + cross.sh "$CROSS_RUNNER" "$tree_sitter" "$@" +else + exec "$tree_sitter" "$@" +fi diff --git a/.github/scripts/wasm_stdlib.js b/.github/scripts/wasm_stdlib.js deleted file mode 100644 index e1350094..00000000 --- a/.github/scripts/wasm_stdlib.js +++ /dev/null @@ -1,25 +0,0 @@ -module.exports = async ({ github, context, core }) => { - if (context.eventName !== 'pull_request') return; - - const prNumber = context.payload.pull_request.number; - const owner = context.repo.owner; - const repo = context.repo.repo; - - const { data: files } = await github.rest.pulls.listFiles({ - owner, - repo, - pull_number: prNumber - }); - - const changedFiles = files.map(file => file.filename); - - const wasmStdLibSrc = 'crates/language/wasm/'; - const dirChanged = changedFiles.some(file => file.startsWith(wasmStdLibSrc)); - - if (!dirChanged) return; - - const wasmStdLibHeader = 'lib/src/wasm/wasm-stdlib.h'; - const requiredChanged = changedFiles.includes(wasmStdLibHeader); - - if (!requiredChanged) core.setFailed(`Changes detected in ${wasmStdLibSrc} but ${wasmStdLibHeader} was not modified.`); -}; diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 7caffa14..a0c15e01 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -14,17 +14,17 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v4 - name: Create app token - uses: actions/create-github-app-token@v2 + uses: actions/create-github-app-token@v1 id: app-token with: app-id: ${{ vars.BACKPORT_APP }} private-key: ${{ secrets.BACKPORT_KEY }} - name: Create backport PR - uses: korthout/backport-action@v4 + uses: korthout/backport-action@v3 with: pull_title: "${pull_title}" label_pattern: "^ci:backport ([^ ]+)$" diff --git a/.github/workflows/bindgen.yml b/.github/workflows/bindgen.yml index 1b0d20af..659c0236 100644 --- a/.github/workflows/bindgen.yml +++ b/.github/workflows/bindgen.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v4 - name: Set up stable Rust toolchain uses: actions-rust-lang/setup-rust-toolchain@v1 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5cde3db9..e6fed643 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,5 +1,10 @@ name: Build & Test +env: + CARGO_TERM_COLOR: always + RUSTFLAGS: "-D warnings" + CROSS_DEBUG: 1 + on: workflow_call: inputs: @@ -26,41 +31,38 @@ jobs: - windows-x86 - macos-arm64 - macos-x64 - - wasm32 include: # When adding a new `target`: # 1. Define a new platform alias above - # 2. Add a new record to the matrix map in `crates/cli/npm/install.js` - - { platform: linux-arm64 , target: aarch64-unknown-linux-gnu , os: ubuntu-24.04-arm } - - { platform: linux-arm , target: armv7-unknown-linux-gnueabihf , os: ubuntu-24.04-arm } - - { platform: linux-x64 , target: x86_64-unknown-linux-gnu , os: ubuntu-24.04 } - - { platform: linux-x86 , target: i686-unknown-linux-gnu , os: ubuntu-24.04 } - - { platform: linux-powerpc64 , target: powerpc64-unknown-linux-gnu , os: ubuntu-24.04 } - - { platform: windows-arm64 , target: aarch64-pc-windows-msvc , os: windows-11-arm } - - { platform: windows-x64 , target: x86_64-pc-windows-msvc , os: windows-2025 } - - { platform: windows-x86 , target: i686-pc-windows-msvc , os: windows-2025 } - - { platform: macos-arm64 , target: aarch64-apple-darwin , os: macos-15 } - - { platform: macos-x64 , target: x86_64-apple-darwin , os: macos-15-intel } - - { platform: wasm32 , target: wasm32-unknown-unknown , os: ubuntu-24.04 } + # 2. Add a new record to the matrix map in `cli/npm/install.js` + - { platform: linux-arm64 , target: aarch64-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } + - { platform: linux-arm , target: arm-unknown-linux-gnueabi , os: ubuntu-latest , use-cross: true } + - { platform: linux-x64 , target: x86_64-unknown-linux-gnu , os: ubuntu-22.04 , features: wasm } + - { platform: linux-x86 , target: i686-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } + - { platform: linux-powerpc64 , target: powerpc64-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } + - { platform: windows-arm64 , target: aarch64-pc-windows-msvc , os: windows-latest } + - { platform: windows-x64 , target: x86_64-pc-windows-msvc , os: windows-latest , features: wasm } + - { platform: windows-x86 , target: i686-pc-windows-msvc , os: windows-latest } + - { platform: macos-arm64 , target: aarch64-apple-darwin , os: macos-latest , features: wasm } + - { platform: macos-x64 , target: x86_64-apple-darwin , os: macos-13 , features: wasm } - # Extra features - - { platform: linux-arm64 , features: wasm } - - { platform: linux-x64 , features: wasm } - - { platform: macos-arm64 , features: wasm } - - { platform: macos-x64 , features: wasm } + # Cross compilers for C library + - { platform: linux-arm64 , cc: aarch64-linux-gnu-gcc , ar: aarch64-linux-gnu-ar } + - { platform: linux-arm , cc: arm-linux-gnueabi-gcc , ar: arm-linux-gnueabi-ar } + - { platform: linux-x86 , cc: i686-linux-gnu-gcc , ar: i686-linux-gnu-ar } + - { platform: linux-powerpc64 , cc: powerpc64-linux-gnu-gcc , ar: powerpc64-linux-gnu-ar } - # Cross-compilation - - { platform: linux-arm , cross: true } - - { platform: linux-x86 , cross: true } - - { platform: linux-powerpc64 , cross: true } + # Prevent race condition (see #2041) + - { platform: windows-x64 , rust-test-threads: 1 } + - { platform: windows-x86 , rust-test-threads: 1 } - # Compile-only - - { platform: wasm32 , no-run: true } + # Can't natively run CLI on Github runner's host + - { platform: windows-arm64 , no-run: true } env: - CARGO_TERM_COLOR: always - RUSTFLAGS: -D warnings + BUILD_CMD: cargo + SUFFIX: ${{ contains(matrix.target, 'windows') && '.exe' || '' }} defaults: run: @@ -68,28 +70,13 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v4 - - name: Set up cross-compilation - if: matrix.cross - run: | - for target in armv7-unknown-linux-gnueabihf i686-unknown-linux-gnu powerpc64-unknown-linux-gnu; do - camel_target=${target//-/_}; target_cc=${target/-unknown/} - printf 'CC_%s=%s\n' "$camel_target" "${target_cc/v7/}-gcc" - printf 'AR_%s=%s\n' "$camel_target" "${target_cc/v7/}-ar" - printf 'CARGO_TARGET_%s_LINKER=%s\n' "${camel_target^^}" "${target_cc/v7/}-gcc" - done >> $GITHUB_ENV - { - printf 'CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_RUNNER=qemu-arm -L /usr/arm-linux-gnueabihf\n' - printf 'CARGO_TARGET_POWERPC64_UNKNOWN_LINUX_GNU_RUNNER=qemu-ppc64 -L /usr/powerpc64-linux-gnu\n' - } >> $GITHUB_ENV - - - name: Get emscripten version - if: contains(matrix.features, 'wasm') - run: printf 'EMSCRIPTEN_VERSION=%s\n' "$(> $GITHUB_ENV + - name: Read Emscripten version + run: printf 'EMSCRIPTEN_VERSION=%s\n' "$(> $GITHUB_ENV - name: Install Emscripten - if: contains(matrix.features, 'wasm') + if: ${{ !matrix.no-run && !matrix.use-cross }} uses: mymindstorm/setup-emsdk@v14 with: version: ${{ env.EMSCRIPTEN_VERSION }} @@ -99,82 +86,63 @@ jobs: with: target: ${{ matrix.target }} - - name: Install cross-compilation toolchain - if: matrix.cross + - name: Install cross + if: ${{ matrix.use-cross }} run: | - sudo apt-get update -qy - if [[ $PLATFORM == linux-arm ]]; then - sudo apt-get install -qy {binutils,gcc}-arm-linux-gnueabihf qemu-user - elif [[ $PLATFORM == linux-x86 ]]; then - sudo apt-get install -qy {binutils,gcc}-i686-linux-gnu - elif [[ $PLATFORM == linux-powerpc64 ]]; then - sudo apt-get install -qy {binutils,gcc}-powerpc64-linux-gnu qemu-user + if [ ! -x "$(command -v cross)" ]; then + # TODO: Remove 'RUSTFLAGS=""' once https://github.com/cross-rs/cross/issues/1561 is resolved + RUSTFLAGS="" cargo install cross --git https://github.com/cross-rs/cross fi - env: - PLATFORM: ${{ matrix.platform }} - - name: Install MinGW and Clang (Windows x64 MSYS2) - if: matrix.platform == 'windows-x64' - uses: msys2/setup-msys2@v2 - with: - update: true - install: | - mingw-w64-x86_64-toolchain - mingw-w64-x86_64-clang - mingw-w64-x86_64-make - mingw-w64-x86_64-cmake - - # TODO: Remove RUSTFLAGS="--cap-lints allow" once we use a wasmtime release that addresses - # the `mismatched-lifetime-syntaxes` lint - - name: Build wasmtime library (Windows x64 MSYS2) - if: contains(matrix.features, 'wasm') && matrix.platform == 'windows-x64' + - name: Configure cross + if: ${{ matrix.use-cross }} run: | - mkdir -p target - WASMTIME_VERSION=$(cargo metadata --format-version=1 --locked --features wasm | \ - jq -r '.packages[] | select(.name == "wasmtime-c-api-impl") | .version') - curl -LSs "$WASMTIME_REPO/archive/refs/tags/v${WASMTIME_VERSION}.tar.gz" | tar xzf - -C target - cd target/wasmtime-${WASMTIME_VERSION} - cmake -S crates/c-api -B target/c-api \ - -DCMAKE_INSTALL_PREFIX="$PWD/artifacts" \ - -DWASMTIME_DISABLE_ALL_FEATURES=ON \ - -DWASMTIME_FEATURE_CRANELIFT=ON \ - -DWASMTIME_TARGET='x86_64-pc-windows-gnu' - cmake --build target/c-api && cmake --install target/c-api - printf 'CMAKE_PREFIX_PATH=%s\n' "$PWD/artifacts" >> $GITHUB_ENV - env: - WASMTIME_REPO: https://github.com/bytecodealliance/wasmtime - RUSTFLAGS: ${{ env.RUSTFLAGS }} --cap-lints allow + printf '%s\n' > Cross.toml \ + '[target.${{ matrix.target }}]' \ + 'image = "ghcr.io/cross-rs/${{ matrix.target }}:edge"' \ + '[build]' \ + 'pre-build = [' \ + ' "dpkg --add-architecture $CROSS_DEB_ARCH",' \ + ' "curl -fsSL https://deb.nodesource.com/setup_22.x | bash -",' \ + ' "apt-get update && apt-get -y install libssl-dev nodejs"' \ + ']' + cat - Cross.toml <<< 'Cross.toml:' + printf '%s\n' >> $GITHUB_ENV \ + "CROSS_CONFIG=$PWD/Cross.toml" \ + "CROSS_IMAGE=ghcr.io/cross-rs/${{ matrix.target }}:edge" - - name: Build C library (Windows x64 MSYS2 CMake) - if: matrix.platform == 'windows-x64' - shell: msys2 {0} + - name: Set up environment + env: + RUST_TEST_THREADS: ${{ matrix.rust-test-threads }} + USE_CROSS: ${{ matrix.use-cross }} + TARGET: ${{ matrix.target }} + CC: ${{ matrix.cc }} + AR: ${{ matrix.ar }} run: | - cmake -G Ninja -S . -B build/static \ - -DBUILD_SHARED_LIBS=OFF \ - -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \ - -DTREE_SITTER_FEATURE_WASM=$WASM \ - -DCMAKE_C_COMPILER=clang - cmake --build build/static + PATH="$PWD/.github/scripts:$PATH" + printf '%s/.github/scripts\n' "$PWD" >> $GITHUB_PATH - cmake -G Ninja -S . -B build/shared \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \ - -DTREE_SITTER_FEATURE_WASM=$WASM \ - -DCMAKE_C_COMPILER=clang - cmake --build build/shared - rm -rf \ - build/{static,shared} \ - "${CMAKE_PREFIX_PATH}/artifacts" \ - target/wasmtime-${WASMTIME_VERSION} - env: - WASM: ${{ contains(matrix.features, 'wasm') && 'ON' || 'OFF' }} + printf '%s\n' >> $GITHUB_ENV \ + 'TREE_SITTER=tree-sitter.sh' \ + "TARGET=$TARGET" \ + "ROOT=$PWD" + + [[ -n $RUST_TEST_THREADS ]] && \ + printf 'RUST_TEST_THREADS=%s\n' "$RUST_TEST_THREADS" >> $GITHUB_ENV + + [[ -n $CC ]] && printf 'CC=%s\n' "$CC" >> $GITHUB_ENV + [[ -n $AR ]] && printf 'AR=%s\n' "$AR" >> $GITHUB_ENV + + if [[ $USE_CROSS == true ]]; then + printf 'BUILD_CMD=cross\n' >> $GITHUB_ENV + runner=$(cross.sh bash -c "env | sed -n 's/^CARGO_TARGET_.*_RUNNER=//p'") + [[ -n $runner ]] && printf 'CROSS_RUNNER=%s\n' "$runner" >> $GITHUB_ENV + fi # TODO: Remove RUSTFLAGS="--cap-lints allow" once we use a wasmtime release that addresses # the `mismatched-lifetime-syntaxes` lint - name: Build wasmtime library - if: contains(matrix.features, 'wasm') + if: ${{ !matrix.use-cross && contains(matrix.features, 'wasm') }} run: | mkdir -p target WASMTIME_VERSION=$(cargo metadata --format-version=1 --locked --features wasm | \ @@ -190,47 +158,37 @@ jobs: printf 'CMAKE_PREFIX_PATH=%s\n' "$PWD/artifacts" >> $GITHUB_ENV env: WASMTIME_REPO: https://github.com/bytecodealliance/wasmtime - RUSTFLAGS: ${{ env.RUSTFLAGS }} --cap-lints allow + RUSTFLAGS: "--cap-lints allow" - name: Build C library (make) - if: runner.os != 'Windows' - run: | - if [[ $PLATFORM == linux-arm ]]; then - CC=arm-linux-gnueabihf-gcc; AR=arm-linux-gnueabihf-ar - elif [[ $PLATFORM == linux-x86 ]]; then - CC=i686-linux-gnu-gcc; AR=i686-linux-gnu-ar - elif [[ $PLATFORM == linux-powerpc64 ]]; then - CC=powerpc64-linux-gnu-gcc; AR=powerpc64-linux-gnu-ar - else - CC=gcc; AR=ar - fi - make -j CFLAGS="$CFLAGS" CC=$CC AR=$AR + if: ${{ runner.os != 'Windows' }} + run: make.sh -j CFLAGS="$CFLAGS" env: - PLATFORM: ${{ matrix.platform }} CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types - name: Build C library (CMake) - if: "!matrix.cross" + if: ${{ !matrix.use-cross }} run: | - cmake -S . -B build/static \ + cmake -S lib -B build/static \ -DBUILD_SHARED_LIBS=OFF \ -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \ -DTREE_SITTER_FEATURE_WASM=$WASM cmake --build build/static --verbose - cmake -S . -B build/shared \ + cmake -S lib -B build/shared \ -DBUILD_SHARED_LIBS=ON \ -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \ -DTREE_SITTER_FEATURE_WASM=$WASM cmake --build build/shared --verbose env: - CC: ${{ contains(matrix.platform, 'linux') && 'clang' || '' }} + CC: ${{ contains(matrix.target, 'linux') && 'clang' || '' }} WASM: ${{ contains(matrix.features, 'wasm') && 'ON' || 'OFF' }} - - name: Build Wasm library - if: contains(matrix.features, 'wasm') + - name: Build wasm library + # No reason to build on the same Github runner hosts many times + if: ${{ !matrix.no-run && !matrix.use-cross }} shell: bash run: | cd lib/binding_web @@ -241,71 +199,70 @@ jobs: npm run build:debug - name: Check no_std builds - if: inputs.run-test && !matrix.no-run - working-directory: lib + if: ${{ !matrix.no-run && inputs.run-test }} shell: bash - run: cargo check --no-default-features --target='${{ matrix.target }}' + run: | + cd lib + $BUILD_CMD check --no-default-features - name: Build target - run: cargo build --release --target='${{ matrix.target }}' --features='${{ matrix.features }}' $PACKAGE - env: - PACKAGE: ${{ matrix.platform == 'wasm32' && '-p tree-sitter' || '' }} + run: $BUILD_CMD build --release --target=${{ matrix.target }} --features=${{ matrix.features }} - name: Cache fixtures id: cache - if: inputs.run-test && !matrix.no-run + if: ${{ !matrix.no-run && inputs.run-test }} uses: ./.github/actions/cache - name: Fetch fixtures - if: inputs.run-test && !matrix.no-run - run: cargo run -p xtask --target='${{ matrix.target }}' -- fetch-fixtures + if: ${{ !matrix.no-run && inputs.run-test }} + run: $BUILD_CMD run -p xtask -- fetch-fixtures - name: Generate fixtures - if: inputs.run-test && !matrix.no-run && steps.cache.outputs.cache-hit != 'true' - run: cargo run -p xtask --target='${{ matrix.target }}' -- generate-fixtures + if: ${{ !matrix.no-run && inputs.run-test && steps.cache.outputs.cache-hit != 'true' }} + run: $BUILD_CMD run -p xtask -- generate-fixtures - name: Generate Wasm fixtures - if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && steps.cache.outputs.cache-hit != 'true' - run: cargo run -p xtask --target='${{ matrix.target }}' -- generate-fixtures --wasm + if: ${{ !matrix.no-run && !matrix.use-cross && inputs.run-test && steps.cache.outputs.cache-hit != 'true' }} + run: $BUILD_CMD run -p xtask -- generate-fixtures --wasm - name: Run main tests - if: inputs.run-test && !matrix.no-run - run: cargo test --target='${{ matrix.target }}' --features='${{ matrix.features }}' + if: ${{ !matrix.no-run && inputs.run-test }} + run: $BUILD_CMD test --target=${{ matrix.target }} --features=${{ matrix.features }} - - name: Run Wasm tests - if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') - run: cargo run -p xtask --target='${{ matrix.target }}' -- test-wasm + - name: Run wasm tests + if: ${{ !matrix.no-run && !matrix.use-cross && inputs.run-test }} + run: $BUILD_CMD run -p xtask -- test-wasm + + - name: Run benchmarks + # Cross-compiled benchmarks are pointless + if: ${{ !matrix.no-run && !matrix.use-cross && inputs.run-test }} + run: $BUILD_CMD bench benchmark -p tree-sitter-cli --target=${{ matrix.target }} - name: Upload CLI artifact - if: "!matrix.no-run" - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v4 with: name: tree-sitter.${{ matrix.platform }} - path: target/${{ matrix.target }}/release/tree-sitter${{ contains(matrix.target, 'windows') && '.exe' || '' }} + path: target/${{ matrix.target }}/release/tree-sitter${{ env.SUFFIX }} if-no-files-found: error retention-days: 7 - name: Upload Wasm artifacts - if: matrix.platform == 'linux-x64' - uses: actions/upload-artifact@v6 + if: ${{ matrix.platform == 'linux-x64' }} + uses: actions/upload-artifact@v4 with: name: tree-sitter.wasm path: | - lib/binding_web/web-tree-sitter.js - lib/binding_web/web-tree-sitter.js.map - lib/binding_web/web-tree-sitter.cjs - lib/binding_web/web-tree-sitter.cjs.map - lib/binding_web/web-tree-sitter.wasm - lib/binding_web/web-tree-sitter.wasm.map - lib/binding_web/debug/web-tree-sitter.cjs - lib/binding_web/debug/web-tree-sitter.cjs.map - lib/binding_web/debug/web-tree-sitter.js - lib/binding_web/debug/web-tree-sitter.js.map - lib/binding_web/debug/web-tree-sitter.wasm - lib/binding_web/debug/web-tree-sitter.wasm.map - lib/binding_web/lib/*.c - lib/binding_web/lib/*.h - lib/binding_web/lib/*.ts - lib/binding_web/src/*.ts + lib/binding_web/tree-sitter.js + lib/binding_web/tree-sitter.js.map + lib/binding_web/tree-sitter.cjs + lib/binding_web/tree-sitter.cjs.map + lib/binding_web/tree-sitter.wasm + lib/binding_web/tree-sitter.wasm.map + lib/binding_web/debug/tree-sitter.cjs + lib/binding_web/debug/tree-sitter.cjs.map + lib/binding_web/debug/tree-sitter.js + lib/binding_web/debug/tree-sitter.js.map + lib/binding_web/debug/tree-sitter.wasm + lib/binding_web/debug/tree-sitter.wasm.map if-no-files-found: error retention-days: 7 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a60c93f4..01fb165b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v4 - name: Set up stable Rust toolchain uses: actions-rust-lang/setup-rust-toolchain@v1 @@ -44,6 +44,3 @@ jobs: build: uses: ./.github/workflows/build.yml - - check-wasm-stdlib: - uses: ./.github/workflows/wasm_stdlib.yml diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 0e4baebf..ae258043 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -3,7 +3,6 @@ on: push: branches: [master] paths: [docs/**] - workflow_dispatch: jobs: deploy-docs: @@ -16,7 +15,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v4 - name: Set up Rust uses: actions-rust-lang/setup-rust-toolchain@v1 @@ -26,7 +25,7 @@ jobs: GH_TOKEN: ${{ github.token }} run: | jq_expr='.assets[] | select(.name | contains("x86_64-unknown-linux-gnu")) | .browser_download_url' - url=$(gh api repos/rust-lang/mdbook/releases/tags/v0.4.52 --jq "$jq_expr") + url=$(gh api repos/rust-lang/mdbook/releases/latest --jq "$jq_expr") mkdir mdbook curl -sSL "$url" | tar -xz -C mdbook printf '%s/mdbook\n' "$PWD" >> "$GITHUB_PATH" @@ -41,7 +40,7 @@ jobs: uses: actions/configure-pages@v5 - name: Upload artifact - uses: actions/upload-pages-artifact@v4 + uses: actions/upload-pages-artifact@v3 with: path: docs/book diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml new file mode 100644 index 00000000..9019fedc --- /dev/null +++ b/.github/workflows/emscripten.yml @@ -0,0 +1,30 @@ +name: Update Emscripten + +on: + pull_request: + types: [opened, synchronize] + +permissions: + contents: write + pull-requests: read + +jobs: + update-emscripten: + if: github.actor == 'dependabot[bot]' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up stable Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + + - name: Run emscripten update xtask + run: | + git config --global user.name "dependabot[bot]" + git config --global user.email "49699333+dependabot[bot]@users.noreply.github.com" + cargo xtask upgrade-emscripten + + - name: Push updated version + run: git push origin HEAD:$GITHUB_HEAD_REF diff --git a/.github/workflows/nvim_ts.yml b/.github/workflows/nvim_ts.yml index 88e3371f..4bf39366 100644 --- a/.github/workflows/nvim_ts.yml +++ b/.github/workflows/nvim_ts.yml @@ -3,10 +3,7 @@ name: nvim-treesitter parser tests on: pull_request: paths: - - 'crates/cli/**' - - 'crates/config/**' - - 'crates/generate/**' - - 'crates/loader/**' + - 'cli/**' - '.github/workflows/nvim_ts.yml' workflow_dispatch: @@ -16,7 +13,7 @@ concurrency: jobs: check_compilation: - timeout-minutes: 30 + timeout-minutes: 20 strategy: fail-fast: false matrix: @@ -28,9 +25,9 @@ jobs: NVIM: ${{ matrix.os == 'windows-latest' && 'nvim-win64\\bin\\nvim.exe' || 'nvim' }} NVIM_TS_DIR: nvim-treesitter steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: repository: nvim-treesitter/nvim-treesitter path: ${{ env.NVIM_TS_DIR }} @@ -58,7 +55,7 @@ jobs: - if: matrix.type == 'build' name: Compile parsers - run: $NVIM -l ./scripts/install-parsers.lua --max-jobs=10 + run: $NVIM -l ./scripts/install-parsers.lua working-directory: ${{ env.NVIM_TS_DIR }} shell: bash diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4f6f9d47..02a59699 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,15 +17,13 @@ jobs: runs-on: ubuntu-latest needs: build permissions: - id-token: write - attestations: write contents: write steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v4 - name: Download build artifacts - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: path: artifacts @@ -35,13 +33,26 @@ jobs: - name: Prepare release artifacts run: | - mkdir -p target web - mv artifacts/tree-sitter.wasm/* web/ + mkdir -p target + mv artifacts/tree-sitter.wasm/* target/ - tar -czf target/web-tree-sitter.tar.gz -C web . + # Rename files + mv target/tree-sitter.js target/web-tree-sitter.js + mv target/tree-sitter.js.map target/web-tree-sitter.js.map + mv target/tree-sitter.cjs target/web-tree-sitter.cjs + mv target/tree-sitter.cjs.map target/web-tree-sitter.cjs.map + mv target/tree-sitter.wasm target/web-tree-sitter.wasm + mv target/tree-sitter.wasm.map target/web-tree-sitter.wasm.map + + mv target/debug/tree-sitter.js target/web-tree-sitter-debug.js + mv target/debug/tree-sitter.js.map target/web-tree-sitter-debug.js.map + mv target/debug/tree-sitter.cjs target/web-tree-sitter-debug.cjs + mv target/debug/tree-sitter.cjs.map target/web-tree-sitter-debug.cjs.map + mv target/debug/tree-sitter.wasm target/web-tree-sitter-debug.wasm + mv target/debug/tree-sitter.wasm.map target/web-tree-sitter-debug.wasm.map + rm -rf target/debug rm -r artifacts/tree-sitter.wasm - for platform in $(cd artifacts; ls | sed 's/^tree-sitter\.//'); do exe=$(ls artifacts/tree-sitter.$platform/tree-sitter*) gzip --stdout --name $exe > target/tree-sitter-$platform.gz @@ -49,65 +60,57 @@ jobs: rm -rf artifacts ls -l target/ - - name: Generate attestations - uses: actions/attest-build-provenance@v3 - with: - subject-path: | - target/tree-sitter-*.gz - target/web-tree-sitter.tar.gz - - name: Create release run: |- - gh release create $GITHUB_REF_NAME \ + gh release create ${{ github.ref_name }} \ target/tree-sitter-*.gz \ - target/web-tree-sitter.tar.gz + target/web-tree-sitter.js \ + target/web-tree-sitter.js.map \ + target/web-tree-sitter.cjs \ + target/web-tree-sitter.cjs.map \ + target/web-tree-sitter.wasm \ + target/web-tree-sitter.wasm.map \ + target/web-tree-sitter-debug.js \ + target/web-tree-sitter-debug.js.map \ + target/web-tree-sitter-debug.cjs \ + target/web-tree-sitter-debug.cjs.map \ + target/web-tree-sitter-debug.wasm \ + target/web-tree-sitter-debug.wasm.map env: GH_TOKEN: ${{ github.token }} crates_io: name: Publish packages to Crates.io runs-on: ubuntu-latest - environment: crates - permissions: - id-token: write - contents: read needs: release steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v4 - name: Set up Rust uses: actions-rust-lang/setup-rust-toolchain@v1 - - name: Set up registry token - id: auth - uses: rust-lang/crates-io-auth-action@v1 - - name: Publish crates to Crates.io uses: katyo/publish-crates@v2 with: - registry-token: ${{ steps.auth.outputs.token }} + registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }} npm: name: Publish packages to npmjs.com runs-on: ubuntu-latest - environment: npm - permissions: - id-token: write - contents: read needs: release strategy: fail-fast: false matrix: - directory: [crates/cli/npm, lib/binding_web] + directory: [cli/npm, lib/binding_web] steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v4 - name: Set up Node - uses: actions/setup-node@v6 + uses: actions/setup-node@v4 with: - node-version: 24 + node-version: 20 registry-url: https://registry.npmjs.org - name: Set up Rust @@ -122,8 +125,9 @@ jobs: npm run build:debug CJS=true npm run build CJS=true npm run build:debug - npm run build:dts - name: Publish to npmjs.com working-directory: ${{ matrix.directory }} run: npm publish + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.github/workflows/response.yml b/.github/workflows/response.yml index 54dd2021..576b9474 100644 --- a/.github/workflows/response.yml +++ b/.github/workflows/response.yml @@ -17,13 +17,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout script - uses: actions/checkout@v6 + uses: actions/checkout@v4 with: sparse-checkout: .github/scripts/close_unresponsive.js sparse-checkout-cone-mode: false - name: Run script - uses: actions/github-script@v8 + uses: actions/github-script@v7 with: script: | const script = require('./.github/scripts/close_unresponsive.js') @@ -35,13 +35,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout script - uses: actions/checkout@v6 + uses: actions/checkout@v4 with: sparse-checkout: .github/scripts/remove_response_label.js sparse-checkout-cone-mode: false - name: Run script - uses: actions/github-script@v8 + uses: actions/github-script@v7 with: script: | const script = require('./.github/scripts/remove_response_label.js') diff --git a/.github/workflows/reviewers_remove.yml b/.github/workflows/reviewers_remove.yml index 3a389ed4..b99f0caa 100644 --- a/.github/workflows/reviewers_remove.yml +++ b/.github/workflows/reviewers_remove.yml @@ -12,13 +12,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout script - uses: actions/checkout@v6 + uses: actions/checkout@v4 with: sparse-checkout: .github/scripts/reviewers_remove.js sparse-checkout-cone-mode: false - name: Run script - uses: actions/github-script@v8 + uses: actions/github-script@v7 with: script: | const script = require('./.github/scripts/reviewers_remove.js') diff --git a/.github/workflows/sanitize.yml b/.github/workflows/sanitize.yml index 2f8851dc..875b8278 100644 --- a/.github/workflows/sanitize.yml +++ b/.github/workflows/sanitize.yml @@ -15,7 +15,7 @@ jobs: TREE_SITTER: ${{ github.workspace }}/target/release/tree-sitter steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v4 - name: Install UBSAN library run: sudo apt-get update -y && sudo apt-get install -y libubsan1 diff --git a/.github/workflows/spam.yml b/.github/workflows/spam.yml deleted file mode 100644 index eb1d4a46..00000000 --- a/.github/workflows/spam.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Close as spam - -on: - issues: - types: [labeled] - pull_request_target: - types: [labeled] - -permissions: - issues: write - pull-requests: write - -jobs: - spam: - runs-on: ubuntu-latest - if: github.event.label.name == 'spam' - steps: - - name: Checkout script - uses: actions/checkout@v6 - with: - sparse-checkout: .github/scripts/close_spam.js - sparse-checkout-cone-mode: false - - - name: Run script - uses: actions/github-script@v8 - with: - script: | - const script = require('./.github/scripts/close_spam.js') - await script({github, context}) diff --git a/.github/workflows/wasm_exports.yml b/.github/workflows/wasm_exports.yml index af48cf8a..febceb44 100644 --- a/.github/workflows/wasm_exports.yml +++ b/.github/workflows/wasm_exports.yml @@ -1,24 +1,23 @@ -name: Check Wasm Exports +name: Check WASM Exports on: pull_request: paths: - lib/include/tree_sitter/api.h - lib/binding_web/** - - xtask/src/** push: branches: [master] paths: - lib/include/tree_sitter/api.h - lib/binding_rust/bindings.rs - - CMakeLists.txt + - lib/CMakeLists.txt jobs: check-wasm-exports: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v4 - name: Set up stable Rust toolchain uses: actions-rust-lang/setup-rust-toolchain@v1 @@ -33,9 +32,9 @@ jobs: env: CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types - - name: Build Wasm Library + - name: Build WASM Library working-directory: lib/binding_web run: npm ci && npm run build:debug - - name: Check Wasm exports + - name: Check WASM exports run: cargo xtask check-wasm-exports diff --git a/.github/workflows/wasm_stdlib.yml b/.github/workflows/wasm_stdlib.yml deleted file mode 100644 index adec8411..00000000 --- a/.github/workflows/wasm_stdlib.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Check Wasm Stdlib build - -on: - workflow_call: - -jobs: - check: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v6 - - - name: Check directory changes - uses: actions/github-script@v8 - with: - script: | - const scriptPath = `${process.env.GITHUB_WORKSPACE}/.github/scripts/wasm_stdlib.js`; - const script = require(scriptPath); - return script({ github, context, core }); diff --git a/.gitignore b/.gitignore index ca47139e..333b718a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,10 @@ log*.html -.direnv .idea *.xcodeproj .vscode .cache .zig-cache -.direnv profile* fuzz-results @@ -26,7 +24,6 @@ docs/assets/js/tree-sitter.js *.dylib *.so *.so.[0-9]* -*.dll *.o *.obj *.exp @@ -36,5 +33,3 @@ docs/assets/js/tree-sitter.js .build build zig-* - -/result diff --git a/.zed/settings.json b/.zed/settings.json deleted file mode 100644 index 15f15d9a..00000000 --- a/.zed/settings.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "lsp": { - "rust-analyzer": { - "initialization_options": { - "cargo": { - "features": "all" - } - } - } - } -} diff --git a/Cargo.lock b/Cargo.lock index ae7803c8..39cec39c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3,19 +3,28 @@ version = 3 [[package]] -name = "addr2line" -version = "0.24.2" +name = "adler2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ - "gimli", + "cfg-if", + "once_cell", + "version_check", + "zerocopy", ] [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] @@ -37,9 +46,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.21" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" dependencies = [ "anstyle", "anstyle-parse", @@ -52,59 +61,50 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.13" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anstyle-parse" -version = "0.2.7" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.5" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.11" +version = "3.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" dependencies = [ "anstyle", - "once_cell_polyfill", - "windows-sys 0.61.2", + "once_cell", + "windows-sys 0.59.0", ] [[package]] name = "anyhow" -version = "1.0.100" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" - -[[package]] -name = "ar_archive_writer" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0c269894b6fe5e9d7ada0cf69b5bf847ff35bc25fc271f08e1d080fce80339a" -dependencies = [ - "object 0.32.2", -] +checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" [[package]] name = "arbitrary" -version = "1.4.2" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" [[package]] name = "ascii" @@ -113,12 +113,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" [[package]] -name = "bindgen" -version = "0.72.1" +name = "base64" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bindgen" +version = "0.71.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.8.0", "cexpr", "clang-sys", "itertools 0.13.0", @@ -140,24 +146,24 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.10.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" [[package]] name = "block2" -version = "0.6.2" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +checksum = "2c132eebf10f5cad5289222520a4a058514204aed6d791f1cf4fe8088b82d15f" dependencies = [ "objc2", ] [[package]] name = "bstr" -version = "1.12.1" +version = "1.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +checksum = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0" dependencies = [ "memchr", "regex-automata", @@ -166,32 +172,39 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.19.1" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" dependencies = [ "allocator-api2", ] [[package]] name = "bytemuck" -version = "1.24.0" +version = "1.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" +checksum = "ef657dfab802224e671f5818e9a4935f9b1957ed18e58292690cc39e7a4092a3" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" [[package]] name = "cc" -version = "1.2.53" +version = "1.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "755d2fce177175ffca841e9a06afdb2c4ab0f593d53b4dee48147dfaade85932" +checksum = "13208fcbb66eaeffe09b99fffbe1af420f00a7b35aa99ad683dfc1aa76145229" dependencies = [ - "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -212,9 +225,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.4" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cfg_aliases" @@ -236,14 +249,14 @@ checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", - "libloading 0.8.9", + "libloading", ] [[package]] name = "clap" -version = "4.5.54" +version = "4.5.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6e6ff9dcd79cff5cd969a17a545d79e84ab086e444102a591e288a8aa3ce394" +checksum = "769b0145982b4b48713e01ec42d61614425f27b7058bda7180a3a41f30104796" dependencies = [ "clap_builder", "clap_derive", @@ -251,9 +264,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.54" +version = "4.5.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa42cf4d2b7a41bc8f663a7cab4031ebafa1bf3875705bfaf8466dc60ab52c00" +checksum = "1b26884eb4b57140e4d2d93652abfa49498b938b3c9179f9fc487b0acc3edad7" dependencies = [ "anstream", "anstyle", @@ -263,18 +276,18 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.5.65" +version = "4.5.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "430b4dc2b5e3861848de79627b2bedc9f3342c7da5173a14eaa5d0f8dc18ae5d" +checksum = "33a7e468e750fa4b6be660e8b5651ad47372e8fb114030b594c2d75d48c5ffd0" dependencies = [ "clap", ] [[package]] name = "clap_complete_nushell" -version = "4.5.10" +version = "4.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "685bc86fd34b7467e0532a4f8435ab107960d69a243785ef0275e571b35b641a" +checksum = "c6a8b1593457dfc2fe539002b795710d022dc62a65bf15023f039f9760c7b18a" dependencies = [ "clap", "clap_complete", @@ -282,9 +295,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.49" +version = "4.5.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" +checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c" dependencies = [ "heck", "proc-macro2", @@ -294,24 +307,21 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.6" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "cobs" -version = "0.3.0" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" -dependencies = [ - "thiserror 2.0.17", -] +checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" [[package]] name = "colorchoice" -version = "1.0.4" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "combine" @@ -325,9 +335,9 @@ dependencies = [ [[package]] name = "console" -version = "0.15.11" +version = "0.15.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +checksum = "ea3c6ecd8059b57859df5c69830340ed3c41d30e3da0c1cbed90a96ac853041b" dependencies = [ "encode_unicode", "libc", @@ -337,19 +347,39 @@ dependencies = [ ] [[package]] -name = "convert_case" -version = "0.10.0" +name = "cookie" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" dependencies = [ - "unicode-segmentation", + "percent-encoding", + "time", + "version_check", +] + +[[package]] +name = "cookie_store" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eac901828f88a5241ee0600950ab981148a18f2f756900ffba1b125ca6a3ef9" +dependencies = [ + "cookie", + "document-features", + "idna", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "time", + "url", ] [[package]] name = "core-foundation" -version = "0.10.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63" dependencies = [ "core-foundation-sys", "libc", @@ -361,38 +391,20 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "cranelift-assembler-x64" -version = "0.120.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5023e06632d8f351c2891793ccccfe4aef957954904392434038745fb6f1f68" -dependencies = [ - "cranelift-assembler-x64-meta", -] - -[[package]] -name = "cranelift-assembler-x64-meta" -version = "0.120.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c4012b4c8c1f6eb05c0a0a540e3e1ee992631af51aa2bbb3e712903ce4fd65" -dependencies = [ - "cranelift-srcgen", -] - [[package]] name = "cranelift-bforest" -version = "0.120.2" +version = "0.116.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d6d883b4942ef3a7104096b8bc6f2d1a41393f159ac8de12aed27b25d67f895" +checksum = "e15d04a0ce86cb36ead88ad68cf693ffd6cda47052b9e0ac114bc47fd9cd23c4" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-bitset" -version = "0.120.2" +version = "0.116.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db7b2ee9eec6ca8a716d900d5264d678fb2c290c58c46c8da7f94ee268175d17" +checksum = "7c6e3969a7ce267259ce244b7867c5d3bc9e65b0a87e81039588dfdeaede9f34" dependencies = [ "serde", "serde_derive", @@ -400,12 +412,11 @@ dependencies = [ [[package]] name = "cranelift-codegen" -version = "0.120.2" +version = "0.116.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aeda0892577afdce1ac2e9a983a55f8c5b87a59334e1f79d8f735a2d7ba4f4b4" +checksum = "2c22032c4cb42558371cf516bb47f26cdad1819d3475c133e93c49f50ebf304e" dependencies = [ "bumpalo", - "cranelift-assembler-x64", "cranelift-bforest", "cranelift-bitset", "cranelift-codegen-meta", @@ -414,9 +425,8 @@ dependencies = [ "cranelift-entity", "cranelift-isle", "gimli", - "hashbrown 0.15.5", + "hashbrown 0.14.5", "log", - "pulley-interpreter", "regalloc2", "rustc-hash", "serde", @@ -426,36 +436,33 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.120.2" +version = "0.116.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e461480d87f920c2787422463313326f67664e68108c14788ba1676f5edfcd15" +checksum = "c904bc71c61b27fc57827f4a1379f29de64fe95653b620a3db77d59655eee0b8" dependencies = [ - "cranelift-assembler-x64-meta", "cranelift-codegen-shared", - "cranelift-srcgen", - "pulley-interpreter", ] [[package]] name = "cranelift-codegen-shared" -version = "0.120.2" +version = "0.116.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976584d09f200c6c84c4b9ff7af64fc9ad0cb64dffa5780991edd3fe143a30a1" +checksum = "40180f5497572f644ce88c255480981ae2ec1d7bb4d8e0c0136a13b87a2f2ceb" [[package]] name = "cranelift-control" -version = "0.120.2" +version = "0.116.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46d43d70f4e17c545aa88dbf4c84d4200755d27c6e3272ebe4de65802fa6a955" +checksum = "26d132c6d0bd8a489563472afc171759da0707804a65ece7ceb15a8c6d7dd5ef" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.120.2" +version = "0.116.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d75418674520cb400c8772bfd6e11a62736c78fc1b6e418195696841d1bf91f1" +checksum = "4b2d0d9618275474fbf679dd018ac6e009acbd6ae6850f6a67be33fb3b00b323" dependencies = [ "cranelift-bitset", "serde", @@ -464,9 +471,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.120.2" +version = "0.116.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c8b1a91c86687a344f3c52dd6dfb6e50db0dfa7f2e9c7711b060b3623e1fdeb" +checksum = "4fac41e16729107393174b0c9e3730fb072866100e1e64e80a1a963b2e484d57" dependencies = [ "cranelift-codegen", "log", @@ -476,32 +483,26 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.120.2" +version = "0.116.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711baa4e3432d4129295b39ec2b4040cc1b558874ba0a37d08e832e857db7285" +checksum = "1ca20d576e5070044d0a72a9effc2deacf4d6aa650403189d8ea50126483944d" [[package]] name = "cranelift-native" -version = "0.120.2" +version = "0.116.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41c83e8666e3bcc5ffeaf6f01f356f0e1f9dcd69ce5511a1efd7ca5722001a3f" +checksum = "b8dee82f3f1f2c4cba9177f1cc5e350fe98764379bcd29340caa7b01f85076c7" dependencies = [ "cranelift-codegen", "libc", "target-lexicon", ] -[[package]] -name = "cranelift-srcgen" -version = "0.120.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e3f4d783a55c64266d17dc67d2708852235732a100fc40dd9f1051adc64d7b" - [[package]] name = "crc32fast" -version = "1.5.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ "cfg-if", ] @@ -518,13 +519,21 @@ dependencies = [ [[package]] name = "ctrlc" -version = "3.5.1" +version = "3.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73736a89c4aff73035ba2ed2e565061954da00d4970fc9ac25dcc85a2a20d790" +checksum = "90eeab0aa92f3f9b4e87f258c72b139c207d251f9cbc1080a0086b86a8870dd3" dependencies = [ - "dispatch2", "nix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", +] + +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", ] [[package]] @@ -547,18 +556,6 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" -[[package]] -name = "dispatch2" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89a09f22a6c6069a18470eb92d2298acf25463f14256d24778e1230d789a2aec" -dependencies = [ - "bitflags 2.10.0", - "block2", - "libc", - "objc2", -] - [[package]] name = "displaydoc" version = "0.2.5" @@ -571,22 +568,19 @@ dependencies = [ ] [[package]] -name = "dunce" -version = "1.0.5" +name = "document-features" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" - -[[package]] -name = "dyn-clone" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +checksum = "cb6969eaabd2421f8a2775cfd2471a2b634372b4a25d41e3bd647b79912850a0" +dependencies = [ + "litrs", +] [[package]] name = "either" -version = "1.15.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] name = "embedded-io" @@ -617,28 +611,29 @@ dependencies = [ [[package]] name = "equivalent" -version = "1.0.2" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.14" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] name = "etcetera" -version = "0.11.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de48cc4d1c1d97a20fd819def54b890cadde72ed3ad0c614822a0a433361be96" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" dependencies = [ "cfg-if", - "windows-sys 0.61.2", + "home", + "windows-sys 0.48.0", ] [[package]] @@ -655,18 +650,34 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "file-id" -version = "0.2.3" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1fc6a637b6dc58414714eddd9170ff187ecb0933d4c7024d1abbd23a3cc26e9" +checksum = "6bc904b9bbefcadbd8e3a9fb0d464a9b979de6324c03b3c663e8994f46a5be36" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] -name = "find-msvc-tools" -version = "0.1.8" +name = "filetime" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" +dependencies = [ + "cfg-if", + "libc", + "libredox", + "windows-sys 0.59.0", +] + +[[package]] +name = "flate2" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] [[package]] name = "fnv" @@ -676,21 +687,15 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "foldhash" -version = "0.1.5" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" - -[[package]] -name = "foldhash" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" [[package]] name = "form_urlencoded" -version = "1.2.2" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] @@ -701,7 +706,7 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c29c30684418547d476f0b48e84f4821639119c483b1eccd566c8cd0cd05f521" dependencies = [ - "rustix 0.38.44", + "rustix", "windows-sys 0.52.0", ] @@ -725,27 +730,15 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.16" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", "wasi", ] -[[package]] -name = "getrandom" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" -dependencies = [ - "cfg-if", - "libc", - "r-efi", - "wasip2", -] - [[package]] name = "gimli" version = "0.31.1" @@ -758,30 +751,43 @@ dependencies = [ ] [[package]] -name = "glob" -version = "0.3.3" +name = "git2" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +checksum = "3fda788993cc341f69012feba8bf45c0ba4f3291fcc08e214b4d5a7332d88aff" +dependencies = [ + "bitflags 2.8.0", + "libc", + "libgit2-sys", + "log", + "openssl-probe", + "openssl-sys", + "url", +] + +[[package]] +name = "glob" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "hashbrown" -version = "0.15.5" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ - "foldhash 0.1.5", - "serde", + "ahash", ] [[package]] name = "hashbrown" -version = "0.16.1" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" dependencies = [ - "allocator-api2", - "equivalent", - "foldhash 0.2.0", + "foldhash", + "serde", ] [[package]] @@ -790,6 +796,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "home" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "html-escape" version = "0.2.13" @@ -799,6 +814,23 @@ dependencies = [ "utf8-width", ] +[[package]] +name = "http" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" + [[package]] name = "httpdate" version = "1.0.3" @@ -807,22 +839,21 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "icu_collections" -version = "2.1.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" dependencies = [ "displaydoc", - "potential_utf", "yoke", "zerofrom", "zerovec", ] [[package]] -name = "icu_locale_core" -version = "2.1.1" +name = "icu_locid" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" dependencies = [ "displaydoc", "litemap", @@ -832,71 +863,109 @@ dependencies = [ ] [[package]] -name = "icu_normalizer" -version = "2.1.1" +name = "icu_locid_transform" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", "icu_collections", "icu_normalizer_data", "icu_properties", "icu_provider", "smallvec", + "utf16_iter", + "utf8_iter", + "write16", "zerovec", ] [[package]] name = "icu_normalizer_data" -version = "2.1.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" [[package]] name = "icu_properties" -version = "2.1.2" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" dependencies = [ + "displaydoc", "icu_collections", - "icu_locale_core", + "icu_locid_transform", "icu_properties_data", "icu_provider", - "zerotrie", + "tinystr", "zerovec", ] [[package]] name = "icu_properties_data" -version = "2.1.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" [[package]] name = "icu_provider" -version = "2.1.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" dependencies = [ "displaydoc", - "icu_locale_core", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", "writeable", "yoke", "zerofrom", - "zerotrie", "zerovec", ] [[package]] -name = "ident_case" -version = "1.0.1" +name = "icu_provider_macros" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "id-arena" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" [[package]] name = "idna" -version = "1.1.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" dependencies = [ "idna_adapter", "smallvec", @@ -905,9 +974,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" dependencies = [ "icu_normalizer", "icu_properties", @@ -915,24 +984,20 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.12.1" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" +checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.15.2", "serde", - "serde_core", ] [[package]] name = "indoc" -version = "2.0.7" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" -dependencies = [ - "rustversion", -] +checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" [[package]] name = "inotify" @@ -940,7 +1005,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f37dccff2791ab604f9babef0ba14fbe0be30bd368dc541e2b08d07c8aa908f3" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.8.0", "inotify-sys", "libc", ] @@ -956,9 +1021,18 @@ dependencies = [ [[package]] name = "is_terminal_polyfill" -version = "1.70.2" +version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] [[package]] name = "itertools" @@ -969,20 +1043,11 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" -dependencies = [ - "either", -] - [[package]] name = "itoa" -version = "1.0.17" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] name = "jni" @@ -1007,10 +1072,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" [[package]] -name = "js-sys" -version = "0.3.83" +name = "jobserver" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ "once_cell", "wasm-bindgen", @@ -1018,9 +1092,9 @@ dependencies = [ [[package]] name = "kqueue" -version = "1.1.1" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac30106d7dce88daf4a3fcb4879ea939476d5074a9b7ddd0fb97fa4bed5596a" +checksum = "7447f1ca1b7b563588a205fe93dea8df60fd981423a768bc1c0ded35ed147d0c" dependencies = [ "kqueue-sys", "libc", @@ -1037,42 +1111,83 @@ dependencies = [ ] [[package]] -name = "leb128fmt" -version = "0.1.0" +name = "leb128" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" +checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "libc" -version = "0.2.178" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] -name = "libloading" -version = "0.8.9" +name = "libgit2-sys" +version = "0.18.0+1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +checksum = "e1a117465e7e1597e8febea8bb0c410f1c7fb93b1e1cddf34363f8390367ffec" dependencies = [ - "cfg-if", - "windows-link", + "cc", + "libc", + "libssh2-sys", + "libz-sys", + "openssl-sys", + "pkg-config", ] [[package]] name = "libloading" -version = "0.9.0" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", - "windows-link", + "windows-targets 0.48.5", ] [[package]] name = "libm" -version = "0.2.15" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" + +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.8.0", + "libc", + "redox_syscall", +] + +[[package]] +name = "libssh2-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dc8a030b787e2119a731f1951d6a773e2280c660f8ec4b0f5e1505a386e71ee" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "libz-sys" +version = "1.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9b68e50e6e0b26f672573834882eb57759f6db9b3be2ea3c35c91188bb4eaa" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] [[package]] name = "linux-raw-sys" @@ -1081,45 +1196,45 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] -name = "linux-raw-sys" -version = "0.11.0" +name = "litemap" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" +checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" [[package]] -name = "litemap" -version = "0.8.1" +name = "litrs" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5" [[package]] name = "log" -version = "0.4.29" +version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" [[package]] name = "mach2" -version = "0.4.3" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44" +checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" dependencies = [ "libc", ] [[package]] name = "memchr" -version = "2.7.6" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memfd" -version = "0.6.5" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad38eb12aea514a0466ea40a80fd8cc83637065948eb4a426e4aa46261175227" +checksum = "b2cffa4ad52c6f791f4f8b15f0c05f9824b2ced1160e88cc393d64fff9a8ac64" dependencies = [ - "rustix 1.1.3", + "rustix", ] [[package]] @@ -1129,15 +1244,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] -name = "mio" -version = "1.1.1" +name = "miniz_oxide" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "b8402cab7aefae129c6977bb0ff1b8fd9a04eb5b51efc50a70bea51cda0c7924" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ "libc", "log", "wasi", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1148,11 +1272,11 @@ checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" [[package]] name = "nix" -version = "0.30.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.8.0", "cfg-if", "cfg_aliases", "libc", @@ -1170,11 +1294,12 @@ dependencies = [ [[package]] name = "notify" -version = "8.2.0" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3" +checksum = "2fee8403b3d66ac7b26aee6e40a897d85dc5ce26f44da36b8b73e987cc52e943" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.8.0", + "filetime", "fsevent-sys", "inotify", "kqueue", @@ -1183,14 +1308,14 @@ dependencies = [ "mio", "notify-types", "walkdir", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] name = "notify-debouncer-full" -version = "0.6.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "375bd3a138be7bfeff3480e4a623df4cbfb55b79df617c055cd810ba466fa078" +checksum = "d2d88b1a7538054351c8258338df7c931a590513fb3745e8c15eb9ff4199b8d1" dependencies = [ "file-id", "log", @@ -1206,11 +1331,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e0826a989adedc2a244799e823aece04662b66609d96af8dff7ac6df9a8925d" [[package]] -name = "objc2" -version = "0.6.3" +name = "num-conv" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c2599ce0ec54857b29ce62166b0ed9b4f6f1a70ccc9a71165b6154caca8c05" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "objc-sys" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb91bdd390c7ce1a8607f35f3ca7151b65afc0ff5ff3b34fa350f7d7c7e4310" + +[[package]] +name = "objc2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46a785d4eeff09c14c487497c162e92766fbb3e4059a71840cecc03d9a50b804" dependencies = [ + "objc-sys", "objc2-encode", ] @@ -1222,23 +1360,16 @@ checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" [[package]] name = "objc2-foundation" -version = "0.3.2" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +checksum = "0ee638a5da3799329310ad4cfa62fbf045d5f56e3ef5ba4149e7452dcf89d5a8" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.8.0", + "block2", + "libc", "objc2", ] -[[package]] -name = "object" -version = "0.32.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" -dependencies = [ - "memchr", -] - [[package]] name = "object" version = "0.36.7" @@ -1246,63 +1377,52 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" dependencies = [ "crc32fast", - "hashbrown 0.15.5", + "hashbrown 0.15.2", "indexmap", "memchr", ] [[package]] name = "once_cell" -version = "1.21.3" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] -name = "once_cell_polyfill" -version = "1.70.2" +name = "openssl-probe" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] -name = "pathdiff" -version = "0.2.3" +name = "openssl-sys" +version = "0.9.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" +checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "path-slash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e91099d4268b0e11973f036e885d652fb0b21fedcf69738c627f94db6a44f42" [[package]] name = "percent-encoding" -version = "2.3.2" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" - -[[package]] -name = "phf" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" -dependencies = [ - "phf_shared", - "serde", -] - -[[package]] -name = "phf_generator" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" -dependencies = [ - "fastrand", - "phf_shared", -] - -[[package]] -name = "phf_shared" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" -dependencies = [ - "siphasher", -] +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pin-project-lite" @@ -1311,10 +1431,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] -name = "postcard" -version = "1.1.3" +name = "pkg-config" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" + +[[package]] +name = "postcard" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "170a2601f67cc9dba8edd8c4870b15f71a6a2dc196daec8c83f72b59dff628a8" dependencies = [ "cobs", "embedded-io 0.4.0", @@ -1323,19 +1449,16 @@ dependencies = [ ] [[package]] -name = "potential_utf" -version = "0.1.4" +name = "powerfmt" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" -dependencies = [ - "zerovec", -] +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.21" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" dependencies = [ "zerocopy", ] @@ -1352,68 +1475,53 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.37" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +checksum = "6924ced06e1f7dfe3fa48d57b9f74f55d8915f5036121bef647ef4b204895fac" dependencies = [ "proc-macro2", "syn", ] -[[package]] -name = "proc-macro-crate" -version = "3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" -dependencies = [ - "toml_edit", -] - [[package]] name = "proc-macro2" -version = "1.0.104" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9695f8df41bb4f3d222c95a67532365f569318332d03d5f3f67f37b20e6ebdf0" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" dependencies = [ "unicode-ident", ] [[package]] name = "psm" -version = "0.1.28" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d11f2fedc3b7dafdc2851bc52f277377c5473d378859be234bc7ebb593144d01" +checksum = "200b9ff220857e53e184257720a14553b2f4aa02577d2ed9842d45d4b9654810" dependencies = [ - "ar_archive_writer", "cc", ] [[package]] name = "pulley-interpreter" -version = "33.0.2" +version = "29.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "986beaef947a51d17b42b0ea18ceaa88450d35b6994737065ed505c39172db71" +checksum = "62d95f8575df49a2708398182f49a888cf9dc30210fb1fd2df87c889edcee75d" dependencies = [ "cranelift-bitset", "log", + "sptr", "wasmtime-math", ] [[package]] name = "quote" -version = "1.0.42" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - [[package]] name = "rand" version = "0.8.5" @@ -1441,38 +1549,27 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.16", + "getrandom", ] [[package]] -name = "ref-cast" -version = "1.0.25" +name = "redox_syscall" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ - "ref-cast-impl", -] - -[[package]] -name = "ref-cast-impl" -version = "1.0.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" -dependencies = [ - "proc-macro2", - "quote", - "syn", + "bitflags 2.8.0", ] [[package]] name = "regalloc2" -version = "0.12.2" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5216b1837de2149f8bc8e6d5f88a9326b63b8c836ed58ce4a0a29ec736a59734" +checksum = "145c1c267e14f20fb0f88aa76a1c5ffec42d592c1d28b3cd9148ae35916158d3" dependencies = [ "allocator-api2", "bumpalo", - "hashbrown 0.15.5", + "hashbrown 0.15.2", "log", "rustc-hash", "smallvec", @@ -1480,9 +1577,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.2" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", @@ -1492,9 +1589,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.13" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", @@ -1503,84 +1600,38 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.8" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" - -[[package]] -name = "relative-path" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bca40a312222d8ba74837cb474edef44b37f561da5f773981007a10bbaa992b0" -dependencies = [ - "serde", -] +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "rgb" -version = "0.8.52" +version = "0.8.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6a884d2998352bb4daf0183589aec883f16a6da1f4dde84d8e2e9a5409a1ce" +checksum = "57397d16646700483b67d2dd6511d79318f9d057fdbd21a4066aeac8b41d310a" dependencies = [ "bytemuck", ] [[package]] -name = "rquickjs" -version = "0.11.0" +name = "ring" +version = "0.17.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c50dc6d6c587c339edb4769cf705867497a2baf0eca8b4645fa6ecd22f02c77a" +checksum = "70ac5d832aa16abd7d1def883a8545280c20a60f523a370aa3a9617c2b8550ee" dependencies = [ - "rquickjs-core", - "rquickjs-macro", -] - -[[package]] -name = "rquickjs-core" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8bf7840285c321c3ab20e752a9afb95548c75cd7f4632a0627cea3507e310c1" -dependencies = [ - "hashbrown 0.16.1", - "phf", - "relative-path", - "rquickjs-sys", -] - -[[package]] -name = "rquickjs-macro" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7106215ff41a5677b104906a13e1a440b880f4b6362b5dc4f3978c267fad2b80" -dependencies = [ - "convert_case", - "fnv", - "ident_case", - "indexmap", - "phf_generator", - "phf_shared", - "proc-macro-crate", - "proc-macro2", - "quote", - "rquickjs-core", - "syn", -] - -[[package]] -name = "rquickjs-sys" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27344601ef27460e82d6a4e1ecb9e7e99f518122095f3c51296da8e9be2b9d83" -dependencies = [ - "bindgen", "cc", + "cfg-if", + "getrandom", + "libc", + "untrusted", + "windows-sys 0.52.0", ] [[package]] name = "rustc-hash" -version = "2.1.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" [[package]] name = "rustix" @@ -1588,31 +1639,59 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.8.0", "errno", "libc", - "linux-raw-sys 0.4.15", + "linux-raw-sys", "windows-sys 0.59.0", ] [[package]] -name = "rustix" -version = "1.1.3" +name = "rustls" +version = "0.23.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +checksum = "8f287924602bf649d949c63dc8ac8b235fa5387d394020705b80c4eb597ce5b8" dependencies = [ - "bitflags 2.10.0", - "errno", - "libc", - "linux-raw-sys 0.11.0", - "windows-sys 0.61.2", + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", ] [[package]] -name = "rustversion" -version = "1.0.22" +name = "rustls-pemfile" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" + +[[package]] +name = "rustls-webpki" +version = "0.102.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "same-file" @@ -1623,76 +1702,29 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "schemars" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54e910108742c57a770f492731f99be216a52fadd361b06c8fb59d74ccc267d2" -dependencies = [ - "dyn-clone", - "ref-cast", - "schemars_derive", - "serde", - "serde_json", -] - -[[package]] -name = "schemars_derive" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4908ad288c5035a8eb12cfdf0d49270def0a268ee162b75eeee0f85d155a7c45" -dependencies = [ - "proc-macro2", - "quote", - "serde_derive_internals", - "syn", -] - [[package]] name = "semver" -version = "1.0.27" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "f79dfe2d285b0488816f30e700a7438c5a73d816b5b7d3ac72fbc48b0d185e03" dependencies = [ "serde", - "serde_core", ] [[package]] name = "serde" -version = "1.0.228" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" -dependencies = [ - "serde_core", - "serde_derive", -] - -[[package]] -name = "serde_core" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_derive_internals" -version = "0.29.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", @@ -1701,23 +1733,31 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "930cfb6e6abf99298aaad7d29abbef7a9999a9a8806a40088f55f0dcec03146b" dependencies = [ "indexmap", "itoa", "memchr", + "ryu", + "serde", +] + +[[package]] +name = "serde_spanned" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +dependencies = [ "serde", - "serde_core", - "zmij", ] [[package]] name = "shell-words" -version = "1.1.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" +checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" [[package]] name = "shlex" @@ -1731,23 +1771,17 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" -[[package]] -name = "siphasher" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" - [[package]] name = "smallbitvec" -version = "2.6.0" +version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d31d263dd118560e1a492922182ab6ca6dc1d03a3bf54e7699993f31a4150e3f" +checksum = "fcc3fc564a4b53fd1e8589628efafe57602d91bde78be18186b5f61e8faea470" [[package]] name = "smallvec" -version = "1.15.1" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" dependencies = [ "serde", ] @@ -1760,9 +1794,9 @@ checksum = "3b9b39299b249ad65f3b7e96443bad61c02ca5cd3589f46cb6d610a0fd6c0d6a" [[package]] name = "stable_deref_trait" -version = "1.2.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "streaming-iterator" @@ -1777,10 +1811,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] -name = "syn" -version = "2.0.112" +name = "subtle" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21f182278bf2d2bcb3c88b1b08a37df029d71ce3d3ae26168e3c653b213b99d4" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.96" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" dependencies = [ "proc-macro2", "quote", @@ -1789,9 +1829,9 @@ dependencies = [ [[package]] name = "synstructure" -version = "0.13.2" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", @@ -1800,21 +1840,22 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.13.4" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1dd07eb858a2067e2f3c7155d54e929265c264e6f37efe3ee7a8d1b5a1dd0ba" +checksum = "dc12939a1c9b9d391e0b7135f72fd30508b73450753e28341fed159317582a77" [[package]] name = "tempfile" -version = "3.24.0" +version = "3.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" +checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704" dependencies = [ + "cfg-if", "fastrand", - "getrandom 0.3.4", + "getrandom", "once_cell", - "rustix 1.1.3", - "windows-sys 0.61.2", + "rustix", + "windows-sys 0.59.0", ] [[package]] @@ -1837,11 +1878,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.17" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" dependencies = [ - "thiserror-impl 2.0.17", + "thiserror-impl 2.0.11", ] [[package]] @@ -1857,9 +1898,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.17" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" dependencies = [ "proc-macro2", "quote", @@ -1868,11 +1909,43 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.1.9" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" dependencies = [ "cfg-if", + "once_cell", +] + +[[package]] +name = "time" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" +dependencies = [ + "num-conv", + "time-core", ] [[package]] @@ -1889,41 +1962,45 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.2" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" dependencies = [ "displaydoc", "zerovec", ] [[package]] -name = "toml_datetime" -version = "0.7.5+spec-1.1.0" +name = "toml" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" dependencies = [ - "serde_core", + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", ] [[package]] name = "toml_edit" -version = "0.23.10+spec-1.0.0" +version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c8b9f757e028cee9fa244aea147aab2a9ec09d5325a9b01e0a49730c2b5269" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ "indexmap", + "serde", + "serde_spanned", "toml_datetime", - "toml_parser", - "winnow", -] - -[[package]] -name = "toml_parser" -version = "1.0.6+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3198b4b0a8e11f09dd03e133c0280504d0801269e9afa46362ffde1cbeebf44" -dependencies = [ "winnow", ] @@ -1935,9 +2012,9 @@ checksum = "ea68304e134ecd095ac6c3574494fc62b909f416c4fca77e440530221e549d3d" [[package]] name = "tracing" -version = "0.1.44" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ "pin-project-lite", "tracing-attributes", @@ -1946,9 +2023,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.31" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", @@ -1957,16 +2034,16 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.36" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", ] [[package]] name = "tree-sitter" -version = "0.27.0" +version = "0.25.9" dependencies = [ "bindgen", "cc", @@ -1980,7 +2057,7 @@ dependencies = [ [[package]] name = "tree-sitter-cli" -version = "0.27.0" +version = "0.25.9" dependencies = [ "ansi_colours", "anstyle", @@ -1989,29 +2066,33 @@ dependencies = [ "clap", "clap_complete", "clap_complete_nushell", - "crc32fast", "ctor", "ctrlc", "dialoguer", "encoding_rs", + "filetime", "glob", "heck", "html-escape", + "indexmap", "indoc", "log", "memchr", "pretty_assertions", "rand", "regex", - "schemars", + "regex-syntax", + "rustc-hash", "semver", "serde", + "serde_derive", "serde_json", "similar", + "smallbitvec", "streaming-iterator", "tempfile", - "thiserror 2.0.17", "tiny_http", + "topological-sort", "tree-sitter", "tree-sitter-config", "tree-sitter-generate", @@ -2020,90 +2101,90 @@ dependencies = [ "tree-sitter-tags", "tree-sitter-tests-proc-macro", "unindent", + "url", "walkdir", - "wasmparser 0.243.0", + "wasmparser 0.224.0", "webbrowser", "widestring", ] [[package]] name = "tree-sitter-config" -version = "0.27.0" +version = "0.25.9" dependencies = [ + "anyhow", "etcetera", - "log", "serde", "serde_json", - "thiserror 2.0.17", ] [[package]] name = "tree-sitter-generate" -version = "0.27.0" +version = "0.25.9" dependencies = [ - "bitflags 2.10.0", - "dunce", + "anyhow", + "heck", "indexmap", "indoc", "log", - "pathdiff", "regex", "regex-syntax", - "rquickjs", "rustc-hash", "semver", "serde", "serde_json", "smallbitvec", - "tempfile", - "thiserror 2.0.17", + "thiserror 2.0.11", "topological-sort", + "tree-sitter", + "url", ] [[package]] name = "tree-sitter-highlight" -version = "0.27.0" +version = "0.25.9" dependencies = [ "regex", "streaming-iterator", - "thiserror 2.0.17", + "thiserror 2.0.11", "tree-sitter", ] [[package]] name = "tree-sitter-language" -version = "0.1.7" +version = "0.1.5" [[package]] name = "tree-sitter-loader" -version = "0.27.0" +version = "0.25.9" dependencies = [ + "anyhow", "cc", "etcetera", "fs4", "indoc", - "libloading 0.9.0", - "log", + "libloading", "once_cell", + "path-slash", "regex", "semver", "serde", "serde_json", "tempfile", - "thiserror 2.0.17", "tree-sitter", "tree-sitter-highlight", "tree-sitter-tags", + "url", ] [[package]] name = "tree-sitter-tags" -version = "0.27.0" +version = "0.25.9" dependencies = [ "memchr", "regex", "streaming-iterator", - "thiserror 2.0.17", + "thiserror 2.0.11", "tree-sitter", ] @@ -2113,38 +2194,80 @@ version = "0.0.0" dependencies = [ "proc-macro2", "quote", + "rand", "syn", ] [[package]] name = "unicode-ident" -version = "1.0.22" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" - -[[package]] -name = "unicode-segmentation" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +checksum = "11cd88e12b17c6494200a9c1b683a04fcac9573ed74cd1b62aeb2727c5592243" [[package]] name = "unicode-width" -version = "0.2.2" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "unindent" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e860613aec8c8643c53fb4a3ea3bed801ac09d982b75cb5a1c579e410ef043c2" +dependencies = [ + "base64", + "cc", + "cookie_store", + "flate2", + "log", + "once_cell", + "percent-encoding", + "rustls", + "rustls-pemfile", + "rustls-pki-types", + "serde", + "serde_json", + "ureq-proto", + "utf-8", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" +checksum = "319fa2a136b0a13deb093ce09407b39918ecceb68abfc688d9bf9353cf840b94" +dependencies = [ + "base64", + "http", + "httparse", + "log", +] [[package]] name = "url" -version = "2.5.7" +version = "2.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", "idna", @@ -2153,10 +2276,22 @@ dependencies = [ ] [[package]] -name = "utf8-width" -version = "0.1.8" +name = "utf-8" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1292c0d970b54115d14f2492fe0170adf21d68a1de108eebc51c1df4f346a091" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8-width" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3" [[package]] name = "utf8_iter" @@ -2170,6 +2305,18 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "walkdir" version = "2.5.0" @@ -2182,49 +2329,29 @@ dependencies = [ [[package]] name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" +version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - -[[package]] -name = "wasip2" -version = "1.0.1+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" -dependencies = [ - "wit-bindgen", -] +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.106" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", "once_cell", - "rustversion", "wasm-bindgen-macro", - "wasm-bindgen-shared", ] [[package]] -name = "wasm-bindgen-macro" -version = "0.2.106" +name = "wasm-bindgen-backend" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", + "log", "proc-macro2", "quote", "syn", @@ -2232,32 +2359,55 @@ dependencies = [ ] [[package]] -name = "wasm-bindgen-shared" -version = "0.2.106" +name = "wasm-bindgen-macro" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" dependencies = [ "unicode-ident", ] [[package]] name = "wasm-encoder" -version = "0.229.0" +version = "0.221.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38ba1d491ecacb085a2552025c10a675a6fddcbd03b1fc9b36c536010ce265d2" +checksum = "c17a3bd88f2155da63a1f2fcb8a56377a24f0b6dfed12733bb5f544e86f690c5" dependencies = [ - "leb128fmt", - "wasmparser 0.229.0", + "leb128", + "wasmparser 0.221.2", ] [[package]] name = "wasmparser" -version = "0.229.0" +version = "0.221.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc3b1f053f5d41aa55640a1fa9b6d1b8a9e4418d118ce308d20e24ff3575a8c" +checksum = "9845c470a2e10b61dd42c385839cdd6496363ed63b5c9e420b5488b77bd22083" dependencies = [ - "bitflags 2.10.0", - "hashbrown 0.15.5", + "bitflags 2.8.0", + "hashbrown 0.15.2", "indexmap", "semver", "serde", @@ -2265,12 +2415,12 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.243.0" +version = "0.224.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6d8db401b0528ec316dfbe579e6ab4152d61739cfe076706d2009127970159d" +checksum = "65881a664fdd43646b647bb27bf186ab09c05bf56779d40aed4c6dce47d423f5" dependencies = [ - "bitflags 2.10.0", - "hashbrown 0.15.5", + "bitflags 2.8.0", + "hashbrown 0.15.2", "indexmap", "semver", "serde", @@ -2278,46 +2428,47 @@ dependencies = [ [[package]] name = "wasmprinter" -version = "0.229.0" +version = "0.221.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d25dac01892684a99b8fbfaf670eb6b56edea8a096438c75392daeb83156ae2e" +checksum = "a80742ff1b9e6d8c231ac7c7247782c6fc5bce503af760bca071811e5fc9ee56" dependencies = [ "anyhow", "termcolor", - "wasmparser 0.229.0", + "wasmparser 0.221.2", ] [[package]] name = "wasmtime" -version = "33.0.2" +version = "29.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57373e1d8699662fb791270ac5dfac9da5c14f618ecf940cdb29dc3ad9472a3c" +checksum = "11976a250672556d1c4c04c6d5d7656ac9192ac9edc42a4587d6c21460010e69" dependencies = [ - "addr2line", "anyhow", - "bitflags 2.10.0", + "bitflags 2.8.0", "bumpalo", "cc", "cfg-if", - "hashbrown 0.15.5", + "hashbrown 0.14.5", "indexmap", "libc", "log", "mach2", "memfd", - "object 0.36.7", + "object", "once_cell", + "paste", "postcard", "psm", "pulley-interpreter", - "rustix 1.1.3", + "rustix", "serde", "serde_derive", "smallvec", "sptr", "target-lexicon", - "wasmparser 0.229.0", + "wasmparser 0.221.2", "wasmtime-asm-macros", + "wasmtime-component-macro", "wasmtime-cranelift", "wasmtime-environ", "wasmtime-fiber", @@ -2331,18 +2482,18 @@ dependencies = [ [[package]] name = "wasmtime-asm-macros" -version = "33.0.2" +version = "29.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd0fc91372865167a695dc98d0d6771799a388a7541d3f34e939d0539d6583de" +checksum = "1f178b0d125201fbe9f75beaf849bd3e511891f9e45ba216a5b620802ccf64f2" dependencies = [ "cfg-if", ] [[package]] name = "wasmtime-c-api-impl" -version = "33.0.2" +version = "29.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46db556f1dccdd88e0672bd407162ab0036b72e5eccb0f4398d8251cba32dba1" +checksum = "ea30cef3608f2de5797c7bbb94c1ba4f3676d9a7f81ae86ced1b512e2766ed0c" dependencies = [ "anyhow", "log", @@ -2353,19 +2504,40 @@ dependencies = [ [[package]] name = "wasmtime-c-api-macros" -version = "33.0.2" +version = "29.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "315cc6bc8cdc66f296accb26d7625ae64c1c7b6da6f189e8a72ce6594bf7bd36" +checksum = "022a79ebe1124d5d384d82463d7e61c6b4dd857d81f15cb8078974eeb86db65b" dependencies = [ "proc-macro2", "quote", ] [[package]] -name = "wasmtime-cranelift" -version = "33.0.2" +name = "wasmtime-component-macro" +version = "29.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2bd72f0a6a0ffcc6a184ec86ac35c174e48ea0e97bbae277c8f15f8bf77a566" +checksum = "d74de6592ed945d0a602f71243982a304d5d02f1e501b638addf57f42d57dfaf" +dependencies = [ + "anyhow", + "proc-macro2", + "quote", + "syn", + "wasmtime-component-util", + "wasmtime-wit-bindgen", + "wit-parser", +] + +[[package]] +name = "wasmtime-component-util" +version = "29.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707dc7b3c112ab5a366b30cfe2fb5b2f8e6a0f682f16df96a5ec582bfe6f056e" + +[[package]] +name = "wasmtime-cranelift" +version = "29.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "366be722674d4bf153290fbcbc4d7d16895cc82fb3e869f8d550ff768f9e9e87" dependencies = [ "anyhow", "cfg-if", @@ -2375,23 +2547,22 @@ dependencies = [ "cranelift-frontend", "cranelift-native", "gimli", - "itertools 0.14.0", + "itertools 0.12.1", "log", - "object 0.36.7", - "pulley-interpreter", + "object", "smallvec", "target-lexicon", - "thiserror 2.0.17", - "wasmparser 0.229.0", + "thiserror 1.0.69", + "wasmparser 0.221.2", "wasmtime-environ", "wasmtime-versioned-export-macros", ] [[package]] name = "wasmtime-environ" -version = "33.0.2" +version = "29.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6187bb108a23eb25d2a92aa65d6c89fb5ed53433a319038a2558567f3011ff2" +checksum = "cdadc1af7097347aa276a4f008929810f726b5b46946971c660b6d421e9994ad" dependencies = [ "anyhow", "cranelift-bitset", @@ -2399,27 +2570,27 @@ dependencies = [ "gimli", "indexmap", "log", - "object 0.36.7", + "object", "postcard", "serde", "serde_derive", "smallvec", "target-lexicon", "wasm-encoder", - "wasmparser 0.229.0", + "wasmparser 0.221.2", "wasmprinter", ] [[package]] name = "wasmtime-fiber" -version = "33.0.2" +version = "29.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc8965d2128c012329f390e24b8b2758dd93d01bf67e1a1a0dd3d8fd72f56873" +checksum = "ccba90d4119f081bca91190485650730a617be1fff5228f8c4757ce133d21117" dependencies = [ "anyhow", "cc", "cfg-if", - "rustix 1.1.3", + "rustix", "wasmtime-asm-macros", "wasmtime-versioned-export-macros", "windows-sys 0.59.0", @@ -2427,9 +2598,9 @@ dependencies = [ [[package]] name = "wasmtime-jit-icache-coherence" -version = "33.0.2" +version = "29.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7af0e940cb062a45c0b3f01a926f77da5947149e99beb4e3dd9846d5b8f11619" +checksum = "ec5e8552e01692e6c2e5293171704fed8abdec79d1a6995a0870ab190e5747d1" dependencies = [ "anyhow", "cfg-if", @@ -2439,24 +2610,24 @@ dependencies = [ [[package]] name = "wasmtime-math" -version = "33.0.2" +version = "29.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acfca360e719dda9a27e26944f2754ff2fd5bad88e21919c42c5a5f38ddd93cb" +checksum = "29210ec2aa25e00f4d54605cedaf080f39ec01a872c5bd520ad04c67af1dde17" dependencies = [ "libm", ] [[package]] name = "wasmtime-slab" -version = "33.0.2" +version = "29.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48e240559cada55c4b24af979d5f6c95e0029f5772f32027ec3c62b258aaff65" +checksum = "fcb5821a96fa04ac14bc7b158bb3d5cd7729a053db5a74dad396cd513a5e5ccf" [[package]] name = "wasmtime-versioned-export-macros" -version = "33.0.2" +version = "29.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0963c1438357a3d8c0efe152b4ef5259846c1cf8b864340270744fe5b3bae5e" +checksum = "86ff86db216dc0240462de40c8290887a613dddf9685508eb39479037ba97b5b" dependencies = [ "proc-macro2", "quote", @@ -2465,26 +2636,38 @@ dependencies = [ [[package]] name = "wasmtime-winch" -version = "33.0.2" +version = "29.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc3b117d03d6eeabfa005a880c5c22c06503bb8820f3aa2e30f0e8d87b6752f" +checksum = "fdbabfb8f20502d5e1d81092b9ead3682ae59988487aafcd7567387b7a43cf8f" dependencies = [ "anyhow", "cranelift-codegen", "gimli", - "object 0.36.7", + "object", "target-lexicon", - "wasmparser 0.229.0", + "wasmparser 0.221.2", "wasmtime-cranelift", "wasmtime-environ", "winch-codegen", ] [[package]] -name = "web-sys" -version = "0.3.83" +name = "wasmtime-wit-bindgen" +version = "29.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" +checksum = "8358319c2dd1e4db79e3c1c5d3a5af84956615343f9f89f4e4996a36816e06e6" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "wit-parser", +] + +[[package]] +name = "web-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" dependencies = [ "js-sys", "wasm-bindgen", @@ -2492,11 +2675,13 @@ dependencies = [ [[package]] name = "webbrowser" -version = "1.0.6" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00f1243ef785213e3a32fa0396093424a3a6ea566f9948497e5a2309261a4c97" +checksum = "ea9fe1ebb156110ff855242c1101df158b822487e4957b0556d9ffce9db0f535" dependencies = [ + "block2", "core-foundation", + "home", "jni", "log", "ndk-context", @@ -2507,45 +2692,47 @@ dependencies = [ ] [[package]] -name = "widestring" -version = "1.2.1" +name = "webpki-roots" +version = "0.26.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" +checksum = "5d642ff16b7e79272ae451b7322067cdc17cadf68c23264be9d94a32319efe7e" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "widestring" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7219d36b6eac893fa81e84ebe06485e7dcbb616177469b142df14f1f4deb1311" [[package]] name = "winapi-util" -version = "0.1.11" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] name = "winch-codegen" -version = "33.0.2" +version = "29.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7914c296fbcef59d1b89a15e82384d34dc9669bc09763f2ef068a28dd3a64ebf" +checksum = "2f849ef2c5f46cb0a20af4b4487aaa239846e52e2c03f13fa3c784684552859c" dependencies = [ "anyhow", - "cranelift-assembler-x64", "cranelift-codegen", "gimli", "regalloc2", "smallvec", "target-lexicon", - "thiserror 2.0.17", - "wasmparser 0.229.0", + "thiserror 1.0.69", + "wasmparser 0.221.2", "wasmtime-cranelift", "wasmtime-environ", ] -[[package]] -name = "windows-link" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" - [[package]] name = "windows-sys" version = "0.45.0" @@ -2555,6 +2742,15 @@ dependencies = [ "windows-targets 0.42.2", ] +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -2573,24 +2769,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", -] - -[[package]] -name = "windows-sys" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" -dependencies = [ - "windows-link", -] - [[package]] name = "windows-targets" version = "0.42.2" @@ -2606,6 +2784,21 @@ dependencies = [ "windows_x86_64_msvc 0.42.2", ] +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -2615,30 +2808,13 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", + "windows_i686_gnullvm", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -2647,15 +2823,15 @@ checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.6" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.53.1" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" @@ -2665,15 +2841,15 @@ checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] name = "windows_aarch64_msvc" -version = "0.52.6" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.53.1" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" @@ -2683,15 +2859,15 @@ checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] name = "windows_i686_gnu" -version = "0.52.6" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.53.1" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" @@ -2699,12 +2875,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -2713,15 +2883,15 @@ checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" [[package]] name = "windows_i686_msvc" -version = "0.52.6" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.53.1" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" @@ -2731,15 +2901,15 @@ checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" [[package]] name = "windows_x86_64_gnu" -version = "0.52.6" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.53.1" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" @@ -2749,15 +2919,15 @@ checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.6" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.53.1" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" @@ -2765,38 +2935,56 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winnow" -version = "0.7.14" +version = "0.6.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" +checksum = "c8d71a593cc5c42ad7876e2c1fda56f314f3754c084128833e64f1345ff8a03a" dependencies = [ "memchr", ] [[package]] -name = "wit-bindgen" -version = "0.46.0" +name = "wit-parser" +version = "0.221.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" +checksum = "fbe1538eea6ea5ddbe5defd0dc82539ad7ba751e1631e9185d24a931f0a5adc8" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser 0.221.2", +] + +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" [[package]] name = "writeable" -version = "0.6.2" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" [[package]] name = "xtask" @@ -2805,17 +2993,18 @@ dependencies = [ "anstyle", "anyhow", "bindgen", + "cc", "clap", - "etcetera", + "git2", "indoc", "notify", "notify-debouncer-full", "regex", - "schemars", "semver", + "serde", "serde_json", - "tree-sitter-cli", - "tree-sitter-loader", + "toml", + "ureq", ] [[package]] @@ -2826,10 +3015,11 @@ checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "yoke" -version = "0.8.1" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" dependencies = [ + "serde", "stable_deref_trait", "yoke-derive", "zerofrom", @@ -2837,9 +3027,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.1" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", @@ -2849,18 +3039,19 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.31" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ + "byteorder", "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.31" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", @@ -2869,18 +3060,18 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.6" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.6" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", @@ -2890,26 +3081,15 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.8.2" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" - -[[package]] -name = "zerotrie" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", -] +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" [[package]] name = "zerovec" -version = "0.11.5" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" dependencies = [ "yoke", "zerofrom", @@ -2918,17 +3098,11 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", "syn", ] - -[[package]] -name = "zmij" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9747e91771f56fd7893e1164abd78febd14a670ceec257caad15e051de35f06" diff --git a/Cargo.toml b/Cargo.toml index ca0d644a..ace5381d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,26 +1,26 @@ [workspace] -default-members = ["crates/cli"] +default-members = ["cli"] members = [ - "crates/cli", - "crates/config", - "crates/generate", - "crates/highlight", - "crates/loader", - "crates/tags", - "crates/xtask", - "crates/language", + "cli", + "cli/config", + "cli/generate", + "cli/loader", "lib", + "lib/language", + "tags", + "highlight", + "xtask", ] resolver = "2" [workspace.package] -version = "0.27.0" +version = "0.25.9" authors = [ "Max Brunsfeld ", "Amaan Qureshi ", ] edition = "2021" -rust-version = "1.85" +rust-version = "1.82" homepage = "https://tree-sitter.github.io/tree-sitter" repository = "https://github.com/tree-sitter/tree-sitter" license = "MIT" @@ -103,61 +103,62 @@ codegen-units = 256 [workspace.dependencies] ansi_colours = "1.2.3" -anstyle = "1.0.13" -anyhow = "1.0.100" -bstr = "1.12.0" -cc = "1.2.53" -clap = { version = "4.5.54", features = [ +anstyle = "1.0.10" +anyhow = "1.0.95" +bstr = "1.11.3" +cc = "1.2.10" +clap = { version = "4.5.27", features = [ "cargo", "derive", "env", "help", - "string", "unstable-styles", ] } -clap_complete = "4.5.65" -clap_complete_nushell = "4.5.10" -crc32fast = "1.5.0" +clap_complete = "4.5.42" +clap_complete_nushell = "4.5.5" ctor = "0.2.9" -ctrlc = { version = "3.5.0", features = ["termination"] } +ctrlc = { version = "3.4.5", features = ["termination"] } dialoguer = { version = "0.11.0", features = ["fuzzy-select"] } -etcetera = "0.11.0" +etcetera = "0.8.0" +filetime = "0.2.25" fs4 = "0.12.0" -glob = "0.3.3" +git2 = "0.20.0" +glob = "0.3.2" heck = "0.5.0" html-escape = "0.2.13" -indexmap = "2.12.1" -indoc = "2.0.6" -libloading = "0.9.0" -log = { version = "0.4.28", features = ["std"] } -memchr = "2.7.6" -once_cell = "1.21.3" +indexmap = "2.7.1" +indoc = "2.0.5" +libloading = "0.8.6" +log = { version = "0.4.25", features = ["std"] } +memchr = "2.7.4" +once_cell = "1.20.2" +path-slash = "0.2.1" pretty_assertions = "1.4.1" rand = "0.8.5" -regex = "1.11.3" -regex-syntax = "0.8.6" -rustc-hash = "2.1.1" -schemars = "1.0.5" -semver = { version = "1.0.27", features = ["serde"] } -serde = { version = "1.0.219", features = ["derive"] } -serde_json = { version = "1.0.149", features = ["preserve_order"] } +regex = "1.11.1" +regex-syntax = "0.8.5" +rustc-hash = "2.1.0" +semver = { version = "1.0.25", features = ["serde"] } +serde = { version = "1.0.217", features = ["derive"] } +serde_derive = "1.0.217" +serde_json = { version = "1.0.137", features = ["preserve_order"] } similar = "2.7.0" -smallbitvec = "2.6.0" +smallbitvec = "2.5.3" streaming-iterator = "0.1.9" -tempfile = "3.23.0" -thiserror = "2.0.17" +tempfile = "3.15.0" +thiserror = "2.0.11" tiny_http = "0.12.0" +toml = "0.8.19" topological-sort = "0.2.2" -unindent = "0.2.4" +unindent = "0.2.3" +url = { version = "2.5.4", features = ["serde"] } walkdir = "2.5.0" -wasmparser = "0.243.0" -webbrowser = "1.0.5" +wasmparser = "0.224.0" +webbrowser = "1.0.3" -tree-sitter = { version = "0.27.0", path = "./lib" } -tree-sitter-generate = { version = "0.27.0", path = "./crates/generate" } -tree-sitter-loader = { version = "0.27.0", path = "./crates/loader" } -tree-sitter-config = { version = "0.27.0", path = "./crates/config" } -tree-sitter-highlight = { version = "0.27.0", path = "./crates/highlight" } -tree-sitter-tags = { version = "0.27.0", path = "./crates/tags" } - -tree-sitter-language = { version = "0.1", path = "./crates/language" } +tree-sitter = { version = "0.25.9", path = "./lib" } +tree-sitter-generate = { version = "0.25.9", path = "./cli/generate" } +tree-sitter-loader = { version = "0.25.9", path = "./cli/loader" } +tree-sitter-config = { version = "0.25.9", path = "./cli/config" } +tree-sitter-highlight = { version = "0.25.9", path = "./highlight" } +tree-sitter-tags = { version = "0.25.9", path = "./tags" } diff --git a/LICENSE b/LICENSE index 971b81f9..451fe1d2 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2018 Max Brunsfeld +Copyright (c) 2018-2024 Max Brunsfeld Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Makefile b/Makefile index 2098d275..3f5f5a4b 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,8 @@ -VERSION := 0.27.0 +ifeq ($(OS),Windows_NT) +$(error Windows is not supported) +endif + +VERSION := 0.25.9 DESCRIPTION := An incremental parsing system for programming tools HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/ @@ -6,7 +10,6 @@ HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/ PREFIX ?= /usr/local INCLUDEDIR ?= $(PREFIX)/include LIBDIR ?= $(PREFIX)/lib -BINDIR ?= $(PREFIX)/bin PCLIBDIR ?= $(LIBDIR)/pkgconfig # collect sources @@ -24,7 +27,7 @@ OBJ := $(SRC:.c=.o) ARFLAGS := rcs CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types override CFLAGS += -std=c11 -fPIC -fvisibility=hidden -override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE -D_BSD_SOURCE -D_DARWIN_C_SOURCE +override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE override CFLAGS += -Ilib/src -Ilib/src/wasm -Ilib/include # ABI versioning @@ -32,25 +35,20 @@ SONAME_MAJOR := $(word 1,$(subst ., ,$(VERSION))) SONAME_MINOR := $(word 2,$(subst ., ,$(VERSION))) # OS-specific bits -MACHINE := $(shell $(CC) -dumpmachine) - -ifneq ($(findstring darwin,$(MACHINE)),) +ifneq ($(findstring darwin,$(shell $(CC) -dumpmachine)),) SOEXT = dylib SOEXTVER_MAJOR = $(SONAME_MAJOR).$(SOEXT) SOEXTVER = $(SONAME_MAJOR).$(SONAME_MINOR).$(SOEXT) LINKSHARED += -dynamiclib -Wl,-install_name,$(LIBDIR)/libtree-sitter.$(SOEXTVER) -else ifneq ($(findstring mingw32,$(MACHINE)),) - SOEXT = dll - LINKSHARED += -s -shared -Wl,--out-implib,libtree-sitter.dll.a else SOEXT = so SOEXTVER_MAJOR = $(SOEXT).$(SONAME_MAJOR) SOEXTVER = $(SOEXT).$(SONAME_MAJOR).$(SONAME_MINOR) LINKSHARED += -shared -Wl,-soname,libtree-sitter.$(SOEXTVER) +endif ifneq ($(filter $(shell uname),FreeBSD NetBSD DragonFly),) PCLIBDIR := $(PREFIX)/libdata/pkgconfig endif -endif all: libtree-sitter.a libtree-sitter.$(SOEXT) tree-sitter.pc @@ -63,10 +61,6 @@ ifneq ($(STRIP),) $(STRIP) $@ endif -ifneq ($(findstring mingw32,$(MACHINE)),) -libtree-sitter.dll.a: libtree-sitter.$(SOEXT) -endif - tree-sitter.pc: lib/tree-sitter.pc.in sed -e 's|@PROJECT_VERSION@|$(VERSION)|' \ -e 's|@CMAKE_INSTALL_LIBDIR@|$(LIBDIR:$(PREFIX)/%=%)|' \ @@ -75,27 +69,17 @@ tree-sitter.pc: lib/tree-sitter.pc.in -e 's|@PROJECT_HOMEPAGE_URL@|$(HOMEPAGE_URL)|' \ -e 's|@CMAKE_INSTALL_PREFIX@|$(PREFIX)|' $< > $@ -shared: libtree-sitter.$(SOEXT) - -static: libtree-sitter.a - clean: - $(RM) $(OBJ) tree-sitter.pc libtree-sitter.a libtree-sitter.$(SOEXT) libtree-stitter.dll.a + $(RM) $(OBJ) tree-sitter.pc libtree-sitter.a libtree-sitter.$(SOEXT) install: all install -d '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter '$(DESTDIR)$(PCLIBDIR)' '$(DESTDIR)$(LIBDIR)' install -m644 lib/include/tree_sitter/api.h '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/api.h install -m644 tree-sitter.pc '$(DESTDIR)$(PCLIBDIR)'/tree-sitter.pc install -m644 libtree-sitter.a '$(DESTDIR)$(LIBDIR)'/libtree-sitter.a -ifneq ($(findstring mingw32,$(MACHINE)),) - install -d '$(DESTDIR)$(BINDIR)' - install -m755 libtree-sitter.dll '$(DESTDIR)$(BINDIR)'/libtree-sitter.dll - install -m755 libtree-sitter.dll.a '$(DESTDIR)$(LIBDIR)'/libtree-sitter.dll.a -else install -m755 libtree-sitter.$(SOEXT) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXTVER) - cd '$(DESTDIR)$(LIBDIR)' && ln -sf libtree-sitter.$(SOEXTVER) libtree-sitter.$(SOEXTVER_MAJOR) - cd '$(DESTDIR)$(LIBDIR)' && ln -sf libtree-sitter.$(SOEXTVER_MAJOR) libtree-sitter.$(SOEXT) -endif + ln -sf libtree-sitter.$(SOEXTVER) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXTVER_MAJOR) + ln -sf libtree-sitter.$(SOEXTVER_MAJOR) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXT) uninstall: $(RM) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.a \ @@ -104,9 +88,8 @@ uninstall: '$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXT) \ '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/api.h \ '$(DESTDIR)$(PCLIBDIR)'/tree-sitter.pc - rmdir '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter -.PHONY: all shared static install uninstall clean +.PHONY: all install uninstall clean ##### Dev targets ##### diff --git a/Package.swift b/Package.swift index fb6c6e95..572ba684 100644 --- a/Package.swift +++ b/Package.swift @@ -27,8 +27,6 @@ let package = Package( .headerSearchPath("src"), .define("_POSIX_C_SOURCE", to: "200112L"), .define("_DEFAULT_SOURCE"), - .define("_BSD_SOURCE"), - .define("_DARWIN_C_SOURCE"), ]), ], cLanguageStandard: .c11 diff --git a/README.md b/README.md index b347c880..d378215e 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,8 @@ Tree-sitter is a parser generator tool and an incremental parsing library. It ca ## Links - [Documentation](https://tree-sitter.github.io) - [Rust binding](lib/binding_rust/README.md) -- [Wasm binding](lib/binding_web/README.md) -- [Command-line interface](crates/cli/README.md) +- [WASM binding](lib/binding_web/README.md) +- [Command-line interface](cli/README.md) [discord]: https://img.shields.io/discord/1063097320771698699?logo=discord&label=discord [matrix]: https://img.shields.io/matrix/tree-sitter-chat%3Amatrix.org?logo=matrix&label=matrix diff --git a/build.zig b/build.zig index 9bb1e818..66a448cb 100644 --- a/build.zig +++ b/build.zig @@ -40,8 +40,6 @@ pub fn build(b: *std.Build) !void { lib.root_module.addCMacro("_POSIX_C_SOURCE", "200112L"); lib.root_module.addCMacro("_DEFAULT_SOURCE", ""); - lib.root_module.addCMacro("_BSD_SOURCE", ""); - lib.root_module.addCMacro("_DARWIN_C_SOURCE", ""); if (wasm) { if (b.lazyDependency(wasmtimeDep(target.result), .{})) |wasmtime| { diff --git a/build.zig.zon b/build.zig.zon index 4ef5de16..998a88eb 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,7 +1,7 @@ .{ .name = .tree_sitter, .fingerprint = 0x841224b447ac0d4f, - .version = "0.27.0", + .version = "0.25.9", .minimum_zig_version = "0.14.1", .paths = .{ "build.zig", @@ -13,83 +13,63 @@ }, .dependencies = .{ .wasmtime_c_api_aarch64_android = .{ - .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-android-c-api.tar.xz", - .hash = "N-V-__8AAIfPIgdw2YnV3QyiFQ2NHdrxrXzzCdjYJyxJDOta", + .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-aarch64-android-c-api.tar.xz", + .hash = "N-V-__8AAC3KCQZMd5ea2CkcbjldaVqCT7BT_9_rLMId6V__", .lazy = true, }, .wasmtime_c_api_aarch64_linux = .{ - .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-linux-c-api.tar.xz", - .hash = "N-V-__8AAIt97QZi7Pf7nNJ2mVY6uxA80Klyuvvtop3pLMRK", + .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-aarch64-linux-c-api.tar.xz", + .hash = "N-V-__8AAGUY3gU6jj2CNJAYb7HiMNVPV1FIcTCI6RSSYwXu", .lazy = true, }, .wasmtime_c_api_aarch64_macos = .{ - .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-macos-c-api.tar.xz", - .hash = "N-V-__8AAAO48QQf91w9RmmUDHTja8DrXZA1n6Bmc8waW3qe", - .lazy = true, - }, - .wasmtime_c_api_aarch64_musl = .{ - .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-musl-c-api.tar.xz", - .hash = "N-V-__8AAI196wa9pwADoA2RbCDp5F7bKQg1iOPq6gIh8-FH", + .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-aarch64-macos-c-api.tar.xz", + .hash = "N-V-__8AAM1GMARD6LGQebhVsSZ0uePUoo3Fw5nEO2L764vf", .lazy = true, }, .wasmtime_c_api_aarch64_windows = .{ - .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-windows-c-api.zip", - .hash = "N-V-__8AAC9u4wXfqd1Q6XyQaC8_DbQZClXux60Vu5743N05", - .lazy = true, - }, - .wasmtime_c_api_armv7_linux = .{ - .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-armv7-linux-c-api.tar.xz", - .hash = "N-V-__8AAHXe8gWs3s83Cc5G6SIq0_jWxj8fGTT5xG4vb6-x", - .lazy = true, - }, - .wasmtime_c_api_i686_linux = .{ - .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-i686-linux-c-api.tar.xz", - .hash = "N-V-__8AAN2pzgUUfulRCYnipSfis9IIYHoTHVlieLRmKuct", - .lazy = true, - }, - .wasmtime_c_api_i686_windows = .{ - .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-i686-windows-c-api.zip", - .hash = "N-V-__8AAJu0YAUUTFBLxFIOi-MSQVezA6MMkpoFtuaf2Quf", + .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-aarch64-windows-c-api.zip", + .hash = "N-V-__8AAH8a_wQ7oAeVVsaJcoOZhKTMkHIBc_XjDyLlHp2x", .lazy = true, }, .wasmtime_c_api_riscv64gc_linux = .{ - .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-riscv64gc-linux-c-api.tar.xz", - .hash = "N-V-__8AAG8m-gc3E3AIImtTZ3l1c7HC6HUWazQ9OH5KACX4", + .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-riscv64gc-linux-c-api.tar.xz", + .hash = "N-V-__8AAN2cuQadBwMc8zJxv0sMY99Ae1Nc1dZcZAK9b4DZ", .lazy = true, }, .wasmtime_c_api_s390x_linux = .{ - .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-s390x-linux-c-api.tar.xz", - .hash = "N-V-__8AAH314gd-gE4IBp2uvAL3gHeuW1uUZjMiLLeUdXL_", + .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-s390x-linux-c-api.tar.xz", + .hash = "N-V-__8AAPevngYz99mwT0KQY9my2ax1p6APzgLEJeV4II9U", .lazy = true, }, .wasmtime_c_api_x86_64_android = .{ - .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-android-c-api.tar.xz", - .hash = "N-V-__8AAIPNRwfNkznebrcGb0IKUe7f35bkuZEYOjcx6q3f", + .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-x86_64-android-c-api.tar.xz", + .hash = "N-V-__8AABHIEgaTyzPfjgnnCy0dwJiXoDiJFblCkYOJsQvy", .lazy = true, }, .wasmtime_c_api_x86_64_linux = .{ - .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-linux-c-api.tar.xz", - .hash = "N-V-__8AAI8EDwcyTtk_Afhk47SEaqfpoRqGkJeZpGs69ChF", + .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-x86_64-linux-c-api.tar.xz", + .hash = "N-V-__8AALUN5AWSEDRulL9u-OJJ-l0_GoT5UFDtGWZayEIq", .lazy = true, }, .wasmtime_c_api_x86_64_macos = .{ - .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-macos-c-api.tar.xz", - .hash = "N-V-__8AAGtGNgVaOpHSxC22IjrampbRIy6lLwscdcAE8nG1", + .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-x86_64-macos-c-api.tar.xz", + .hash = "N-V-__8AANUeXwSPh13TqJCSSFdi87GEcHs8zK6FqE4v_TjB", .lazy = true, }, .wasmtime_c_api_x86_64_mingw = .{ - .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-mingw-c-api.zip", - .hash = "N-V-__8AAPS2PAbVix50L6lnddlgazCPTz3whLUFk1qnRtnZ", + .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-x86_64-mingw-c-api.zip", + .hash = "N-V-__8AALundgW-p1ffOnd7bsYyL8SY5OziDUZu7cXio2EL", .lazy = true, }, .wasmtime_c_api_x86_64_musl = .{ - .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-musl-c-api.tar.xz", - .hash = "N-V-__8AAF-WEQe0nzvi09PgusM5i46FIuCKJmIDWUleWgQ3", + .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-x86_64-musl-c-api.tar.xz", + .hash = "N-V-__8AALMZ5wXJWW5qY-3MMjTAYR0MusckvzCsmg-69ALH", .lazy = true, }, .wasmtime_c_api_x86_64_windows = .{ - .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-windows-c-api.zip", - .hash = "N-V-__8AAKGNXwbpJQsn0_6kwSIVDDWifSg8cBzf7T2RzsC9", + .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-x86_64-windows-c-api.zip", + .hash = "N-V-__8AAG-uVQVEDMsB1ymJzxpHcoiXo1_I3TFnPM5Zjy1i", .lazy = true, }, }, diff --git a/crates/cli/Cargo.toml b/cli/Cargo.toml similarity index 84% rename from crates/cli/Cargo.toml rename to cli/Cargo.toml index c10b4652..45aa5833 100644 --- a/crates/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -8,18 +8,14 @@ rust-version.workspace = true readme = "README.md" homepage.workspace = true repository.workspace = true -documentation = "https://docs.rs/tree-sitter-cli" license.workspace = true keywords.workspace = true categories.workspace = true -include = ["build.rs", "README.md", "LICENSE", "benches/*", "src/**"] +include = ["build.rs", "README.md", "benches/*", "src/**"] [lints] workspace = true -[lib] -path = "src/tree_sitter_cli.rs" - [[bin]] name = "tree-sitter" path = "src/main.rs" @@ -30,9 +26,7 @@ name = "benchmark" harness = false [features] -default = ["qjs-rt"] wasm = ["tree-sitter/wasm", "tree-sitter-loader/wasm"] -qjs-rt = ["tree-sitter-generate/qjs-rt"] [dependencies] ansi_colours.workspace = true @@ -42,26 +36,31 @@ bstr.workspace = true clap.workspace = true clap_complete.workspace = true clap_complete_nushell.workspace = true -crc32fast.workspace = true ctor.workspace = true ctrlc.workspace = true dialoguer.workspace = true +filetime.workspace = true glob.workspace = true heck.workspace = true html-escape.workspace = true +indexmap.workspace = true indoc.workspace = true log.workspace = true memchr.workspace = true rand.workspace = true regex.workspace = true -schemars.workspace = true +regex-syntax.workspace = true +rustc-hash.workspace = true semver.workspace = true serde.workspace = true +serde_derive.workspace = true serde_json.workspace = true similar.workspace = true +smallbitvec.workspace = true streaming-iterator.workspace = true -thiserror.workspace = true tiny_http.workspace = true +topological-sort.workspace = true +url.workspace = true walkdir.workspace = true wasmparser.workspace = true webbrowser.workspace = true @@ -75,7 +74,7 @@ tree-sitter-tags.workspace = true [dev-dependencies] encoding_rs = "0.8.35" -widestring = "1.2.1" +widestring = "1.1.0" tree_sitter_proc_macro = { path = "src/tests/proc_macro", package = "tree-sitter-tests-proc-macro" } tempfile.workspace = true diff --git a/crates/cli/README.md b/cli/README.md similarity index 81% rename from crates/cli/README.md rename to cli/README.md index e3ef899e..5a399f08 100644 --- a/crates/cli/README.md +++ b/cli/README.md @@ -7,8 +7,7 @@ [npmjs.com]: https://www.npmjs.org/package/tree-sitter-cli [npmjs.com badge]: https://img.shields.io/npm/v/tree-sitter-cli.svg?color=%23BF4A4A -The Tree-sitter CLI allows you to develop, test, and use Tree-sitter grammars from the command line. It works on `MacOS`, -`Linux`, and `Windows`. +The Tree-sitter CLI allows you to develop, test, and use Tree-sitter grammars from the command line. It works on `MacOS`, `Linux`, and `Windows`. ### Installation @@ -35,11 +34,9 @@ The `tree-sitter` binary itself has no dependencies, but specific commands have ### Commands -* `generate` - The `tree-sitter generate` command will generate a Tree-sitter parser based on the grammar in the current - working directory. See [the documentation] for more information. +* `generate` - The `tree-sitter generate` command will generate a Tree-sitter parser based on the grammar in the current working directory. See [the documentation] for more information. -* `test` - The `tree-sitter test` command will run the unit tests for the Tree-sitter parser in the current working directory. - See [the documentation] for more information. +* `test` - The `tree-sitter test` command will run the unit tests for the Tree-sitter parser in the current working directory. See [the documentation] for more information. * `parse` - The `tree-sitter parse` command will parse a file (or list of files) using Tree-sitter parsers. diff --git a/crates/cli/benches/benchmark.rs b/cli/benches/benchmark.rs similarity index 86% rename from crates/cli/benches/benchmark.rs rename to cli/benches/benchmark.rs index 3edda96e..943390c6 100644 --- a/crates/cli/benches/benchmark.rs +++ b/cli/benches/benchmark.rs @@ -8,7 +8,6 @@ use std::{ }; use anyhow::Context; -use log::info; use tree_sitter::{Language, Parser, Query}; use tree_sitter_loader::{CompileConfig, Loader}; @@ -72,8 +71,6 @@ static EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR: LazyLock< }); fn main() { - tree_sitter_cli::logger::init(); - let max_path_length = EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR .values() .flat_map(|(e, q)| { @@ -84,7 +81,7 @@ fn main() { .max() .unwrap_or(0); - info!("Benchmarking with {} repetitions", *REPETITION_COUNT); + eprintln!("Benchmarking with {} repetitions", *REPETITION_COUNT); let mut parser = Parser::new(); let mut all_normal_speeds = Vec::new(); @@ -101,11 +98,11 @@ fn main() { } } - info!("\nLanguage: {language_name}"); + eprintln!("\nLanguage: {language_name}"); let language = get_language(language_path); parser.set_language(&language).unwrap(); - info!(" Constructing Queries"); + eprintln!(" Constructing Queries"); for path in query_paths { if let Some(filter) = EXAMPLE_FILTER.as_ref() { if !path.to_str().unwrap().contains(filter.as_str()) { @@ -120,7 +117,7 @@ fn main() { }); } - info!(" Parsing Valid Code:"); + eprintln!(" Parsing Valid Code:"); let mut normal_speeds = Vec::new(); for example_path in example_paths { if let Some(filter) = EXAMPLE_FILTER.as_ref() { @@ -134,7 +131,7 @@ fn main() { })); } - info!(" Parsing Invalid Code (mismatched languages):"); + eprintln!(" Parsing Invalid Code (mismatched languages):"); let mut error_speeds = Vec::new(); for (other_language_path, (example_paths, _)) in EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR.iter() @@ -155,30 +152,30 @@ fn main() { } if let Some((average_normal, worst_normal)) = aggregate(&normal_speeds) { - info!(" Average Speed (normal): {average_normal} bytes/ms"); - info!(" Worst Speed (normal): {worst_normal} bytes/ms"); + eprintln!(" Average Speed (normal): {average_normal} bytes/ms"); + eprintln!(" Worst Speed (normal): {worst_normal} bytes/ms"); } if let Some((average_error, worst_error)) = aggregate(&error_speeds) { - info!(" Average Speed (errors): {average_error} bytes/ms"); - info!(" Worst Speed (errors): {worst_error} bytes/ms"); + eprintln!(" Average Speed (errors): {average_error} bytes/ms"); + eprintln!(" Worst Speed (errors): {worst_error} bytes/ms"); } all_normal_speeds.extend(normal_speeds); all_error_speeds.extend(error_speeds); } - info!("\n Overall"); + eprintln!("\n Overall"); if let Some((average_normal, worst_normal)) = aggregate(&all_normal_speeds) { - info!(" Average Speed (normal): {average_normal} bytes/ms"); - info!(" Worst Speed (normal): {worst_normal} bytes/ms"); + eprintln!(" Average Speed (normal): {average_normal} bytes/ms"); + eprintln!(" Worst Speed (normal): {worst_normal} bytes/ms"); } if let Some((average_error, worst_error)) = aggregate(&all_error_speeds) { - info!(" Average Speed (errors): {average_error} bytes/ms"); - info!(" Worst Speed (errors): {worst_error} bytes/ms"); + eprintln!(" Average Speed (errors): {average_error} bytes/ms"); + eprintln!(" Worst Speed (errors): {worst_error} bytes/ms"); } - info!(""); + eprintln!(); } fn aggregate(speeds: &[usize]) -> Option<(usize, usize)> { @@ -197,6 +194,12 @@ fn aggregate(speeds: &[usize]) -> Option<(usize, usize)> { } fn parse(path: &Path, max_path_length: usize, mut action: impl FnMut(&[u8])) -> usize { + eprint!( + " {:width$}\t", + path.file_name().unwrap().to_str().unwrap(), + width = max_path_length + ); + let source_code = fs::read(path) .with_context(|| format!("Failed to read {}", path.display())) .unwrap(); @@ -207,9 +210,8 @@ fn parse(path: &Path, max_path_length: usize, mut action: impl FnMut(&[u8])) -> let duration = time.elapsed() / (*REPETITION_COUNT as u32); let duration_ns = duration.as_nanos(); let speed = ((source_code.len() as u128) * 1_000_000) / duration_ns; - info!( - " {:max_path_length$}\ttime {:>7.2} ms\t\tspeed {speed:>6} bytes/ms", - path.file_name().unwrap().to_str().unwrap(), + eprintln!( + "time {:>7.2} ms\t\tspeed {speed:>6} bytes/ms", (duration_ns as f64) / 1e6, ); speed as usize diff --git a/crates/cli/build.rs b/cli/build.rs similarity index 92% rename from crates/cli/build.rs rename to cli/build.rs index 45f54634..0081395e 100644 --- a/crates/cli/build.rs +++ b/cli/build.rs @@ -52,9 +52,9 @@ fn main() { fn web_playground_files_present() -> bool { let paths = [ - "../../docs/src/assets/js/playground.js", - "../../lib/binding_web/web-tree-sitter.js", - "../../lib/binding_web/web-tree-sitter.wasm", + "../docs/src/assets/js/playground.js", + "../lib/binding_web/tree-sitter.js", + "../lib/binding_web/tree-sitter.wasm", ]; paths.iter().all(|p| Path::new(p).exists()) diff --git a/crates/config/Cargo.toml b/cli/config/Cargo.toml similarity index 77% rename from crates/config/Cargo.toml rename to cli/config/Cargo.toml index 641b434b..e332db77 100644 --- a/crates/config/Cargo.toml +++ b/cli/config/Cargo.toml @@ -8,20 +8,15 @@ rust-version.workspace = true readme = "README.md" homepage.workspace = true repository.workspace = true -documentation = "https://docs.rs/tree-sitter-config" license.workspace = true keywords.workspace = true categories.workspace = true -[lib] -path = "src/tree_sitter_config.rs" - [lints] workspace = true [dependencies] +anyhow.workspace = true etcetera.workspace = true -log.workspace = true serde.workspace = true serde_json.workspace = true -thiserror.workspace = true diff --git a/crates/config/README.md b/cli/config/README.md similarity index 100% rename from crates/config/README.md rename to cli/config/README.md diff --git a/crates/config/src/tree_sitter_config.rs b/cli/config/src/lib.rs similarity index 66% rename from crates/config/src/tree_sitter_config.rs rename to cli/config/src/lib.rs index 17b1d384..bca9163f 100644 --- a/crates/config/src/tree_sitter_config.rs +++ b/cli/config/src/lib.rs @@ -1,54 +1,11 @@ -#![cfg_attr(not(any(test, doctest)), doc = include_str!("../README.md"))] +#![doc = include_str!("../README.md")] -use std::{ - env, fs, - path::{Path, PathBuf}, -}; +use std::{env, fs, path::PathBuf}; +use anyhow::{Context, Result}; use etcetera::BaseStrategy as _; -use log::warn; use serde::{Deserialize, Serialize}; use serde_json::Value; -use thiserror::Error; - -pub type ConfigResult = Result; - -#[derive(Debug, Error)] -pub enum ConfigError { - #[error("Bad JSON config {0} -- {1}")] - ConfigRead(String, serde_json::Error), - #[error(transparent)] - HomeDir(#[from] etcetera::HomeDirError), - #[error(transparent)] - IO(IoError), - #[error(transparent)] - Serialization(#[from] serde_json::Error), -} - -#[derive(Debug, Error)] -pub struct IoError { - pub error: std::io::Error, - pub path: Option, -} - -impl IoError { - fn new(error: std::io::Error, path: Option<&Path>) -> Self { - Self { - error, - path: path.map(|p| p.to_string_lossy().to_string()), - } - } -} - -impl std::fmt::Display for IoError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.error)?; - if let Some(ref path) = self.path { - write!(f, " ({path})")?; - } - Ok(()) - } -} /// Holds the contents of tree-sitter's configuration file. /// @@ -65,7 +22,7 @@ pub struct Config { } impl Config { - pub fn find_config_file() -> ConfigResult> { + pub fn find_config_file() -> Result> { if let Ok(path) = env::var("TREE_SITTER_DIR") { let mut path = PathBuf::from(path); path.push("config.json"); @@ -88,14 +45,10 @@ impl Config { .join("tree-sitter") .join("config.json"); if legacy_apple_path.is_file() { - let xdg_dir = xdg_path.parent().unwrap(); - fs::create_dir_all(xdg_dir) - .map_err(|e| ConfigError::IO(IoError::new(e, Some(xdg_dir))))?; - fs::rename(&legacy_apple_path, &xdg_path).map_err(|e| { - ConfigError::IO(IoError::new(e, Some(legacy_apple_path.as_path()))) - })?; - warn!( - "Your config.json file has been automatically migrated from \"{}\" to \"{}\"", + fs::create_dir_all(xdg_path.parent().unwrap())?; + fs::rename(&legacy_apple_path, &xdg_path)?; + println!( + "Warning: your config.json file has been automatically migrated from \"{}\" to \"{}\"", legacy_apple_path.display(), xdg_path.display() ); @@ -113,7 +66,7 @@ impl Config { Ok(None) } - fn xdg_config_file() -> ConfigResult { + fn xdg_config_file() -> Result { let xdg_path = etcetera::choose_base_strategy()? .config_dir() .join("tree-sitter") @@ -130,7 +83,7 @@ impl Config { /// [`etcetera::choose_base_strategy`](https://docs.rs/etcetera/*/etcetera/#basestrategy) /// - `$HOME/.tree-sitter/config.json` as a fallback from where tree-sitter _used_ to store /// its configuration - pub fn load(path: Option) -> ConfigResult { + pub fn load(path: Option) -> Result { let location = if let Some(path) = path { path } else if let Some(path) = Self::find_config_file()? { @@ -140,9 +93,9 @@ impl Config { }; let content = fs::read_to_string(&location) - .map_err(|e| ConfigError::IO(IoError::new(e, Some(location.as_path()))))?; + .with_context(|| format!("Failed to read {}", location.to_string_lossy()))?; let config = serde_json::from_str(&content) - .map_err(|e| ConfigError::ConfigRead(location.to_string_lossy().to_string(), e))?; + .with_context(|| format!("Bad JSON config {}", location.to_string_lossy()))?; Ok(Self { location, config }) } @@ -152,7 +105,7 @@ impl Config { /// disk. /// /// (Note that this is typically only done by the `tree-sitter init-config` command.) - pub fn initial() -> ConfigResult { + pub fn initial() -> Result { let location = if let Ok(path) = env::var("TREE_SITTER_DIR") { let mut path = PathBuf::from(path); path.push("config.json"); @@ -165,20 +118,17 @@ impl Config { } /// Saves this configuration to the file that it was originally loaded from. - pub fn save(&self) -> ConfigResult<()> { + pub fn save(&self) -> Result<()> { let json = serde_json::to_string_pretty(&self.config)?; - let config_dir = self.location.parent().unwrap(); - fs::create_dir_all(config_dir) - .map_err(|e| ConfigError::IO(IoError::new(e, Some(config_dir))))?; - fs::write(&self.location, json) - .map_err(|e| ConfigError::IO(IoError::new(e, Some(self.location.as_path()))))?; + fs::create_dir_all(self.location.parent().unwrap())?; + fs::write(&self.location, json)?; Ok(()) } /// Parses a component-specific configuration from the configuration file. The type `C` must /// be [deserializable](https://docs.rs/serde/*/serde/trait.Deserialize.html) from a JSON /// object, and must only include the fields relevant to that component. - pub fn get(&self) -> ConfigResult + pub fn get(&self) -> Result where C: for<'de> Deserialize<'de>, { @@ -189,7 +139,7 @@ impl Config { /// Adds a component-specific configuration to the configuration file. The type `C` must be /// [serializable](https://docs.rs/serde/*/serde/trait.Serialize.html) into a JSON object, and /// must only include the fields relevant to that component. - pub fn add(&mut self, config: C) -> ConfigResult<()> + pub fn add(&mut self, config: C) -> Result<()> where C: Serialize, { diff --git a/crates/cli/eslint/index.js b/cli/eslint/index.js similarity index 100% rename from crates/cli/eslint/index.js rename to cli/eslint/index.js diff --git a/crates/cli/eslint/package-lock.json b/cli/eslint/package-lock.json similarity index 99% rename from crates/cli/eslint/package-lock.json rename to cli/eslint/package-lock.json index 60559b10..44266c4c 100644 --- a/crates/cli/eslint/package-lock.json +++ b/cli/eslint/package-lock.json @@ -305,9 +305,9 @@ "peer": true }, "node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", "license": "MIT", "peer": true, "dependencies": { @@ -805,9 +805,9 @@ "peer": true }, "node_modules/js-yaml": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", - "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", "license": "MIT", "peer": true, "dependencies": { diff --git a/crates/cli/eslint/package.json b/cli/eslint/package.json similarity index 86% rename from crates/cli/eslint/package.json rename to cli/eslint/package.json index a3ef0c10..a3ba26c9 100644 --- a/crates/cli/eslint/package.json +++ b/cli/eslint/package.json @@ -21,9 +21,5 @@ }, "peerDependencies": { "eslint": ">= 9" - }, - "scripts": { - "prepack": "cp ../../../LICENSE .", - "postpack": "rm LICENSE" } } diff --git a/crates/generate/Cargo.toml b/cli/generate/Cargo.toml similarity index 60% rename from crates/generate/Cargo.toml rename to cli/generate/Cargo.toml index e55be890..d3e6b224 100644 --- a/crates/generate/Cargo.toml +++ b/cli/generate/Cargo.toml @@ -8,44 +8,30 @@ rust-version.workspace = true readme = "README.md" homepage.workspace = true repository.workspace = true -documentation = "https://docs.rs/tree-sitter-generate" license.workspace = true keywords.workspace = true categories.workspace = true -[lib] -path = "src/generate.rs" - [lints] workspace = true -[features] -default = ["qjs-rt"] -load = ["dep:semver"] -qjs-rt = ["load", "rquickjs", "pathdiff"] - [dependencies] -bitflags = "2.9.4" -dunce = "1.0.5" +anyhow.workspace = true +heck.workspace = true indexmap.workspace = true indoc.workspace = true log.workspace = true -pathdiff = { version = "0.2.3", optional = true } regex.workspace = true regex-syntax.workspace = true -rquickjs = { version = "0.11.0", optional = true, features = [ - "bindgen", - "loader", - "macro", - "phf", -] } rustc-hash.workspace = true -semver = { workspace = true, optional = true } +semver.workspace = true serde.workspace = true serde_json.workspace = true smallbitvec.workspace = true thiserror.workspace = true topological-sort.workspace = true -[dev-dependencies] -tempfile.workspace = true +tree-sitter.workspace = true + +[target.'cfg(windows)'.dependencies] +url.workspace = true diff --git a/crates/generate/README.md b/cli/generate/README.md similarity index 100% rename from crates/generate/README.md rename to cli/generate/README.md diff --git a/crates/generate/src/build_tables/build_lex_table.rs b/cli/generate/src/build_tables/build_lex_table.rs similarity index 99% rename from crates/generate/src/build_tables/build_lex_table.rs rename to cli/generate/src/build_tables/build_lex_table.rs index 9d0d4fb7..2de6c0f8 100644 --- a/crates/generate/src/build_tables/build_lex_table.rs +++ b/cli/generate/src/build_tables/build_lex_table.rs @@ -3,7 +3,7 @@ use std::{ mem, }; -use log::debug; +use log::info; use super::{coincident_tokens::CoincidentTokenIndex, token_conflicts::TokenConflictMap}; use crate::{ @@ -176,7 +176,7 @@ impl<'a> LexTableBuilder<'a> { let (state_id, is_new) = self.add_state(nfa_states, eof_valid); if is_new { - debug!( + info!( "entry point state: {state_id}, tokens: {:?}", tokens .iter() diff --git a/crates/generate/src/build_tables/build_parse_table.rs b/cli/generate/src/build_tables/build_parse_table.rs similarity index 97% rename from crates/generate/src/build_tables/build_parse_table.rs rename to cli/generate/src/build_tables/build_parse_table.rs index 66f29609..9f43f0bd 100644 --- a/crates/generate/src/build_tables/build_parse_table.rs +++ b/cli/generate/src/build_tables/build_parse_table.rs @@ -5,7 +5,6 @@ use std::{ }; use indexmap::{map::Entry, IndexMap}; -use log::warn; use rustc_hash::FxHasher; use serde::Serialize; use thiserror::Error; @@ -77,11 +76,9 @@ pub enum ParseTableBuilderError { "The non-terminal rule `{0}` is used in a non-terminal `extra` rule, which is not allowed." )] ImproperNonTerminalExtra(String), - #[error("State count `{0}` exceeds the max value {max}.", max=u16::MAX)] - StateCount(usize), } -#[derive(Default, Debug, Serialize, Error)] +#[derive(Default, Debug, Serialize)] pub struct ConflictError { pub symbol_sequence: Vec, pub conflicting_lookahead: String, @@ -89,7 +86,7 @@ pub struct ConflictError { pub possible_resolutions: Vec, } -#[derive(Default, Debug, Serialize, Error)] +#[derive(Default, Debug, Serialize)] pub struct Interpretation { pub preceding_symbols: Vec, pub variable_name: String, @@ -108,7 +105,7 @@ pub enum Resolution { AddConflict { symbols: Vec }, } -#[derive(Debug, Serialize, Error)] +#[derive(Debug, Serialize)] pub struct AmbiguousExtraError { pub parent_symbols: Vec, } @@ -238,6 +235,9 @@ impl std::fmt::Display for AmbiguousExtraError { } } +impl std::error::Error for ConflictError {} +impl std::error::Error for AmbiguousExtraError {} + impl<'a> ParseTableBuilder<'a> { fn new( syntax_grammar: &'a SyntaxGrammar, @@ -346,21 +346,17 @@ impl<'a> ParseTableBuilder<'a> { } if !self.actual_conflicts.is_empty() { - warn!( - "unnecessary conflicts:\n {}", - &self - .actual_conflicts - .iter() - .map(|conflict| { - conflict - .iter() - .map(|symbol| format!("`{}`", self.symbol_name(symbol))) - .collect::>() - .join(", ") - }) - .collect::>() - .join("\n ") - ); + println!("Warning: unnecessary conflicts"); + for conflict in &self.actual_conflicts { + println!( + " {}", + conflict + .iter() + .map(|symbol| format!("`{}`", self.symbol_name(symbol))) + .collect::>() + .join(", ") + ); + } } Ok((self.parse_table, self.parse_state_info_by_id)) @@ -860,9 +856,9 @@ impl<'a> ParseTableBuilder<'a> { for symbol in preceding_symbols { conflict_error .symbol_sequence - .push(self.symbol_name(symbol)); + .push(self.symbol_name(symbol).to_string()); } - conflict_error.conflicting_lookahead = self.symbol_name(&conflicting_lookahead); + conflict_error.conflicting_lookahead = self.symbol_name(&conflicting_lookahead).to_string(); let interpretations = conflicting_items .iter() @@ -870,7 +866,7 @@ impl<'a> ParseTableBuilder<'a> { let preceding_symbols = preceding_symbols .iter() .take(preceding_symbols.len() - item.step_index as usize) - .map(|symbol| self.symbol_name(symbol)) + .map(|symbol| self.symbol_name(symbol).to_string()) .collect::>(); let variable_name = self.syntax_grammar.variables[item.variable_index as usize] @@ -881,7 +877,7 @@ impl<'a> ParseTableBuilder<'a> { .production .steps .iter() - .map(|step| self.symbol_name(&step.symbol)) + .map(|step| self.symbol_name(&step.symbol).to_string()) .collect::>(); let precedence = match item.precedence() { @@ -897,7 +893,7 @@ impl<'a> ParseTableBuilder<'a> { production_step_symbols, step_index: item.step_index, done: item.is_done(), - conflicting_lookahead: self.symbol_name(&conflicting_lookahead), + conflicting_lookahead: self.symbol_name(&conflicting_lookahead).to_string(), precedence, associativity, } diff --git a/crates/generate/src/build_tables/coincident_tokens.rs b/cli/generate/src/build_tables/coincident_tokens.rs similarity index 100% rename from crates/generate/src/build_tables/coincident_tokens.rs rename to cli/generate/src/build_tables/coincident_tokens.rs diff --git a/crates/generate/src/build_tables/item.rs b/cli/generate/src/build_tables/item.rs similarity index 99% rename from crates/generate/src/build_tables/item.rs rename to cli/generate/src/build_tables/item.rs index cd70ce74..0beb3e3c 100644 --- a/crates/generate/src/build_tables/item.rs +++ b/cli/generate/src/build_tables/item.rs @@ -204,7 +204,7 @@ impl fmt::Display for ParseItemDisplay<'_> { || step.reserved_word_set_id != ReservedWordSetId::default() { write!(f, " (")?; - if !step.precedence.is_none() { + if step.precedence.is_none() { write!(f, " {}", step.precedence)?; } if let Some(associativity) = step.associativity { diff --git a/crates/generate/src/build_tables/item_set_builder.rs b/cli/generate/src/build_tables/item_set_builder.rs similarity index 99% rename from crates/generate/src/build_tables/item_set_builder.rs rename to cli/generate/src/build_tables/item_set_builder.rs index 44e05702..a0bf02db 100644 --- a/crates/generate/src/build_tables/item_set_builder.rs +++ b/cli/generate/src/build_tables/item_set_builder.rs @@ -81,7 +81,7 @@ impl<'a> ParseItemSetBuilder<'a> { .insert(symbol, ReservedWordSetId::default()); } - // The FIRST set of a non-terminal `i` is the union of the FIRST sets + // The FIRST set of a non-terminal `i` is the union of the the FIRST sets // of all the symbols that appear at the beginnings of i's productions. Some // of these symbols may themselves be non-terminals, so this is a recursive // definition. diff --git a/crates/generate/src/build_tables/minimize_parse_table.rs b/cli/generate/src/build_tables/minimize_parse_table.rs similarity index 97% rename from crates/generate/src/build_tables/minimize_parse_table.rs rename to cli/generate/src/build_tables/minimize_parse_table.rs index 6c26f1c4..86932121 100644 --- a/crates/generate/src/build_tables/minimize_parse_table.rs +++ b/cli/generate/src/build_tables/minimize_parse_table.rs @@ -3,7 +3,7 @@ use std::{ mem, }; -use log::debug; +use log::info; use super::token_conflicts::TokenConflictMap; use crate::{ @@ -11,7 +11,6 @@ use crate::{ grammars::{LexicalGrammar, SyntaxGrammar, VariableType}, rules::{AliasMap, Symbol, TokenSet}, tables::{GotoAction, ParseAction, ParseState, ParseStateId, ParseTable, ParseTableEntry}, - OptLevel, }; pub fn minimize_parse_table( @@ -21,7 +20,6 @@ pub fn minimize_parse_table( simple_aliases: &AliasMap, token_conflict_map: &TokenConflictMap, keywords: &TokenSet, - optimizations: OptLevel, ) { let mut minimizer = Minimizer { parse_table, @@ -31,9 +29,7 @@ pub fn minimize_parse_table( keywords, simple_aliases, }; - if optimizations.contains(OptLevel::MergeStates) { - minimizer.merge_compatible_states(); - } + minimizer.merge_compatible_states(); minimizer.remove_unit_reductions(); minimizer.remove_unused_states(); minimizer.reorder_states_by_descending_size(); @@ -248,7 +244,7 @@ impl Minimizer<'_> { let group1 = group_ids_by_state_id[*s1]; let group2 = group_ids_by_state_id[*s2]; if group1 != group2 { - debug!( + info!( "split states {} {} - successors for {} are split: {s1} {s2}", state1.id, state2.id, @@ -269,7 +265,7 @@ impl Minimizer<'_> { let group1 = group_ids_by_state_id[*s1]; let group2 = group_ids_by_state_id[*s2]; if group1 != group2 { - debug!( + info!( "split states {} {} - successors for {} are split: {s1} {s2}", state1.id, state2.id, @@ -299,14 +295,16 @@ impl Minimizer<'_> { let actions1 = &entry1.actions; let actions2 = &entry2.actions; if actions1.len() != actions2.len() { - debug!( + info!( "split states {state_id1} {state_id2} - differing action counts for token {}", self.symbol_name(token) ); return true; } - for (action1, action2) in actions1.iter().zip(actions2.iter()) { + for (i, action1) in actions1.iter().enumerate() { + let action2 = &actions2[i]; + // Two shift actions are equivalent if their destinations are in the same group. if let ( ParseAction::Shift { @@ -324,13 +322,13 @@ impl Minimizer<'_> { if group1 == group2 && is_repetition1 == is_repetition2 { continue; } - debug!( + info!( "split states {state_id1} {state_id2} - successors for {} are split: {s1} {s2}", self.symbol_name(token), ); return true; } else if action1 != action2 { - debug!( + info!( "split states {state_id1} {state_id2} - unequal actions for {}", self.symbol_name(token), ); @@ -349,14 +347,14 @@ impl Minimizer<'_> { new_token: Symbol, ) -> bool { if new_token == Symbol::end_of_nonterminal_extra() { - debug!("split states {left_id} {right_id} - end of non-terminal extra",); + info!("split states {left_id} {right_id} - end of non-terminal extra",); return true; } // Do not add external tokens; they could conflict lexically with any of the state's // existing lookahead tokens. if new_token.is_external() { - debug!( + info!( "split states {left_id} {right_id} - external token {}", self.symbol_name(&new_token), ); @@ -375,7 +373,7 @@ impl Minimizer<'_> { .iter() .any(|external| external.corresponding_internal_token == Some(new_token)) { - debug!( + info!( "split states {left_id} {right_id} - internal/external token {}", self.symbol_name(&new_token), ); @@ -401,7 +399,7 @@ impl Minimizer<'_> { .token_conflict_map .does_match_same_string(new_token.index, token.index) { - debug!( + info!( "split states {} {} - token {} conflicts with {}", left_id, right_id, diff --git a/crates/generate/src/build_tables.rs b/cli/generate/src/build_tables/mod.rs similarity index 94% rename from crates/generate/src/build_tables.rs rename to cli/generate/src/build_tables/mod.rs index 8c6ef2a4..f5709419 100644 --- a/crates/generate/src/build_tables.rs +++ b/cli/generate/src/build_tables/mod.rs @@ -11,7 +11,7 @@ use std::collections::{BTreeSet, HashMap}; pub use build_lex_table::LARGE_CHARACTER_RANGE_COUNT; use build_parse_table::BuildTableResult; pub use build_parse_table::ParseTableBuilderError; -use log::{debug, info}; +use log::info; use self::{ build_lex_table::build_lex_table, @@ -27,7 +27,6 @@ use crate::{ node_types::VariableInfo, rules::{AliasMap, Symbol, SymbolType, TokenSet}, tables::{LexTable, ParseAction, ParseTable, ParseTableEntry}, - OptLevel, }; pub struct Tables { @@ -44,7 +43,6 @@ pub fn build_tables( variable_info: &[VariableInfo], inlines: &InlinedProductionMap, report_symbol_name: Option<&str>, - optimizations: OptLevel, ) -> BuildTableResult { let item_set_builder = ParseItemSetBuilder::new(syntax_grammar, lexical_grammar, inlines); let following_tokens = @@ -80,7 +78,6 @@ pub fn build_tables( simple_aliases, &token_conflict_map, &keywords, - optimizations, ); let lex_tables = build_lex_table( &mut parse_table, @@ -103,10 +100,6 @@ pub fn build_tables( ); } - if parse_table.states.len() > u16::MAX as usize { - Err(ParseTableBuilderError::StateCount(parse_table.states.len()))?; - } - Ok(Tables { parse_table, main_lex_table: lex_tables.main_lex_table, @@ -179,7 +172,7 @@ fn populate_error_state( if conflicts_with_other_tokens { None } else { - debug!( + info!( "error recovery - token {} has no conflicts", lexical_grammar.variables[i].name ); @@ -205,14 +198,14 @@ fn populate_error_state( !coincident_token_index.contains(symbol, *t) && token_conflict_map.does_conflict(symbol.index, t.index) }) { - debug!( + info!( "error recovery - exclude token {} because of conflict with {}", lexical_grammar.variables[i].name, lexical_grammar.variables[t.index].name ); continue; } } - debug!( + info!( "error recovery - include token {}", lexical_grammar.variables[i].name ); @@ -345,7 +338,7 @@ fn identify_keywords( && token_conflict_map.does_match_same_string(i, word_token.index) && !token_conflict_map.does_match_different_string(i, word_token.index) { - debug!( + info!( "Keywords - add candidate {}", lexical_grammar.variables[i].name ); @@ -364,7 +357,7 @@ fn identify_keywords( if other_token != *token && token_conflict_map.does_match_same_string(other_token.index, token.index) { - debug!( + info!( "Keywords - exclude {} because it matches the same string as {}", lexical_grammar.variables[token.index].name, lexical_grammar.variables[other_token.index].name @@ -406,7 +399,7 @@ fn identify_keywords( word_token.index, other_index, ) { - debug!( + info!( "Keywords - exclude {} because of conflict with {}", lexical_grammar.variables[token.index].name, lexical_grammar.variables[other_index].name @@ -415,7 +408,7 @@ fn identify_keywords( } } - debug!( + info!( "Keywords - include {}", lexical_grammar.variables[token.index].name, ); @@ -487,14 +480,14 @@ fn report_state_info<'a>( .max() .unwrap(); for (symbol, states) in &symbols_with_state_indices { - info!( + eprintln!( "{:width$}\t{}", syntax_grammar.variables[symbol.index].name, states.len(), width = max_symbol_name_length ); } - info!(""); + eprintln!(); let state_indices = if report_symbol_name == "*" { Some(&all_state_indices) @@ -517,25 +510,20 @@ fn report_state_info<'a>( for state_index in state_indices { let id = parse_table.states[state_index].id; let (preceding_symbols, item_set) = &parse_state_info[id]; - info!("state index: {state_index}"); - info!("state id: {id}"); - info!( - "symbol sequence: {}", - preceding_symbols - .iter() - .map(|symbol| { - if symbol.is_terminal() { - lexical_grammar.variables[symbol.index].name.clone() - } else if symbol.is_external() { - syntax_grammar.external_tokens[symbol.index].name.clone() - } else { - syntax_grammar.variables[symbol.index].name.clone() - } - }) - .collect::>() - .join(" ") - ); - info!( + eprintln!("state index: {state_index}"); + eprintln!("state id: {id}"); + eprint!("symbol sequence:"); + for symbol in preceding_symbols { + let name = if symbol.is_terminal() { + &lexical_grammar.variables[symbol.index].name + } else if symbol.is_external() { + &syntax_grammar.external_tokens[symbol.index].name + } else { + &syntax_grammar.variables[symbol.index].name + }; + eprint!(" {name}"); + } + eprintln!( "\nitems:\n{}", item::ParseItemSetDisplay(item_set, syntax_grammar, lexical_grammar), ); diff --git a/crates/generate/src/build_tables/token_conflicts.rs b/cli/generate/src/build_tables/token_conflicts.rs similarity index 99% rename from crates/generate/src/build_tables/token_conflicts.rs rename to cli/generate/src/build_tables/token_conflicts.rs index d72effd4..bacac1b4 100644 --- a/crates/generate/src/build_tables/token_conflicts.rs +++ b/cli/generate/src/build_tables/token_conflicts.rs @@ -28,7 +28,7 @@ pub struct TokenConflictMap<'a> { impl<'a> TokenConflictMap<'a> { /// Create a token conflict map based on a lexical grammar, which describes the structure - /// of each token, and a `following_token` map, which indicates which tokens may be appear + /// each token, and a `following_token` map, which indicates which tokens may be appear /// immediately after each other token. /// /// This analyzes the possible kinds of overlap between each pair of tokens and stores diff --git a/crates/generate/src/dedup.rs b/cli/generate/src/dedup.rs similarity index 100% rename from crates/generate/src/dedup.rs rename to cli/generate/src/dedup.rs diff --git a/crates/generate/src/dsl.js b/cli/generate/src/dsl.js similarity index 96% rename from crates/generate/src/dsl.js rename to cli/generate/src/dsl.js index f522fd7f..faaace05 100644 --- a/crates/generate/src/dsl.js +++ b/cli/generate/src/dsl.js @@ -70,7 +70,7 @@ function prec(number, rule) { }; } -prec.left = function (number, rule) { +prec.left = function(number, rule) { if (rule == null) { rule = number; number = 0; @@ -92,7 +92,7 @@ prec.left = function (number, rule) { }; } -prec.right = function (number, rule) { +prec.right = function(number, rule) { if (rule == null) { rule = number; number = 0; @@ -114,7 +114,7 @@ prec.right = function (number, rule) { }; } -prec.dynamic = function (number, rule) { +prec.dynamic = function(number, rule) { checkPrecedence(number); checkArguments( arguments, @@ -184,7 +184,7 @@ function token(value) { }; } -token.immediate = function (value) { +token.immediate = function(value) { checkArguments(arguments, arguments.length, token.immediate, 'token.immediate', '', 'literal'); return { type: "IMMEDIATE_TOKEN", @@ -517,7 +517,6 @@ function checkPrecedence(value) { } function getEnv(name) { - if (globalThis.native) return globalThis.__ts_grammar_path; if (globalThis.process) return process.env[name]; // Node/Bun if (globalThis.Deno) return Deno.env.get(name); // Deno throw Error("Unsupported JS runtime"); @@ -538,23 +537,14 @@ globalThis.grammar = grammar; globalThis.field = field; globalThis.RustRegex = RustRegex; -const grammarPath = getEnv("TREE_SITTER_GRAMMAR_PATH"); -let result = await import(grammarPath); -let grammarObj = result.default?.grammar ?? result.grammar; - -if (globalThis.native && !grammarObj) { - grammarObj = module.exports.grammar; -} - +const result = await import(getEnv("TREE_SITTER_GRAMMAR_PATH")); const object = { "$schema": "https://tree-sitter.github.io/tree-sitter/assets/schemas/grammar.schema.json", - ...grammarObj, + ...(result.default?.grammar ?? result.grammar) }; const output = JSON.stringify(object); -if (globalThis.native) { - globalThis.output = output; -} else if (globalThis.process) { // Node/Bun +if (globalThis.process) { // Node/Bun process.stdout.write(output); } else if (globalThis.Deno) { // Deno Deno.stdout.writeSync(new TextEncoder().encode(output)); diff --git a/cli/generate/src/grammar_files.rs b/cli/generate/src/grammar_files.rs new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/cli/generate/src/grammar_files.rs @@ -0,0 +1 @@ + diff --git a/crates/generate/src/grammars.rs b/cli/generate/src/grammars.rs similarity index 100% rename from crates/generate/src/grammars.rs rename to cli/generate/src/grammars.rs diff --git a/crates/generate/src/generate.rs b/cli/generate/src/lib.rs similarity index 55% rename from crates/generate/src/generate.rs rename to cli/generate/src/lib.rs index 6a005637..95bcb0a9 100644 --- a/crates/generate/src/generate.rs +++ b/cli/generate/src/lib.rs @@ -1,40 +1,32 @@ -use std::{collections::BTreeMap, sync::LazyLock}; -#[cfg(feature = "load")] use std::{ env, fs, io::Write, path::{Path, PathBuf}, process::{Command, Stdio}, + sync::LazyLock, }; -use bitflags::bitflags; -use log::warn; -use node_types::VariableInfo; +use anyhow::Result; use regex::{Regex, RegexBuilder}; -use rules::{Alias, Symbol}; -#[cfg(feature = "load")] use semver::Version; -#[cfg(feature = "load")] -use serde::Deserialize; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use thiserror::Error; mod build_tables; mod dedup; +mod grammar_files; mod grammars; mod nfa; mod node_types; pub mod parse_grammar; mod prepare_grammar; -#[cfg(feature = "qjs-rt")] -mod quickjs; mod render; mod rules; mod tables; use build_tables::build_tables; pub use build_tables::ParseTableBuilderError; -use grammars::{InlinedProductionMap, InputGrammar, LexicalGrammar, SyntaxGrammar}; +use grammars::InputGrammar; pub use node_types::{SuperTypeCycleError, VariableInfoError}; use parse_grammar::parse_grammar; pub use parse_grammar::ParseGrammarError; @@ -50,29 +42,13 @@ static JSON_COMMENT_REGEX: LazyLock = LazyLock::new(|| { .unwrap() }); -struct JSONOutput { - #[cfg(feature = "load")] - node_types_json: String, - syntax_grammar: SyntaxGrammar, - lexical_grammar: LexicalGrammar, - inlines: InlinedProductionMap, - simple_aliases: BTreeMap, - variable_info: Vec, -} - struct GeneratedParser { c_code: String, - #[cfg(feature = "load")] node_types_json: String, } -// NOTE: This constant must be kept in sync with the definition of -// `TREE_SITTER_LANGUAGE_VERSION` in `lib/include/tree_sitter/api.h`. -const LANGUAGE_VERSION: usize = 15; - pub const ALLOC_HEADER: &str = include_str!("templates/alloc.h"); pub const ARRAY_HEADER: &str = include_str!("templates/array.h"); -pub const PARSER_HEADER: &str = include_str!("parser.h.inc"); pub type GenerateResult = Result; @@ -80,9 +56,8 @@ pub type GenerateResult = Result; pub enum GenerateError { #[error("Error with specified path -- {0}")] GrammarPath(String), - #[error(transparent)] - IO(IoError), - #[cfg(feature = "load")] + #[error("{0}")] + IO(String), #[error(transparent)] LoadGrammarFile(#[from] LoadGrammarError), #[error(transparent)] @@ -93,42 +68,20 @@ pub enum GenerateError { VariableInfo(#[from] VariableInfoError), #[error(transparent)] BuildTables(#[from] ParseTableBuilderError), - #[cfg(feature = "load")] #[error(transparent)] ParseVersion(#[from] ParseVersionError), #[error(transparent)] SuperTypeCycle(#[from] SuperTypeCycleError), } -#[derive(Debug, Error, Serialize)] -pub struct IoError { - pub error: String, - pub path: Option, -} - -impl IoError { - fn new(error: &std::io::Error, path: Option<&Path>) -> Self { - Self { - error: error.to_string(), - path: path.map(|p| p.to_string_lossy().to_string()), - } +impl From for GenerateError { + fn from(value: std::io::Error) -> Self { + Self::IO(value.to_string()) } } -impl std::fmt::Display for IoError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.error)?; - if let Some(ref path) = self.path { - write!(f, " ({path})")?; - } - Ok(()) - } -} - -#[cfg(feature = "load")] pub type LoadGrammarFileResult = Result; -#[cfg(feature = "load")] #[derive(Debug, Error, Serialize)] pub enum LoadGrammarError { #[error("Path to a grammar file with `.js` or `.json` extension is required")] @@ -136,26 +89,29 @@ pub enum LoadGrammarError { #[error("Failed to load grammar.js -- {0}")] LoadJSGrammarFile(#[from] JSError), #[error("Failed to load grammar.json -- {0}")] - IO(IoError), + IO(String), #[error("Unknown grammar file extension: {0:?}")] FileExtension(PathBuf), } -#[cfg(feature = "load")] +impl From for LoadGrammarError { + fn from(value: std::io::Error) -> Self { + Self::IO(value.to_string()) + } +} + #[derive(Debug, Error, Serialize)] pub enum ParseVersionError { #[error("{0}")] Version(String), #[error("{0}")] JSON(String), - #[error(transparent)] - IO(IoError), + #[error("{0}")] + IO(String), } -#[cfg(feature = "load")] pub type JSResult = Result; -#[cfg(feature = "load")] #[derive(Debug, Error, Serialize)] pub enum JSError { #[error("Failed to run `{runtime}` -- {error}")] @@ -164,138 +120,85 @@ pub enum JSError { JSRuntimeUtf8 { runtime: String, error: String }, #[error("`{runtime}` process exited with status {code}")] JSRuntimeExit { runtime: String, code: i32 }, - #[error("Failed to open stdin for `{runtime}`")] - JSRuntimeStdin { runtime: String }, - #[error("Failed to write {item} to `{runtime}`'s stdin -- {error}")] - JSRuntimeWrite { - runtime: String, - item: String, - error: String, - }, - #[error("Failed to read output from `{runtime}` -- {error}")] - JSRuntimeRead { runtime: String, error: String }, - #[error(transparent)] - IO(IoError), - #[cfg(feature = "qjs-rt")] - #[error("Failed to get relative path")] - RelativePath, + #[error("{0}")] + IO(String), #[error("Could not parse this package's version as semver -- {0}")] Semver(String), #[error("Failed to serialze grammar JSON -- {0}")] Serialzation(String), - #[cfg(feature = "qjs-rt")] - #[error("QuickJS error: {0}")] - QuickJS(String), } -#[cfg(feature = "load")] +impl From for JSError { + fn from(value: std::io::Error) -> Self { + Self::IO(value.to_string()) + } +} + impl From for JSError { fn from(value: serde_json::Error) -> Self { Self::Serialzation(value.to_string()) } } -#[cfg(feature = "load")] impl From for JSError { fn from(value: semver::Error) -> Self { Self::Semver(value.to_string()) } } -#[cfg(feature = "qjs-rt")] -impl From for JSError { - fn from(value: rquickjs::Error) -> Self { - Self::QuickJS(value.to_string()) - } -} - -bitflags! { - #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] - pub struct OptLevel: u32 { - const MergeStates = 1 << 0; - } -} - -impl Default for OptLevel { - fn default() -> Self { - Self::MergeStates - } -} - -#[cfg(feature = "load")] -#[allow(clippy::too_many_arguments)] -pub fn generate_parser_in_directory( - repo_path: T, - out_path: Option, - grammar_path: Option, +pub fn generate_parser_in_directory( + repo_path: &Path, + out_path: Option<&str>, + grammar_path: Option<&str>, mut abi_version: usize, report_symbol_name: Option<&str>, js_runtime: Option<&str>, - generate_parser: bool, - optimizations: OptLevel, -) -> GenerateResult<()> -where - T: Into, - U: Into, - V: Into, -{ - let mut repo_path: PathBuf = repo_path.into(); +) -> GenerateResult<()> { + let mut repo_path = repo_path.to_owned(); + let mut grammar_path = grammar_path; // Populate a new empty grammar directory. - let grammar_path = if let Some(path) = grammar_path { - let path_buf: PathBuf = path.into(); - if !path_buf + if let Some(path) = grammar_path { + let path = PathBuf::from(path); + if !path .try_exists() .map_err(|e| GenerateError::GrammarPath(e.to_string()))? { - fs::create_dir_all(&path_buf) - .map_err(|e| GenerateError::IO(IoError::new(&e, Some(path_buf.as_path()))))?; - repo_path = path_buf; - repo_path.join("grammar.js") - } else { - path_buf + fs::create_dir_all(&path)?; + grammar_path = None; + repo_path = path; } - } else { - repo_path.join("grammar.js") - }; + } + + let grammar_path = grammar_path.map_or_else(|| repo_path.join("grammar.js"), PathBuf::from); // Read the grammar file. let grammar_json = load_grammar_file(&grammar_path, js_runtime)?; - let src_path = out_path.map_or_else(|| repo_path.join("src"), |p| p.into()); + let src_path = out_path.map_or_else(|| repo_path.join("src"), PathBuf::from); let header_path = src_path.join("tree_sitter"); - // Ensure that the output directory exists - fs::create_dir_all(&src_path) - .map_err(|e| GenerateError::IO(IoError::new(&e, Some(src_path.as_path()))))?; + // Ensure that the output directories exist. + fs::create_dir_all(&src_path)?; + fs::create_dir_all(&header_path)?; if grammar_path.file_name().unwrap() != "grammar.json" { - fs::write(src_path.join("grammar.json"), &grammar_json) - .map_err(|e| GenerateError::IO(IoError::new(&e, Some(src_path.as_path()))))?; + fs::write(src_path.join("grammar.json"), &grammar_json).map_err(|e| { + GenerateError::IO(format!( + "Failed to write grammar.json to {} -- {e}", + src_path.display() + )) + })?; } - // If our job is only to generate `grammar.json` and not `parser.c`, stop here. + // Parse and preprocess the grammar. let input_grammar = parse_grammar(&grammar_json)?; - if !generate_parser { - let node_types_json = generate_node_types_from_grammar(&input_grammar)?.node_types_json; - write_file(&src_path.join("node-types.json"), node_types_json)?; - return Ok(()); - } - let semantic_version = read_grammar_version(&repo_path)?; if semantic_version.is_none() && abi_version > ABI_VERSION_MIN { - warn!( - concat!( - "No `tree-sitter.json` file found in your grammar, ", - "this file is required to generate with ABI {}. ", - "Using ABI version {} instead.\n", - "This file can be set up with `tree-sitter init`. ", - "For more information, see https://tree-sitter.github.io/tree-sitter/cli/init." - ), - abi_version, ABI_VERSION_MIN - ); + println!("Warning: No `tree-sitter.json` file found in your grammar, this file is required to generate with ABI {abi_version}. Using ABI version {ABI_VERSION_MIN} instead."); + println!("This file can be set up with `tree-sitter init`. For more information, see https://tree-sitter.github.io/tree-sitter/cli/init."); abi_version = ABI_VERSION_MIN; } @@ -308,16 +211,13 @@ where abi_version, semantic_version.map(|v| (v.major as u8, v.minor as u8, v.patch as u8)), report_symbol_name, - optimizations, )?; write_file(&src_path.join("parser.c"), c_code)?; write_file(&src_path.join("node-types.json"), node_types_json)?; - fs::create_dir_all(&header_path) - .map_err(|e| GenerateError::IO(IoError::new(&e, Some(header_path.as_path()))))?; write_file(&header_path.join("alloc.h"), ALLOC_HEADER)?; write_file(&header_path.join("array.h"), ARRAY_HEADER)?; - write_file(&header_path.join("parser.h"), PARSER_HEADER)?; + write_file(&header_path.join("parser.h"), tree_sitter::PARSER_HEADER)?; Ok(()) } @@ -330,54 +230,29 @@ pub fn generate_parser_for_grammar( let input_grammar = parse_grammar(&grammar_json)?; let parser = generate_parser_for_grammar_with_opts( &input_grammar, - LANGUAGE_VERSION, + tree_sitter::LANGUAGE_VERSION, semantic_version, None, - OptLevel::empty(), )?; Ok((input_grammar.name, parser.c_code)) } -fn generate_node_types_from_grammar(input_grammar: &InputGrammar) -> GenerateResult { - let (syntax_grammar, lexical_grammar, inlines, simple_aliases) = - prepare_grammar(input_grammar)?; - let variable_info = - node_types::get_variable_info(&syntax_grammar, &lexical_grammar, &simple_aliases)?; - - #[cfg(feature = "load")] - let node_types_json = node_types::generate_node_types_json( - &syntax_grammar, - &lexical_grammar, - &simple_aliases, - &variable_info, - )?; - Ok(JSONOutput { - #[cfg(feature = "load")] - node_types_json: serde_json::to_string_pretty(&node_types_json).unwrap(), - syntax_grammar, - lexical_grammar, - inlines, - simple_aliases, - variable_info, - }) -} - fn generate_parser_for_grammar_with_opts( input_grammar: &InputGrammar, abi_version: usize, semantic_version: Option<(u8, u8, u8)>, report_symbol_name: Option<&str>, - optimizations: OptLevel, ) -> GenerateResult { - let JSONOutput { - syntax_grammar, - lexical_grammar, - inlines, - simple_aliases, - variable_info, - #[cfg(feature = "load")] - node_types_json, - } = generate_node_types_from_grammar(input_grammar)?; + let (syntax_grammar, lexical_grammar, inlines, simple_aliases) = + prepare_grammar(input_grammar)?; + let variable_info = + node_types::get_variable_info(&syntax_grammar, &lexical_grammar, &simple_aliases)?; + let node_types_json = node_types::generate_node_types_json( + &syntax_grammar, + &lexical_grammar, + &simple_aliases, + &variable_info, + )?; let supertype_symbol_map = node_types::get_supertype_symbol_map(&syntax_grammar, &simple_aliases, &variable_info); let tables = build_tables( @@ -387,7 +262,6 @@ fn generate_parser_for_grammar_with_opts( &variable_info, &inlines, report_symbol_name, - optimizations, )?; let c_code = render_c_code( &input_grammar.name, @@ -401,8 +275,7 @@ fn generate_parser_for_grammar_with_opts( ); Ok(GeneratedParser { c_code, - #[cfg(feature = "load")] - node_types_json, + node_types_json: serde_json::to_string_pretty(&node_types_json).unwrap(), }) } @@ -411,7 +284,6 @@ fn generate_parser_for_grammar_with_opts( /// If the file is not found in the current directory or any of its parent directories, this will /// return `None` to maintain backwards compatibility. If the file is found but the version cannot /// be parsed as semver, this will return an error. -#[cfg(feature = "load")] fn read_grammar_version(repo_path: &Path) -> Result, ParseVersionError> { #[derive(Deserialize)] struct TreeSitterJson { @@ -430,8 +302,9 @@ fn read_grammar_version(repo_path: &Path) -> Result, ParseVersio let json = path .exists() .then(|| { - let contents = fs::read_to_string(path.as_path()) - .map_err(|e| ParseVersionError::IO(IoError::new(&e, Some(path.as_path()))))?; + let contents = fs::read_to_string(path.as_path()).map_err(|e| { + ParseVersionError::IO(format!("Failed to read `{}` -- {e}", path.display())) + })?; serde_json::from_str::(&contents).map_err(|e| { ParseVersionError::JSON(format!("Failed to parse `{}` -- {e}", path.display())) }) @@ -455,7 +328,6 @@ fn read_grammar_version(repo_path: &Path) -> Result, ParseVersio } } -#[cfg(feature = "load")] pub fn load_grammar_file( grammar_path: &Path, js_runtime: Option<&str>, @@ -465,26 +337,18 @@ pub fn load_grammar_file( } match grammar_path.extension().and_then(|e| e.to_str()) { Some("js") => Ok(load_js_grammar_file(grammar_path, js_runtime)?), - Some("json") => Ok(fs::read_to_string(grammar_path) - .map_err(|e| LoadGrammarError::IO(IoError::new(&e, Some(grammar_path))))?), + Some("json") => Ok(fs::read_to_string(grammar_path)?), _ => Err(LoadGrammarError::FileExtension(grammar_path.to_owned()))?, } } -#[cfg(feature = "load")] fn load_js_grammar_file(grammar_path: &Path, js_runtime: Option<&str>) -> JSResult { - let grammar_path = dunce::canonicalize(grammar_path) - .map_err(|e| JSError::IO(IoError::new(&e, Some(grammar_path))))?; + let grammar_path = fs::canonicalize(grammar_path)?; - #[cfg(feature = "qjs-rt")] - if js_runtime == Some("native") { - return quickjs::execute_native_runtime(&grammar_path); - } - - // The "file:///" prefix is incompatible with the quickjs runtime, but is required - // for node and bun #[cfg(windows)] - let grammar_path = PathBuf::from(format!("file:///{}", grammar_path.display())); + let grammar_path = url::Url::from_file_path(grammar_path) + .expect("Failed to convert path to URL") + .to_string(); let js_runtime = js_runtime.unwrap_or("node"); @@ -515,9 +379,7 @@ fn load_js_grammar_file(grammar_path: &Path, js_runtime: Option<&str>) -> JSResu let mut js_stdin = js_process .stdin .take() - .ok_or_else(|| JSError::JSRuntimeStdin { - runtime: js_runtime.to_string(), - })?; + .ok_or_else(|| JSError::IO(format!("Failed to open stdin for `{js_runtime}`")))?; let cli_version = Version::parse(env!("CARGO_PKG_VERSION"))?; write!( @@ -527,27 +389,23 @@ fn load_js_grammar_file(grammar_path: &Path, js_runtime: Option<&str>) -> JSResu globalThis.TREE_SITTER_CLI_VERSION_PATCH = {};", cli_version.major, cli_version.minor, cli_version.patch, ) - .map_err(|e| JSError::JSRuntimeWrite { - runtime: js_runtime.to_string(), - item: "tree-sitter version".to_string(), - error: e.to_string(), + .map_err(|e| { + JSError::IO(format!( + "Failed to write tree-sitter version to `{js_runtime}`'s stdin -- {e}" + )) + })?; + js_stdin.write(include_bytes!("./dsl.js")).map_err(|e| { + JSError::IO(format!( + "Failed to write grammar dsl to `{js_runtime}`'s stdin -- {e}" + )) })?; - js_stdin - .write(include_bytes!("./dsl.js")) - .map_err(|e| JSError::JSRuntimeWrite { - runtime: js_runtime.to_string(), - item: "grammar dsl".to_string(), - error: e.to_string(), - })?; drop(js_stdin); let output = js_process .wait_with_output() - .map_err(|e| JSError::JSRuntimeRead { - runtime: js_runtime.to_string(), - error: e.to_string(), - })?; + .map_err(|e| JSError::IO(format!("Failed to read output from `{js_runtime}` -- {e}")))?; match output.status.code() { + None => panic!("`{js_runtime}` process was killed"), Some(0) => { let stdout = String::from_utf8(output.stdout).map_err(|e| JSError::JSRuntimeUtf8 { runtime: js_runtime.to_string(), @@ -562,15 +420,9 @@ fn load_js_grammar_file(grammar_path: &Path, js_runtime: Option<&str>) -> JSResu grammar_json = &stdout[pos + 1..]; let mut stdout = std::io::stdout().lock(); - stdout - .write_all(node_output.as_bytes()) - .map_err(|e| JSError::IO(IoError::new(&e, None)))?; - stdout - .write_all(b"\n") - .map_err(|e| JSError::IO(IoError::new(&e, None)))?; - stdout - .flush() - .map_err(|e| JSError::IO(IoError::new(&e, None)))?; + stdout.write_all(node_output.as_bytes())?; + stdout.write_all(b"\n")?; + stdout.flush()?; } Ok(serde_json::to_string_pretty(&serde_json::from_str::< @@ -581,41 +433,10 @@ fn load_js_grammar_file(grammar_path: &Path, js_runtime: Option<&str>) -> JSResu runtime: js_runtime.to_string(), code, }), - None => Err(JSError::JSRuntimeExit { - runtime: js_runtime.to_string(), - code: -1, - }), } } -#[cfg(feature = "load")] pub fn write_file(path: &Path, body: impl AsRef<[u8]>) -> GenerateResult<()> { - fs::write(path, body).map_err(|e| GenerateError::IO(IoError::new(&e, Some(path)))) -} - -#[cfg(test)] -mod tests { - use super::{LANGUAGE_VERSION, PARSER_HEADER}; - #[test] - fn test_language_versions_are_in_sync() { - let api_h = include_str!("../../../lib/include/tree_sitter/api.h"); - let api_language_version = api_h - .lines() - .find_map(|line| { - line.trim() - .strip_prefix("#define TREE_SITTER_LANGUAGE_VERSION ") - .and_then(|v| v.parse::().ok()) - }) - .expect("Failed to find TREE_SITTER_LANGUAGE_VERSION definition in api.h"); - assert_eq!(LANGUAGE_VERSION, api_language_version); - } - - #[test] - fn test_parser_header_in_sync() { - let parser_h = include_str!("../../../lib/src/parser.h"); - assert!( - parser_h == PARSER_HEADER, - "parser.h.inc is out of sync with lib/src/parser.h. Run: cp lib/src/parser.h crates/generate/src/parser.h.inc" - ); - } + fs::write(path, body) + .map_err(|e| GenerateError::IO(format!("Failed to write {:?} -- {e}", path.file_name()))) } diff --git a/crates/generate/src/nfa.rs b/cli/generate/src/nfa.rs similarity index 99% rename from crates/generate/src/nfa.rs rename to cli/generate/src/nfa.rs index eecbc40b..9a63cb97 100644 --- a/crates/generate/src/nfa.rs +++ b/cli/generate/src/nfa.rs @@ -434,7 +434,6 @@ impl Nfa { } pub fn last_state_id(&self) -> u32 { - assert!(!self.states.is_empty()); self.states.len() as u32 - 1 } } diff --git a/crates/generate/src/node_types.rs b/cli/generate/src/node_types.rs similarity index 98% rename from crates/generate/src/node_types.rs rename to cli/generate/src/node_types.rs index 2dde0c49..07be52c6 100644 --- a/crates/generate/src/node_types.rs +++ b/cli/generate/src/node_types.rs @@ -1,5 +1,6 @@ -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; +use anyhow::Result; use serde::Serialize; use thiserror::Error; @@ -29,7 +30,6 @@ pub struct VariableInfo { } #[derive(Debug, Serialize, PartialEq, Eq, Default, PartialOrd, Ord)] -#[cfg(feature = "load")] pub struct NodeInfoJSON { #[serde(rename = "type")] kind: String, @@ -47,7 +47,6 @@ pub struct NodeInfoJSON { } #[derive(Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord, Hash)] -#[cfg(feature = "load")] pub struct NodeTypeJSON { #[serde(rename = "type")] kind: String, @@ -55,7 +54,6 @@ pub struct NodeTypeJSON { } #[derive(Debug, Serialize, PartialEq, Eq, PartialOrd, Ord)] -#[cfg(feature = "load")] pub struct FieldInfoJSON { multiple: bool, required: bool, @@ -69,7 +67,6 @@ pub struct ChildQuantity { multiple: bool, } -#[cfg(feature = "load")] impl Default for FieldInfoJSON { fn default() -> Self { Self { @@ -105,7 +102,7 @@ impl ChildQuantity { } } - const fn append(&mut self, other: Self) { + fn append(&mut self, other: Self) { if other.exists { if self.exists || other.multiple { self.multiple = true; @@ -117,7 +114,7 @@ impl ChildQuantity { } } - const fn union(&mut self, other: Self) -> bool { + fn union(&mut self, other: Self) -> bool { let mut result = false; if !self.exists && other.exists { result = true; @@ -377,11 +374,11 @@ pub fn get_variable_info( fn get_aliases_by_symbol( syntax_grammar: &SyntaxGrammar, default_aliases: &AliasMap, -) -> HashMap>> { +) -> HashMap>> { let mut aliases_by_symbol = HashMap::new(); for (symbol, alias) in default_aliases { aliases_by_symbol.insert(*symbol, { - let mut aliases = BTreeSet::new(); + let mut aliases = HashSet::new(); aliases.insert(Some(alias.clone())); aliases }); @@ -390,7 +387,7 @@ fn get_aliases_by_symbol( if !default_aliases.contains_key(extra_symbol) { aliases_by_symbol .entry(*extra_symbol) - .or_insert_with(BTreeSet::new) + .or_insert_with(HashSet::new) .insert(None); } } @@ -399,7 +396,7 @@ fn get_aliases_by_symbol( for step in &production.steps { aliases_by_symbol .entry(step.symbol) - .or_insert_with(BTreeSet::new) + .or_insert_with(HashSet::new) .insert( step.alias .as_ref() @@ -444,7 +441,6 @@ pub fn get_supertype_symbol_map( supertype_symbol_map } -#[cfg(feature = "load")] pub type SuperTypeCycleResult = Result; #[derive(Debug, Error, Serialize)] @@ -466,7 +462,6 @@ impl std::fmt::Display for SuperTypeCycleError { } } -#[cfg(feature = "load")] pub fn generate_node_types_json( syntax_grammar: &SyntaxGrammar, lexical_grammar: &LexicalGrammar, @@ -530,7 +525,7 @@ pub fn generate_node_types_json( let aliases_by_symbol = get_aliases_by_symbol(syntax_grammar, default_aliases); - let empty = BTreeSet::new(); + let empty = HashSet::new(); let extra_names = syntax_grammar .extra_symbols .iter() @@ -589,7 +584,7 @@ pub fn generate_node_types_json( } else if !syntax_grammar.variables_to_inline.contains(&symbol) { // If a rule is aliased under multiple names, then its information // contributes to multiple entries in the final JSON. - for alias in aliases_by_symbol.get(&symbol).unwrap_or(&BTreeSet::new()) { + for alias in aliases_by_symbol.get(&symbol).unwrap_or(&HashSet::new()) { let kind; let is_named; if let Some(alias) = alias { @@ -783,15 +778,11 @@ pub fn generate_node_types_json( a_is_leaf.cmp(&b_is_leaf) }) .then_with(|| a.kind.cmp(&b.kind)) - .then_with(|| a.named.cmp(&b.named)) - .then_with(|| a.root.cmp(&b.root)) - .then_with(|| a.extra.cmp(&b.extra)) }); result.dedup(); Ok(result) } -#[cfg(feature = "load")] fn process_supertypes(info: &mut FieldInfoJSON, subtype_map: &[(NodeTypeJSON, Vec)]) { for (supertype, subtypes) in subtype_map { if info.types.contains(supertype) { @@ -828,17 +819,17 @@ fn extend_sorted<'a, T>(vec: &mut Vec, values: impl IntoIterator, + content: Box, named: bool, value: String, }, @@ -33,46 +34,46 @@ enum RuleJSON { name: String, }, CHOICE { - members: Vec, + members: Vec, }, FIELD { name: String, - content: Box, + content: Box, }, SEQ { - members: Vec, + members: Vec, }, REPEAT { - content: Box, + content: Box, }, REPEAT1 { - content: Box, + content: Box, }, PREC_DYNAMIC { value: i32, - content: Box, + content: Box, }, PREC_LEFT { value: PrecedenceValueJSON, - content: Box, + content: Box, }, PREC_RIGHT { value: PrecedenceValueJSON, - content: Box, + content: Box, }, PREC { value: PrecedenceValueJSON, - content: Box, + content: Box, }, TOKEN { - content: Box, + content: Box, }, IMMEDIATE_TOKEN { - content: Box, + content: Box, }, RESERVED { context_name: String, - content: Box, + content: Box, }, } @@ -280,13 +281,7 @@ pub(crate) fn parse_grammar(input: &str) -> ParseGrammarResult { _ => false, }; if matches_empty { - warn!( - concat!( - "Named extra rule `{}` matches the empty string. ", - "Inline this to avoid infinite loops while parsing." - ), - name - ); + eprintln!("Warning: Named extra rule `{name}` matches the empty string. Inline this to avoid infinite loops while parsing."); } } variables.push(Variable { @@ -347,7 +342,7 @@ fn parse_rule(json: RuleJSON, is_token: bool) -> ParseGrammarResult { } else { // silently ignore unicode flags if c != 'u' && c != 'v' { - warn!("unsupported flag {c}"); + eprintln!("Warning: unsupported flag {c}"); } false } diff --git a/crates/generate/src/prepare_grammar/expand_repeats.rs b/cli/generate/src/prepare_grammar/expand_repeats.rs similarity index 100% rename from crates/generate/src/prepare_grammar/expand_repeats.rs rename to cli/generate/src/prepare_grammar/expand_repeats.rs diff --git a/crates/generate/src/prepare_grammar/expand_tokens.rs b/cli/generate/src/prepare_grammar/expand_tokens.rs similarity index 99% rename from crates/generate/src/prepare_grammar/expand_tokens.rs rename to cli/generate/src/prepare_grammar/expand_tokens.rs index acfb9ba3..2762b41c 100644 --- a/crates/generate/src/prepare_grammar/expand_tokens.rs +++ b/cli/generate/src/prepare_grammar/expand_tokens.rs @@ -1,3 +1,4 @@ +use anyhow::Result; use regex_syntax::{ hir::{Class, Hir, HirKind}, ParserBuilder, @@ -26,7 +27,7 @@ pub enum ExpandTokensError { "The rule `{0}` matches the empty string. Tree-sitter does not support syntactic rules that match the empty string unless they are used only as the grammar's start rule. -" + " )] EmptyString(String), #[error(transparent)] @@ -188,7 +189,7 @@ impl NfaBuilder { } Rule::String(s) => { for c in s.chars().rev() { - self.push_advance(CharacterSet::from_char(c), next_state_id); + self.push_advance(CharacterSet::empty().add_char(c), next_state_id); next_state_id = self.nfa.last_state_id(); } Ok(!s.is_empty()) diff --git a/crates/generate/src/prepare_grammar/extract_default_aliases.rs b/cli/generate/src/prepare_grammar/extract_default_aliases.rs similarity index 99% rename from crates/generate/src/prepare_grammar/extract_default_aliases.rs rename to cli/generate/src/prepare_grammar/extract_default_aliases.rs index cc977362..68ea1e48 100644 --- a/crates/generate/src/prepare_grammar/extract_default_aliases.rs +++ b/cli/generate/src/prepare_grammar/extract_default_aliases.rs @@ -69,7 +69,9 @@ pub(super) fn extract_default_aliases( SymbolType::External => &mut external_status_list[symbol.index], SymbolType::NonTerminal => &mut non_terminal_status_list[symbol.index], SymbolType::Terminal => &mut terminal_status_list[symbol.index], - SymbolType::End | SymbolType::EndOfNonTerminalExtra => panic!("Unexpected end token"), + SymbolType::End | SymbolType::EndOfNonTerminalExtra => { + panic!("Unexpected end token") + } }; status.appears_unaliased = true; } diff --git a/crates/generate/src/prepare_grammar/extract_tokens.rs b/cli/generate/src/prepare_grammar/extract_tokens.rs similarity index 95% rename from crates/generate/src/prepare_grammar/extract_tokens.rs rename to cli/generate/src/prepare_grammar/extract_tokens.rs index a7b4f227..579ded06 100644 --- a/crates/generate/src/prepare_grammar/extract_tokens.rs +++ b/cli/generate/src/prepare_grammar/extract_tokens.rs @@ -1,5 +1,6 @@ use std::collections::HashMap; +use anyhow::Result; use serde::Serialize; use thiserror::Error; @@ -152,7 +153,7 @@ pub(super) fn extract_tokens( } } - let mut external_tokens = Vec::with_capacity(grammar.external_tokens.len()); + let mut external_tokens = Vec::new(); for external_token in grammar.external_tokens { let rule = symbol_replacer.replace_symbols_in_rule(&external_token.rule); if let Rule::Symbol(symbol) = rule { @@ -180,7 +181,8 @@ pub(super) fn extract_tokens( } } - let word_token = if let Some(token) = grammar.word_token { + let mut word_token = None; + if let Some(token) = grammar.word_token { let token = symbol_replacer.replace_symbol(token); if token.is_non_terminal() { let word_token_variable = &variables[token.index]; @@ -195,10 +197,8 @@ pub(super) fn extract_tokens( conflicting_symbol_name, }))?; } - Some(token) - } else { - None - }; + word_token = Some(token); + } let mut reserved_word_contexts = Vec::with_capacity(grammar.reserved_word_sets.len()); for reserved_word_context in grammar.reserved_word_sets { @@ -212,12 +212,7 @@ pub(super) fn extract_tokens( { reserved_words.push(Symbol::terminal(index)); } else { - let rule = if let Rule::Metadata { rule, .. } = &reserved_rule { - rule.as_ref() - } else { - &reserved_rule - }; - let token_name = match rule { + let token_name = match &reserved_rule { Rule::String(s) => s.clone(), Rule::Pattern(p, _) => p.clone(), _ => "unknown".to_string(), @@ -285,11 +280,10 @@ impl TokenExtractor { let mut params = params.clone(); params.is_token = false; - let string_value = if let Rule::String(value) = rule.as_ref() { - Some(value) - } else { - None - }; + let mut string_value = None; + if let Rule::String(value) = rule.as_ref() { + string_value = Some(value); + } let rule_to_extract = if params == MetadataParams::default() { rule.as_ref() @@ -590,13 +584,14 @@ mod test { ]); grammar.external_tokens = vec![Variable::named("rule_1", Rule::non_terminal(1))]; - let result = extract_tokens(grammar); - assert!(result.is_err(), "Expected an error but got no error"); - let err = result.err().unwrap(); - assert_eq!( - err.to_string(), - "Rule 'rule_1' cannot be used as both an external token and a non-terminal rule" - ); + match extract_tokens(grammar) { + Err(e) => { + assert_eq!(e.to_string(), "Rule 'rule_1' cannot be used as both an external token and a non-terminal rule"); + } + _ => { + panic!("Expected an error but got no error"); + } + } } #[test] diff --git a/crates/generate/src/prepare_grammar/flatten_grammar.rs b/cli/generate/src/prepare_grammar/flatten_grammar.rs similarity index 99% rename from crates/generate/src/prepare_grammar/flatten_grammar.rs rename to cli/generate/src/prepare_grammar/flatten_grammar.rs index 3bec17bf..cb0f1dae 100644 --- a/crates/generate/src/prepare_grammar/flatten_grammar.rs +++ b/cli/generate/src/prepare_grammar/flatten_grammar.rs @@ -1,5 +1,6 @@ use std::collections::HashMap; +use anyhow::Result; use serde::Serialize; use thiserror::Error; diff --git a/crates/generate/src/prepare_grammar/intern_symbols.rs b/cli/generate/src/prepare_grammar/intern_symbols.rs similarity index 94% rename from crates/generate/src/prepare_grammar/intern_symbols.rs rename to cli/generate/src/prepare_grammar/intern_symbols.rs index 92f0f095..41a5b56e 100644 --- a/crates/generate/src/prepare_grammar/intern_symbols.rs +++ b/cli/generate/src/prepare_grammar/intern_symbols.rs @@ -1,4 +1,4 @@ -use log::warn; +use anyhow::Result; use serde::Serialize; use thiserror::Error; @@ -95,15 +95,14 @@ pub(super) fn intern_symbols(grammar: &InputGrammar) -> InternSymbolsResult { fn intern_rule(&self, rule: &Rule, name: Option<&str>) -> InternSymbolsResult { match rule { Rule::Choice(elements) => { - self.check_single(elements, name, "choice"); + self.check_single(elements, name); let mut result = Vec::with_capacity(elements.len()); for element in elements { result.push(self.intern_rule(element, name)?); @@ -140,7 +139,7 @@ impl Interner<'_> { Ok(Rule::Choice(result)) } Rule::Seq(elements) => { - self.check_single(elements, name, "seq"); + self.check_single(elements, name); let mut result = Vec::with_capacity(elements.len()); for element in elements { result.push(self.intern_rule(element, name)?); @@ -184,10 +183,10 @@ impl Interner<'_> { // In the case of a seq or choice rule of 1 element in a hidden rule, weird // inconsistent behavior with queries can occur. So we should warn the user about it. - fn check_single(&self, elements: &[Rule], name: Option<&str>, kind: &str) { + fn check_single(&self, elements: &[Rule], name: Option<&str>) { if elements.len() == 1 && matches!(elements[0], Rule::String(_) | Rule::Pattern(_, _)) { - warn!( - "rule {} contains a `{kind}` rule with a single element. This is unnecessary.", + eprintln!( + "Warning: rule {} contains a `seq` or `choice` rule with a single element. This is unnecessary.", name.unwrap_or_default() ); } @@ -278,9 +277,10 @@ mod tests { fn test_grammar_with_undefined_symbols() { let result = intern_symbols(&build_grammar(vec![Variable::named("x", Rule::named("y"))])); - assert!(result.is_err(), "Expected an error but got none"); - let e = result.err().unwrap(); - assert_eq!(e.to_string(), "Undefined symbol `y`"); + match result { + Err(e) => assert_eq!(e.to_string(), "Undefined symbol `y`"), + _ => panic!("Expected an error but got none"), + } } fn build_grammar(variables: Vec) -> InputGrammar { diff --git a/crates/generate/src/prepare_grammar.rs b/cli/generate/src/prepare_grammar/mod.rs similarity index 99% rename from crates/generate/src/prepare_grammar.rs rename to cli/generate/src/prepare_grammar/mod.rs index 58e0869c..10940ac8 100644 --- a/crates/generate/src/prepare_grammar.rs +++ b/cli/generate/src/prepare_grammar/mod.rs @@ -12,6 +12,7 @@ use std::{ mem, }; +use anyhow::Result; pub use expand_tokens::ExpandTokensError; pub use extract_tokens::ExtractTokensError; pub use flatten_grammar::FlattenGrammarError; @@ -267,7 +268,7 @@ fn validate_precedences(grammar: &InputGrammar) -> ValidatePrecedenceResult<()> if let Precedence::Name(n) = ¶ms.precedence { if !names.contains(n) { Err(UndeclaredPrecedenceError { - precedence: n.clone(), + precedence: n.to_string(), rule: rule_name.to_string(), })?; } diff --git a/crates/generate/src/prepare_grammar/process_inlines.rs b/cli/generate/src/prepare_grammar/process_inlines.rs similarity index 97% rename from crates/generate/src/prepare_grammar/process_inlines.rs rename to cli/generate/src/prepare_grammar/process_inlines.rs index 460d2359..085e6732 100644 --- a/crates/generate/src/prepare_grammar/process_inlines.rs +++ b/cli/generate/src/prepare_grammar/process_inlines.rs @@ -1,5 +1,6 @@ use std::collections::HashMap; +use anyhow::Result; use serde::Serialize; use thiserror::Error; @@ -70,13 +71,12 @@ impl InlinedProductionMapBuilder { let production_map = production_indices_by_step_id .into_iter() .map(|(step_id, production_indices)| { - let production = - core::ptr::from_ref::(step_id.variable_index.map_or_else( - || &productions[step_id.production_index], - |variable_index| { - &grammar.variables[variable_index].productions[step_id.production_index] - }, - )); + let production = step_id.variable_index.map_or_else( + || &productions[step_id.production_index], + |variable_index| { + &grammar.variables[variable_index].productions[step_id.production_index] + }, + ) as *const Production; ((production, step_id.step_index as u32), production_indices) }) .collect(); @@ -549,9 +549,10 @@ mod tests { ..Default::default() }; - let result = process_inlines(&grammar, &lexical_grammar); - assert!(result.is_err(), "expected an error, but got none"); - let err = result.err().unwrap(); - assert_eq!(err.to_string(), "Token `something` cannot be inlined",); + if let Err(error) = process_inlines(&grammar, &lexical_grammar) { + assert_eq!(error.to_string(), "Token `something` cannot be inlined"); + } else { + panic!("expected an error, but got none"); + } } } diff --git a/crates/generate/src/render.rs b/cli/generate/src/render.rs similarity index 99% rename from crates/generate/src/render.rs rename to cli/generate/src/render.rs index bcfc832e..2fd45d82 100644 --- a/crates/generate/src/render.rs +++ b/cli/generate/src/render.rs @@ -5,7 +5,6 @@ use std::{ mem::swap, }; -use crate::LANGUAGE_VERSION; use indoc::indoc; use super::{ @@ -22,10 +21,10 @@ use super::{ const SMALL_STATE_THRESHOLD: usize = 64; pub const ABI_VERSION_MIN: usize = 14; -pub const ABI_VERSION_MAX: usize = LANGUAGE_VERSION; +pub const ABI_VERSION_MAX: usize = tree_sitter::LANGUAGE_VERSION; const ABI_VERSION_WITH_RESERVED_WORDS: usize = 15; +const BUILD_VERSION: &str = env!("CARGO_PKG_VERSION"); -#[clippy::format_args] macro_rules! add { ($this: tt, $($arg: tt)*) => {{ $this.buffer.write_fmt(format_args!($($arg)*)).unwrap(); @@ -34,15 +33,12 @@ macro_rules! add { macro_rules! add_whitespace { ($this:tt) => {{ - // 4 bytes per char, 2 spaces per indent level - $this.buffer.reserve(4 * 2 * $this.indent_level); for _ in 0..$this.indent_level { write!(&mut $this.buffer, " ").unwrap(); } }}; } -#[clippy::format_args] macro_rules! add_line { ($this: tt, $($arg: tt)*) => { add_whitespace!($this); @@ -325,7 +321,10 @@ impl Generator { } fn add_header(&mut self) { - add_line!(self, "/* Automatically @generated by tree-sitter */",); + add_line!( + self, + "/* Automatically @generated by tree-sitter v{BUILD_VERSION} */", + ); add_line!(self, ""); } @@ -690,14 +689,13 @@ impl Generator { flat_field_map.push((field_name.clone(), *location)); } } - let field_map_len = flat_field_map.len(); field_map_ids.push(( self.get_field_map_id( - flat_field_map, + flat_field_map.clone(), &mut flat_field_maps, &mut next_flat_field_map_index, ), - field_map_len, + flat_field_map.len(), )); } } @@ -965,7 +963,10 @@ impl Generator { large_char_set_ix = Some(char_set_ix); } - let line_break = format!("\n{}", " ".repeat(self.indent_level + 2)); + let mut line_break = "\n".to_string(); + for _ in 0..self.indent_level + 2 { + line_break.push_str(" "); + } let has_positive_condition = large_char_set_ix.is_some() || !asserted_chars.is_empty(); let has_negative_condition = !negated_chars.is_empty(); diff --git a/crates/generate/src/rules.rs b/cli/generate/src/rules.rs similarity index 98% rename from crates/generate/src/rules.rs rename to cli/generate/src/rules.rs index 05a0c426..cd4aa482 100644 --- a/crates/generate/src/rules.rs +++ b/cli/generate/src/rules.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, fmt}; +use std::{collections::HashMap, fmt}; use serde::Serialize; use smallbitvec::SmallBitVec; @@ -34,7 +34,7 @@ pub enum Precedence { Name(String), } -pub type AliasMap = BTreeMap; +pub type AliasMap = HashMap; #[derive(Clone, Debug, Default, PartialEq, Eq, Hash, Serialize)] pub struct MetadataParams { @@ -60,15 +60,15 @@ pub enum Rule { Pattern(String, String), NamedSymbol(String), Symbol(Symbol), - Choice(Vec), + Choice(Vec), Metadata { params: MetadataParams, - rule: Box, + rule: Box, }, - Repeat(Box), - Seq(Vec), + Repeat(Box), + Seq(Vec), Reserved { - rule: Box, + rule: Box, context_name: String, }, } diff --git a/crates/generate/src/tables.rs b/cli/generate/src/tables.rs similarity index 100% rename from crates/generate/src/tables.rs rename to cli/generate/src/tables.rs diff --git a/crates/generate/src/templates/alloc.h b/cli/generate/src/templates/alloc.h similarity index 100% rename from crates/generate/src/templates/alloc.h rename to cli/generate/src/templates/alloc.h diff --git a/crates/generate/src/templates/array.h b/cli/generate/src/templates/array.h similarity index 100% rename from crates/generate/src/templates/array.h rename to cli/generate/src/templates/array.h diff --git a/crates/loader/Cargo.toml b/cli/loader/Cargo.toml similarity index 88% rename from crates/loader/Cargo.toml rename to cli/loader/Cargo.toml index cecae218..e42274d9 100644 --- a/crates/loader/Cargo.toml +++ b/cli/loader/Cargo.toml @@ -8,7 +8,6 @@ rust-version.workspace = true readme = "README.md" homepage.workspace = true repository.workspace = true -documentation = "https://docs.rs/tree-sitter-loader" license.workspace = true keywords.workspace = true categories.workspace = true @@ -17,9 +16,6 @@ categories.workspace = true all-features = true rustdoc-args = ["--cfg", "docsrs"] -[lib] -path = "src/loader.rs" - [lints] workspace = true @@ -28,19 +24,20 @@ wasm = ["tree-sitter/wasm"] default = ["tree-sitter-highlight", "tree-sitter-tags"] [dependencies] +anyhow.workspace = true cc.workspace = true etcetera.workspace = true fs4.workspace = true indoc.workspace = true libloading.workspace = true -log.workspace = true once_cell.workspace = true +path-slash.workspace = true regex.workspace = true semver.workspace = true serde.workspace = true serde_json.workspace = true tempfile.workspace = true -thiserror.workspace = true +url.workspace = true tree-sitter = { workspace = true } tree-sitter-highlight = { workspace = true, optional = true } diff --git a/crates/loader/README.md b/cli/loader/README.md similarity index 100% rename from crates/loader/README.md rename to cli/loader/README.md diff --git a/crates/loader/build.rs b/cli/loader/build.rs similarity index 58% rename from crates/loader/build.rs rename to cli/loader/build.rs index 5ffc889a..b01f01bb 100644 --- a/crates/loader/build.rs +++ b/cli/loader/build.rs @@ -7,4 +7,7 @@ fn main() { "cargo:rustc-env=BUILD_HOST={}", std::env::var("HOST").unwrap() ); + + let emscripten_version = std::fs::read_to_string("emscripten-version").unwrap(); + println!("cargo:rustc-env=EMSCRIPTEN_VERSION={emscripten_version}"); } diff --git a/cli/loader/emscripten-version b/cli/loader/emscripten-version new file mode 100644 index 00000000..7d666cb2 --- /dev/null +++ b/cli/loader/emscripten-version @@ -0,0 +1 @@ +4.0.4 \ No newline at end of file diff --git a/crates/loader/src/loader.rs b/cli/loader/src/lib.rs similarity index 55% rename from crates/loader/src/loader.rs rename to cli/loader/src/lib.rs index e7584378..a2ae3eb1 100644 --- a/crates/loader/src/loader.rs +++ b/cli/loader/src/lib.rs @@ -1,4 +1,4 @@ -#![cfg_attr(not(any(test, doctest)), doc = include_str!("../README.md"))] +#![doc = include_str!("../README.md")] #![cfg_attr(docsrs, feature(doc_cfg))] #[cfg(any(feature = "tree-sitter-highlight", feature = "tree-sitter-tags"))] @@ -7,254 +7,43 @@ use std::ops::Range; use std::sync::Mutex; use std::{ collections::HashMap, - env, fs, - hash::{Hash as _, Hasher as _}, + env, + ffi::{OsStr, OsString}, + fs, io::{BufRead, BufReader}, marker::PhantomData, mem, path::{Path, PathBuf}, process::Command, sync::LazyLock, - time::{SystemTime, SystemTimeError}, + time::SystemTime, }; +use anyhow::{anyhow, Context, Error, Result}; use etcetera::BaseStrategy as _; use fs4::fs_std::FileExt; +use indoc::indoc; use libloading::{Library, Symbol}; -use log::{error, info, warn}; use once_cell::unsync::OnceCell; +use path_slash::PathBufExt as _; use regex::{Regex, RegexBuilder}; use semver::Version; use serde::{Deserialize, Deserializer, Serialize}; -use thiserror::Error; use tree_sitter::Language; #[cfg(any(feature = "tree-sitter-highlight", feature = "tree-sitter-tags"))] use tree_sitter::QueryError; #[cfg(feature = "tree-sitter-highlight")] use tree_sitter::QueryErrorKind; -#[cfg(feature = "wasm")] -use tree_sitter::WasmError; #[cfg(feature = "tree-sitter-highlight")] use tree_sitter_highlight::HighlightConfiguration; #[cfg(feature = "tree-sitter-tags")] use tree_sitter_tags::{Error as TagsError, TagsConfiguration}; +use url::Url; static GRAMMAR_NAME_REGEX: LazyLock = LazyLock::new(|| Regex::new(r#""name":\s*"(.*?)""#).unwrap()); -const WASI_SDK_VERSION: &str = include_str!("../wasi-sdk-version").trim_ascii(); - -pub type LoaderResult = Result; - -#[derive(Debug, Error)] -pub enum LoaderError { - #[error(transparent)] - Compiler(CompilerError), - #[error("Parser compilation failed.\nStdout: {0}\nStderr: {1}")] - Compilation(String, String), - #[error("Failed to execute curl for {0} -- {1}")] - Curl(String, std::io::Error), - #[error("Failed to load language in current directory:\n{0}")] - CurrentDirectoryLoad(Box), - #[error("External file path {0} is outside of parser directory {1}")] - ExternalFile(String, String), - #[error("Failed to extract archive {0} to {1}")] - Extraction(String, String), - #[error("Failed to load language for file name {0}:\n{1}")] - FileNameLoad(String, Box), - #[error("Failed to parse the language name from grammar.json at {0}")] - GrammarJSON(String), - #[error(transparent)] - HomeDir(#[from] etcetera::HomeDirError), - #[error(transparent)] - IO(IoError), - #[error(transparent)] - Library(LibraryError), - #[error("Failed to compare binary and source timestamps:\n{0}")] - ModifiedTime(Box), - #[error("No language found")] - NoLanguage, - #[error(transparent)] - Query(LoaderQueryError), - #[error(transparent)] - ScannerSymbols(ScannerSymbolError), - #[error("Failed to load language for scope '{0}':\n{1}")] - ScopeLoad(String, Box), - #[error(transparent)] - Serialization(#[from] serde_json::Error), - #[error(transparent)] - Symbol(SymbolError), - #[error(transparent)] - Tags(#[from] TagsError), - #[error("Failed to execute tar for {0} -- {1}")] - Tar(String, std::io::Error), - #[error(transparent)] - Time(#[from] SystemTimeError), - #[error("Unknown scope '{0}'")] - UnknownScope(String), - #[error("Failed to download wasi-sdk from {0}")] - WasiSDKDownload(String), - #[error(transparent)] - WasiSDKClang(#[from] WasiSDKClangError), - #[error("Unsupported platform for wasi-sdk")] - WasiSDKPlatform, - #[cfg(feature = "wasm")] - #[error(transparent)] - Wasm(#[from] WasmError), - #[error("Failed to run wasi-sdk clang -- {0}")] - WasmCompiler(std::io::Error), - #[error("wasi-sdk clang command failed: {0}")] - WasmCompilation(String), -} - -#[derive(Debug, Error)] -pub struct CompilerError { - pub error: std::io::Error, - pub command: Box, -} - -impl std::fmt::Display for CompilerError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "Failed to execute the C compiler with the following command:\n{:?}\nError: {}", - *self.command, self.error - )?; - Ok(()) - } -} - -#[derive(Debug, Error)] -pub struct IoError { - pub error: std::io::Error, - pub path: Option, -} - -impl IoError { - fn new(error: std::io::Error, path: Option<&Path>) -> Self { - Self { - error, - path: path.map(|p| p.to_string_lossy().to_string()), - } - } -} - -impl std::fmt::Display for IoError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.error)?; - if let Some(ref path) = self.path { - write!(f, " ({path})")?; - } - Ok(()) - } -} - -#[derive(Debug, Error)] -pub struct LibraryError { - pub error: libloading::Error, - pub path: String, -} - -impl std::fmt::Display for LibraryError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "Error opening dynamic library {} -- {}", - self.path, self.error - )?; - Ok(()) - } -} - -#[derive(Debug, Error)] -pub struct LoaderQueryError { - pub error: QueryError, - pub file: Option, -} - -impl std::fmt::Display for LoaderQueryError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - if let Some(ref path) = self.file { - writeln!(f, "Error in query file {path}:")?; - } - write!(f, "{}", self.error)?; - Ok(()) - } -} - -#[derive(Debug, Error)] -pub struct SymbolError { - pub error: libloading::Error, - pub symbol_name: String, - pub path: String, -} - -impl std::fmt::Display for SymbolError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "Failed to load symbol {} from {} -- {}", - self.symbol_name, self.path, self.error - )?; - Ok(()) - } -} - -#[derive(Debug, Error)] -pub struct ScannerSymbolError { - pub missing: Vec, -} - -impl std::fmt::Display for ScannerSymbolError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - writeln!( - f, - "Missing required functions in the external scanner, parsing won't work without these!\n" - )?; - for symbol in &self.missing { - writeln!(f, " `{symbol}`")?; - } - writeln!( - f, - "You can read more about this at https://tree-sitter.github.io/tree-sitter/creating-parsers/4-external-scanners\n" - )?; - Ok(()) - } -} - -#[derive(Debug, Error)] -pub struct WasiSDKClangError { - pub wasi_sdk_dir: String, - pub possible_executables: Vec<&'static str>, - pub download: bool, -} - -impl std::fmt::Display for WasiSDKClangError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - if self.download { - write!( - f, - "Failed to find clang executable in downloaded wasi-sdk at '{}'.", - self.wasi_sdk_dir - )?; - } else { - write!(f, "TREE_SITTER_WASI_SDK_PATH is set to '{}', but no clang executable found in 'bin/' directory.", self.wasi_sdk_dir)?; - } - - let possible_exes = self.possible_executables.join(", "); - write!(f, " Looked for: {possible_exes}.")?; - - Ok(()) - } -} - -pub const DEFAULT_HIGHLIGHTS_QUERY_FILE_NAME: &str = "highlights.scm"; - -pub const DEFAULT_INJECTIONS_QUERY_FILE_NAME: &str = "injections.scm"; - -pub const DEFAULT_LOCALS_QUERY_FILE_NAME: &str = "locals.scm"; - -pub const DEFAULT_TAGS_QUERY_FILE_NAME: &str = "tags.scm"; +pub const EMSCRIPTEN_TAG: &str = concat!("docker.io/emscripten/emsdk:", env!("EMSCRIPTEN_VERSION")); #[derive(Default, Deserialize, Serialize)] pub struct Config { @@ -287,17 +76,6 @@ impl PathsJSON { const fn is_empty(&self) -> bool { matches!(self, Self::Empty) } - - /// Represent this set of paths as a string that can be included in templates - #[must_use] - pub fn to_variable_value<'a>(&'a self, default: &'a PathBuf) -> &'a str { - match self { - Self::Empty => Some(default), - Self::Single(path_buf) => Some(path_buf), - Self::Multiple(paths) => paths.first(), - } - .map_or("", |path| path.as_os_str().to_str().unwrap_or("")) - } } #[derive(Serialize, Deserialize, Clone)] @@ -370,10 +148,9 @@ pub struct TreeSitterJSON { } impl TreeSitterJSON { - pub fn from_file(path: &Path) -> LoaderResult { - let path = path.join("tree-sitter.json"); - Ok(serde_json::from_str(&fs::read_to_string(&path).map_err( - |e| LoaderError::IO(IoError::new(e, Some(path.as_path()))), + pub fn from_file(path: &Path) -> Result { + Ok(serde_json::from_str(&fs::read_to_string( + path.join("tree-sitter.json"), )?)?) } @@ -441,16 +218,19 @@ pub struct Author { #[derive(Serialize, Deserialize)] pub struct Links { - pub repository: String, + pub repository: Url, #[serde(skip_serializing_if = "Option::is_none")] - pub funding: Option, + pub funding: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub homepage: Option, } -#[derive(Serialize, Deserialize, Clone)] +#[derive(Serialize, Deserialize)] #[serde(default)] pub struct Bindings { pub c: bool, pub go: bool, + #[serde(skip)] pub java: bool, #[serde(skip)] pub kotlin: bool, @@ -461,61 +241,6 @@ pub struct Bindings { pub zig: bool, } -impl Bindings { - /// return available languages and its default enabled state. - #[must_use] - pub const fn languages(&self) -> [(&'static str, bool); 8] { - [ - ("c", true), - ("go", true), - ("java", false), - // Comment out Kotlin until the bindings are actually available. - // ("kotlin", false), - ("node", true), - ("python", true), - ("rust", true), - ("swift", true), - ("zig", false), - ] - } - - /// construct Bindings from a language list. If a language isn't supported, its name will be put on the error part. - pub fn with_enabled_languages<'a, I>(languages: I) -> Result - where - I: Iterator, - { - let mut out = Self { - c: false, - go: false, - java: false, - kotlin: false, - node: false, - python: false, - rust: false, - swift: false, - zig: false, - }; - - for v in languages { - match v { - "c" => out.c = true, - "go" => out.go = true, - "java" => out.java = true, - // Comment out Kotlin until the bindings are actually available. - // "kotlin" => out.kotlin = true, - "node" => out.node = true, - "python" => out.python = true, - "rust" => out.rust = true, - "swift" => out.swift = true, - "zig" => out.zig = true, - unsupported => return Err(unsupported), - } - } - - Ok(out) - } -} - impl Default for Bindings { fn default() -> Self { Self { @@ -578,6 +303,7 @@ impl Config { } const BUILD_TARGET: &str = env!("BUILD_TARGET"); +const BUILD_HOST: &str = env!("BUILD_HOST"); pub struct LanguageConfiguration<'a> { pub scope: Option, @@ -658,7 +384,7 @@ impl<'a> CompileConfig<'a> { unsafe impl Sync for Loader {} impl Loader { - pub fn new() -> LoaderResult { + pub fn new() -> Result { let parser_lib_path = if let Ok(path) = env::var("TREE_SITTER_LIBDIR") { PathBuf::from(path) } else { @@ -667,9 +393,7 @@ impl Loader { .cache_dir() // `$HOME/Library/Caches/` .join("tree-sitter"); if legacy_apple_path.exists() && legacy_apple_path.is_dir() { - std::fs::remove_dir_all(&legacy_apple_path).map_err(|e| { - LoaderError::IO(IoError::new(e, Some(legacy_apple_path.as_path()))) - })?; + std::fs::remove_dir_all(legacy_apple_path)?; } } @@ -719,19 +443,17 @@ impl Loader { self.highlight_names.lock().unwrap().clone() } - pub fn find_all_languages(&mut self, config: &Config) -> LoaderResult<()> { + pub fn find_all_languages(&mut self, config: &Config) -> Result<()> { if config.parser_directories.is_empty() { - warn!(concat!( - "You have not configured any parser directories!\n", - "Please run `tree-sitter init-config` and edit the resulting\n", - "configuration file to indicate where we should look for\n", - "language grammars.\n" - )); + eprintln!("Warning: You have not configured any parser directories!"); + eprintln!("Please run `tree-sitter init-config` and edit the resulting"); + eprintln!("configuration file to indicate where we should look for"); + eprintln!("language grammars.\n"); } for parser_container_dir in &config.parser_directories { if let Ok(entries) = fs::read_dir(parser_container_dir) { for entry in entries { - let entry = entry.map_err(|e| LoaderError::IO(IoError::new(e, None)))?; + let entry = entry?; if let Some(parser_dir_name) = entry.file_name().to_str() { if parser_dir_name.starts_with("tree-sitter-") { self.find_language_configurations_at_path( @@ -747,7 +469,7 @@ impl Loader { Ok(()) } - pub fn languages_at_path(&mut self, path: &Path) -> LoaderResult> { + pub fn languages_at_path(&mut self, path: &Path) -> Result> { if let Ok(configurations) = self.find_language_configurations_at_path(path, true) { let mut language_ids = configurations .iter() @@ -758,14 +480,14 @@ impl Loader { language_ids .into_iter() .map(|(id, name)| Ok((self.language_for_id(id)?, name))) - .collect::>>() + .collect::>>() } else { Ok(Vec::new()) } } #[must_use] - pub fn get_all_language_configurations(&self) -> Vec<(&LanguageConfiguration<'static>, &Path)> { + pub fn get_all_language_configurations(&self) -> Vec<(&LanguageConfiguration, &Path)> { self.language_configurations .iter() .map(|c| (c, self.languages_by_id[c.language_id].0.as_ref())) @@ -775,7 +497,7 @@ impl Loader { pub fn language_configuration_for_scope( &self, scope: &str, - ) -> LoaderResult)>> { + ) -> Result> { for configuration in &self.language_configurations { if configuration.scope.as_ref().is_some_and(|s| s == scope) { let language = self.language_for_id(configuration.language_id)?; @@ -788,19 +510,14 @@ impl Loader { pub fn language_configuration_for_first_line_regex( &self, path: &Path, - ) -> LoaderResult)>> { + ) -> Result> { self.language_configuration_ids_by_first_line_regex .iter() .try_fold(None, |_, (regex, ids)| { if let Some(regex) = Self::regex(Some(regex)) { - let file = fs::File::open(path) - .map_err(|e| LoaderError::IO(IoError::new(e, Some(path))))?; + let file = fs::File::open(path)?; let reader = BufReader::new(file); - let first_line = reader - .lines() - .next() - .transpose() - .map_err(|e| LoaderError::IO(IoError::new(e, Some(path))))?; + let first_line = reader.lines().next().transpose()?; if let Some(first_line) = first_line { if regex.is_match(&first_line) && !ids.is_empty() { let configuration = &self.language_configurations[ids[0]]; @@ -817,7 +534,7 @@ impl Loader { pub fn language_configuration_for_file_name( &self, path: &Path, - ) -> LoaderResult)>> { + ) -> Result> { // Find all the language configurations that match this file name // or a suffix of the file name. let configuration_ids = path @@ -844,8 +561,8 @@ impl Loader { // If multiple language configurations match, then determine which // one to use by applying the configurations' content regexes. else { - let file_contents = - fs::read(path).map_err(|e| LoaderError::IO(IoError::new(e, Some(path))))?; + let file_contents = fs::read(path) + .with_context(|| format!("Failed to read path {}", path.display()))?; let file_contents = String::from_utf8_lossy(&file_contents); let mut best_score = -2isize; let mut best_configuration_id = None; @@ -889,7 +606,7 @@ impl Loader { pub fn language_configuration_for_injection_string( &self, string: &str, - ) -> LoaderResult)>> { + ) -> Result> { let mut best_match_length = 0; let mut best_match_position = None; for (i, configuration) in self.language_configurations.iter().enumerate() { @@ -916,11 +633,11 @@ impl Loader { pub fn language_for_configuration( &self, configuration: &LanguageConfiguration, - ) -> LoaderResult { + ) -> Result { self.language_for_id(configuration.language_id) } - fn language_for_id(&self, id: usize) -> LoaderResult { + fn language_for_id(&self, id: usize) -> Result { let (path, language, externals) = &self.languages_by_id[id]; language .get_or_try_init(|| { @@ -939,25 +656,25 @@ impl Loader { grammar_path: &Path, output_path: PathBuf, flags: &[&str], - ) -> LoaderResult<()> { + ) -> Result<()> { let src_path = grammar_path.join("src"); let mut config = CompileConfig::new(&src_path, None, Some(output_path)); config.flags = flags; self.load_language_at_path(config).map(|_| ()) } - pub fn load_language_at_path(&self, mut config: CompileConfig) -> LoaderResult { + pub fn load_language_at_path(&self, mut config: CompileConfig) -> Result { let grammar_path = config.src_path.join("grammar.json"); config.name = Self::grammar_json_name(&grammar_path)?; self.load_language_at_path_with_name(config) } - pub fn load_language_at_path_with_name( - &self, - mut config: CompileConfig, - ) -> LoaderResult { - let mut lib_name = config.name.clone(); - let language_fn_name = format!("tree_sitter_{}", config.name.replace('-', "_")); + pub fn load_language_at_path_with_name(&self, mut config: CompileConfig) -> Result { + let mut lib_name = config.name.to_string(); + let language_fn_name = format!( + "tree_sitter_{}", + replace_dashes_with_underscores(&config.name) + ); if self.debug_build { lib_name.push_str(".debug._"); } @@ -968,9 +685,7 @@ impl Loader { } if config.output_path.is_none() { - fs::create_dir_all(&self.parser_lib_path).map_err(|e| { - LoaderError::IO(IoError::new(e, Some(self.parser_lib_path.as_path()))) - })?; + fs::create_dir_all(&self.parser_lib_path)?; } let mut recompile = self.force_rebuild || config.output_path.is_some(); // if specified, always recompile @@ -1004,7 +719,8 @@ impl Loader { ); if !recompile { - recompile = needs_recompile(&output_path, &paths_to_check)?; + recompile = needs_recompile(&output_path, &paths_to_check) + .with_context(|| "Failed to compare source and binary timestamps")?; } #[cfg(feature = "wasm")] @@ -1012,82 +728,68 @@ impl Loader { if recompile { self.compile_parser_to_wasm( &config.name, + None, config.src_path, config .scanner_path .as_ref() .and_then(|p| p.strip_prefix(config.src_path).ok()), &output_path, + false, )?; } - let wasm_bytes = fs::read(&output_path) - .map_err(|e| LoaderError::IO(IoError::new(e, Some(output_path.as_path()))))?; + let wasm_bytes = fs::read(&output_path)?; return Ok(wasm_store.load_language(&config.name, &wasm_bytes)?); } - // Create a unique lock path based on the output path hash to prevent - // interference when multiple processes build the same grammar (by name) - // to different output locations - let lock_hash = { - let mut hasher = std::hash::DefaultHasher::new(); - output_path.hash(&mut hasher); - format!("{:x}", hasher.finish()) - }; - let lock_path = if env::var("CROSS_RUNNER").is_ok() { tempfile::tempdir() - .expect("create a temp dir") + .unwrap() .path() - .to_path_buf() + .join("tree-sitter") + .join("lock") + .join(format!("{}.lock", config.name)) } else { - etcetera::choose_base_strategy()?.cache_dir() - } - .join("tree-sitter") - .join("lock") - .join(format!("{}-{lock_hash}.lock", config.name)); + etcetera::choose_base_strategy()? + .cache_dir() + .join("tree-sitter") + .join("lock") + .join(format!("{}.lock", config.name)) + }; if let Ok(lock_file) = fs::OpenOptions::new().write(true).open(&lock_path) { recompile = false; if lock_file.try_lock_exclusive().is_err() { // if we can't acquire the lock, another process is compiling the parser, wait for // it and don't recompile - lock_file - .lock_exclusive() - .map_err(|e| LoaderError::IO(IoError::new(e, Some(lock_path.as_path()))))?; + lock_file.lock_exclusive()?; recompile = false; } else { // if we can acquire the lock, check if the lock file is older than 30 seconds, a // run that was interrupted and left the lock file behind should not block // subsequent runs - let time = lock_file - .metadata() - .map_err(|e| LoaderError::IO(IoError::new(e, Some(lock_path.as_path()))))? - .modified() - .map_err(|e| LoaderError::IO(IoError::new(e, Some(lock_path.as_path()))))? - .elapsed()? - .as_secs(); + let time = lock_file.metadata()?.modified()?.elapsed()?.as_secs(); if time > 30 { - fs::remove_file(&lock_path) - .map_err(|e| LoaderError::IO(IoError::new(e, Some(lock_path.as_path()))))?; + fs::remove_file(&lock_path)?; recompile = true; } } } if recompile { - let parent_path = lock_path.parent().unwrap(); - fs::create_dir_all(parent_path) - .map_err(|e| LoaderError::IO(IoError::new(e, Some(parent_path))))?; + fs::create_dir_all(lock_path.parent().unwrap()).with_context(|| { + format!( + "Failed to create directory {}", + lock_path.parent().unwrap().display() + ) + })?; let lock_file = fs::OpenOptions::new() .create(true) .truncate(true) .write(true) - .open(&lock_path) - .map_err(|e| LoaderError::IO(IoError::new(e, Some(lock_path.as_path()))))?; - lock_file - .lock_exclusive() - .map_err(|e| LoaderError::IO(IoError::new(e, Some(lock_path.as_path()))))?; + .open(&lock_path)?; + lock_file.lock_exclusive()?; self.compile_parser_to_dylib(&config, &lock_file, &lock_path)?; @@ -1096,46 +798,12 @@ impl Loader { } } - // Ensure the dynamic library exists before trying to load it. This can - // happen in race conditions where we couldn't acquire the lock because - // another process was compiling but it still hasn't finished by the - // time we reach this point, so the output file still doesn't exist. - // - // Instead of allowing the `load_language` call below to fail, return a - // clearer error to the user here. - if !output_path.exists() { - let msg = format!( - "Dynamic library `{}` not found after build attempt. \ - Are you running multiple processes building to the same output location?", - output_path.display() - ); - - Err(LoaderError::IO(IoError::new( - std::io::Error::new(std::io::ErrorKind::NotFound, msg), - Some(output_path.as_path()), - )))?; - } - - Self::load_language(&output_path, &language_fn_name) - } - - pub fn load_language(path: &Path, function_name: &str) -> LoaderResult { - let library = unsafe { Library::new(path) }.map_err(|e| { - LoaderError::Library(LibraryError { - error: e, - path: path.to_string_lossy().to_string(), - }) - })?; + let library = unsafe { Library::new(&output_path) } + .with_context(|| format!("Error opening dynamic library {}", output_path.display()))?; let language = unsafe { let language_fn = library - .get:: Language>>(function_name.as_bytes()) - .map_err(|e| { - LoaderError::Symbol(SymbolError { - error: e, - symbol_name: function_name.to_string(), - path: path.to_string_lossy().to_string(), - }) - })?; + .get:: Language>>(language_fn_name.as_bytes()) + .with_context(|| format!("Failed to load symbol {language_fn_name}"))?; language_fn() }; mem::forget(library); @@ -1147,16 +815,13 @@ impl Loader { config: &CompileConfig, lock_file: &fs::File, lock_path: &Path, - ) -> LoaderResult<()> { + ) -> Result<(), Error> { let mut cc_config = cc::Build::new(); cc_config .cargo_metadata(false) .cargo_warnings(false) .target(BUILD_TARGET) - // BUILD_TARGET from the build environment becomes a runtime host for cc. - // Otherwise, when cross compiled, cc will keep looking for a cross-compiler - // on the target system instead of the native compiler. - .host(BUILD_TARGET) + .host(BUILD_HOST) .debug(self.debug_build) .file(&config.parser_path) .includes(&config.header_paths) @@ -1185,27 +850,12 @@ impl Loader { let output_path = config.output_path.as_ref().unwrap(); - let temp_dir = if compiler.is_like_msvc() { + if compiler.is_like_msvc() { let out = format!("-out:{}", output_path.to_str().unwrap()); command.arg(if self.debug_build { "-LDd" } else { "-LD" }); command.arg("-utf-8"); - - // Windows creates intermediate files when compiling (.exp, .lib, .obj), which causes - // issues when multiple processes are compiling in the same directory. This creates a - // temporary directory for those files to go into, which is deleted after compilation. - let temp_dir = output_path.parent().unwrap().join(format!( - "tmp_{}_{:?}", - std::process::id(), - std::thread::current().id() - )); - std::fs::create_dir_all(&temp_dir).unwrap(); - - command.arg(format!("/Fo{}\\", temp_dir.display())); command.args(cc_config.get_files()); command.arg("-link").arg(out); - command.arg(format!("/IMPLIB:{}.lib", temp_dir.join("temp").display())); - - Some(temp_dir) } else { command.arg("-Werror=implicit-function-declaration"); if cfg!(any(target_os = "macos", target_os = "ios")) { @@ -1217,48 +867,33 @@ impl Loader { } command.args(cc_config.get_files()); command.arg("-o").arg(output_path); - - None - }; - - let output = command.output().map_err(|e| { - LoaderError::Compiler(CompilerError { - error: e, - command: Box::new(command), - }) - })?; - - if let Some(temp_dir) = temp_dir { - let _ = fs::remove_dir_all(temp_dir); } - FileExt::unlock(lock_file) - .map_err(|e| LoaderError::IO(IoError::new(e, Some(lock_path))))?; - fs::remove_file(lock_path) - .map_err(|e| LoaderError::IO(IoError::new(e, Some(lock_path))))?; + let output = command.output().with_context(|| { + format!("Failed to execute the C compiler with the following command:\n{command:?}") + })?; + + FileExt::unlock(lock_file)?; + fs::remove_file(lock_path)?; if output.status.success() { Ok(()) } else { - Err(LoaderError::Compilation( - String::from_utf8_lossy(&output.stdout).to_string(), - String::from_utf8_lossy(&output.stderr).to_string(), + Err(anyhow!( + "Parser compilation failed.\nStdout: {}\nStderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) )) } } #[cfg(unix)] - fn check_external_scanner(&self, name: &str, library_path: &Path) -> LoaderResult<()> { + fn check_external_scanner(&self, name: &str, library_path: &Path) -> Result<()> { let prefix = if cfg!(any(target_os = "macos", target_os = "ios")) { "_" } else { "" }; - let section = if cfg!(all(target_arch = "powerpc64", target_os = "linux")) { - " D " - } else { - " T " - }; let mut must_have = vec![ format!("{prefix}tree_sitter_{name}_external_scanner_create"), format!("{prefix}tree_sitter_{name}_external_scanner_destroy"), @@ -1267,25 +902,25 @@ impl Loader { format!("{prefix}tree_sitter_{name}_external_scanner_scan"), ]; - let nm_cmd = env::var("NM").unwrap_or_else(|_| "nm".to_owned()); - let command = Command::new(nm_cmd) - .arg("--defined-only") + let command = Command::new("nm") + .arg("-W") + .arg("-U") .arg(library_path) .output(); if let Ok(output) = command { if output.status.success() { let mut found_non_static = false; for line in String::from_utf8_lossy(&output.stdout).lines() { - if line.contains(section) { + if line.contains(" T ") { if let Some(function_name) = line.split_whitespace().collect::>().get(2) { if !line.contains("tree_sitter_") { if !found_non_static { found_non_static = true; - warn!("Found non-static non-tree-sitter functions in the external scanner"); + eprintln!("Warning: Found non-static non-tree-sitter functions in the external scannner"); } - warn!(" `{function_name}`"); + eprintln!(" `{function_name}`"); } else { must_have.retain(|f| f != function_name); } @@ -1293,30 +928,35 @@ impl Loader { } } if found_non_static { - warn!(concat!( - "Consider making these functions static, they can cause conflicts ", - "when another tree-sitter project uses the same function name." - )); + eprintln!("Consider making these functions static, they can cause conflicts when another tree-sitter project uses the same function name"); } if !must_have.is_empty() { - return Err(LoaderError::ScannerSymbols(ScannerSymbolError { - missing: must_have, - })); + let missing = must_have + .iter() + .map(|f| format!(" `{f}`")) + .collect::>() + .join("\n"); + + return Err(anyhow!(format!( + indoc! {" + Missing required functions in the external scanner, parsing won't work without these! + + {} + + You can read more about this at https://tree-sitter.github.io/tree-sitter/creating-parsers/4-external-scanners + "}, + missing, + ))); } } - } else { - warn!( - "Failed to run `nm` to verify symbols in {}", - library_path.display() - ); } Ok(()) } #[cfg(windows)] - fn check_external_scanner(&self, _name: &str, _library_path: &Path) -> LoaderResult<()> { + fn check_external_scanner(&self, _name: &str, _library_path: &Path) -> Result<()> { // TODO: there's no nm command on windows, whoever wants to implement this can and should :) // let mut must_have = vec![ @@ -1333,184 +973,142 @@ impl Loader { pub fn compile_parser_to_wasm( &self, language_name: &str, + root_path: Option<&Path>, src_path: &Path, scanner_filename: Option<&Path>, output_path: &Path, - ) -> LoaderResult<()> { - let clang_executable = self.ensure_wasi_sdk_exists()?; + force_docker: bool, + ) -> Result<(), Error> { + #[derive(PartialEq, Eq)] + enum EmccSource { + Native, + Docker, + Podman, + } - let mut command = Command::new(&clang_executable); - command.current_dir(src_path).args([ + let root_path = root_path.unwrap_or(src_path); + let emcc_name = if cfg!(windows) { "emcc.bat" } else { "emcc" }; + + // Order of preference: emscripten > docker > podman > error + let source = if !force_docker && Command::new(emcc_name).output().is_ok() { + EmccSource::Native + } else if Command::new("docker") + .output() + .is_ok_and(|out| out.status.success()) + { + EmccSource::Docker + } else if Command::new("podman") + .arg("--version") + .output() + .is_ok_and(|out| out.status.success()) + { + EmccSource::Podman + } else { + return Err(anyhow!( + "You must have either emcc, docker, or podman on your PATH to run this command" + )); + }; + + let mut command = match source { + EmccSource::Native => { + let mut command = Command::new(emcc_name); + command.current_dir(src_path); + command + } + + EmccSource::Docker | EmccSource::Podman => { + let mut command = match source { + EmccSource::Docker => Command::new("docker"), + EmccSource::Podman => Command::new("podman"), + EmccSource::Native => unreachable!(), + }; + command.args(["run", "--rm"]); + + // The working directory is the directory containing the parser itself + let workdir = if root_path == src_path { + PathBuf::from("/src") + } else { + let mut path = PathBuf::from("/src"); + path.push(src_path.strip_prefix(root_path).unwrap()); + path + }; + command.args(["--workdir", &workdir.to_slash_lossy()]); + + // Mount the root directory as a volume, which is the repo root + let mut volume_string = OsString::from(&root_path); + volume_string.push(":/src:Z"); + command.args([OsStr::new("--volume"), &volume_string]); + + // In case `docker` is an alias to `podman`, ensure that podman + // mounts the current directory as writable by the container + // user which has the same uid as the host user. Setting the + // podman-specific variable is more reliable than attempting to + // detect whether `docker` is an alias for `podman`. + // see https://docs.podman.io/en/latest/markdown/podman-run.1.html#userns-mode + command.env("PODMAN_USERNS", "keep-id"); + + // Get the current user id so that files created in the docker container will have + // the same owner. + #[cfg(unix)] + { + #[link(name = "c")] + extern "C" { + fn getuid() -> u32; + } + // don't need to set user for podman since PODMAN_USERNS=keep-id is already set + if source == EmccSource::Docker { + let user_id = unsafe { getuid() }; + command.args(["--user", &user_id.to_string()]); + } + }; + + // Run `emcc` in a container using the `emscripten-slim` image + command.args([EMSCRIPTEN_TAG, "emcc"]); + command + } + }; + + let output_name = "output.wasm"; + + command.args([ "-o", - output_path.to_str().unwrap(), - "-fPIC", - "-shared", - if self.debug_build { "-g" } else { "-Os" }, - format!("-Wl,--export=tree_sitter_{language_name}").as_str(), - "-Wl,--allow-undefined", - "-Wl,--no-entry", - "-nostdlib", + output_name, + "-Os", + "-s", + "WASM=1", + "-s", + "SIDE_MODULE=2", + "-s", + "TOTAL_MEMORY=33554432", + "-s", + "NODEJS_CATCH_EXIT=0", + "-s", + &format!("EXPORTED_FUNCTIONS=[\"_tree_sitter_{language_name}\"]"), "-fno-exceptions", "-fvisibility=hidden", "-I", ".", - "parser.c", ]); if let Some(scanner_filename) = scanner_filename { command.arg(scanner_filename); } - let output = command.output().map_err(LoaderError::WasmCompiler)?; - - if !output.status.success() { - return Err(LoaderError::WasmCompilation( - String::from_utf8_lossy(&output.stderr).to_string(), - )); + command.arg("parser.c"); + let status = command + .spawn() + .with_context(|| "Failed to run emcc command")? + .wait()?; + if !status.success() { + return Err(anyhow!("emcc command failed")); } + fs::rename(src_path.join(output_name), output_path) + .context("failed to rename wasm output file")?; + Ok(()) } - /// Extracts a tar.gz archive with `tar`, stripping the first path component. - fn extract_tar_gz_with_strip( - &self, - archive_path: &Path, - destination: &Path, - ) -> LoaderResult<()> { - let status = Command::new("tar") - .arg("-xzf") - .arg(archive_path) - .arg("--strip-components=1") - .arg("-C") - .arg(destination) - .status() - .map_err(|e| LoaderError::Tar(archive_path.to_string_lossy().to_string(), e))?; - - if !status.success() { - return Err(LoaderError::Extraction( - archive_path.to_string_lossy().to_string(), - destination.to_string_lossy().to_string(), - )); - } - - Ok(()) - } - - /// This ensures that the wasi-sdk is available, downloading and extracting it if necessary, - /// and returns the path to the `clang` executable. - /// - /// If `TREE_SITTER_WASI_SDK_PATH` is set, it will use that path to look for the clang executable. - fn ensure_wasi_sdk_exists(&self) -> LoaderResult { - let possible_executables = if cfg!(windows) { - vec![ - "clang.exe", - "wasm32-unknown-wasi-clang.exe", - "wasm32-wasi-clang.exe", - ] - } else { - vec!["clang", "wasm32-unknown-wasi-clang", "wasm32-wasi-clang"] - }; - - if let Ok(wasi_sdk_path) = std::env::var("TREE_SITTER_WASI_SDK_PATH") { - let wasi_sdk_dir = PathBuf::from(wasi_sdk_path); - - for exe in &possible_executables { - let clang_exe = wasi_sdk_dir.join("bin").join(exe); - if clang_exe.exists() { - return Ok(clang_exe); - } - } - - return Err(LoaderError::WasiSDKClang(WasiSDKClangError { - wasi_sdk_dir: wasi_sdk_dir.to_string_lossy().to_string(), - possible_executables, - download: false, - })); - } - - let cache_dir = etcetera::choose_base_strategy()? - .cache_dir() - .join("tree-sitter"); - fs::create_dir_all(&cache_dir) - .map_err(|e| LoaderError::IO(IoError::new(e, Some(cache_dir.as_path()))))?; - - let wasi_sdk_dir = cache_dir.join("wasi-sdk"); - - for exe in &possible_executables { - let clang_exe = wasi_sdk_dir.join("bin").join(exe); - if clang_exe.exists() { - return Ok(clang_exe); - } - } - - fs::create_dir_all(&wasi_sdk_dir) - .map_err(|e| LoaderError::IO(IoError::new(e, Some(wasi_sdk_dir.as_path()))))?; - - let arch_os = if cfg!(target_os = "macos") { - if cfg!(target_arch = "aarch64") { - "arm64-macos" - } else { - "x86_64-macos" - } - } else if cfg!(target_os = "windows") { - if cfg!(target_arch = "aarch64") { - "arm64-windows" - } else { - "x86_64-windows" - } - } else if cfg!(target_os = "linux") { - if cfg!(target_arch = "aarch64") { - "arm64-linux" - } else { - "x86_64-linux" - } - } else { - return Err(LoaderError::WasiSDKPlatform); - }; - - let sdk_filename = format!("wasi-sdk-{WASI_SDK_VERSION}-{arch_os}.tar.gz"); - let wasi_sdk_major_version = WASI_SDK_VERSION - .trim_end_matches(char::is_numeric) // trim minor version... - .trim_end_matches('.'); // ...and '.' separator - let sdk_url = format!( - "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-{wasi_sdk_major_version}/{sdk_filename}", - ); - - info!("Downloading wasi-sdk from {sdk_url}..."); - let temp_tar_path = cache_dir.join(sdk_filename); - - let status = Command::new("curl") - .arg("-f") - .arg("-L") - .arg("-o") - .arg(&temp_tar_path) - .arg(&sdk_url) - .status() - .map_err(|e| LoaderError::Curl(sdk_url.clone(), e))?; - - if !status.success() { - return Err(LoaderError::WasiSDKDownload(sdk_url)); - } - - info!("Extracting wasi-sdk to {}...", wasi_sdk_dir.display()); - self.extract_tar_gz_with_strip(&temp_tar_path, &wasi_sdk_dir)?; - - fs::remove_file(temp_tar_path).ok(); - for exe in &possible_executables { - let clang_exe = wasi_sdk_dir.join("bin").join(exe); - if clang_exe.exists() { - return Ok(clang_exe); - } - } - - Err(LoaderError::WasiSDKClang(WasiSDKClangError { - wasi_sdk_dir: wasi_sdk_dir.to_string_lossy().to_string(), - possible_executables, - download: true, - })) - } - #[must_use] #[cfg(feature = "tree-sitter-highlight")] pub fn highlight_config_for_injection_string<'a>( @@ -1519,15 +1117,15 @@ impl Loader { ) -> Option<&'a HighlightConfiguration> { match self.language_configuration_for_injection_string(string) { Err(e) => { - error!("Failed to load language for injection string '{string}': {e}",); + eprintln!("Failed to load language for injection string '{string}': {e}",); None } Ok(None) => None, Ok(Some((language, configuration))) => { match configuration.highlight_config(language, None) { Err(e) => { - error!( - "Failed to load higlight config for injection string '{string}': {e}" + eprintln!( + "Failed to load property sheet for injection string '{string}': {e}", ); None } @@ -1539,9 +1137,7 @@ impl Loader { } #[must_use] - pub fn get_language_configuration_in_current_path( - &self, - ) -> Option<&LanguageConfiguration<'static>> { + pub fn get_language_configuration_in_current_path(&self) -> Option<&LanguageConfiguration> { self.language_configuration_in_current_path .map(|i| &self.language_configurations[i]) } @@ -1550,121 +1146,113 @@ impl Loader { &mut self, parser_path: &Path, set_current_path_config: bool, - ) -> LoaderResult<&[LanguageConfiguration<'static>]> { + ) -> Result<&[LanguageConfiguration]> { let initial_language_configuration_count = self.language_configurations.len(); - match TreeSitterJSON::from_file(parser_path) { - Ok(config) => { - let language_count = self.languages_by_id.len(); - for grammar in config.grammars { - // Determine the path to the parser directory. This can be specified in - // the tree-sitter.json, but defaults to the directory containing the - // tree-sitter.json. - let language_path = - parser_path.join(grammar.path.unwrap_or(PathBuf::from("."))); + let ts_json = TreeSitterJSON::from_file(parser_path); + if let Ok(config) = ts_json { + let language_count = self.languages_by_id.len(); + for grammar in config.grammars { + // Determine the path to the parser directory. This can be specified in + // the tree-sitter.json, but defaults to the directory containing the + // tree-sitter.json. + let language_path = parser_path.join(grammar.path.unwrap_or(PathBuf::from("."))); - // Determine if a previous language configuration in this package.json file - // already uses the same language. - let mut language_id = None; - for (id, (path, _, _)) in - self.languages_by_id.iter().enumerate().skip(language_count) - { - if language_path == *path { - language_id = Some(id); - } - } - - // If not, add a new language path to the list. - let language_id = if let Some(language_id) = language_id { - language_id - } else { - self.languages_by_id.push(( - language_path, - OnceCell::new(), - grammar - .external_files - .clone() - .into_vec() - .map(|files| { - files - .into_iter() - .map(|path| { - let path = parser_path.join(path); - // prevent p being above/outside of parser_path - if path.starts_with(parser_path) { - Ok(path) - } else { - Err(LoaderError::ExternalFile( - path.to_string_lossy().to_string(), - parser_path.to_string_lossy().to_string(), - )) - } - }) - .collect::>>() - }) - .transpose()?, - )); - self.languages_by_id.len() - 1 - }; - - let configuration = LanguageConfiguration { - root_path: parser_path.to_path_buf(), - language_name: grammar.name, - scope: Some(grammar.scope), - language_id, - file_types: grammar.file_types.unwrap_or_default(), - content_regex: Self::regex(grammar.content_regex.as_deref()), - first_line_regex: Self::regex(grammar.first_line_regex.as_deref()), - injection_regex: Self::regex(grammar.injection_regex.as_deref()), - injections_filenames: grammar.injections.into_vec(), - locals_filenames: grammar.locals.into_vec(), - tags_filenames: grammar.tags.into_vec(), - highlights_filenames: grammar.highlights.into_vec(), - #[cfg(feature = "tree-sitter-highlight")] - highlight_config: OnceCell::new(), - #[cfg(feature = "tree-sitter-tags")] - tags_config: OnceCell::new(), - #[cfg(feature = "tree-sitter-highlight")] - highlight_names: &self.highlight_names, - #[cfg(feature = "tree-sitter-highlight")] - use_all_highlight_names: self.use_all_highlight_names, - _phantom: PhantomData, - }; - - for file_type in &configuration.file_types { - self.language_configuration_ids_by_file_type - .entry(file_type.clone()) - .or_default() - .push(self.language_configurations.len()); - } - if let Some(first_line_regex) = &configuration.first_line_regex { - self.language_configuration_ids_by_first_line_regex - .entry(first_line_regex.to_string()) - .or_default() - .push(self.language_configurations.len()); - } - - self.language_configurations.push(unsafe { - mem::transmute::, LanguageConfiguration<'static>>( - configuration, - ) - }); - - if set_current_path_config - && self.language_configuration_in_current_path.is_none() - { - self.language_configuration_in_current_path = - Some(self.language_configurations.len() - 1); + // Determine if a previous language configuration in this package.json file + // already uses the same language. + let mut language_id = None; + for (id, (path, _, _)) in + self.languages_by_id.iter().enumerate().skip(language_count) + { + if language_path == *path { + language_id = Some(id); } } + + // If not, add a new language path to the list. + let language_id = if let Some(language_id) = language_id { + language_id + } else { + self.languages_by_id.push(( + language_path, + OnceCell::new(), + grammar.external_files.clone().into_vec().map(|files| { + files.into_iter() + .map(|path| { + let path = parser_path.join(path); + // prevent p being above/outside of parser_path + if path.starts_with(parser_path) { + Ok(path) + } else { + Err(anyhow!("External file path {path:?} is outside of parser directory {parser_path:?}")) + } + }) + .collect::>>() + }).transpose()?, + )); + self.languages_by_id.len() - 1 + }; + + let configuration = LanguageConfiguration { + root_path: parser_path.to_path_buf(), + language_name: grammar.name, + scope: Some(grammar.scope), + language_id, + file_types: grammar.file_types.unwrap_or_default(), + content_regex: Self::regex(grammar.content_regex.as_deref()), + first_line_regex: Self::regex(grammar.first_line_regex.as_deref()), + injection_regex: Self::regex(grammar.injection_regex.as_deref()), + injections_filenames: grammar.injections.into_vec(), + locals_filenames: grammar.locals.into_vec(), + tags_filenames: grammar.tags.into_vec(), + highlights_filenames: grammar.highlights.into_vec(), + #[cfg(feature = "tree-sitter-highlight")] + highlight_config: OnceCell::new(), + #[cfg(feature = "tree-sitter-tags")] + tags_config: OnceCell::new(), + #[cfg(feature = "tree-sitter-highlight")] + highlight_names: &self.highlight_names, + #[cfg(feature = "tree-sitter-highlight")] + use_all_highlight_names: self.use_all_highlight_names, + _phantom: PhantomData, + }; + + for file_type in &configuration.file_types { + self.language_configuration_ids_by_file_type + .entry(file_type.to_string()) + .or_default() + .push(self.language_configurations.len()); + } + if let Some(first_line_regex) = &configuration.first_line_regex { + self.language_configuration_ids_by_first_line_regex + .entry(first_line_regex.to_string()) + .or_default() + .push(self.language_configurations.len()); + } + + self.language_configurations.push(unsafe { + mem::transmute::, LanguageConfiguration<'static>>( + configuration, + ) + }); + + if set_current_path_config && self.language_configuration_in_current_path.is_none() + { + self.language_configuration_in_current_path = + Some(self.language_configurations.len() - 1); + } } - Err(LoaderError::Serialization(e)) => { - warn!( - "Failed to parse {} -- {e}", - parser_path.join("tree-sitter.json").display() - ); + } else if let Err(e) = ts_json { + match e.downcast_ref::() { + // This is noisy, and not really an issue. + Some(e) if e.kind() == std::io::ErrorKind::NotFound => {} + _ => { + eprintln!( + "Warning: Failed to parse {} -- {e}", + parser_path.join("tree-sitter.json").display() + ); + } } - _ => {} } // If we didn't find any language configurations in the tree-sitter.json file, @@ -1714,86 +1302,86 @@ impl Loader { pattern.and_then(|r| RegexBuilder::new(r).multi_line(true).build().ok()) } - fn grammar_json_name(grammar_path: &Path) -> LoaderResult { - let file = fs::File::open(grammar_path) - .map_err(|e| LoaderError::IO(IoError::new(e, Some(grammar_path))))?; + fn grammar_json_name(grammar_path: &Path) -> Result { + let file = fs::File::open(grammar_path).with_context(|| { + format!("Failed to open grammar.json at {}", grammar_path.display()) + })?; let first_three_lines = BufReader::new(file) .lines() .take(3) - .collect::, std::io::Error>>() - .map_err(|_| LoaderError::GrammarJSON(grammar_path.to_string_lossy().to_string()))? + .collect::, _>>() + .with_context(|| { + format!( + "Failed to read the first three lines of grammar.json at {}", + grammar_path.display() + ) + })? .join("\n"); let name = GRAMMAR_NAME_REGEX .captures(&first_three_lines) .and_then(|c| c.get(1)) - .ok_or_else(|| LoaderError::GrammarJSON(grammar_path.to_string_lossy().to_string()))?; + .ok_or_else(|| { + anyhow!( + "Failed to parse the language name from grammar.json at {}", + grammar_path.display() + ) + })?; Ok(name.as_str().to_string()) } pub fn select_language( &mut self, - path: Option<&Path>, + path: &Path, current_dir: &Path, scope: Option<&str>, - // path to dynamic library, name of language - lib_info: Option<&(PathBuf, &str)>, - ) -> LoaderResult { - if let Some((ref lib_path, language_name)) = lib_info { - let language_fn_name = format!("tree_sitter_{}", language_name.replace('-', "_")); - Self::load_language(lib_path, &language_fn_name) - } else if let Some(scope) = scope { + ) -> Result { + if let Some(scope) = scope { if let Some(config) = self .language_configuration_for_scope(scope) - .map_err(|e| LoaderError::ScopeLoad(scope.to_string(), Box::new(e)))? + .with_context(|| format!("Failed to load language for scope '{scope}'"))? { Ok(config.0) } else { - Err(LoaderError::UnknownScope(scope.to_string())) + Err(anyhow!("Unknown scope '{scope}'")) } - } else if let Some((lang, _)) = if let Some(path) = path { - self.language_configuration_for_file_name(path) - .map_err(|e| { - LoaderError::FileNameLoad( - path.file_name().unwrap().to_string_lossy().to_string(), - Box::new(e), - ) - })? - } else { - None - } { + } else if let Some((lang, _)) = self + .language_configuration_for_file_name(path) + .with_context(|| { + format!( + "Failed to load language for file name {}", + path.file_name().unwrap().to_string_lossy() + ) + })? + { Ok(lang) } else if let Some(id) = self.language_configuration_in_current_path { Ok(self.language_for_id(self.language_configurations[id].language_id)?) } else if let Some(lang) = self .languages_at_path(current_dir) - .map_err(|e| LoaderError::CurrentDirectoryLoad(Box::new(e)))? + .with_context(|| "Failed to load language in current directory")? .first() .cloned() { Ok(lang.0) - } else if let Some(lang) = if let Some(path) = path { - self.language_configuration_for_first_line_regex(path)? - } else { - None - } { + } else if let Some(lang) = self.language_configuration_for_first_line_regex(path)? { Ok(lang.0) } else { - Err(LoaderError::NoLanguage) + Err(anyhow!("No language found")) } } - pub const fn debug_build(&mut self, flag: bool) { + pub fn debug_build(&mut self, flag: bool) { self.debug_build = flag; } - pub const fn sanitize_build(&mut self, flag: bool) { + pub fn sanitize_build(&mut self, flag: bool) { self.sanitize_build = flag; } - pub const fn force_rebuild(&mut self, rebuild: bool) { + pub fn force_rebuild(&mut self, rebuild: bool) { self.force_rebuild = rebuild; } @@ -1816,27 +1404,27 @@ impl LanguageConfiguration<'_> { &self, language: Language, paths: Option<&[PathBuf]>, - ) -> LoaderResult> { + ) -> Result> { let (highlights_filenames, injections_filenames, locals_filenames) = match paths { Some(paths) => ( Some( paths .iter() - .filter(|p| p.ends_with(DEFAULT_HIGHLIGHTS_QUERY_FILE_NAME)) + .filter(|p| p.ends_with("highlights.scm")) .cloned() .collect::>(), ), Some( paths .iter() - .filter(|p| p.ends_with(DEFAULT_TAGS_QUERY_FILE_NAME)) + .filter(|p| p.ends_with("tags.scm")) .cloned() .collect::>(), ), Some( paths .iter() - .filter(|p| p.ends_with(DEFAULT_LOCALS_QUERY_FILE_NAME)) + .filter(|p| p.ends_with("locals.scm")) .cloned() .collect::>(), ), @@ -1851,7 +1439,7 @@ impl LanguageConfiguration<'_> { } else { self.highlights_filenames.as_deref() }, - DEFAULT_HIGHLIGHTS_QUERY_FILE_NAME, + "highlights.scm", )?; let (injections_query, injection_ranges) = self.read_queries( if injections_filenames.is_some() { @@ -1859,7 +1447,7 @@ impl LanguageConfiguration<'_> { } else { self.injections_filenames.as_deref() }, - DEFAULT_INJECTIONS_QUERY_FILE_NAME, + "injections.scm", )?; let (locals_query, locals_ranges) = self.read_queries( if locals_filenames.is_some() { @@ -1867,7 +1455,7 @@ impl LanguageConfiguration<'_> { } else { self.locals_filenames.as_deref() }, - DEFAULT_LOCALS_QUERY_FILE_NAME, + "locals.scm", )?; if highlights_query.is_empty() { @@ -1881,9 +1469,7 @@ impl LanguageConfiguration<'_> { &locals_query, ) .map_err(|error| match error.kind { - QueryErrorKind::Language => { - LoaderError::Query(LoaderQueryError { error, file: None }) - } + QueryErrorKind::Language => Error::from(error), _ => { if error.offset < injections_query.len() { Self::include_path_in_query_error( @@ -1926,15 +1512,13 @@ impl LanguageConfiguration<'_> { } #[cfg(feature = "tree-sitter-tags")] - pub fn tags_config(&self, language: Language) -> LoaderResult> { + pub fn tags_config(&self, language: Language) -> Result> { self.tags_config .get_or_try_init(|| { - let (tags_query, tags_ranges) = self - .read_queries(self.tags_filenames.as_deref(), DEFAULT_TAGS_QUERY_FILE_NAME)?; - let (locals_query, locals_ranges) = self.read_queries( - self.locals_filenames.as_deref(), - DEFAULT_LOCALS_QUERY_FILE_NAME, - )?; + let (tags_query, tags_ranges) = + self.read_queries(self.tags_filenames.as_deref(), "tags.scm")?; + let (locals_query, locals_ranges) = + self.read_queries(self.locals_filenames.as_deref(), "locals.scm")?; if tags_query.is_empty() { Ok(None) } else { @@ -1972,7 +1556,7 @@ impl LanguageConfiguration<'_> { ranges: &[(PathBuf, Range)], source: &str, start_offset: usize, - ) -> LoaderError { + ) -> Error { let offset_within_section = error.offset - start_offset; let (path, range) = ranges .iter() @@ -1982,10 +1566,7 @@ impl LanguageConfiguration<'_> { error.row = source[range.start..offset_within_section] .matches('\n') .count(); - LoaderError::Query(LoaderQueryError { - error, - file: Some(path.to_string_lossy().to_string()), - }) + Error::from(error).context(format!("Error in query file {}", path.display())) } #[allow(clippy::type_complexity)] @@ -1994,7 +1575,7 @@ impl LanguageConfiguration<'_> { &self, paths: Option<&[PathBuf]>, default_path: &str, - ) -> LoaderResult<(String, Vec<(PathBuf, Range)>)> { + ) -> Result<(String, Vec<(PathBuf, Range)>)> { let mut query = String::new(); let mut path_ranges = Vec::new(); if let Some(paths) = paths { @@ -2002,29 +1583,25 @@ impl LanguageConfiguration<'_> { let abs_path = self.root_path.join(path); let prev_query_len = query.len(); query += &fs::read_to_string(&abs_path) - .map_err(|e| LoaderError::IO(IoError::new(e, Some(abs_path.as_path()))))?; + .with_context(|| format!("Failed to read query file {}", path.display()))?; path_ranges.push((path.clone(), prev_query_len..query.len())); } } else { // highlights.scm is needed to test highlights, and tags.scm to test tags - if default_path == DEFAULT_HIGHLIGHTS_QUERY_FILE_NAME - || default_path == DEFAULT_TAGS_QUERY_FILE_NAME - { - warn!( - concat!( - "You should add a `{}` entry pointing to the {} path in the `tree-sitter` ", - "object in the grammar's tree-sitter.json file. See more here: ", - "https://tree-sitter.github.io/tree-sitter/3-syntax-highlighting#query-paths" - ), - default_path.replace(".scm", ""), - default_path + if default_path == "highlights.scm" || default_path == "tags.scm" { + eprintln!( + indoc! {" + Warning: you should add a `{}` entry pointing to the highlights path in the `tree-sitter` object in the grammar's tree-sitter.json file. + See more here: https://tree-sitter.github.io/tree-sitter/3-syntax-highlighting#query-paths + "}, + default_path.replace(".scm", "") ); } let queries_path = self.root_path.join("queries"); let path = queries_path.join(default_path); if path.exists() { query = fs::read_to_string(&path) - .map_err(|e| LoaderError::IO(IoError::new(e, Some(path.as_path()))))?; + .with_context(|| format!("Failed to read query file {}", path.display()))?; path_ranges.push((PathBuf::from(default_path), 0..query.len())); } } @@ -2033,22 +1610,32 @@ impl LanguageConfiguration<'_> { } } -fn needs_recompile(lib_path: &Path, paths_to_check: &[PathBuf]) -> LoaderResult { +fn needs_recompile(lib_path: &Path, paths_to_check: &[PathBuf]) -> Result { if !lib_path.exists() { return Ok(true); } - let lib_mtime = mtime(lib_path).map_err(|e| LoaderError::ModifiedTime(Box::new(e)))?; + let lib_mtime = mtime(lib_path) + .with_context(|| format!("Failed to read mtime of {}", lib_path.display()))?; for path in paths_to_check { - if mtime(path).map_err(|e| LoaderError::ModifiedTime(Box::new(e)))? > lib_mtime { + if mtime(path)? > lib_mtime { return Ok(true); } } Ok(false) } -fn mtime(path: &Path) -> LoaderResult { - fs::metadata(path) - .map_err(|e| LoaderError::IO(IoError::new(e, Some(path))))? - .modified() - .map_err(|e| LoaderError::IO(IoError::new(e, Some(path)))) +fn mtime(path: &Path) -> Result { + Ok(fs::metadata(path)?.modified()?) +} + +fn replace_dashes_with_underscores(name: &str) -> String { + let mut result = String::with_capacity(name.len()); + for c in name.chars() { + if c == '-' { + result.push('_'); + } else { + result.push(c); + } + } + result } diff --git a/crates/cli/npm/.gitignore b/cli/npm/.gitignore similarity index 100% rename from crates/cli/npm/.gitignore rename to cli/npm/.gitignore diff --git a/crates/cli/npm/cli.js b/cli/npm/cli.js similarity index 100% rename from crates/cli/npm/cli.js rename to cli/npm/cli.js diff --git a/crates/cli/npm/dsl.d.ts b/cli/npm/dsl.d.ts similarity index 99% rename from crates/cli/npm/dsl.d.ts rename to cli/npm/dsl.d.ts index accdb95f..3ad9ea2a 100644 --- a/crates/cli/npm/dsl.d.ts +++ b/cli/npm/dsl.d.ts @@ -29,7 +29,6 @@ type Rule = | PrecRule | Repeat1Rule | RepeatRule - | ReservedRule | SeqRule | StringRule | SymbolRule diff --git a/crates/cli/npm/install.js b/cli/npm/install.js similarity index 100% rename from crates/cli/npm/install.js rename to cli/npm/install.js diff --git a/crates/cli/npm/package.json b/cli/npm/package.json similarity index 89% rename from crates/cli/npm/package.json rename to cli/npm/package.json index d49abd46..5e4af444 100644 --- a/crates/cli/npm/package.json +++ b/cli/npm/package.json @@ -1,6 +1,6 @@ { "name": "tree-sitter-cli", - "version": "0.27.0", + "version": "0.25.9", "author": { "name": "Max Brunsfeld", "email": "maxbrunsfeld@gmail.com" @@ -27,7 +27,7 @@ }, "scripts": { "install": "node install.js", - "prepack": "cp ../../../LICENSE ../README.md .", + "prepack": "cp ../../LICENSE ../README.md .", "postpack": "rm LICENSE README.md" }, "bin": { diff --git a/crates/cli/src/fuzz/allocations.rs b/cli/src/fuzz/allocations.rs similarity index 73% rename from crates/cli/src/fuzz/allocations.rs rename to cli/src/fuzz/allocations.rs index ca0c0860..9d7c91ab 100644 --- a/crates/cli/src/fuzz/allocations.rs +++ b/cli/src/fuzz/allocations.rs @@ -40,11 +40,7 @@ extern "C" { fn free(ptr: *mut c_void); } -pub fn record(f: impl FnOnce() -> T) -> T { - record_checked(f).unwrap() -} - -pub fn record_checked(f: impl FnOnce() -> T) -> Result { +pub fn record(f: impl FnOnce() -> T) -> Result { RECORDER.with(|recorder| { recorder.enabled.store(true, SeqCst); recorder.allocation_count.store(0, SeqCst); @@ -97,34 +93,19 @@ fn record_dealloc(ptr: *mut c_void) { }); } -/// # Safety -/// -/// The caller must ensure that the returned pointer is eventually -/// freed by calling `ts_record_free`. -#[must_use] -pub unsafe extern "C" fn ts_record_malloc(size: usize) -> *mut c_void { +unsafe extern "C" fn ts_record_malloc(size: usize) -> *mut c_void { let result = malloc(size); record_alloc(result); result } -/// # Safety -/// -/// The caller must ensure that the returned pointer is eventually -/// freed by calling `ts_record_free`. -#[must_use] -pub unsafe extern "C" fn ts_record_calloc(count: usize, size: usize) -> *mut c_void { +unsafe extern "C" fn ts_record_calloc(count: usize, size: usize) -> *mut c_void { let result = calloc(count, size); record_alloc(result); result } -/// # Safety -/// -/// The caller must ensure that the returned pointer is eventually -/// freed by calling `ts_record_free`. -#[must_use] -pub unsafe extern "C" fn ts_record_realloc(ptr: *mut c_void, size: usize) -> *mut c_void { +unsafe extern "C" fn ts_record_realloc(ptr: *mut c_void, size: usize) -> *mut c_void { let result = realloc(ptr, size); if ptr.is_null() { record_alloc(result); @@ -135,11 +116,7 @@ pub unsafe extern "C" fn ts_record_realloc(ptr: *mut c_void, size: usize) -> *mu result } -/// # Safety -/// -/// The caller must ensure that `ptr` was allocated by a previous call -/// to `ts_record_malloc`, `ts_record_calloc`, or `ts_record_realloc`. -pub unsafe extern "C" fn ts_record_free(ptr: *mut c_void) { +unsafe extern "C" fn ts_record_free(ptr: *mut c_void) { record_dealloc(ptr); free(ptr); } diff --git a/crates/cli/src/fuzz/corpus_test.rs b/cli/src/fuzz/corpus_test.rs similarity index 98% rename from crates/cli/src/fuzz/corpus_test.rs rename to cli/src/fuzz/corpus_test.rs index e95ab283..8807d9a9 100644 --- a/crates/cli/src/fuzz/corpus_test.rs +++ b/cli/src/fuzz/corpus_test.rs @@ -23,7 +23,7 @@ pub fn check_consistent_sizes(tree: &Tree, input: &[u8]) { let mut some_child_has_changes = false; let mut actual_named_child_count = 0; for i in 0..node.child_count() { - let child = node.child(i as u32).unwrap(); + let child = node.child(i).unwrap(); assert!(child.start_byte() >= last_child_end_byte); assert!(child.start_position() >= last_child_end_point); check(child, line_offsets); diff --git a/crates/cli/src/fuzz/edits.rs b/cli/src/fuzz/edits.rs similarity index 100% rename from crates/cli/src/fuzz/edits.rs rename to cli/src/fuzz/edits.rs diff --git a/crates/cli/src/fuzz.rs b/cli/src/fuzz/mod.rs similarity index 86% rename from crates/cli/src/fuzz.rs rename to cli/src/fuzz/mod.rs index 39ad7691..04b4910d 100644 --- a/crates/cli/src/fuzz.rs +++ b/cli/src/fuzz/mod.rs @@ -1,11 +1,5 @@ -use std::{ - collections::HashMap, - env, fs, - path::{Path, PathBuf}, - sync::LazyLock, -}; +use std::{collections::HashMap, env, fs, path::Path, sync::LazyLock}; -use log::{error, info}; use rand::Rng; use regex::Regex; use tree_sitter::{Language, Parser}; @@ -25,7 +19,7 @@ use crate::{ random::Rand, }, parse::perform_edit, - test::{parse_tests, strip_sexp_fields, DiffKey, TestDiff, TestEntry}, + test::{parse_tests, print_diff, print_diff_key, strip_sexp_fields, TestEntry}, }; pub static LOG_ENABLED: LazyLock = LazyLock::new(|| env::var("TREE_SITTER_LOG").is_ok()); @@ -63,14 +57,14 @@ pub fn new_seed() -> usize { int_env_var("TREE_SITTER_SEED").unwrap_or_else(|| { let mut rng = rand::thread_rng(); let seed = rng.gen::(); - info!("Seed: {seed}"); + eprintln!("Seed: {seed}"); seed }) } pub struct FuzzOptions { pub skipped: Option>, - pub subdir: Option, + pub subdir: Option, pub edits: usize, pub iterations: usize, pub include: Option, @@ -109,12 +103,12 @@ pub fn fuzz_language_corpus( let corpus_dir = grammar_dir.join(subdir).join("test").join("corpus"); if !corpus_dir.exists() || !corpus_dir.is_dir() { - error!("No corpus directory found, ensure that you have a `test/corpus` directory in your grammar directory with at least one test file."); + eprintln!("No corpus directory found, ensure that you have a `test/corpus` directory in your grammar directory with at least one test file."); return; } if std::fs::read_dir(&corpus_dir).unwrap().count() == 0 { - error!("No corpus files found in `test/corpus`, ensure that you have at least one test file in your corpus directory."); + eprintln!("No corpus files found in `test/corpus`, ensure that you have at least one test file in your corpus directory."); return; } @@ -150,7 +144,7 @@ pub fn fuzz_language_corpus( let dump_edits = env::var("TREE_SITTER_DUMP_EDITS").is_ok(); if log_seed { - info!(" start seed: {start_seed}"); + println!(" start seed: {start_seed}"); } println!(); @@ -164,7 +158,7 @@ pub fn fuzz_language_corpus( println!(" {test_index}. {test_name}"); - let passed = allocations::record_checked(|| { + let passed = allocations::record(|| { let mut log_session = None; let mut parser = get_parser(&mut log_session, "log.html"); parser.set_language(language).unwrap(); @@ -183,8 +177,8 @@ pub fn fuzz_language_corpus( if actual_output != test.output { println!("Incorrect initial parse for {test_name}"); - DiffKey::print(); - println!("{}", TestDiff::new(&actual_output, &test.output)); + print_diff_key(); + print_diff(&actual_output, &test.output, true); println!(); return false; } @@ -192,7 +186,7 @@ pub fn fuzz_language_corpus( true }) .unwrap_or_else(|e| { - error!("{e}"); + eprintln!("Error: {e}"); false }); @@ -208,7 +202,7 @@ pub fn fuzz_language_corpus( for trial in 0..options.iterations { let seed = start_seed + trial; - let passed = allocations::record_checked(|| { + let passed = allocations::record(|| { let mut rand = Rand::new(seed); let mut log_session = None; let mut parser = get_parser(&mut log_session, "log.html"); @@ -217,7 +211,7 @@ pub fn fuzz_language_corpus( let mut input = test.input.clone(); if options.log_graphs { - info!("{}\n", String::from_utf8_lossy(&input)); + eprintln!("{}\n", String::from_utf8_lossy(&input)); } // Perform a random series of edits and reparse. @@ -230,7 +224,7 @@ pub fn fuzz_language_corpus( } if log_seed { - info!(" {test_index}.{trial:<2} seed: {seed}"); + println!(" {test_index}.{trial:<2} seed: {seed}"); } if dump_edits { @@ -244,7 +238,7 @@ pub fn fuzz_language_corpus( } if options.log_graphs { - info!("{}\n", String::from_utf8_lossy(&input)); + eprintln!("{}\n", String::from_utf8_lossy(&input)); } set_included_ranges(&mut parser, &input, test.template_delimiters); @@ -253,7 +247,7 @@ pub fn fuzz_language_corpus( // Check that the new tree is consistent. check_consistent_sizes(&tree2, &input); if let Err(message) = check_changed_ranges(&tree, &tree2, &input) { - error!("\nUnexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n",); + println!("\nUnexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n",); return false; } @@ -262,7 +256,7 @@ pub fn fuzz_language_corpus( perform_edit(&mut tree2, &mut input, &edit).unwrap(); } if options.log_graphs { - info!("{}\n", String::from_utf8_lossy(&input)); + eprintln!("{}\n", String::from_utf8_lossy(&input)); } set_included_ranges(&mut parser, &test.input, test.template_delimiters); @@ -276,8 +270,8 @@ pub fn fuzz_language_corpus( if actual_output != test.output && !test.error { println!("Incorrect parse for {test_name} - seed {seed}"); - DiffKey::print(); - println!("{}", TestDiff::new(&actual_output, &test.output)); + print_diff_key(); + print_diff(&actual_output, &test.output, true); println!(); return false; } @@ -285,13 +279,13 @@ pub fn fuzz_language_corpus( // Check that the edited tree is consistent. check_consistent_sizes(&tree3, &input); if let Err(message) = check_changed_ranges(&tree2, &tree3, &input) { - error!("Unexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n"); + println!("Unexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n"); return false; } true }).unwrap_or_else(|e| { - error!("{e}"); + eprintln!("Error: {e}"); false }); @@ -303,17 +297,17 @@ pub fn fuzz_language_corpus( } if failure_count != 0 { - info!("{failure_count} {language_name} corpus tests failed fuzzing"); + eprintln!("{failure_count} {language_name} corpus tests failed fuzzing"); } skipped.retain(|_, v| *v == 0); if !skipped.is_empty() { - info!("Non matchable skip definitions:"); + println!("Non matchable skip definitions:"); for k in skipped.keys() { - info!(" {k}"); + println!(" {k}"); } - panic!("Non matchable skip definitions need to be removed"); + panic!("Non matchable skip definitions needs to be removed"); } } diff --git a/crates/cli/src/fuzz/random.rs b/cli/src/fuzz/random.rs similarity index 100% rename from crates/cli/src/fuzz/random.rs rename to cli/src/fuzz/random.rs diff --git a/crates/cli/src/fuzz/scope_sequence.rs b/cli/src/fuzz/scope_sequence.rs similarity index 100% rename from crates/cli/src/fuzz/scope_sequence.rs rename to cli/src/fuzz/scope_sequence.rs diff --git a/crates/cli/src/highlight.rs b/cli/src/highlight.rs similarity index 98% rename from crates/cli/src/highlight.rs rename to cli/src/highlight.rs index a2cb01cf..abb64020 100644 --- a/crates/cli/src/highlight.rs +++ b/cli/src/highlight.rs @@ -12,7 +12,6 @@ use std::{ use ansi_colours::{ansi256_from_rgb, rgb_from_ansi256}; use anstyle::{Ansi256Color, AnsiColor, Color, Effects, RgbColor}; use anyhow::Result; -use log::{info, warn}; use serde::{ser::SerializeMap, Deserialize, Deserializer, Serialize, Serializer}; use serde_json::{json, Value}; use tree_sitter_highlight::{HighlightConfiguration, HighlightEvent, Highlighter, HtmlRenderer}; @@ -349,17 +348,19 @@ pub fn highlight( config.nonconformant_capture_names(&HashSet::new()) }; if names.is_empty() { - info!("All highlight captures conform to standards."); + eprintln!("All highlight captures conform to standards."); } else { - warn!( - "Non-standard highlight {} detected:\n* {}", + eprintln!( + "Non-standard highlight {} detected:", if names.len() > 1 { "captures" } else { "capture" - }, - names.join("\n* ") + } ); + for name in names { + eprintln!("* {name}"); + } } } @@ -450,7 +451,7 @@ pub fn highlight( } if opts.print_time { - info!("Time: {}ms", time.elapsed().as_millis()); + eprintln!("Time: {}ms", time.elapsed().as_millis()); } Ok(()) diff --git a/crates/cli/src/init.rs b/cli/src/init.rs similarity index 57% rename from crates/cli/src/init.rs rename to cli/src/init.rs index 70ca25af..709612af 100644 --- a/crates/cli/src/init.rs +++ b/cli/src/init.rs @@ -5,20 +5,14 @@ use std::{ }; use anyhow::{anyhow, Context, Result}; -use crc32fast::hash as crc32; use heck::{ToKebabCase, ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase}; use indoc::{formatdoc, indoc}; -use log::info; -use rand::{thread_rng, Rng}; use semver::Version; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use tree_sitter_generate::write_file; -use tree_sitter_loader::{ - Author, Bindings, Grammar, Links, Metadata, PathsJSON, TreeSitterJSON, - DEFAULT_HIGHLIGHTS_QUERY_FILE_NAME, DEFAULT_INJECTIONS_QUERY_FILE_NAME, - DEFAULT_LOCALS_QUERY_FILE_NAME, DEFAULT_TAGS_QUERY_FILE_NAME, -}; +use tree_sitter_loader::{Author, Bindings, Grammar, Links, Metadata, PathsJSON, TreeSitterJSON}; +use url::Url; const CLI_VERSION: &str = env!("CARGO_PKG_VERSION"); const CLI_VERSION_PLACEHOLDER: &str = "CLI_VERSION"; @@ -36,12 +30,9 @@ const PARSER_CLASS_NAME_PLACEHOLDER: &str = "PARSER_CLASS_NAME"; const PARSER_DESCRIPTION_PLACEHOLDER: &str = "PARSER_DESCRIPTION"; const PARSER_LICENSE_PLACEHOLDER: &str = "PARSER_LICENSE"; -const PARSER_NS_PLACEHOLDER: &str = "PARSER_NS"; -const PARSER_NS_CLEANED_PLACEHOLDER: &str = "PARSER_NS_CLEANED"; const PARSER_URL_PLACEHOLDER: &str = "PARSER_URL"; const PARSER_URL_STRIPPED_PLACEHOLDER: &str = "PARSER_URL_STRIPPED"; const PARSER_VERSION_PLACEHOLDER: &str = "PARSER_VERSION"; -const PARSER_FINGERPRINT_PLACEHOLDER: &str = "PARSER_FINGERPRINT"; const AUTHOR_NAME_PLACEHOLDER: &str = "PARSER_AUTHOR_NAME"; const AUTHOR_EMAIL_PLACEHOLDER: &str = "PARSER_AUTHOR_EMAIL"; @@ -60,22 +51,12 @@ const AUTHOR_BLOCK_RS: &str = "\nauthors = ["; const AUTHOR_NAME_PLACEHOLDER_RS: &str = "PARSER_AUTHOR_NAME"; const AUTHOR_EMAIL_PLACEHOLDER_RS: &str = " PARSER_AUTHOR_EMAIL"; -const AUTHOR_BLOCK_JAVA: &str = "\n "; -const AUTHOR_NAME_PLACEHOLDER_JAVA: &str = "\n PARSER_AUTHOR_NAME"; -const AUTHOR_EMAIL_PLACEHOLDER_JAVA: &str = "\n PARSER_AUTHOR_EMAIL"; -const AUTHOR_URL_PLACEHOLDER_JAVA: &str = "\n PARSER_AUTHOR_URL"; - const AUTHOR_BLOCK_GRAMMAR: &str = "\n * @author "; const AUTHOR_NAME_PLACEHOLDER_GRAMMAR: &str = "PARSER_AUTHOR_NAME"; const AUTHOR_EMAIL_PLACEHOLDER_GRAMMAR: &str = " PARSER_AUTHOR_EMAIL"; const FUNDING_URL_PLACEHOLDER: &str = "FUNDING_URL"; -const HIGHLIGHTS_QUERY_PATH_PLACEHOLDER: &str = "HIGHLIGHTS_QUERY_PATH"; -const INJECTIONS_QUERY_PATH_PLACEHOLDER: &str = "INJECTIONS_QUERY_PATH"; -const LOCALS_QUERY_PATH_PLACEHOLDER: &str = "LOCALS_QUERY_PATH"; -const TAGS_QUERY_PATH_PLACEHOLDER: &str = "TAGS_QUERY_PATH"; - const GRAMMAR_JS_TEMPLATE: &str = include_str!("./templates/grammar.js"); const PACKAGE_JSON_TEMPLATE: &str = include_str!("./templates/package.json"); const GITIGNORE_TEMPLATE: &str = include_str!("./templates/gitignore"); @@ -114,18 +95,32 @@ const TEST_BINDING_PY_TEMPLATE: &str = include_str!("./templates/test_binding.py const PACKAGE_SWIFT_TEMPLATE: &str = include_str!("./templates/package.swift"); const TESTS_SWIFT_TEMPLATE: &str = include_str!("./templates/tests.swift"); -const POM_XML_TEMPLATE: &str = include_str!("./templates/pom.xml"); -const BINDING_JAVA_TEMPLATE: &str = include_str!("./templates/binding.java"); -const TEST_JAVA_TEMPLATE: &str = include_str!("./templates/test.java"); - const BUILD_ZIG_TEMPLATE: &str = include_str!("./templates/build.zig"); const BUILD_ZIG_ZON_TEMPLATE: &str = include_str!("./templates/build.zig.zon"); const ROOT_ZIG_TEMPLATE: &str = include_str!("./templates/root.zig"); const TEST_ZIG_TEMPLATE: &str = include_str!("./templates/test.zig"); -pub const TREE_SITTER_JSON_SCHEMA: &str = +const TREE_SITTER_JSON_SCHEMA: &str = "https://tree-sitter.github.io/tree-sitter/assets/schemas/config.schema.json"; +#[must_use] +pub fn path_in_ignore(repo_path: &Path) -> bool { + [ + "bindings", + "build", + "examples", + "node_modules", + "queries", + "script", + "src", + "target", + "test", + "types", + ] + .iter() + .any(|dir| repo_path.ends_with(dir)) +} + #[derive(Serialize, Deserialize, Clone)] pub struct JsonConfigOpts { pub name: String, @@ -133,9 +128,9 @@ pub struct JsonConfigOpts { pub title: String, pub description: String, #[serde(skip_serializing_if = "Option::is_none")] - pub repository: Option, + pub repository: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub funding: Option, + pub funding: Option, pub scope: String, pub file_types: Vec, pub version: Version, @@ -144,9 +139,7 @@ pub struct JsonConfigOpts { #[serde(skip_serializing_if = "Option::is_none")] pub email: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub url: Option, - pub namespace: Option, - pub bindings: Bindings, + pub url: Option, } impl JsonConfigOpts { @@ -178,17 +171,22 @@ impl JsonConfigOpts { authors: Some(vec![Author { name: self.author, email: self.email, - url: self.url, + url: self.url.map(|url| url.to_string()), }]), links: Some(Links { repository: self.repository.unwrap_or_else(|| { - format!("https://github.com/tree-sitter/tree-sitter-{}", self.name) + Url::parse(&format!( + "https://github.com/tree-sitter/tree-sitter-{}", + self.name + )) + .expect("Failed to parse default repository URL") }), funding: self.funding, + homepage: None, }), - namespace: self.namespace, + namespace: None, }, - bindings: self.bindings, + bindings: Bindings::default(), } } } @@ -209,8 +207,6 @@ impl Default for JsonConfigOpts { author: String::new(), email: None, url: None, - namespace: None, - bindings: Bindings::default(), } } } @@ -227,11 +223,6 @@ struct GenerateOpts<'a> { camel_parser_name: &'a str, title_parser_name: &'a str, class_name: &'a str, - highlights_query_path: &'a str, - injections_query_path: &'a str, - locals_query_path: &'a str, - tags_query_path: &'a str, - namespace: Option<&'a str>, } pub fn generate_grammar_files( @@ -282,11 +273,6 @@ pub fn generate_grammar_files( .clone() .unwrap_or_else(|| format!("TreeSitter{}", language_name.to_upper_camel_case())); - let default_highlights_path = Path::new("queries").join(DEFAULT_HIGHLIGHTS_QUERY_FILE_NAME); - let default_injections_path = Path::new("queries").join(DEFAULT_INJECTIONS_QUERY_FILE_NAME); - let default_locals_path = Path::new("queries").join(DEFAULT_LOCALS_QUERY_FILE_NAME); - let default_tags_path = Path::new("queries").join(DEFAULT_TAGS_QUERY_FILE_NAME); - let generate_opts = GenerateOpts { author_name: authors .map(|a| a.first().map(|a| a.name.as_str())) @@ -308,24 +294,11 @@ pub fn generate_grammar_files( .metadata .links .as_ref() - .and_then(|l| l.funding.as_deref()), + .and_then(|l| l.funding.as_ref().map(|f| f.as_str())), version: &tree_sitter_config.metadata.version, camel_parser_name: &camel_name, title_parser_name: &title_name, class_name: &class_name, - highlights_query_path: tree_sitter_config.grammars[0] - .highlights - .to_variable_value(&default_highlights_path), - injections_query_path: tree_sitter_config.grammars[0] - .injections - .to_variable_value(&default_injections_path), - locals_query_path: tree_sitter_config.grammars[0] - .locals - .to_variable_value(&default_locals_path), - tags_query_path: tree_sitter_config.grammars[0] - .tags - .to_variable_value(&default_tags_path), - namespace: tree_sitter_config.metadata.namespace.as_deref(), }; // Create package.json @@ -341,9 +314,9 @@ pub fn generate_grammar_files( ) }, |path| { - let mut contents = fs::read_to_string(path)? + let contents = fs::read_to_string(path)? .replace( - r#""node-addon-api": "^8.3.1""#, + r#""node-addon-api": "^8.3.1"#, r#""node-addon-api": "^8.5.0""#, ) .replace( @@ -352,18 +325,9 @@ pub fn generate_grammar_files( "tree-sitter-cli":"#}, indoc! {r#" "prebuildify": "^6.0.1", - "tree-sitter": "^0.25.0", + "tree-sitter": "^0.22.4", "tree-sitter-cli":"#}, ); - if !contents.contains("module") { - info!("Migrating package.json to ESM"); - contents = contents.replace( - r#""repository":"#, - indoc! {r#" - "type": "module", - "repository":"#}, - ); - } write_file(path, contents)?; Ok(()) }, @@ -371,21 +335,9 @@ pub fn generate_grammar_files( // Do not create a grammar.js file in a repo with multiple language configs if !tree_sitter_config.has_multiple_language_configs() { - missing_path_else( - repo_path.join("grammar.js"), - allow_update, - |path| generate_file(path, GRAMMAR_JS_TEMPLATE, language_name, &generate_opts), - |path| { - let mut contents = fs::read_to_string(path)?; - if contents.contains("module.exports") { - info!("Migrating grammars.js to ESM"); - contents = contents.replace("module.exports =", "export default"); - write_file(path, contents)?; - } - - Ok(()) - }, - )?; + missing_path(repo_path.join("grammar.js"), |path| { + generate_file(path, GRAMMAR_JS_TEMPLATE, language_name, &generate_opts) + })?; } // Write .gitignore file @@ -394,16 +346,10 @@ pub fn generate_grammar_files( allow_update, |path| generate_file(path, GITIGNORE_TEMPLATE, language_name, &generate_opts), |path| { - let mut contents = fs::read_to_string(path)?; + let contents = fs::read_to_string(path)?; if !contents.contains("Zig artifacts") { - info!("Adding zig entries to .gitignore"); - contents.push('\n'); - contents.push_str(indoc! {" - # Zig artifacts - .zig-cache/ - zig-cache/ - zig-out/ - "}); + eprintln!("Replacing .gitignore"); + generate_file(path, GITIGNORE_TEMPLATE, language_name, &generate_opts)?; } Ok(()) }, @@ -416,13 +362,8 @@ pub fn generate_grammar_files( |path| generate_file(path, GITATTRIBUTES_TEMPLATE, language_name, &generate_opts), |path| { let mut contents = fs::read_to_string(path)?; - let c_bindings_entry = "bindings/c/* "; - if contents.contains(c_bindings_entry) { - info!("Updating c bindings entry in .gitattributes"); - contents = contents.replace(c_bindings_entry, "bindings/c/** "); - } + contents = contents.replace("bindings/c/* ", "bindings/c/** "); if !contents.contains("Zig bindings") { - info!("Adding zig entries to .gitattributes"); contents.push('\n'); contents.push_str(indoc! {" # Zig bindings @@ -445,131 +386,13 @@ pub fn generate_grammar_files( // Generate Rust bindings if tree_sitter_config.bindings.rust { missing_path(bindings_dir.join("rust"), create_dir)?.apply(|path| { - missing_path_else(path.join("lib.rs"), allow_update, |path| { + missing_path(path.join("lib.rs"), |path| { generate_file(path, LIB_RS_TEMPLATE, language_name, &generate_opts) - }, |path| { - let mut contents = fs::read_to_string(path)?; - if !contents.contains("#[cfg(with_highlights_query)]") { - info!("Updating query constants in bindings/rust/lib.rs"); - let replacement = indoc! {r#" - #[cfg(with_highlights_query)] - /// The syntax highlighting query for this grammar. - pub const HIGHLIGHTS_QUERY: &str = include_str!("../../HIGHLIGHTS_QUERY_PATH"); - - #[cfg(with_injections_query)] - /// The language injection query for this grammar. - pub const INJECTIONS_QUERY: &str = include_str!("../../INJECTIONS_QUERY_PATH"); - - #[cfg(with_locals_query)] - /// The local variable query for this grammar. - pub const LOCALS_QUERY: &str = include_str!("../../LOCALS_QUERY_PATH"); - - #[cfg(with_tags_query)] - /// The symbol tagging query for this grammar. - pub const TAGS_QUERY: &str = include_str!("../../TAGS_QUERY_PATH"); - "#} - .replace("HIGHLIGHTS_QUERY_PATH", generate_opts.highlights_query_path) - .replace("INJECTIONS_QUERY_PATH", generate_opts.injections_query_path) - .replace("LOCALS_QUERY_PATH", generate_opts.locals_query_path) - .replace("TAGS_QUERY_PATH", generate_opts.tags_query_path); - contents = contents - .replace( - indoc! {r#" - // NOTE: uncomment these to include any queries that this grammar contains: - - // pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm"); - // pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm"); - // pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm"); - // pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm"); - "#}, - &replacement, - ); - } - write_file(path, contents)?; - Ok(()) })?; - missing_path_else( - path.join("build.rs"), - allow_update, - |path| generate_file(path, BUILD_RS_TEMPLATE, language_name, &generate_opts), - |path| { - let mut contents = fs::read_to_string(path)?; - if !contents.contains("wasm32-unknown-unknown") { - info!("Adding wasm32-unknown-unknown target to bindings/rust/build.rs"); - let replacement = indoc!{r#" - c_config.flag("-utf-8"); - - if std::env::var("TARGET").unwrap() == "wasm32-unknown-unknown" { - let Ok(wasm_headers) = std::env::var("DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS") else { - panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS must be set by the language crate"); - }; - let Ok(wasm_src) = - std::env::var("DEP_TREE_SITTER_LANGUAGE_WASM_SRC").map(std::path::PathBuf::from) - else { - panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_SRC must be set by the language crate"); - }; - - c_config.include(&wasm_headers); - c_config.files([ - wasm_src.join("stdio.c"), - wasm_src.join("stdlib.c"), - wasm_src.join("string.c"), - ]); - } - "#} - .lines() - .map(|line| if line.is_empty() { line.to_string() } else { format!(" {line}") }) - .collect::>() - .join("\n"); - - contents = contents.replace(r#" c_config.flag("-utf-8");"#, &replacement); - } - - // Introduce configuration variables for dynamic query inclusion - if !contents.contains("with_highlights_query") { - info!("Adding support for dynamic query inclusion to bindings/rust/build.rs"); - let replaced = indoc! {r#" - c_config.compile("tree-sitter-KEBAB_PARSER_NAME"); - }"#} - .replace("KEBAB_PARSER_NAME", &language_name.to_kebab_case()); - - let replacement = indoc! {r#" - c_config.compile("tree-sitter-KEBAB_PARSER_NAME"); - - println!("cargo:rustc-check-cfg=cfg(with_highlights_query)"); - if !"HIGHLIGHTS_QUERY_PATH".is_empty() && std::path::Path::new("HIGHLIGHTS_QUERY_PATH").exists() { - println!("cargo:rustc-cfg=with_highlights_query"); - } - println!("cargo:rustc-check-cfg=cfg(with_injections_query)"); - if !"INJECTIONS_QUERY_PATH".is_empty() && std::path::Path::new("INJECTIONS_QUERY_PATH").exists() { - println!("cargo:rustc-cfg=with_injections_query"); - } - println!("cargo:rustc-check-cfg=cfg(with_locals_query)"); - if !"LOCALS_QUERY_PATH".is_empty() && std::path::Path::new("LOCALS_QUERY_PATH").exists() { - println!("cargo:rustc-cfg=with_locals_query"); - } - println!("cargo:rustc-check-cfg=cfg(with_tags_query)"); - if !"TAGS_QUERY_PATH".is_empty() && std::path::Path::new("TAGS_QUERY_PATH").exists() { - println!("cargo:rustc-cfg=with_tags_query"); - } - }"#} - .replace("KEBAB_PARSER_NAME", &language_name.to_kebab_case()) - .replace("HIGHLIGHTS_QUERY_PATH", generate_opts.highlights_query_path) - .replace("INJECTIONS_QUERY_PATH", generate_opts.injections_query_path) - .replace("LOCALS_QUERY_PATH", generate_opts.locals_query_path) - .replace("TAGS_QUERY_PATH", generate_opts.tags_query_path); - - contents = contents.replace( - &replaced, - &replacement, - ); - } - - write_file(path, contents)?; - Ok(()) - }, - )?; + missing_path(path.join("build.rs"), |path| { + generate_file(path, BUILD_RS_TEMPLATE, language_name, &generate_opts) + })?; missing_path_else( repo_path.join("Cargo.toml"), @@ -585,7 +408,6 @@ pub fn generate_grammar_files( |path| { let contents = fs::read_to_string(path)?; if contents.contains("\"LICENSE\"") { - info!("Adding LICENSE entry to bindings/rust/Cargo.toml"); write_file(path, contents.replace("\"LICENSE\"", "\"/LICENSE\""))?; } Ok(()) @@ -605,53 +427,26 @@ pub fn generate_grammar_files( |path| generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts), |path| { let contents = fs::read_to_string(path)?; - if !contents.contains("Object.defineProperty") { - info!("Replacing index.js"); + if !contents.contains("bun") { + eprintln!("Replacing index.js"); generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts)?; } Ok(()) }, )?; - missing_path_else( - path.join("index.d.ts"), - allow_update, - |path| generate_file(path, INDEX_D_TS_TEMPLATE, language_name, &generate_opts), - |path| { - let contents = fs::read_to_string(path)?; - if !contents.contains("export default binding") { - info!("Replacing index.d.ts"); - generate_file(path, INDEX_D_TS_TEMPLATE, language_name, &generate_opts)?; - } - Ok(()) - }, - )?; + missing_path(path.join("index.d.ts"), |path| { + generate_file(path, INDEX_D_TS_TEMPLATE, language_name, &generate_opts) + })?; - missing_path_else( - path.join("binding_test.js"), - allow_update, - |path| { - generate_file( - path, - BINDING_TEST_JS_TEMPLATE, - language_name, - &generate_opts, - ) - }, - |path| { - let contents = fs::read_to_string(path)?; - if !contents.contains("import") { - info!("Replacing binding_test.js"); - generate_file( - path, - BINDING_TEST_JS_TEMPLATE, - language_name, - &generate_opts, - )?; - } - Ok(()) - }, - )?; + missing_path(path.join("binding_test.js"), |path| { + generate_file( + path, + BINDING_TEST_JS_TEMPLATE, + language_name, + &generate_opts, + ) + })?; missing_path(path.join("binding.cc"), |path| { generate_file(path, JS_BINDING_CC_TEMPLATE, language_name, &generate_opts) @@ -664,7 +459,6 @@ pub fn generate_grammar_files( |path| { let contents = fs::read_to_string(path)?; if contents.contains("fs.exists(") { - info!("Replacing `fs.exists` calls in binding.gyp"); write_file(path, contents.replace("fs.exists(", "fs.existsSync("))?; } Ok(()) @@ -677,17 +471,14 @@ pub fn generate_grammar_files( // Generate C bindings if tree_sitter_config.bindings.c { - let kebab_case_name = language_name.to_kebab_case(); missing_path(bindings_dir.join("c"), create_dir)?.apply(|path| { - let header_name = format!("tree-sitter-{kebab_case_name}.h"); - let old_file = &path.join(&header_name); + let old_file = &path.join(format!("tree-sitter-{}.h", language_name.to_kebab_case())); if allow_update && fs::exists(old_file).unwrap_or(false) { - info!("Removing bindings/c/{header_name}"); fs::remove_file(old_file)?; } missing_path(path.join("tree_sitter"), create_dir)?.apply(|include_path| { missing_path( - include_path.join(&header_name), + include_path.join(format!("tree-sitter-{}.h", language_name.to_kebab_case())), |path| { generate_file(path, PARSER_NAME_H_TEMPLATE, language_name, &generate_opts) }, @@ -696,7 +487,7 @@ pub fn generate_grammar_files( })?; missing_path( - path.join(format!("tree-sitter-{kebab_case_name}.pc.in")), + path.join(format!("tree-sitter-{}.pc.in", language_name.to_kebab_case())), |path| { generate_file( path, @@ -714,31 +505,11 @@ pub fn generate_grammar_files( generate_file(path, MAKEFILE_TEMPLATE, language_name, &generate_opts) }, |path| { - let mut contents = fs::read_to_string(path)?; - if !contents.contains("cd '$(DESTDIR)$(LIBDIR)' && ln -sf") { - info!("Replacing Makefile"); - generate_file(path, MAKEFILE_TEMPLATE, language_name, &generate_opts)?; - } else { - let replaced = indoc! {r" - $(PARSER): $(SRC_DIR)/grammar.json - $(TS) generate $^ - "}; - if contents.contains(replaced) { - info!("Adding --no-parser target to Makefile"); - contents = contents - .replace( - replaced, - indoc! {r" - $(SRC_DIR)/grammar.json: grammar.js - $(TS) generate --no-parser $^ - - $(PARSER): $(SRC_DIR)/grammar.json - $(TS) generate $^ - "} - ); - } - write_file(path, contents)?; - } + let contents = fs::read_to_string(path)?.replace( + "-m644 bindings/c/$(LANGUAGE_NAME).h", + "-m644 bindings/c/tree_sitter/$(LANGUAGE_NAME).h" + ); + write_file(path, contents)?; Ok(()) }, )?; @@ -748,8 +519,8 @@ pub fn generate_grammar_files( allow_update, |path| generate_file(path, CMAKELISTS_TXT_TEMPLATE, language_name, &generate_opts), |path| { - let contents = fs::read_to_string(path)?; - let replaced_contents = contents + let mut contents = fs::read_to_string(path)?; + contents = contents .replace("add_custom_target(test", "add_custom_target(ts-test") .replace( &formatdoc! {r#" @@ -769,38 +540,8 @@ pub fn generate_grammar_files( INTERFACE $ $) "} - ).replace( - indoc! {r#" - add_custom_command(OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/src/parser.c" - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/grammar.json" - COMMAND "${TREE_SITTER_CLI}" generate src/grammar.json - --abi=${TREE_SITTER_ABI_VERSION} - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" - COMMENT "Generating parser.c") - "#}, - indoc! {r#" - add_custom_command(OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/src/grammar.json" - "${CMAKE_CURRENT_SOURCE_DIR}/src/node-types.json" - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/grammar.js" - COMMAND "${TREE_SITTER_CLI}" generate grammar.js --no-parser - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" - COMMENT "Generating grammar.json") - - add_custom_command(OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/src/parser.c" - BYPRODUCTS "${CMAKE_CURRENT_SOURCE_DIR}/src/tree_sitter/parser.h" - "${CMAKE_CURRENT_SOURCE_DIR}/src/tree_sitter/alloc.h" - "${CMAKE_CURRENT_SOURCE_DIR}/src/tree_sitter/array.h" - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/grammar.json" - COMMAND "${TREE_SITTER_CLI}" generate src/grammar.json - --abi=${TREE_SITTER_ABI_VERSION} - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" - COMMENT "Generating parser.c") - "#} ); - if !replaced_contents.eq(&contents) { - info!("Updating CMakeLists.txt"); - write_file(path, replaced_contents)?; - } + write_file(path, contents)?; Ok(()) }, )?; @@ -836,8 +577,7 @@ pub fn generate_grammar_files( // Generate Python bindings if tree_sitter_config.bindings.python { missing_path(bindings_dir.join("python"), create_dir)?.apply(|path| { - let snake_case_grammar_name = format!("tree_sitter_{}", language_name.to_snake_case()); - let lang_path = path.join(&snake_case_grammar_name); + let lang_path = path.join(format!("tree_sitter_{}", language_name.to_snake_case())); missing_path(&lang_path, create_dir)?; missing_path_else( @@ -847,7 +587,6 @@ pub fn generate_grammar_files( |path| { let mut contents = fs::read_to_string(path)?; if !contents.contains("PyModuleDef_Init") { - info!("Updating bindings/python/{snake_case_grammar_name}/binding.c"); contents = contents .replace("PyModule_Create", "PyModuleDef_Init") .replace( @@ -880,44 +619,13 @@ pub fn generate_grammar_files( }, )?; - missing_path_else( - lang_path.join("__init__.py"), - allow_update, - |path| { - generate_file(path, INIT_PY_TEMPLATE, language_name, &generate_opts) - }, - |path| { - let contents = fs::read_to_string(path)?; - if !contents.contains("uncomment these to include any queries") { - info!("Replacing __init__.py"); - generate_file(path, INIT_PY_TEMPLATE, language_name, &generate_opts)?; - } - Ok(()) - }, - )?; + missing_path(lang_path.join("__init__.py"), |path| { + generate_file(path, INIT_PY_TEMPLATE, language_name, &generate_opts) + })?; - missing_path_else( - lang_path.join("__init__.pyi"), - allow_update, - |path| generate_file(path, INIT_PYI_TEMPLATE, language_name, &generate_opts), - |path| { - let mut contents = fs::read_to_string(path)?; - if contents.contains("uncomment these to include any queries") { - info!("Replacing __init__.pyi"); - generate_file(path, INIT_PYI_TEMPLATE, language_name, &generate_opts)?; - } else if !contents.contains("CapsuleType") { - info!("Updating __init__.pyi"); - contents = contents - .replace( - "from typing import Final", - "from typing import Final\nfrom typing_extensions import CapsuleType" - ) - .replace("-> object:", "-> CapsuleType:"); - write_file(path, contents)?; - } - Ok(()) - }, - )?; + missing_path(lang_path.join("__init__.pyi"), |path| { + generate_file(path, INIT_PYI_TEMPLATE, language_name, &generate_opts) + })?; missing_path(lang_path.join("py.typed"), |path| { generate_file(path, "", language_name, &generate_opts) // py.typed is empty @@ -938,7 +646,6 @@ pub fn generate_grammar_files( |path| { let mut contents = fs::read_to_string(path)?; if !contents.contains("Parser(Language(") { - info!("Updating Language function in bindings/python/tests/test_binding.py"); contents = contents .replace("tree_sitter.Language(", "Parser(Language(") .replace(".language())\n", ".language()))\n") @@ -959,19 +666,11 @@ pub fn generate_grammar_files( allow_update, |path| generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts), |path| { - let mut contents = fs::read_to_string(path)?; + let contents = fs::read_to_string(path)?; if !contents.contains("build_ext") { - info!("Replacing setup.py"); + eprintln!("Replacing setup.py"); generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts)?; } - if !contents.contains(" and not get_config_var") { - info!("Updating Python free-threading support in setup.py"); - contents = contents.replace( - r#"startswith("cp"):"#, - r#"startswith("cp") and not get_config_var("Py_GIL_DISABLED"):"# - ); - write_file(path, contents)?; - } Ok(()) }, )?; @@ -990,7 +689,6 @@ pub fn generate_grammar_files( |path| { let mut contents = fs::read_to_string(path)?; if !contents.contains("cp310-*") { - info!("Updating dependencies in pyproject.toml"); contents = contents .replace(r#"build = "cp39-*""#, r#"build = "cp310-*""#) .replace(r#"python = ">=3.9""#, r#"python = ">=3.10""#) @@ -1028,18 +726,15 @@ pub fn generate_grammar_files( allow_update, |path| generate_file(path, PACKAGE_SWIFT_TEMPLATE, language_name, &generate_opts), |path| { - let contents = fs::read_to_string(path)?; - let replaced_contents = contents + let mut contents = fs::read_to_string(path)?; + contents = contents .replace( "https://github.com/ChimeHQ/SwiftTreeSitter", "https://github.com/tree-sitter/swift-tree-sitter", ) .replace("version: \"0.8.0\")", "version: \"0.9.0\")") .replace("(url:", "(name: \"SwiftTreeSitter\", url:"); - if !replaced_contents.eq(&contents) { - info!("Updating tree-sitter dependency in Package.swift"); - write_file(path, contents)?; - } + write_file(path, contents)?; Ok(()) }, )?; @@ -1057,7 +752,7 @@ pub fn generate_grammar_files( |path| { let contents = fs::read_to_string(path)?; if !contents.contains("b.pkg_hash.len") { - info!("Replacing build.zig"); + eprintln!("Replacing build.zig"); generate_file(path, BUILD_ZIG_TEMPLATE, language_name, &generate_opts) } else { Ok(()) @@ -1072,7 +767,7 @@ pub fn generate_grammar_files( |path| { let contents = fs::read_to_string(path)?; if !contents.contains(".name = .tree_sitter_") { - info!("Replacing build.zig.zon"); + eprintln!("Replacing build.zig.zon"); generate_file(path, BUILD_ZIG_ZON_TEMPLATE, language_name, &generate_opts) } else { Ok(()) @@ -1088,7 +783,7 @@ pub fn generate_grammar_files( |path| { let contents = fs::read_to_string(path)?; if contents.contains("ts.Language") { - info!("Replacing root.zig"); + eprintln!("Replacing root.zig"); generate_file(path, ROOT_ZIG_TEMPLATE, language_name, &generate_opts) } else { Ok(()) @@ -1104,45 +799,6 @@ pub fn generate_grammar_files( })?; } - // Generate Java bindings - if tree_sitter_config.bindings.java { - missing_path(repo_path.join("pom.xml"), |path| { - generate_file(path, POM_XML_TEMPLATE, language_name, &generate_opts) - })?; - - missing_path(bindings_dir.join("java"), create_dir)?.apply(|path| { - missing_path(path.join("main"), create_dir)?.apply(|path| { - let package_path = generate_opts - .namespace - .unwrap_or("io.github.treesitter") - .replace(['-', '_'], "") - .split('.') - .fold(path.to_path_buf(), |path, dir| path.join(dir)) - .join("jtreesitter") - .join(language_name.to_lowercase().replace('_', "")); - missing_path(package_path, create_dir)?.apply(|path| { - missing_path(path.join(format!("{class_name}.java")), |path| { - generate_file(path, BINDING_JAVA_TEMPLATE, language_name, &generate_opts) - })?; - - Ok(()) - })?; - - Ok(()) - })?; - - missing_path(path.join("test"), create_dir)?.apply(|path| { - missing_path(path.join(format!("{class_name}Test.java")), |path| { - generate_file(path, TEST_JAVA_TEMPLATE, language_name, &generate_opts) - })?; - - Ok(()) - })?; - - Ok(()) - })?; - } - Ok(()) } @@ -1192,15 +848,6 @@ fn generate_file( ) -> Result<()> { let filename = path.file_name().unwrap().to_str().unwrap(); - let lower_parser_name = if path - .extension() - .is_some_and(|e| e.eq_ignore_ascii_case("java")) - { - language_name.to_snake_case().replace('_', "") - } else { - language_name.to_snake_case() - }; - let mut replacement = template .replace( CAMEL_PARSER_NAME_PLACEHOLDER, @@ -1214,11 +861,14 @@ fn generate_file( UPPER_PARSER_NAME_PLACEHOLDER, &language_name.to_shouty_snake_case(), ) + .replace( + LOWER_PARSER_NAME_PLACEHOLDER, + &language_name.to_snake_case(), + ) .replace( KEBAB_PARSER_NAME_PLACEHOLDER, &language_name.to_kebab_case(), ) - .replace(LOWER_PARSER_NAME_PLACEHOLDER, &lower_parser_name) .replace(PARSER_NAME_PLACEHOLDER, language_name) .replace(CLI_VERSION_PLACEHOLDER, CLI_VERSION) .replace(RUST_BINDING_VERSION_PLACEHOLDER, RUST_BINDING_VERSION) @@ -1227,20 +877,7 @@ fn generate_file( PARSER_VERSION_PLACEHOLDER, &generate_opts.version.to_string(), ) - .replace(PARSER_CLASS_NAME_PLACEHOLDER, generate_opts.class_name) - .replace( - HIGHLIGHTS_QUERY_PATH_PLACEHOLDER, - generate_opts.highlights_query_path, - ) - .replace( - INJECTIONS_QUERY_PATH_PLACEHOLDER, - generate_opts.injections_query_path, - ) - .replace( - LOCALS_QUERY_PATH_PLACEHOLDER, - generate_opts.locals_query_path, - ) - .replace(TAGS_QUERY_PATH_PLACEHOLDER, generate_opts.tags_query_path); + .replace(PARSER_CLASS_NAME_PLACEHOLDER, generate_opts.class_name); if let Some(name) = generate_opts.author_name { replacement = replacement.replace(AUTHOR_NAME_PLACEHOLDER, name); @@ -1258,9 +895,6 @@ fn generate_file( "Cargo.toml" => { replacement = replacement.replace(AUTHOR_NAME_PLACEHOLDER_RS, ""); } - "pom.xml" => { - replacement = replacement.replace(AUTHOR_NAME_PLACEHOLDER_JAVA, ""); - } _ => {} } } @@ -1286,52 +920,30 @@ fn generate_file( "Cargo.toml" => { replacement = replacement.replace(AUTHOR_EMAIL_PLACEHOLDER_RS, ""); } - "pom.xml" => { - replacement = replacement.replace(AUTHOR_EMAIL_PLACEHOLDER_JAVA, ""); - } _ => {} } } - match (generate_opts.author_url, filename) { - (Some(url), "package.json" | "pom.xml") => { + if filename == "package.json" { + if let Some(url) = generate_opts.author_url { replacement = replacement.replace(AUTHOR_URL_PLACEHOLDER, url); - } - (None, "package.json") => { + } else { replacement = replacement.replace(AUTHOR_URL_PLACEHOLDER_JS, ""); } - (None, "pom.xml") => { - replacement = replacement.replace(AUTHOR_URL_PLACEHOLDER_JAVA, ""); - } - _ => {} } if generate_opts.author_name.is_none() && generate_opts.author_email.is_none() && generate_opts.author_url.is_none() + && filename == "package.json" { - match filename { - "package.json" => { - if let Some(start_idx) = replacement.find(AUTHOR_BLOCK_JS) { - if let Some(end_idx) = replacement[start_idx..] - .find("},") - .map(|i| i + start_idx + 2) - { - replacement.replace_range(start_idx..end_idx, ""); - } - } + if let Some(start_idx) = replacement.find(AUTHOR_BLOCK_JS) { + if let Some(end_idx) = replacement[start_idx..] + .find("},") + .map(|i| i + start_idx + 2) + { + replacement.replace_range(start_idx..end_idx, ""); } - "pom.xml" => { - if let Some(start_idx) = replacement.find(AUTHOR_BLOCK_JAVA) { - if let Some(end_idx) = replacement[start_idx..] - .find("") - .map(|i| i + start_idx + 12) - { - replacement.replace_range(start_idx..end_idx, ""); - } - } - } - _ => {} } } else if generate_opts.author_name.is_none() && generate_opts.author_email.is_none() { match filename { @@ -1369,60 +981,52 @@ fn generate_file( } } - if let Some(license) = generate_opts.license { - replacement = replacement.replace(PARSER_LICENSE_PLACEHOLDER, license); - } else { - replacement = replacement.replace(PARSER_LICENSE_PLACEHOLDER, "MIT"); + match generate_opts.license { + Some(license) => replacement = replacement.replace(PARSER_LICENSE_PLACEHOLDER, license), + _ => replacement = replacement.replace(PARSER_LICENSE_PLACEHOLDER, "MIT"), } - if let Some(description) = generate_opts.description { - replacement = replacement.replace(PARSER_DESCRIPTION_PLACEHOLDER, description); - } else { - replacement = replacement.replace( - PARSER_DESCRIPTION_PLACEHOLDER, - &format!( - "{} grammar for tree-sitter", - generate_opts.camel_parser_name, - ), - ); - } - - if let Some(repository) = generate_opts.repository { - replacement = replacement - .replace( - PARSER_URL_STRIPPED_PLACEHOLDER, - &repository.replace("https://", "").to_lowercase(), - ) - .replace(PARSER_URL_PLACEHOLDER, &repository.to_lowercase()); - } else { - replacement = replacement - .replace( - PARSER_URL_STRIPPED_PLACEHOLDER, + match generate_opts.description { + Some(description) => { + replacement = replacement.replace(PARSER_DESCRIPTION_PLACEHOLDER, description); + } + _ => { + replacement = replacement.replace( + PARSER_DESCRIPTION_PLACEHOLDER, &format!( - "github.com/tree-sitter/tree-sitter-{}", - language_name.to_lowercase() - ), - ) - .replace( - PARSER_URL_PLACEHOLDER, - &format!( - "https://github.com/tree-sitter/tree-sitter-{}", - language_name.to_lowercase() + "{} grammar for tree-sitter", + generate_opts.camel_parser_name, ), ); + } } - if let Some(namespace) = generate_opts.namespace { - replacement = replacement - .replace( - PARSER_NS_CLEANED_PLACEHOLDER, - &namespace.replace(['-', '_'], ""), - ) - .replace(PARSER_NS_PLACEHOLDER, namespace); - } else { - replacement = replacement - .replace(PARSER_NS_CLEANED_PLACEHOLDER, "io.github.treesitter") - .replace(PARSER_NS_PLACEHOLDER, "io.github.tree-sitter"); + match generate_opts.repository { + Some(repository) => { + replacement = replacement + .replace( + PARSER_URL_STRIPPED_PLACEHOLDER, + &repository.replace("https://", "").to_lowercase(), + ) + .replace(PARSER_URL_PLACEHOLDER, &repository.to_lowercase()); + } + _ => { + replacement = replacement + .replace( + PARSER_URL_STRIPPED_PLACEHOLDER, + &format!( + "github.com/tree-sitter/tree-sitter-{}", + language_name.to_lowercase() + ), + ) + .replace( + PARSER_URL_PLACEHOLDER, + &format!( + "https://github.com/tree-sitter/tree-sitter-{}", + language_name.to_lowercase() + ), + ); + } } if let Some(funding_url) = generate_opts.funding { @@ -1444,18 +1048,6 @@ fn generate_file( } } - if filename == "build.zig.zon" { - let id = thread_rng().gen_range(1u32..0xFFFF_FFFFu32); - let checksum = crc32(format!("tree_sitter_{language_name}").as_bytes()); - replacement = replacement.replace( - PARSER_FINGERPRINT_PLACEHOLDER, - #[cfg(target_endian = "little")] - &format!("0x{checksum:x}{id:x}"), - #[cfg(target_endian = "big")] - &format!("0x{id:x}{checksum:x}"), - ); - } - write_file(path, replacement)?; Ok(()) } diff --git a/crates/cli/src/input.rs b/cli/src/input.rs similarity index 100% rename from crates/cli/src/input.rs rename to cli/src/input.rs diff --git a/crates/cli/src/tree_sitter_cli.rs b/cli/src/lib.rs similarity index 81% rename from crates/cli/src/tree_sitter_cli.rs rename to cli/src/lib.rs index 3960d961..9f42a835 100644 --- a/crates/cli/src/tree_sitter_cli.rs +++ b/cli/src/lib.rs @@ -1,4 +1,4 @@ -#![cfg_attr(not(any(test, doctest)), doc = include_str!("../README.md"))] +#![doc = include_str!("../README.md")] pub mod fuzz; pub mod highlight; @@ -20,5 +20,6 @@ pub mod wasm; #[cfg(test)] mod tests; +// To run compile fail tests #[cfg(doctest)] mod tests; diff --git a/cli/src/logger.rs b/cli/src/logger.rs new file mode 100644 index 00000000..ce4f74a3 --- /dev/null +++ b/cli/src/logger.rs @@ -0,0 +1,30 @@ +use log::{LevelFilter, Log, Metadata, Record}; + +#[allow(dead_code)] +struct Logger { + pub filter: Option, +} + +impl Log for Logger { + fn enabled(&self, _: &Metadata) -> bool { + true + } + + fn log(&self, record: &Record) { + eprintln!( + "[{}] {}", + record + .module_path() + .unwrap_or_default() + .trim_start_matches("rust_tree_sitter_cli::"), + record.args() + ); + } + + fn flush(&self) {} +} + +pub fn init() { + log::set_boxed_logger(Box::new(Logger { filter: None })).unwrap(); + log::set_max_level(LevelFilter::Info); +} diff --git a/crates/cli/src/main.rs b/cli/src/main.rs similarity index 67% rename from crates/cli/src/main.rs rename to cli/src/main.rs index fbf75b8e..ecb638f7 100644 --- a/crates/cli/src/main.rs +++ b/cli/src/main.rs @@ -8,9 +8,8 @@ use anstyle::{AnsiColor, Color, Style}; use anyhow::{anyhow, Context, Result}; use clap::{crate_authors, Args, Command, FromArgMatches as _, Subcommand, ValueEnum}; use clap_complete::generate; -use dialoguer::{theme::ColorfulTheme, Confirm, FuzzySelect, Input, MultiSelect}; +use dialoguer::{theme::ColorfulTheme, Confirm, FuzzySelect, Input}; use heck::ToUpperCamelCase; -use log::{error, info, warn}; use regex::Regex; use semver::Version as SemverVersion; use tree_sitter::{ffi, Parser, Point}; @@ -20,23 +19,20 @@ use tree_sitter_cli::{ LOG_GRAPH_ENABLED, START_SEED, }, highlight::{self, HighlightOptions}, - init::{generate_grammar_files, JsonConfigOpts, TREE_SITTER_JSON_SCHEMA}, + init::{generate_grammar_files, get_root_path, JsonConfigOpts}, input::{get_input, get_tmp_source_file, CliInput}, logger, parse::{self, ParseDebugType, ParseFileOptions, ParseOutput, ParseTheme}, - playground, - query::{self, QueryFileOptions}, + playground, query, tags::{self, TagsOptions}, - test::{self, TestOptions, TestStats, TestSummary}, - test_highlight, test_tags, util, - version::{self, BumpLevel}, - wasm, + test::{self, TestOptions, TestStats}, + test_highlight, test_tags, util, version, wasm, }; use tree_sitter_config::Config; -use tree_sitter_generate::OptLevel; use tree_sitter_highlight::Highlighter; -use tree_sitter_loader::{self as loader, Bindings, TreeSitterJSON}; +use tree_sitter_loader::{self as loader, TreeSitterJSON}; use tree_sitter_tags::TagsContext; +use url::Url; const BUILD_VERSION: &str = env!("CARGO_PKG_VERSION"); const BUILD_SHA: Option<&'static str> = option_env!("BUILD_SHA"); @@ -57,7 +53,7 @@ enum Commands { Parse(Parse), /// Run a parser's tests Test(Test), - /// Display or increment the version of a grammar + /// Increment the version of a grammar Version(Version), /// Fuzz a parser Fuzz(Fuzz), @@ -84,9 +80,6 @@ struct Init { /// Update outdated files #[arg(long, short)] pub update: bool, - /// The path to the tree-sitter grammar directory - #[arg(long, short = 'p')] - pub grammar_path: Option, } #[derive(Args)] @@ -94,10 +87,13 @@ struct Init { struct Generate { /// The path to the grammar file #[arg(index = 1)] - pub grammar_path: Option, + pub grammar_path: Option, /// Show debug log during generation #[arg(long, short)] pub log: bool, + /// Deprecated (no-op) + #[arg(long)] + pub no_bindings: bool, #[arg( long = "abi", value_name = "VERSION", @@ -111,77 +107,50 @@ struct Generate { ) )] pub abi_version: Option, - /// Only generate `grammar.json` and `node-types.json` - #[arg(long)] - pub no_parser: bool, - /// Deprecated: use the `build` command + /// Compile all defined languages in the current dir #[arg(long, short = 'b')] pub build: bool, - /// Deprecated: use the `build` command + /// Compile a parser in debug mode #[arg(long, short = '0')] pub debug_build: bool, - /// Deprecated: use the `build` command + /// The path to the directory containing the parser library #[arg(long, value_name = "PATH")] - pub libdir: Option, + pub libdir: Option, /// The path to output the generated source files #[arg(long, short, value_name = "DIRECTORY")] - pub output: Option, + pub output: Option, /// Produce a report of the states for the given rule, use `-` to report every rule - #[arg(long, conflicts_with = "json", conflicts_with = "json_summary")] - pub report_states_for_rule: Option, - /// Deprecated: use --json-summary - #[arg( - long, - conflicts_with = "json_summary", - conflicts_with = "report_states_for_rule" - )] - pub json: bool, - /// Report conflicts in a JSON format - #[arg( - long, - conflicts_with = "json", - conflicts_with = "report_states_for_rule" - )] - pub json_summary: bool, - /// The name or path of the JavaScript runtime to use for generating parsers - #[cfg(not(feature = "qjs-rt"))] - #[arg( - long, - value_name = "EXECUTABLE", - env = "TREE_SITTER_JS_RUNTIME", - default_value = "node" - )] - pub js_runtime: Option, - - #[cfg(feature = "qjs-rt")] - #[arg( - long, - value_name = "EXECUTABLE", - env = "TREE_SITTER_JS_RUNTIME", - default_value = "node" - )] - /// The name or path of the JavaScript runtime to use for generating parsers, specify `native` - /// to use the native `QuickJS` runtime - pub js_runtime: Option, - - /// Disable optimizations when generating the parser. Currently, this only affects - /// the merging of compatible parse states. #[arg(long)] - pub disable_optimizations: bool, + pub report_states_for_rule: Option, + /// Report conflicts in a JSON format + #[arg(long)] + pub json: bool, + /// The name or path of the JavaScript runtime to use for generating parsers + #[arg( + long, + value_name = "EXECUTABLE", + env = "TREE_SITTER_JS_RUNTIME", + default_value = "node" + )] + pub js_runtime: Option, } #[derive(Args)] #[command(alias = "b")] struct Build { - /// Build a Wasm module instead of a dynamic library + /// Build a WASM module instead of a dynamic library #[arg(short, long)] pub wasm: bool, + /// Run emscripten via docker even if it is installed locally (only if building a WASM module + /// with --wasm) + #[arg(short, long)] + pub docker: bool, /// The path to output the compiled file #[arg(short, long)] - pub output: Option, + pub output: Option, /// The path to the grammar directory #[arg(index = 1, num_args = 1)] - pub path: Option, + pub path: Option, /// Make the parser reuse the same allocator as the library #[arg(long)] pub reuse_allocator: bool, @@ -199,16 +168,6 @@ struct Parse { /// The source file(s) to use #[arg(num_args=1..)] pub paths: Option>, - /// The path to the tree-sitter grammar directory, implies --rebuild - #[arg(long, short = 'p', conflicts_with = "rebuild")] - pub grammar_path: Option, - /// The path to the parser's dynamic library - #[arg(long, short = 'l')] - pub lib_path: Option, - /// If `--lib-path` is used, the name of the language used to extract the - /// library's language function - #[arg(long)] - pub lang_name: Option, /// Select a language by the scope instead of a file extension #[arg(long)] pub scope: Option, @@ -222,8 +181,8 @@ struct Parse { /// Produce the log.html file with debug graphs #[arg(long, short = 'D')] pub debug_graph: bool, - /// Compile parsers to Wasm instead of native dynamic libraries - #[arg(long, hide = cfg!(not(feature = "wasm")))] + /// Compile parsers to wasm instead of native dynamic libraries + #[arg(long)] pub wasm: bool, /// Output the parse data with graphviz dot #[arg(long = "dot")] @@ -235,7 +194,7 @@ struct Parse { #[arg(long = "cst", short = 'c')] pub output_cst: bool, /// Show parsing statistic - #[arg(long, short, conflicts_with = "json", conflicts_with = "json_summary")] + #[arg(long, short)] pub stat: bool, /// Interrupt the parsing process by timeout (µs) #[arg(long)] @@ -260,12 +219,9 @@ struct Parse { /// Open `log.html` in the default browser, if `--debug-graph` is supplied #[arg(long)] pub open_log: bool, - /// Deprecated: use --json-summary - #[arg(long, conflicts_with = "json_summary", conflicts_with = "stat")] - pub json: bool, /// Output parsing results in a JSON format - #[arg(long, short = 'j', conflicts_with = "json", conflicts_with = "stat")] - pub json_summary: bool, + #[arg(long, short = 'j')] + pub json: bool, /// The path to an alternative config.json file #[arg(long)] pub config_path: Option, @@ -297,19 +253,9 @@ struct Test { /// Only run corpus test cases whose name does not match the given regex #[arg(long, short)] pub exclude: Option, - /// Only run corpus test cases from a given filename + /// Only run corpus test cases from from a given filename #[arg(long)] pub file_name: Option, - /// The path to the tree-sitter grammar directory, implies --rebuild - #[arg(long, short = 'p', conflicts_with = "rebuild")] - pub grammar_path: Option, - /// The path to the parser's dynamic library - #[arg(long, short = 'l')] - pub lib_path: Option, - /// If `--lib-path` is used, the name of the language used to extract the - /// library's language function - #[arg(long)] - pub lang_name: Option, /// Update all syntax trees in corpus files with current parser output #[arg(long, short)] pub update: bool, @@ -322,8 +268,8 @@ struct Test { /// Produce the log.html file with debug graphs #[arg(long, short = 'D')] pub debug_graph: bool, - /// Compile parsers to Wasm instead of native dynamic libraries - #[arg(long, hide = cfg!(not(feature = "wasm")))] + /// Compile parsers to wasm instead of native dynamic libraries + #[arg(long)] pub wasm: bool, /// Open `log.html` in the default browser, if `--debug-graph` is supplied #[arg(long)] @@ -343,33 +289,15 @@ struct Test { /// Show only the pass-fail overview tree #[arg(long)] pub overview_only: bool, - /// Output the test summary in a JSON format - #[arg(long)] - pub json_summary: bool, } #[derive(Args)] #[command(alias = "publish")] -/// Display or increment the version of a grammar +/// Increment the version of a grammar struct Version { + #[arg(num_args = 1)] /// The version to bump to - #[arg( - conflicts_with = "bump", - long_help = "\ - The version to bump to\n\ - \n\ - Examples:\n \ - tree-sitter version: display the current version\n \ - tree-sitter version : bump to specified version\n \ - tree-sitter version --bump : automatic bump" - )] - pub version: Option, - /// The path to the tree-sitter grammar directory - #[arg(long, short = 'p')] - pub grammar_path: Option, - /// Automatically bump from the current version - #[arg(long, value_enum, conflicts_with = "version")] - pub bump: Option, + pub version: SemverVersion, } #[derive(Args)] @@ -380,17 +308,7 @@ struct Fuzz { pub skip: Option>, /// Subdirectory to the language #[arg(long)] - pub subdir: Option, - /// The path to the tree-sitter grammar directory, implies --rebuild - #[arg(long, short = 'p', conflicts_with = "rebuild")] - pub grammar_path: Option, - /// The path to the parser's dynamic library - #[arg(long)] - pub lib_path: Option, - /// If `--lib-path` is used, the name of the language used to extract the - /// library's language function - #[arg(long)] - pub lang_name: Option, + pub subdir: Option, /// Maximum number of edits to perform per fuzz test #[arg(long)] pub edits: Option, @@ -420,16 +338,6 @@ struct Query { /// Path to a file with queries #[arg(index = 1, required = true)] query_path: PathBuf, - /// The path to the tree-sitter grammar directory, implies --rebuild - #[arg(long, short = 'p', conflicts_with = "rebuild")] - pub grammar_path: Option, - /// The path to the parser's dynamic library - #[arg(long, short = 'l')] - pub lib_path: Option, - /// If `--lib-path` is used, the name of the language used to extract the - /// library's language function - #[arg(long)] - pub lang_name: Option, /// Measure execution time #[arg(long, short)] pub time: bool, @@ -448,14 +356,6 @@ struct Query { /// The range of rows in which the query will be executed #[arg(long)] pub row_range: Option, - /// The range of byte offsets in which the query will be executed. Only the matches that are fully contained within the provided - /// byte range will be returned. - #[arg(long)] - pub containing_byte_range: Option, - /// The range of rows in which the query will be executed. Only the matches that are fully contained within the provided row range - /// will be returned. - #[arg(long)] - pub containing_row_range: Option, /// Select a language by the scope instead of a file extension #[arg(long)] pub scope: Option, @@ -472,9 +372,6 @@ struct Query { #[arg(long, short = 'n')] #[clap(conflicts_with = "paths", conflicts_with = "paths_file")] pub test_number: Option, - /// Force rebuild the parser - #[arg(short, long)] - pub rebuild: bool, } #[derive(Args)] @@ -510,9 +407,6 @@ struct Highlight { /// The source file(s) to use #[arg(num_args = 1..)] pub paths: Option>, - /// The path to the tree-sitter grammar directory, implies --rebuild - #[arg(long, short = 'p', conflicts_with = "rebuild")] - pub grammar_path: Option, /// The path to an alternative config.json file #[arg(long)] pub config_path: Option, @@ -520,9 +414,6 @@ struct Highlight { #[arg(long, short = 'n')] #[clap(conflicts_with = "paths", conflicts_with = "paths_file")] pub test_number: Option, - /// Force rebuild the parser - #[arg(short, long)] - pub rebuild: bool, } #[derive(Args)] @@ -542,9 +433,6 @@ struct Tags { /// The source file(s) to use #[arg(num_args = 1..)] pub paths: Option>, - /// The path to the tree-sitter grammar directory, implies --rebuild - #[arg(long, short = 'p', conflicts_with = "rebuild")] - pub grammar_path: Option, /// The path to an alternative config.json file #[arg(long)] pub config_path: Option, @@ -552,9 +440,6 @@ struct Tags { #[arg(long, short = 'n')] #[clap(conflicts_with = "paths", conflicts_with = "paths_file")] pub test_number: Option, - /// Force rebuild the parser - #[arg(short, long)] - pub rebuild: bool, } #[derive(Args)] @@ -563,12 +448,9 @@ struct Playground { /// Don't open in default browser #[arg(long, short)] pub quiet: bool, - /// Path to the directory containing the grammar and Wasm files + /// Path to the directory containing the grammar and wasm files #[arg(long)] - pub grammar_path: Option, - /// Export playground files to specified directory instead of serving them - #[arg(long, short)] - pub export: Option, + pub grammar_path: Option, } #[derive(Args)] @@ -597,20 +479,6 @@ pub enum Shell { Nushell, } -/// Complete `action` if the wasm feature is enabled, otherwise return an error -macro_rules! checked_wasm { - ($action:block) => { - #[cfg(feature = "wasm")] - { - $action - } - #[cfg(not(feature = "wasm"))] - { - Err(anyhow!("--wasm flag specified, but this build of tree-sitter-cli does not include the wasm feature"))?; - } - }; -} - impl InitConfig { fn run() -> Result<()> { if let Ok(Some(config_path)) = Config::find_config_file() { @@ -623,7 +491,7 @@ impl InitConfig { config.add(tree_sitter_loader::Config::initial())?; config.add(tree_sitter_cli::highlight::ThemeConfig::default())?; config.save()?; - info!( + println!( "Saved initial configuration to {}", config.location.display() ); @@ -688,10 +556,15 @@ impl Init { }; let repository = |name: &str| { - Input::::with_theme(&ColorfulTheme::default()) + Input::::with_theme(&ColorfulTheme::default()) .with_prompt("Repository URL") .allow_empty(true) - .default(format!("https://github.com/tree-sitter/tree-sitter-{name}")) + .default( + Url::parse(&format!( + "https://github.com/tree-sitter/tree-sitter-{name}" + )) + .expect("Failed to parse default repository URL"), + ) .show_default(false) .interact_text() }; @@ -700,8 +573,18 @@ impl Init { Input::::with_theme(&ColorfulTheme::default()) .with_prompt("Funding URL") .allow_empty(true) + .validate_with(|input: &String| { + if input.trim().is_empty() + || Url::parse(input) + .is_ok_and(|u| u.scheme() == "http" || u.scheme() == "https") + { + Ok(()) + } else { + Err("The URL must start with 'http://' or 'https://'") + } + }) .interact_text() - .map(|e| Some(e.trim().to_string())) + .map(|e| (!e.trim().is_empty()).then(|| Url::parse(&e).unwrap())) }; let scope = |name: &str| { @@ -768,31 +651,15 @@ impl Init { Input::::with_theme(&ColorfulTheme::default()) .with_prompt("Author URL") .allow_empty(true) + .validate_with(|input: &String| -> Result<(), &str> { + if input.trim().is_empty() || Url::parse(input).is_ok() { + Ok(()) + } else { + Err("This is not a valid URL") + } + }) .interact_text() - .map(|e| Some(e.trim().to_string())) - }; - - let namespace = || { - Input::::with_theme(&ColorfulTheme::default()) - .with_prompt("Package namespace") - .default("io.github.tree-sitter".to_string()) - .allow_empty(true) - .interact() - }; - - let bindings = || { - let languages = Bindings::default().languages(); - - let enabled = MultiSelect::new() - .with_prompt("Bindings") - .items_checked(&languages) - .interact()? - .into_iter() - .map(|i| languages[i].0); - - let out = Bindings::with_enabled_languages(enabled) - .expect("unexpected unsupported language"); - anyhow::Ok(out) + .map(|e| (!e.trim().is_empty()).then(|| Url::parse(&e).unwrap())) }; let choices = [ @@ -809,8 +676,6 @@ impl Init { "author", "email", "url", - "namespace", - "bindings", "exit", ]; @@ -830,8 +695,6 @@ impl Init { "author" => opts.author = author()?, "email" => opts.email = email()?, "url" => opts.url = url()?, - "namespace" => opts.namespace = Some(namespace()?), - "bindings" => opts.bindings = bindings()?, "exit" => break, _ => unreachable!(), } @@ -845,7 +708,7 @@ impl Init { // Loop for editing the configuration loop { - info!( + println!( "Your current configuration:\n{}", serde_json::to_string_pretty(&opts)? ); @@ -867,26 +730,10 @@ impl Init { (opts.name.clone(), Some(opts)) } else { - let old_config = fs::read_to_string(current_dir.join("tree-sitter.json")) - .with_context(|| "Failed to read tree-sitter.json")?; - - let mut json = serde_json::from_str::(&old_config)?; - if json.schema.is_none() { - json.schema = Some(TREE_SITTER_JSON_SCHEMA.to_string()); - } - - let new_config = format!("{}\n", serde_json::to_string_pretty(&json)?); - // Write the re-serialized config back, as newly added optional boolean fields - // will be included with explicit `false`s rather than implict `null`s - if self.update && !old_config.trim().eq(new_config.trim()) { - info!("Updating tree-sitter.json"); - fs::write( - current_dir.join("tree-sitter.json"), - serde_json::to_string_pretty(&json)?, - ) - .with_context(|| "Failed to write tree-sitter.json")?; - } - + let mut json = serde_json::from_str::( + &fs::read_to_string(current_dir.join("tree-sitter.json")) + .with_context(|| "Failed to read tree-sitter.json")?, + )?; (json.grammars.swap_remove(0).name, None) }; @@ -903,8 +750,11 @@ impl Init { impl Generate { fn run(self, mut loader: loader::Loader, current_dir: &Path) -> Result<()> { + if self.no_bindings { + eprint!("The --no-bindings flag is no longer used and will be removed in v0.25.0"); + } if self.log { - logger::enable_debug(); + logger::init(); } let abi_version = self.abi_version @@ -916,14 +766,6 @@ impl Generate { version.parse().expect("invalid abi version flag") } }); - - let json_summary = if self.json { - warn!("--json is deprecated, use --json-summary instead"); - true - } else { - self.json_summary - }; - if let Err(err) = tree_sitter_generate::generate_parser_in_directory( current_dir, self.output.as_deref(), @@ -931,14 +773,8 @@ impl Generate { abi_version, self.report_states_for_rule.as_deref(), self.js_runtime.as_deref(), - !self.no_parser, - if self.disable_optimizations { - OptLevel::empty() - } else { - OptLevel::default() - }, ) { - if json_summary { + if self.json { eprintln!("{}", serde_json::to_string_pretty(&err)?); // Exit early to prevent errors from being printed a second time in the caller std::process::exit(1); @@ -948,9 +784,8 @@ impl Generate { } } if self.build { - warn!("--build is deprecated, use the `build` command"); if let Some(path) = self.libdir { - loader = loader::Loader::with_parser_lib_path(path); + loader = loader::Loader::with_parser_lib_path(PathBuf::from(path)); } loader.debug_build(self.debug_build); loader.languages_at_path(current_dir)?; @@ -961,31 +796,27 @@ impl Generate { impl Build { fn run(self, mut loader: loader::Loader, current_dir: &Path) -> Result<()> { - let grammar_path = current_dir.join(self.path.unwrap_or_default()); - - loader.debug_build(self.debug); + let grammar_path = current_dir.join(self.path.as_deref().unwrap_or_default()); if self.wasm { let output_path = self.output.map(|path| current_dir.join(path)); - wasm::compile_language_to_wasm(&loader, &grammar_path, current_dir, output_path)?; + let root_path = get_root_path(&grammar_path.join("tree-sitter.json"))?; + wasm::compile_language_to_wasm( + &loader, + Some(&root_path), + &grammar_path, + current_dir, + output_path, + self.docker, + )?; } else { let output_path = if let Some(ref path) = self.output { let path = Path::new(path); - let full_path = if path.is_absolute() { + if path.is_absolute() { path.to_path_buf() } else { current_dir.join(path) - }; - let parent_path = full_path - .parent() - .context("Output path must have a parent")?; - let name = full_path - .file_name() - .context("Ouput path must have a filename")?; - fs::create_dir_all(parent_path).context("Failed to create output path")?; - let mut canon_path = parent_path.canonicalize().context("Invalid output path")?; - canon_path.push(name); - canon_path + } } else { let file_name = grammar_path .file_stem() @@ -1006,11 +837,15 @@ impl Build { (false, false) => &[], }; + loader.debug_build(self.debug); loader.force_rebuild(true); + let config = Config::load(None)?; + let loader_config = config.get()?; + loader.find_all_languages(&loader_config).unwrap(); loader .compile_parser_at_path(&grammar_path, output_path, flags) - .context("Failed to compile parser")?; + .unwrap(); } Ok(()) } @@ -1020,19 +855,13 @@ impl Parse { fn run(self, mut loader: loader::Loader, current_dir: &Path) -> Result<()> { let config = Config::load(self.config_path)?; let color = env::var("NO_COLOR").map_or(true, |v| v != "1"); - let json_summary = if self.json { - warn!("--json is deprecated, use --json-summary instead"); - true - } else { - self.json_summary - }; let output = if self.output_dot { ParseOutput::Dot } else if self.output_xml { ParseOutput::Xml } else if self.output_cst { ParseOutput::Cst - } else if self.quiet || json_summary { + } else if self.quiet || self.json { ParseOutput::Quiet } else { ParseOutput::Normal @@ -1061,16 +890,15 @@ impl Parse { let mut parser = Parser::new(); loader.debug_build(self.debug_build); - loader.force_rebuild(self.rebuild || self.grammar_path.is_some()); + loader.force_rebuild(self.rebuild); + #[cfg(feature = "wasm")] if self.wasm { - checked_wasm!({ - let engine = tree_sitter::wasmtime::Engine::default(); - parser - .set_wasm_store(tree_sitter::WasmStore::new(&engine).unwrap()) - .unwrap(); - loader.use_wasm(&engine); - }); + let engine = tree_sitter::wasmtime::Engine::default(); + parser + .set_wasm_store(tree_sitter::WasmStore::new(&engine).unwrap()) + .unwrap(); + loader.use_wasm(&engine); } let timeout = self.timeout.unwrap_or_default(); @@ -1107,7 +935,7 @@ impl Parse { let mut update_stats = |stats: &mut parse::ParseStats| { let parse_result = stats.parse_summaries.last().unwrap(); - if should_track_stats || json_summary { + if should_track_stats { stats.cumulative_stats.total_parses += 1; if parse_result.successful { stats.cumulative_stats.successful_parses += 1; @@ -1121,11 +949,6 @@ impl Parse { has_error |= !parse_result.successful; }; - if self.lib_path.is_none() && self.lang_name.is_some() { - warn!("--lang-name` specified without --lib-path. This argument will be ignored."); - } - let lib_info = get_lib_info(self.lib_path.as_ref(), self.lang_name.as_ref(), current_dir); - let input = get_input( self.paths_file.as_deref(), self.paths, @@ -1139,19 +962,13 @@ impl Parse { .map(|p| p.to_string_lossy().chars().count()) .max() .unwrap_or(0); - options.stats.source_count = paths.len(); for path in &paths { let path = Path::new(&path); let language = loader - .select_language( - Some(path), - current_dir, - self.scope.as_deref(), - lib_info.as_ref(), - ) + .select_language(path, current_dir, self.scope.as_deref()) .with_context(|| { - anyhow!("Failed to load language for path \"{}\"", path.display()) + anyhow!("Failed to load langauge for path \"{}\"", path.display()) })?; parse::parse_file_at_path( @@ -1173,33 +990,16 @@ impl Parse { } => { let path = get_tmp_source_file(&contents)?; let languages = loader.languages_at_path(current_dir)?; - - let language = if let Some(ref lib_path) = self.lib_path { - &loader - .select_language( - None, - current_dir, - self.scope.as_deref(), - lib_info.as_ref(), - ) - .with_context(|| { - anyhow!( - "Failed to load language for path \"{}\"", - lib_path.display() - ) - })? - } else { - &languages - .iter() - .find(|(_, n)| language_names.contains(&Box::from(n.as_str()))) - .or_else(|| languages.first()) - .map(|(l, _)| l.clone()) - .ok_or_else(|| anyhow!("No language found"))? - }; + let language = languages + .iter() + .find(|(_, n)| language_names.contains(&Box::from(n.as_str()))) + .or_else(|| languages.first()) + .map(|(l, _)| l.clone()) + .ok_or_else(|| anyhow!("No language found"))?; parse::parse_file_at_path( &mut parser, - language, + &language, &path, &name, name.chars().count(), @@ -1215,12 +1015,7 @@ impl Parse { let path = get_tmp_source_file(&contents)?; let name = "stdin"; - let language = loader.select_language( - None, - current_dir, - self.scope.as_deref(), - lib_info.as_ref(), - )?; + let language = loader.select_language(&path, current_dir, None)?; parse::parse_file_at_path( &mut parser, @@ -1238,7 +1033,7 @@ impl Parse { if should_track_stats { println!("\n{}", stats.cumulative_stats); } - if json_summary { + if self.json { println!("{}", serde_json::to_string_pretty(&stats)?); } @@ -1250,28 +1045,6 @@ impl Parse { } } -/// In case an error is encountered, prints out the contents of `test_summary` and -/// propagates the error -fn check_test( - test_result: Result<()>, - test_summary: &TestSummary, - json_summary: bool, -) -> Result<()> { - if let Err(e) = test_result { - if json_summary { - let json_summary = serde_json::to_string_pretty(test_summary) - .expect("Failed to encode summary to JSON"); - println!("{json_summary}"); - } else { - println!("{test_summary}"); - } - - Err(e)?; - } - - Ok(()) -} - impl Test { fn run(self, mut loader: loader::Loader, current_dir: &Path) -> Result<()> { let config = Config::load(self.config_path)?; @@ -1279,56 +1052,36 @@ impl Test { let stat = self.stat.unwrap_or_default(); loader.debug_build(self.debug_build); - loader.force_rebuild(self.rebuild || self.grammar_path.is_some()); + loader.force_rebuild(self.rebuild); let mut parser = Parser::new(); + #[cfg(feature = "wasm")] if self.wasm { - checked_wasm!({ - let engine = tree_sitter::wasmtime::Engine::default(); - parser - .set_wasm_store(tree_sitter::WasmStore::new(&engine).unwrap()) - .unwrap(); - loader.use_wasm(&engine); - }); + let engine = tree_sitter::wasmtime::Engine::default(); + parser + .set_wasm_store(tree_sitter::WasmStore::new(&engine).unwrap()) + .unwrap(); + loader.use_wasm(&engine); } - if self.lib_path.is_none() && self.lang_name.is_some() { - warn!("--lang-name` specified without --lib-path. This argument will be ignored."); - } let languages = loader.languages_at_path(current_dir)?; - let language = if let Some(ref lib_path) = self.lib_path { - let lib_info = - get_lib_info(self.lib_path.as_ref(), self.lang_name.as_ref(), current_dir); - &loader - .select_language(None, current_dir, None, lib_info.as_ref()) - .with_context(|| { - anyhow!( - "Failed to load language for path \"{}\"", - lib_path.display() - ) - })? - } else { - &languages - .first() - .ok_or_else(|| anyhow!("No language found"))? - .0 - }; + let language = &languages + .first() + .ok_or_else(|| anyhow!("No language found"))? + .0; parser.set_language(language)?; let test_dir = current_dir.join("test"); - let mut test_summary = TestSummary::new( - color, - stat, - self.update, - self.overview_only, - self.json_summary, - ); + let mut stats = parse::Stats::default(); // Run the corpus tests. Look for them in `test/corpus`. let test_corpus_dir = test_dir.join("corpus"); if test_corpus_dir.is_dir() { - let opts = TestOptions { + let mut output = String::new(); + let mut rates = Vec::new(); + let mut opts = TestOptions { + output: &mut output, path: test_corpus_dir, debug: self.debug, debug_graph: self.debug_graph, @@ -1339,67 +1092,51 @@ impl Test { open_log: self.open_log, languages: languages.iter().map(|(l, n)| (n.as_str(), l)).collect(), color, + test_num: 1, + parse_rates: &mut rates, + stat_display: stat, + stats: &mut stats, show_fields: self.show_fields, overview_only: self.overview_only, }; - check_test( - test::run_tests_at_path(&mut parser, &opts, &mut test_summary), - &test_summary, - self.json_summary, - )?; - test_summary.test_num = 1; + test::run_tests_at_path(&mut parser, &mut opts)?; + println!("\n{stats}"); } // Check that all of the queries are valid. - let query_dir = current_dir.join("queries"); - check_test( - test::check_queries_at_path(language, &query_dir), - &test_summary, - self.json_summary, - )?; - test_summary.test_num = 1; + test::check_queries_at_path(language, ¤t_dir.join("queries"))?; // Run the syntax highlighting tests. let test_highlight_dir = test_dir.join("highlight"); if test_highlight_dir.is_dir() { let mut highlighter = Highlighter::new(); highlighter.parser = parser; - check_test( - test_highlight::test_highlights( - &loader, - &config.get()?, - &mut highlighter, - &test_highlight_dir, - &mut test_summary, - ), - &test_summary, - self.json_summary, + test_highlight::test_highlights( + &loader, + &config.get()?, + &mut highlighter, + &test_highlight_dir, + color, )?; parser = highlighter.parser; - test_summary.test_num = 1; } let test_tag_dir = test_dir.join("tags"); if test_tag_dir.is_dir() { let mut tags_context = TagsContext::new(); tags_context.parser = parser; - check_test( - test_tags::test_tags( - &loader, - &config.get()?, - &mut tags_context, - &test_tag_dir, - &mut test_summary, - ), - &test_summary, - self.json_summary, + test_tags::test_tags( + &loader, + &config.get()?, + &mut tags_context, + &test_tag_dir, + color, )?; - test_summary.test_num = 1; } // For the rest of the queries, find their tests and run them - for entry in walkdir::WalkDir::new(&query_dir) + for entry in walkdir::WalkDir::new(current_dir.join("queries")) .into_iter() .filter_map(|e| e.ok()) .filter(|e| e.file_type().is_file()) @@ -1422,80 +1159,46 @@ impl Test { }) .collect::>(); if !entries.is_empty() { - test_summary.query_results.add_group(stem); + println!("{stem}:"); } - test_summary.test_num = 1; - let opts = QueryFileOptions::default(); - for entry in &entries { + for entry in entries { let path = entry.path(); - check_test( - query::query_file_at_path( - language, - path, - &path.display().to_string(), - path, - &opts, - Some(&mut test_summary), - ), - &test_summary, - self.json_summary, + query::query_file_at_path( + language, + path, + &path.display().to_string(), + path, + false, + None, + None, + true, + false, + false, + false, )?; } - if !entries.is_empty() { - test_summary.query_results.pop_traversal(); - } } } - test_summary.test_num = 1; - - if self.json_summary { - let json_summary = serde_json::to_string_pretty(&test_summary) - .expect("Failed to encode test summary to JSON"); - println!("{json_summary}"); - } else { - println!("{test_summary}"); - } - Ok(()) } } impl Version { fn run(self, current_dir: PathBuf) -> Result<()> { - Ok(version::Version::new(self.version, current_dir, self.bump).run()?) + version::Version::new(self.version.to_string(), current_dir).run() } } impl Fuzz { fn run(self, mut loader: loader::Loader, current_dir: &Path) -> Result<()> { loader.sanitize_build(true); - loader.force_rebuild(self.rebuild || self.grammar_path.is_some()); + loader.force_rebuild(self.rebuild); - if self.lib_path.is_none() && self.lang_name.is_some() { - warn!("--lang-name` specified without --lib-path. This argument will be ignored."); - } let languages = loader.languages_at_path(current_dir)?; - let (language, language_name) = if let Some(ref lib_path) = self.lib_path { - let lib_info = get_lib_info(Some(lib_path), self.lang_name.as_ref(), current_dir) - .with_context(|| anyhow!("No language name found for {}", lib_path.display()))?; - let lang_name = lib_info.1.to_string(); - &( - loader - .select_language(None, current_dir, None, Some(&lib_info)) - .with_context(|| { - anyhow!( - "Failed to load language for path \"{}\"", - lib_path.display() - ) - })?, - lang_name, - ) - } else { - languages - .first() - .ok_or_else(|| anyhow!("No language found"))? - }; + let (language, language_name) = &languages + .first() + .ok_or_else(|| anyhow!("No language found"))?; let mut fuzz_options = FuzzOptions { skipped: self.skip, @@ -1523,23 +1226,24 @@ impl Query { fn run(self, mut loader: loader::Loader, current_dir: &Path) -> Result<()> { let config = Config::load(self.config_path)?; let loader_config = config.get()?; - loader.force_rebuild(self.rebuild || self.grammar_path.is_some()); loader.find_all_languages(&loader_config)?; let query_path = Path::new(&self.query_path); - let byte_range = parse_range(&self.byte_range, |x| x)?; - let point_range = parse_range(&self.row_range, |row| Point::new(row, 0))?; - let containing_byte_range = parse_range(&self.containing_byte_range, |x| x)?; - let containing_point_range = - parse_range(&self.containing_row_range, |row| Point::new(row, 0))?; + let byte_range = self.byte_range.as_ref().and_then(|range| { + let mut parts = range.split(':'); + let start = parts.next()?.parse().ok()?; + let end = parts.next().unwrap().parse().ok()?; + Some(start..end) + }); + let point_range = self.row_range.as_ref().and_then(|range| { + let mut parts = range.split(':'); + let start = parts.next()?.parse().ok()?; + let end = parts.next().unwrap().parse().ok()?; + Some(Point::new(start, 0)..Point::new(end, 0)) + }); let cancellation_flag = util::cancel_on_signal(); - if self.lib_path.is_none() && self.lang_name.is_some() { - warn!("--lang-name specified without --lib-path. This argument will be ignored."); - } - let lib_info = get_lib_info(self.lib_path.as_ref(), self.lang_name.as_ref(), current_dir); - let input = get_input( self.paths_file.as_deref(), self.paths, @@ -1550,30 +1254,24 @@ impl Query { match input { CliInput::Paths(paths) => { let language = loader.select_language( - Some(Path::new(&paths[0])), + Path::new(&paths[0]), current_dir, self.scope.as_deref(), - lib_info.as_ref(), )?; - let opts = QueryFileOptions { - ordered_captures: self.captures, - byte_range, - point_range, - containing_byte_range, - containing_point_range, - quiet: self.quiet, - print_time: self.time, - stdin: false, - }; for path in paths { query::query_file_at_path( &language, &path, &path.display().to_string(), query_path, - &opts, - None, + self.captures, + byte_range.clone(), + point_range.clone(), + self.test, + self.quiet, + self.time, + false, )?; } } @@ -1584,34 +1282,25 @@ impl Query { } => { let path = get_tmp_source_file(&contents)?; let languages = loader.languages_at_path(current_dir)?; - let language = if let Some(ref lib_path) = self.lib_path { - &loader - .select_language(None, current_dir, None, lib_info.as_ref()) - .with_context(|| { - anyhow!( - "Failed to load language for path \"{}\"", - lib_path.display() - ) - })? - } else { - &languages - .iter() - .find(|(_, n)| language_names.contains(&Box::from(n.as_str()))) - .or_else(|| languages.first()) - .map(|(l, _)| l.clone()) - .ok_or_else(|| anyhow!("No language found"))? - }; - let opts = QueryFileOptions { - ordered_captures: self.captures, + let language = languages + .iter() + .find(|(_, n)| language_names.contains(&Box::from(n.as_str()))) + .or_else(|| languages.first()) + .map(|(l, _)| l.clone()) + .ok_or_else(|| anyhow!("No language found"))?; + query::query_file_at_path( + &language, + &path, + &name, + query_path, + self.captures, byte_range, point_range, - containing_byte_range, - containing_point_range, - quiet: self.quiet, - print_time: self.time, - stdin: true, - }; - query::query_file_at_path(language, &path, &name, query_path, &opts, None)?; + self.test, + self.quiet, + self.time, + true, + )?; fs::remove_file(path)?; } CliInput::Stdin(contents) => { @@ -1619,19 +1308,20 @@ impl Query { println!(); let path = get_tmp_source_file(&contents)?; - let language = - loader.select_language(None, current_dir, None, lib_info.as_ref())?; - let opts = QueryFileOptions { - ordered_captures: self.captures, + let language = loader.select_language(&path, current_dir, None)?; + query::query_file_at_path( + &language, + &path, + "stdin", + query_path, + self.captures, byte_range, point_range, - containing_byte_range, - containing_point_range, - quiet: self.quiet, - print_time: self.time, - stdin: true, - }; - query::query_file_at_path(&language, &path, "stdin", query_path, &opts, None)?; + self.test, + self.quiet, + self.time, + true, + )?; fs::remove_file(path)?; } } @@ -1647,8 +1337,6 @@ impl Highlight { loader.configure_highlights(&theme_config.theme.highlight_names); let loader_config = config.get()?; loader.find_all_languages(&loader_config)?; - loader.force_rebuild(self.rebuild || self.grammar_path.is_some()); - let languages = loader.languages_at_path(current_dir)?; let cancellation_flag = util::cancel_on_signal(); @@ -1693,7 +1381,7 @@ impl Highlight { { (lang, lang_config) } else { - warn!( + eprintln!( "{}", util::lang_not_found_for_path(&path, &loader_config) ); @@ -1714,7 +1402,7 @@ impl Highlight { &options, )?; } else { - warn!( + eprintln!( "No syntax highlighting config found for path {}", path.display() ); @@ -1729,6 +1417,7 @@ impl Highlight { } => { let path = get_tmp_source_file(&contents)?; + let languages = loader.languages_at_path(current_dir)?; let language = languages .iter() .find(|(_, n)| language_names.contains(&Box::from(n.as_str()))) @@ -1744,7 +1433,7 @@ impl Highlight { { highlight::highlight(&loader, &path, &name, highlight_config, false, &options)?; } else { - warn!("No syntax highlighting config found for test {name}"); + eprintln!("No syntax highlighting config found for test {name}"); } fs::remove_file(path)?; } @@ -1759,6 +1448,7 @@ impl Highlight { if let (Some(l), Some(lc)) = (language.clone(), language_configuration) { (l, lc) } else { + let languages = loader.languages_at_path(current_dir)?; let language = languages .first() .map(|(l, _)| l.clone()) @@ -1783,7 +1473,7 @@ impl Highlight { &options, )?; } else { - warn!( + eprintln!( "No syntax highlighting config found for path {}", current_dir.display() ); @@ -1801,7 +1491,6 @@ impl Tags { let config = Config::load(self.config_path)?; let loader_config = config.get()?; loader.find_all_languages(&loader_config)?; - loader.force_rebuild(self.rebuild || self.grammar_path.is_some()); let cancellation_flag = util::cancel_on_signal(); @@ -1842,7 +1531,7 @@ impl Tags { { (lang, lang_config) } else { - warn!( + eprintln!( "{}", util::lang_not_found_for_path(&path, &loader_config) ); @@ -1860,7 +1549,7 @@ impl Tags { &options, )?; } else { - warn!("No tags config found for path {}", path.display()); + eprintln!("No tags config found for path {}", path.display()); } } } @@ -1886,7 +1575,7 @@ impl Tags { if let Some(tags_config) = language_config.tags_config(language)? { tags::generate_tags(&path, &name, tags_config, false, &options)?; } else { - warn!("No tags config found for test {name}"); + eprintln!("No tags config found for test {name}"); } fs::remove_file(path)?; } @@ -1917,7 +1606,7 @@ impl Tags { if let Some(tags_config) = language_config.tags_config(language)? { tags::generate_tags(&path, "stdin", tags_config, false, &options)?; } else { - warn!("No tags config found for path {}", current_dir.display()); + eprintln!("No tags config found for path {}", current_dir.display()); } fs::remove_file(path)?; } @@ -1929,15 +1618,9 @@ impl Tags { impl Playground { fn run(self, current_dir: &Path) -> Result<()> { + let open_in_browser = !self.quiet; let grammar_path = self.grammar_path.as_deref().map_or(current_dir, Path::new); - - if let Some(export_path) = self.export { - playground::export(grammar_path, &export_path)?; - } else { - let open_in_browser = !self.quiet; - playground::serve(grammar_path, open_in_browser)?; - } - + playground::serve(grammar_path, open_in_browser)?; Ok(()) } } @@ -1948,9 +1631,8 @@ impl DumpLanguages { let loader_config = config.get()?; loader.find_all_languages(&loader_config)?; for (configuration, language_path) in loader.get_all_language_configurations() { - info!( + println!( concat!( - "name: {}\n", "scope: {}\n", "parser: {:?}\n", "highlights: {:?}\n", @@ -1958,7 +1640,6 @@ impl DumpLanguages { "content_regex: {:?}\n", "injection_regex: {:?}\n", ), - configuration.language_name, configuration.scope.as_ref().unwrap_or(&String::new()), language_path, configuration.highlights_filenames, @@ -1999,30 +1680,29 @@ fn main() { } } if !err.to_string().is_empty() { - error!("{err:?}"); + eprintln!("{err:?}"); } std::process::exit(1); } } fn run() -> Result<()> { - logger::init(); - let version = BUILD_SHA.map_or_else( || BUILD_VERSION.to_string(), |build_sha| format!("{BUILD_VERSION} ({build_sha})"), ); + let version: &'static str = Box::leak(version.into_boxed_str()); let cli = Command::new("tree-sitter") - .help_template(concat!( - "\n", - "{before-help}{name} {version}\n", - "{author-with-newline}{about-with-newline}\n", - "{usage-heading} {usage}\n", - "\n", - "{all-args}{after-help}\n", - "\n" - )) + .help_template( + "\ +{before-help}{name} {version} +{author-with-newline}{about-with-newline} +{usage-heading} {usage} + +{all-args}{after-help} +", + ) .version(version) .subcommand_required(true) .arg_required_else_help(true) @@ -2032,25 +1712,7 @@ fn run() -> Result<()> { let command = Commands::from_arg_matches(&cli.clone().get_matches())?; - let current_dir = match &command { - Commands::Init(Init { grammar_path, .. }) - | Commands::Parse(Parse { grammar_path, .. }) - | Commands::Test(Test { grammar_path, .. }) - | Commands::Version(Version { grammar_path, .. }) - | Commands::Fuzz(Fuzz { grammar_path, .. }) - | Commands::Query(Query { grammar_path, .. }) - | Commands::Highlight(Highlight { grammar_path, .. }) - | Commands::Tags(Tags { grammar_path, .. }) - | Commands::Playground(Playground { grammar_path, .. }) => grammar_path, - Commands::Build(_) - | Commands::Generate(_) - | Commands::InitConfig(_) - | Commands::DumpLanguages(_) - | Commands::Complete(_) => &None, - } - .as_ref() - .map_or_else(|| env::current_dir().unwrap(), |p| p.clone()); - + let current_dir = env::current_dir().unwrap(); let loader = loader::Loader::new()?; match command { @@ -2104,59 +1766,3 @@ const fn get_styles() -> clap::builder::Styles { ) .placeholder(Style::new().fg_color(Some(Color::Ansi(AnsiColor::White)))) } - -/// Utility to extract the shared library path and language function name from user-provided -/// arguments if present. -fn get_lib_info<'a>( - lib_path: Option<&'a PathBuf>, - language_name: Option<&'a String>, - current_dir: &Path, -) -> Option<(PathBuf, &'a str)> { - if let Some(lib_path) = lib_path { - let absolute_lib_path = if lib_path.is_absolute() { - lib_path.clone() - } else { - current_dir.join(lib_path) - }; - // Use the user-specified name if present, otherwise try to derive it from - // the lib path - match ( - language_name.map(|s| s.as_str()), - lib_path.file_stem().and_then(|s| s.to_str()), - ) { - (Some(name), _) | (None, Some(name)) => Some((absolute_lib_path, name)), - _ => None, - } - } else { - None - } -} - -/// Parse a range string of the form "start:end" into an optional Range. -fn parse_range( - range_str: &Option, - make: impl Fn(usize) -> T, -) -> Result>> { - if let Some(range) = range_str.as_ref() { - let err_msg = format!("Invalid range '{range}', expected 'start:end'"); - let mut parts = range.split(':'); - - let Some(part) = parts.next() else { - Err(anyhow!(err_msg))? - }; - let Ok(start) = part.parse::() else { - Err(anyhow!(err_msg))? - }; - - let Some(part) = parts.next() else { - Err(anyhow!(err_msg))? - }; - let Ok(end) = part.parse::() else { - Err(anyhow!(err_msg))? - }; - - Ok(Some(make(start)..make(end))) - } else { - Ok(None) - } -} diff --git a/crates/cli/src/parse.rs b/cli/src/parse.rs similarity index 87% rename from crates/cli/src/parse.rs rename to cli/src/parse.rs index 3551f556..bc01a908 100644 --- a/crates/cli/src/parse.rs +++ b/cli/src/parse.rs @@ -1,7 +1,6 @@ use std::{ fmt, fs, - io::{self, Write}, - ops::ControlFlow, + io::{self, StdoutLock, Write}, path::{Path, PathBuf}, sync::atomic::{AtomicUsize, Ordering}, time::{Duration, Instant}, @@ -10,17 +9,16 @@ use std::{ use anstyle::{AnsiColor, Color, RgbColor}; use anyhow::{anyhow, Context, Result}; use clap::ValueEnum; -use log::info; -use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use tree_sitter::{ ffi, InputEdit, Language, LogType, ParseOptions, ParseState, Parser, Point, Range, Tree, TreeCursor, }; -use crate::{fuzz::edits::Edit, logger::paint, util}; +use super::util; +use crate::{fuzz::edits::Edit, test::paint}; -#[derive(Debug, Default, Serialize, JsonSchema)] +#[derive(Debug, Default, Serialize)] pub struct Stats { pub successful_parses: usize, pub total_parses: usize, @@ -231,21 +229,10 @@ impl ParseSummary { } } -#[derive(Serialize, Debug)] +#[derive(Serialize, Debug, Default)] pub struct ParseStats { pub parse_summaries: Vec, pub cumulative_stats: Stats, - pub source_count: usize, -} - -impl Default for ParseStats { - fn default() -> Self { - Self { - parse_summaries: Vec::new(), - cumulative_stats: Stats::default(), - source_count: 1, - } - } } #[derive(Serialize, ValueEnum, Debug, Copy, Clone, Default, Eq, PartialEq)] @@ -370,15 +357,15 @@ pub fn parse_file_at_path( let progress_callback = &mut |_: &ParseState| { if let Some(cancellation_flag) = opts.cancellation_flag { if cancellation_flag.load(Ordering::SeqCst) != 0 { - return ControlFlow::Break(()); + return true; } } if opts.timeout > 0 && start_time.elapsed().as_micros() > opts.timeout as u128 { - return ControlFlow::Break(()); + return true; } - ControlFlow::Continue(()) + false }; let parse_opts = ParseOptions::new().progress_callback(progress_callback); @@ -437,7 +424,7 @@ pub fn parse_file_at_path( if let Some(mut tree) = tree { if opts.debug_graph && !opts.edits.is_empty() { - info!("BEFORE:\n{}", String::from_utf8_lossy(&source_code)); + println!("BEFORE:\n{}", String::from_utf8_lossy(&source_code)); } let edit_time = Instant::now(); @@ -447,7 +434,7 @@ pub fn parse_file_at_path( tree = parser.parse(&source_code, Some(&tree)).unwrap(); if opts.debug_graph { - info!("AFTER {i}:\n{}", String::from_utf8_lossy(&source_code)); + println!("AFTER {i}:\n{}", String::from_utf8_lossy(&source_code)); } } let edit_duration = edit_time.elapsed(); @@ -514,23 +501,63 @@ pub fn parse_file_at_path( } if opts.output == ParseOutput::Cst { - render_cst(&source_code, &tree, &mut cursor, opts, &mut stdout)?; + let lossy_source_code = String::from_utf8_lossy(&source_code); + let total_width = lossy_source_code + .lines() + .enumerate() + .map(|(row, col)| { + (row as f64).log10() as usize + (col.len() as f64).log10() as usize + 1 + }) + .max() + .unwrap_or(1); + let mut indent_level = 1; + let mut did_visit_children = false; + let mut in_error = false; + loop { + if did_visit_children { + if cursor.goto_next_sibling() { + did_visit_children = false; + } else if cursor.goto_parent() { + did_visit_children = true; + indent_level -= 1; + if !cursor.node().has_error() { + in_error = false; + } + } else { + break; + } + } else { + cst_render_node( + opts, + &mut cursor, + &source_code, + &mut stdout, + total_width, + indent_level, + in_error, + )?; + if cursor.goto_first_child() { + did_visit_children = false; + indent_level += 1; + if cursor.node().has_error() { + in_error = true; + } + } else { + did_visit_children = true; + } + } + } + cursor.reset(tree.root_node()); + println!(); } if opts.output == ParseOutput::Xml { let mut needs_newline = false; - let mut indent_level = 2; + let mut indent_level = 0; let mut did_visit_children = false; let mut had_named_children = false; let mut tags = Vec::<&str>::new(); - - // If we're parsing the first file, write the header - if opts.stats.parse_summaries.is_empty() { - writeln!(&mut stdout, "")?; - writeln!(&mut stdout, "")?; - } - writeln!(&mut stdout, " ", path.display())?; - + writeln!(&mut stdout, "")?; loop { let node = cursor.node(); let is_named = node.is_named(); @@ -545,7 +572,7 @@ pub fn parse_file_at_path( write!(&mut stdout, "", tag.expect("there is a tag"))?; // we only write a line in the case where it's the last sibling if let Some(parent) = node.parent() { - if parent.child(parent.child_count() as u32 - 1).unwrap() == node { + if parent.child(parent.child_count() - 1).unwrap() == node { stdout.write_all(b"\n")?; } } @@ -609,14 +636,8 @@ pub fn parse_file_at_path( } } } - writeln!(&mut stdout)?; - writeln!(&mut stdout, " ")?; - - // If we parsed the last file, write the closing tag for the `sources` header - if opts.stats.parse_summaries.len() == opts.stats.source_count - 1 { - writeln!(&mut stdout, "")?; - } cursor.reset(tree.root_node()); + println!(); } if opts.output == ParseOutput::Dot { @@ -674,9 +695,10 @@ pub fn parse_file_at_path( width = max_path_length )?; if let Some(node) = first_error { - let node_kind = node.kind(); - let mut node_text = String::with_capacity(node_kind.len()); - for c in node_kind.chars() { + let start = node.start_position(); + let end = node.end_position(); + let mut node_text = String::new(); + for c in node.kind().chars() { if let Some(escaped) = escape_invisible(c) { node_text += escaped; } else { @@ -693,9 +715,6 @@ pub fn parse_file_at_path( } else { write!(&mut stdout, "{node_text}")?; } - - let start = node.start_position(); - let end = node.end_position(); write!( &mut stdout, " [{}, {}] - [{}, {}])", @@ -762,77 +781,12 @@ const fn escape_invisible(c: char) -> Option<&'static str> { }) } -const fn escape_delimiter(c: char) -> Option<&'static str> { - Some(match c { - '`' => "\\`", - '\"' => "\\\"", - _ => return None, - }) -} - -pub fn render_cst<'a, 'b: 'a>( - source_code: &[u8], - tree: &'b Tree, - cursor: &mut TreeCursor<'a>, - opts: &ParseFileOptions, - out: &mut impl Write, -) -> Result<()> { - let lossy_source_code = String::from_utf8_lossy(source_code); - let total_width = lossy_source_code - .lines() - .enumerate() - .map(|(row, col)| (row as f64).log10() as usize + (col.len() as f64).log10() as usize + 1) - .max() - .unwrap_or(1); - let mut indent_level = usize::from(!opts.no_ranges); - let mut did_visit_children = false; - let mut in_error = false; - loop { - if did_visit_children { - if cursor.goto_next_sibling() { - did_visit_children = false; - } else if cursor.goto_parent() { - did_visit_children = true; - indent_level -= 1; - if !cursor.node().has_error() { - in_error = false; - } - } else { - break; - } - } else { - cst_render_node( - opts, - cursor, - source_code, - out, - total_width, - indent_level, - in_error, - )?; - if cursor.goto_first_child() { - did_visit_children = false; - indent_level += 1; - if cursor.node().has_error() { - in_error = true; - } - } else { - did_visit_children = true; - } - } - } - cursor.reset(tree.root_node()); - Ok(()) -} - fn render_node_text(source: &str) -> String { source .chars() .fold(String::with_capacity(source.len()), |mut acc, c| { if let Some(esc) = escape_invisible(c) { acc.push_str(esc); - } else if let Some(esc) = escape_delimiter(c) { - acc.push_str(esc); } else { acc.push(c); } @@ -842,7 +796,7 @@ fn render_node_text(source: &str) -> String { fn write_node_text( opts: &ParseFileOptions, - out: &mut impl Write, + stdout: &mut StdoutLock<'static>, cursor: &TreeCursor, is_named: bool, source: &str, @@ -858,7 +812,7 @@ fn write_node_text( if !is_named { write!( - out, + stdout, "{}{}{}", paint(quote_color, &String::from(quote)), paint(color, &render_node_text(source)), @@ -882,24 +836,35 @@ fn write_node_text( 0 }; let formatted_line = render_line_feed(line, opts); - write!( - out, - "{}{}{}{}{}{}", - if multiline { "\n" } else { " " }, - if multiline && !opts.no_ranges { - render_node_range(opts, cursor, is_named, true, total_width, node_range) - } else { - String::new() - }, - if multiline { - " ".repeat(indent_level + 1) - } else { - String::new() - }, - paint(quote_color, &String::from(quote)), - paint(color, &render_node_text(&formatted_line)), - paint(quote_color, &String::from(quote)), - )?; + if !opts.no_ranges { + write!( + stdout, + "{}{}{}{}{}{}", + if multiline { "\n" } else { "" }, + if multiline { + render_node_range(opts, cursor, is_named, true, total_width, node_range) + } else { + String::new() + }, + if multiline { + " ".repeat(indent_level + 1) + } else { + String::new() + }, + paint(quote_color, &String::from(quote)), + &paint(color, &render_node_text(&formatted_line)), + paint(quote_color, &String::from(quote)), + )?; + } else { + write!( + stdout, + "\n{}{}{}{}", + " ".repeat(indent_level + 1), + paint(quote_color, &String::from(quote)), + &paint(color, &render_node_text(&formatted_line)), + paint(quote_color, &String::from(quote)), + )?; + } } } @@ -953,9 +918,9 @@ fn render_node_range( fn cst_render_node( opts: &ParseFileOptions, - cursor: &TreeCursor, + cursor: &mut TreeCursor, source_code: &[u8], - out: &mut impl Write, + stdout: &mut StdoutLock<'static>, total_width: usize, indent_level: usize, in_error: bool, @@ -964,13 +929,13 @@ fn cst_render_node( let is_named = node.is_named(); if !opts.no_ranges { write!( - out, + stdout, "{}", render_node_range(opts, cursor, is_named, false, total_width, node.range()) )?; } write!( - out, + stdout, "{}{}", " ".repeat(indent_level), if in_error && !node.has_error() { @@ -982,14 +947,14 @@ fn cst_render_node( if is_named { if let Some(field_name) = cursor.field_name() { write!( - out, + stdout, "{}", paint(opts.parse_theme.field, &format!("{field_name}: ")) )?; } if node.has_error() || node.is_error() { - write!(out, "{}", paint(opts.parse_theme.error, "•"))?; + write!(stdout, "{}", paint(opts.parse_theme.error, "•"))?; } let kind_color = if node.is_error() { @@ -999,13 +964,13 @@ fn cst_render_node( } else { opts.parse_theme.node_kind }; - write!(out, "{}", paint(kind_color, node.kind()))?; + write!(stdout, "{} ", paint(kind_color, node.kind()))?; if node.child_count() == 0 { // Node text from a pattern or external scanner write_node_text( opts, - out, + stdout, cursor, is_named, &String::from_utf8_lossy(&source_code[node.start_byte()..node.end_byte()]), @@ -1014,13 +979,17 @@ fn cst_render_node( )?; } } else if node.is_missing() { - write!(out, "{}: ", paint(opts.parse_theme.missing, "MISSING"))?; - write!(out, "\"{}\"", paint(opts.parse_theme.missing, node.kind()))?; + write!(stdout, "{}: ", paint(opts.parse_theme.missing, "MISSING"))?; + write!( + stdout, + "\"{}\"", + paint(opts.parse_theme.missing, node.kind()) + )?; } else { // Terminal literals, like "fn" write_node_text( opts, - out, + stdout, cursor, is_named, node.kind(), @@ -1028,7 +997,7 @@ fn cst_render_node( (total_width, indent_level), )?; } - writeln!(out)?; + writeln!(stdout)?; Ok(()) } diff --git a/cli/src/playground.html b/cli/src/playground.html new file mode 100644 index 00000000..018c4ced --- /dev/null +++ b/cli/src/playground.html @@ -0,0 +1,410 @@ + + + tree-sitter THE_LANGUAGE_NAME + + + + + + + + + + + + + + + + + + + + diff --git a/crates/cli/src/playground.rs b/cli/src/playground.rs similarity index 60% rename from crates/cli/src/playground.rs rename to cli/src/playground.rs index f2c04fed..1fdaa057 100644 --- a/crates/cli/src/playground.rs +++ b/cli/src/playground.rs @@ -7,7 +7,6 @@ use std::{ }; use anyhow::{anyhow, Context, Result}; -use log::{error, info}; use tiny_http::{Header, Response, Server}; use super::wasm; @@ -19,7 +18,7 @@ macro_rules! optional_resource { if let Some(tree_sitter_dir) = tree_sitter_dir { Cow::Owned(fs::read(tree_sitter_dir.join($path)).unwrap()) } else { - Cow::Borrowed(include_bytes!(concat!("../../../", $path))) + Cow::Borrowed(include_bytes!(concat!("../../", $path))) } } @@ -35,91 +34,25 @@ macro_rules! optional_resource { } optional_resource!(get_playground_js, "docs/src/assets/js/playground.js"); -optional_resource!(get_lib_js, "lib/binding_web/web-tree-sitter.js"); -optional_resource!(get_lib_wasm, "lib/binding_web/web-tree-sitter.wasm"); +optional_resource!(get_lib_js, "lib/binding_web/tree-sitter.js"); +optional_resource!(get_lib_wasm, "lib/binding_web/tree-sitter.wasm"); fn get_main_html(tree_sitter_dir: Option<&Path>) -> Cow<'static, [u8]> { tree_sitter_dir.map_or( Cow::Borrowed(include_bytes!("playground.html")), |tree_sitter_dir| { - Cow::Owned(fs::read(tree_sitter_dir.join("crates/cli/src/playground.html")).unwrap()) + Cow::Owned(fs::read(tree_sitter_dir.join("cli/src/playground.html")).unwrap()) }, ) } -pub fn export(grammar_path: &Path, export_path: &Path) -> Result<()> { - let (grammar_name, language_wasm) = wasm::load_language_wasm_file(grammar_path)?; - - fs::create_dir_all(export_path).with_context(|| { - format!( - "Failed to create export directory: {}", - export_path.display() - ) - })?; - - let tree_sitter_dir = env::var("TREE_SITTER_BASE_DIR").map(PathBuf::from).ok(); - - let playground_js = get_playground_js(tree_sitter_dir.as_deref()); - let lib_js = get_lib_js(tree_sitter_dir.as_deref()); - let lib_wasm = get_lib_wasm(tree_sitter_dir.as_deref()); - - let has_local_playground_js = !playground_js.is_empty(); - let has_local_lib_js = !lib_js.is_empty(); - let has_local_lib_wasm = !lib_wasm.is_empty(); - - let mut main_html = str::from_utf8(&get_main_html(tree_sitter_dir.as_deref())) - .unwrap() - .replace("THE_LANGUAGE_NAME", &grammar_name); - - if !has_local_playground_js { - main_html = main_html.replace( - r#""#, - r#""# - ); - } - if !has_local_lib_js { - main_html = main_html.replace( - "import * as TreeSitter from './web-tree-sitter.js';", - "import * as TreeSitter from 'https://tree-sitter.github.io/web-tree-sitter.js';", - ); - } - - fs::write(export_path.join("index.html"), main_html.as_bytes()) - .with_context(|| "Failed to write index.html")?; - - fs::write(export_path.join("tree-sitter-parser.wasm"), language_wasm) - .with_context(|| "Failed to write parser wasm file")?; - - if has_local_playground_js { - fs::write(export_path.join("playground.js"), playground_js) - .with_context(|| "Failed to write playground.js")?; - } - - if has_local_lib_js { - fs::write(export_path.join("web-tree-sitter.js"), lib_js) - .with_context(|| "Failed to write web-tree-sitter.js")?; - } - - if has_local_lib_wasm { - fs::write(export_path.join("web-tree-sitter.wasm"), lib_wasm) - .with_context(|| "Failed to write web-tree-sitter.wasm")?; - } - - println!( - "Exported playground to {}", - export_path.canonicalize()?.display() - ); - - Ok(()) -} - pub fn serve(grammar_path: &Path, open_in_browser: bool) -> Result<()> { let server = get_server()?; let (grammar_name, language_wasm) = wasm::load_language_wasm_file(grammar_path)?; let url = format!("http://{}", server.server_addr()); - info!("Started playground on: {url}"); + println!("Started playground on: {url}"); if open_in_browser && webbrowser::open(&url).is_err() { - error!("Failed to open '{url}' in a web browser"); + eprintln!("Failed to open '{url}' in a web browser"); } let tree_sitter_dir = env::var("TREE_SITTER_BASE_DIR").map(PathBuf::from).ok(); @@ -146,16 +79,16 @@ pub fn serve(grammar_path: &Path, open_in_browser: bool) -> Result<()> { response(&playground_js, &js_header) } } - "/web-tree-sitter.js" => { + "/tree-sitter.js" => { if lib_js.is_empty() { - redirect("https://tree-sitter.github.io/web-tree-sitter.js") + redirect("https://tree-sitter.github.io/tree-sitter.js") } else { response(&lib_js, &js_header) } } - "/web-tree-sitter.wasm" => { + "/tree-sitter.wasm" => { if lib_wasm.is_empty() { - redirect("https://tree-sitter.github.io/web-tree-sitter.wasm") + redirect("https://tree-sitter.github.io/tree-sitter.wasm") } else { response(&lib_wasm, &wasm_header) } diff --git a/crates/cli/src/query.rs b/cli/src/query.rs similarity index 59% rename from crates/cli/src/query.rs rename to cli/src/query.rs index 54674115..81ccea50 100644 --- a/crates/cli/src/query.rs +++ b/cli/src/query.rs @@ -6,35 +6,29 @@ use std::{ time::Instant, }; +use anstyle::AnsiColor; use anyhow::{Context, Result}; -use log::warn; use streaming_iterator::StreamingIterator; use tree_sitter::{Language, Parser, Point, Query, QueryCursor}; use crate::{ query_testing::{self, to_utf8_point}, - test::{TestInfo, TestOutcome, TestResult, TestSummary}, + test::paint, }; -#[derive(Default)] -pub struct QueryFileOptions { - pub ordered_captures: bool, - pub byte_range: Option>, - pub point_range: Option>, - pub containing_byte_range: Option>, - pub containing_point_range: Option>, - pub quiet: bool, - pub print_time: bool, - pub stdin: bool, -} - +#[allow(clippy::too_many_arguments)] pub fn query_file_at_path( language: &Language, path: &Path, name: &str, query_path: &Path, - opts: &QueryFileOptions, - test_summary: Option<&mut TestSummary>, + ordered_captures: bool, + byte_range: Option>, + point_range: Option>, + should_test: bool, + quiet: bool, + print_time: bool, + stdin: bool, ) -> Result<()> { let stdout = io::stdout(); let mut stdout = stdout.lock(); @@ -44,26 +38,19 @@ pub fn query_file_at_path( let query = Query::new(language, &query_source).with_context(|| "Query compilation failed")?; let mut query_cursor = QueryCursor::new(); - if let Some(ref range) = opts.byte_range { - query_cursor.set_byte_range(range.clone()); + if let Some(range) = byte_range { + query_cursor.set_byte_range(range); } - if let Some(ref range) = opts.point_range { - query_cursor.set_point_range(range.clone()); - } - if let Some(ref range) = opts.containing_byte_range { - query_cursor.set_containing_byte_range(range.clone()); - } - if let Some(ref range) = opts.containing_point_range { - query_cursor.set_containing_point_range(range.clone()); + if let Some(range) = point_range { + query_cursor.set_point_range(range); } let mut parser = Parser::new(); parser.set_language(language)?; let mut results = Vec::new(); - let should_test = test_summary.is_some(); - if !should_test && !opts.stdin { + if !should_test && !stdin { writeln!(&mut stdout, "{name}")?; } @@ -72,12 +59,12 @@ pub fn query_file_at_path( let tree = parser.parse(&source_code, None).unwrap(); let start = Instant::now(); - if opts.ordered_captures { + if ordered_captures { let mut captures = query_cursor.captures(&query, tree.root_node(), source_code.as_slice()); while let Some((mat, capture_index)) = captures.next() { let capture = mat.captures[*capture_index]; let capture_name = &query.capture_names()[capture.index as usize]; - if !opts.quiet && !should_test { + if !quiet && !should_test { writeln!( &mut stdout, " pattern: {:>2}, capture: {} - {capture_name}, start: {}, end: {}, text: `{}`", @@ -88,25 +75,23 @@ pub fn query_file_at_path( capture.node.utf8_text(&source_code).unwrap_or("") )?; } - if should_test { - results.push(query_testing::CaptureInfo { - name: (*capture_name).to_string(), - start: to_utf8_point(capture.node.start_position(), source_code.as_slice()), - end: to_utf8_point(capture.node.end_position(), source_code.as_slice()), - }); - } + results.push(query_testing::CaptureInfo { + name: (*capture_name).to_string(), + start: to_utf8_point(capture.node.start_position(), source_code.as_slice()), + end: to_utf8_point(capture.node.end_position(), source_code.as_slice()), + }); } } else { let mut matches = query_cursor.matches(&query, tree.root_node(), source_code.as_slice()); while let Some(m) = matches.next() { - if !opts.quiet && !should_test { + if !quiet && !should_test { writeln!(&mut stdout, " pattern: {}", m.pattern_index)?; } for capture in m.captures { let start = capture.node.start_position(); let end = capture.node.end_position(); let capture_name = &query.capture_names()[capture.index as usize]; - if !opts.quiet && !should_test { + if !quiet && !should_test { if end.row == start.row { writeln!( &mut stdout, @@ -121,52 +106,41 @@ pub fn query_file_at_path( )?; } } - if should_test { - results.push(query_testing::CaptureInfo { - name: (*capture_name).to_string(), - start: to_utf8_point(capture.node.start_position(), source_code.as_slice()), - end: to_utf8_point(capture.node.end_position(), source_code.as_slice()), - }); - } + results.push(query_testing::CaptureInfo { + name: (*capture_name).to_string(), + start: to_utf8_point(capture.node.start_position(), source_code.as_slice()), + end: to_utf8_point(capture.node.end_position(), source_code.as_slice()), + }); } } } if query_cursor.did_exceed_match_limit() { - warn!("Query exceeded maximum number of in-progress captures!"); + writeln!( + &mut stdout, + " WARNING: Query exceeded maximum number of in-progress captures!" + )?; } if should_test { - let path_name = if opts.stdin { + let path_name = if stdin { "stdin" } else { Path::new(&path).file_name().unwrap().to_str().unwrap() }; - // Invariant: `test_summary` will always be `Some` when `should_test` is true - let test_summary = test_summary.unwrap(); match query_testing::assert_expected_captures(&results, path, &mut parser, language) { Ok(assertion_count) => { - test_summary.query_results.add_case(TestResult { - name: path_name.to_string(), - info: TestInfo::AssertionTest { - outcome: TestOutcome::AssertionPassed { assertion_count }, - test_num: test_summary.test_num, - }, - }); + println!( + " ✓ {} ({} assertions)", + paint(Some(AnsiColor::Green), path_name), + assertion_count + ); } Err(e) => { - test_summary.query_results.add_case(TestResult { - name: path_name.to_string(), - info: TestInfo::AssertionTest { - outcome: TestOutcome::AssertionFailed { - error: e.to_string(), - }, - test_num: test_summary.test_num, - }, - }); + println!(" ✗ {}", paint(Some(AnsiColor::Red), path_name)); return Err(e); } } } - if opts.print_time { + if print_time { writeln!(&mut stdout, "{:?}", start.elapsed())?; } diff --git a/crates/cli/src/query_testing.rs b/cli/src/query_testing.rs similarity index 99% rename from crates/cli/src/query_testing.rs rename to cli/src/query_testing.rs index e4923d65..289591f0 100644 --- a/crates/cli/src/query_testing.rs +++ b/cli/src/query_testing.rs @@ -237,8 +237,8 @@ pub fn assert_expected_captures( return Err(anyhow!( "Assertion failed: at {}, found {}, expected {}", found.start, - found.name, assertion.expected_capture_name, + found.name )); } } else { diff --git a/crates/cli/src/tags.rs b/cli/src/tags.rs similarity index 100% rename from crates/cli/src/tags.rs rename to cli/src/tags.rs diff --git a/crates/cli/src/templates/.editorconfig b/cli/src/templates/.editorconfig similarity index 85% rename from crates/cli/src/templates/.editorconfig rename to cli/src/templates/.editorconfig index ff17b12f..65330c40 100644 --- a/crates/cli/src/templates/.editorconfig +++ b/cli/src/templates/.editorconfig @@ -3,11 +3,11 @@ root = true [*] charset = utf-8 -[*.{json,toml,yml,gyp,xml}] +[*.{json,toml,yml,gyp}] indent_style = space indent_size = 2 -[*.{js,ts}] +[*.js] indent_style = space indent_size = 2 @@ -31,10 +31,6 @@ indent_size = 4 indent_style = space indent_size = 4 -[*.java] -indent_style = space -indent_size = 4 - [*.go] indent_style = tab indent_size = 8 diff --git a/crates/cli/src/templates/PARSER_NAME.h b/cli/src/templates/PARSER_NAME.h similarity index 100% rename from crates/cli/src/templates/PARSER_NAME.h rename to cli/src/templates/PARSER_NAME.h diff --git a/crates/cli/src/templates/PARSER_NAME.pc.in b/cli/src/templates/PARSER_NAME.pc.in similarity index 100% rename from crates/cli/src/templates/PARSER_NAME.pc.in rename to cli/src/templates/PARSER_NAME.pc.in diff --git a/cli/src/templates/__init__.py b/cli/src/templates/__init__.py new file mode 100644 index 00000000..fd137b0f --- /dev/null +++ b/cli/src/templates/__init__.py @@ -0,0 +1,42 @@ +"""PARSER_DESCRIPTION""" + +from importlib.resources import files as _files + +from ._binding import language + + +def _get_query(name, file): + query = _files(f"{__package__}.queries") / file + globals()[name] = query.read_text() + return globals()[name] + + +def __getattr__(name): + # NOTE: uncomment these to include any queries that this grammar contains: + + # if name == "HIGHLIGHTS_QUERY": + # return _get_query("HIGHLIGHTS_QUERY", "highlights.scm") + # if name == "INJECTIONS_QUERY": + # return _get_query("INJECTIONS_QUERY", "injections.scm") + # if name == "LOCALS_QUERY": + # return _get_query("LOCALS_QUERY", "locals.scm") + # if name == "TAGS_QUERY": + # return _get_query("TAGS_QUERY", "tags.scm") + + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +__all__ = [ + "language", + # "HIGHLIGHTS_QUERY", + # "INJECTIONS_QUERY", + # "LOCALS_QUERY", + # "TAGS_QUERY", +] + + +def __dir__(): + return sorted(__all__ + [ + "__all__", "__builtins__", "__cached__", "__doc__", "__file__", + "__loader__", "__name__", "__package__", "__path__", "__spec__", + ]) diff --git a/cli/src/templates/__init__.pyi b/cli/src/templates/__init__.pyi new file mode 100644 index 00000000..abf6633f --- /dev/null +++ b/cli/src/templates/__init__.pyi @@ -0,0 +1,10 @@ +from typing import Final + +# NOTE: uncomment these to include any queries that this grammar contains: + +# HIGHLIGHTS_QUERY: Final[str] +# INJECTIONS_QUERY: Final[str] +# LOCALS_QUERY: Final[str] +# TAGS_QUERY: Final[str] + +def language() -> object: ... diff --git a/crates/cli/src/templates/_cargo.toml b/cli/src/templates/_cargo.toml similarity index 100% rename from crates/cli/src/templates/_cargo.toml rename to cli/src/templates/_cargo.toml diff --git a/crates/cli/src/templates/binding.go b/cli/src/templates/binding.go similarity index 100% rename from crates/cli/src/templates/binding.go rename to cli/src/templates/binding.go diff --git a/crates/cli/src/templates/binding.gyp b/cli/src/templates/binding.gyp similarity index 100% rename from crates/cli/src/templates/binding.gyp rename to cli/src/templates/binding.gyp diff --git a/crates/cli/src/templates/binding_test.go b/cli/src/templates/binding_test.go similarity index 100% rename from crates/cli/src/templates/binding_test.go rename to cli/src/templates/binding_test.go diff --git a/cli/src/templates/binding_test.js b/cli/src/templates/binding_test.js new file mode 100644 index 00000000..55becacf --- /dev/null +++ b/cli/src/templates/binding_test.js @@ -0,0 +1,9 @@ +const assert = require("node:assert"); +const { test } = require("node:test"); + +const Parser = require("tree-sitter"); + +test("can load grammar", () => { + const parser = new Parser(); + assert.doesNotThrow(() => parser.setLanguage(require("."))); +}); diff --git a/cli/src/templates/build.rs b/cli/src/templates/build.rs new file mode 100644 index 00000000..5ef59e52 --- /dev/null +++ b/cli/src/templates/build.rs @@ -0,0 +1,21 @@ +fn main() { + let src_dir = std::path::Path::new("src"); + + let mut c_config = cc::Build::new(); + c_config.std("c11").include(src_dir); + + #[cfg(target_env = "msvc")] + c_config.flag("-utf-8"); + + let parser_path = src_dir.join("parser.c"); + c_config.file(&parser_path); + println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap()); + + let scanner_path = src_dir.join("scanner.c"); + if scanner_path.exists() { + c_config.file(&scanner_path); + println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap()); + } + + c_config.compile("tree-sitter-KEBAB_PARSER_NAME"); +} diff --git a/crates/cli/src/templates/build.zig b/cli/src/templates/build.zig similarity index 100% rename from crates/cli/src/templates/build.zig rename to cli/src/templates/build.zig diff --git a/crates/cli/src/templates/build.zig.zon b/cli/src/templates/build.zig.zon similarity index 93% rename from crates/cli/src/templates/build.zig.zon rename to cli/src/templates/build.zig.zon index 0d542675..ef084d23 100644 --- a/crates/cli/src/templates/build.zig.zon +++ b/cli/src/templates/build.zig.zon @@ -1,6 +1,5 @@ .{ .name = .tree_sitter_PARSER_NAME, - .fingerprint = PARSER_FINGERPRINT, .version = "PARSER_VERSION", .dependencies = .{ .tree_sitter = .{ diff --git a/crates/cli/src/templates/cmakelists.cmake b/cli/src/templates/cmakelists.cmake similarity index 81% rename from crates/cli/src/templates/cmakelists.cmake rename to cli/src/templates/cmakelists.cmake index 06acbc8f..34dd8efc 100644 --- a/crates/cli/src/templates/cmakelists.cmake +++ b/cli/src/templates/cmakelists.cmake @@ -19,17 +19,7 @@ include(GNUInstallDirs) find_program(TREE_SITTER_CLI tree-sitter DOC "Tree-sitter CLI") -add_custom_command(OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/src/grammar.json" - "${CMAKE_CURRENT_SOURCE_DIR}/src/node-types.json" - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/grammar.js" - COMMAND "${TREE_SITTER_CLI}" generate grammar.js --no-parser - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" - COMMENT "Generating grammar.json") - add_custom_command(OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/src/parser.c" - BYPRODUCTS "${CMAKE_CURRENT_SOURCE_DIR}/src/tree_sitter/parser.h" - "${CMAKE_CURRENT_SOURCE_DIR}/src/tree_sitter/alloc.h" - "${CMAKE_CURRENT_SOURCE_DIR}/src/tree_sitter/array.h" DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/grammar.json" COMMAND "${TREE_SITTER_CLI}" generate src/grammar.json --abi=${TREE_SITTER_ABI_VERSION} diff --git a/crates/cli/src/templates/gitattributes b/cli/src/templates/gitattributes similarity index 92% rename from crates/cli/src/templates/gitattributes rename to cli/src/templates/gitattributes index 027ac707..7772c942 100644 --- a/crates/cli/src/templates/gitattributes +++ b/cli/src/templates/gitattributes @@ -40,7 +40,3 @@ Package.resolved linguist-generated bindings/zig/* linguist-generated build.zig linguist-generated build.zig.zon linguist-generated - -# Java bindings -pom.xml linguist-generated -bindings/java/** linguist-generated diff --git a/crates/cli/src/templates/gitignore b/cli/src/templates/gitignore similarity index 89% rename from crates/cli/src/templates/gitignore rename to cli/src/templates/gitignore index 7c0cb7f5..87a0c80c 100644 --- a/crates/cli/src/templates/gitignore +++ b/cli/src/templates/gitignore @@ -1,13 +1,16 @@ # Rust artifacts target/ +Cargo.lock # Node artifacts build/ prebuilds/ node_modules/ +package-lock.json # Swift artifacts .build/ +Package.resolved # Go artifacts _obj/ @@ -45,4 +48,3 @@ zig-out/ *.tar.gz *.tgz *.zip -*.jar diff --git a/crates/cli/src/templates/go.mod b/cli/src/templates/go.mod similarity index 100% rename from crates/cli/src/templates/go.mod rename to cli/src/templates/go.mod diff --git a/crates/cli/src/templates/grammar.js b/cli/src/templates/grammar.js similarity index 91% rename from crates/cli/src/templates/grammar.js rename to cli/src/templates/grammar.js index edee3cbc..01586557 100644 --- a/crates/cli/src/templates/grammar.js +++ b/cli/src/templates/grammar.js @@ -7,7 +7,7 @@ /// // @ts-check -export default grammar({ +module.exports = grammar({ name: "LOWER_PARSER_NAME", rules: { diff --git a/cli/src/templates/index.d.ts b/cli/src/templates/index.d.ts new file mode 100644 index 00000000..528e060f --- /dev/null +++ b/cli/src/templates/index.d.ts @@ -0,0 +1,27 @@ +type BaseNode = { + type: string; + named: boolean; +}; + +type ChildNode = { + multiple: boolean; + required: boolean; + types: BaseNode[]; +}; + +type NodeInfo = + | (BaseNode & { + subtypes: BaseNode[]; + }) + | (BaseNode & { + fields: { [name: string]: ChildNode }; + children: ChildNode[]; + }); + +type Language = { + language: unknown; + nodeTypeInfo: NodeInfo[]; +}; + +declare const language: Language; +export = language; diff --git a/cli/src/templates/index.js b/cli/src/templates/index.js new file mode 100644 index 00000000..cbbaa32f --- /dev/null +++ b/cli/src/templates/index.js @@ -0,0 +1,11 @@ +const root = require("path").join(__dirname, "..", ".."); + +module.exports = + typeof process.versions.bun === "string" + // Support `bun build --compile` by being statically analyzable enough to find the .node file at build-time + ? require(`../../prebuilds/${process.platform}-${process.arch}/tree-sitter-KEBAB_PARSER_NAME.node`) + : require("node-gyp-build")(root); + +try { + module.exports.nodeTypeInfo = require("../../src/node-types.json"); +} catch (_) {} diff --git a/crates/cli/src/templates/js-binding.cc b/cli/src/templates/js-binding.cc similarity index 100% rename from crates/cli/src/templates/js-binding.cc rename to cli/src/templates/js-binding.cc diff --git a/crates/cli/src/templates/lib.rs b/cli/src/templates/lib.rs similarity index 71% rename from crates/cli/src/templates/lib.rs rename to cli/src/templates/lib.rs index 1e8c9ca3..8478f488 100644 --- a/crates/cli/src/templates/lib.rs +++ b/cli/src/templates/lib.rs @@ -32,21 +32,12 @@ pub const LANGUAGE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_PARSE /// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers/6-static-node-types pub const NODE_TYPES: &str = include_str!("../../src/node-types.json"); -#[cfg(with_highlights_query)] -/// The syntax highlighting query for this grammar. -pub const HIGHLIGHTS_QUERY: &str = include_str!("../../HIGHLIGHTS_QUERY_PATH"); +// NOTE: uncomment these to include any queries that this grammar contains: -#[cfg(with_injections_query)] -/// The language injection query for this grammar. -pub const INJECTIONS_QUERY: &str = include_str!("../../INJECTIONS_QUERY_PATH"); - -#[cfg(with_locals_query)] -/// The local variable query for this grammar. -pub const LOCALS_QUERY: &str = include_str!("../../LOCALS_QUERY_PATH"); - -#[cfg(with_tags_query)] -/// The symbol tagging query for this grammar. -pub const TAGS_QUERY: &str = include_str!("../../TAGS_QUERY_PATH"); +// pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm"); +// pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm"); +// pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm"); +// pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm"); #[cfg(test)] mod tests { diff --git a/crates/cli/src/templates/makefile b/cli/src/templates/makefile similarity index 75% rename from crates/cli/src/templates/makefile rename to cli/src/templates/makefile index b42dab97..4e8411c5 100644 --- a/crates/cli/src/templates/makefile +++ b/cli/src/templates/makefile @@ -1,3 +1,7 @@ +ifeq ($(OS),Windows_NT) +$(error Windows is not supported) +endif + LANGUAGE_NAME := tree-sitter-KEBAB_PARSER_NAME HOMEPAGE_URL := PARSER_URL VERSION := PARSER_VERSION @@ -12,7 +16,6 @@ PREFIX ?= /usr/local DATADIR ?= $(PREFIX)/share INCLUDEDIR ?= $(PREFIX)/include LIBDIR ?= $(PREFIX)/lib -BINDIR ?= $(PREFIX)/bin PCLIBDIR ?= $(LIBDIR)/pkgconfig # source/object files @@ -29,25 +32,20 @@ SONAME_MAJOR = $(shell sed -n 's/\#define LANGUAGE_VERSION //p' $(PARSER)) SONAME_MINOR = $(word 1,$(subst ., ,$(VERSION))) # OS-specific bits -MACHINE := $(shell $(CC) -dumpmachine) - -ifneq ($(findstring darwin,$(MACHINE)),) +ifeq ($(shell uname),Darwin) SOEXT = dylib SOEXTVER_MAJOR = $(SONAME_MAJOR).$(SOEXT) SOEXTVER = $(SONAME_MAJOR).$(SONAME_MINOR).$(SOEXT) LINKSHARED = -dynamiclib -Wl,-install_name,$(LIBDIR)/lib$(LANGUAGE_NAME).$(SOEXTVER),-rpath,@executable_path/../Frameworks -else ifneq ($(findstring mingw32,$(MACHINE)),) - SOEXT = dll - LINKSHARED += -s -shared -Wl,--out-implib,lib$(LANGUAGE_NAME).dll.a else SOEXT = so SOEXTVER_MAJOR = $(SOEXT).$(SONAME_MAJOR) SOEXTVER = $(SOEXT).$(SONAME_MAJOR).$(SONAME_MINOR) LINKSHARED = -shared -Wl,-soname,lib$(LANGUAGE_NAME).$(SOEXTVER) +endif ifneq ($(filter $(shell uname),FreeBSD NetBSD DragonFly),) PCLIBDIR := $(PREFIX)/libdata/pkgconfig endif -endif all: lib$(LANGUAGE_NAME).a lib$(LANGUAGE_NAME).$(SOEXT) $(LANGUAGE_NAME).pc @@ -60,10 +58,6 @@ ifneq ($(STRIP),) $(STRIP) $@ endif -ifneq ($(findstring mingw32,$(MACHINE)),) -lib$(LANGUAGE_NAME).dll.a: lib$(LANGUAGE_NAME).$(SOEXT) -endif - $(LANGUAGE_NAME).pc: bindings/c/$(LANGUAGE_NAME).pc.in sed -e 's|@PROJECT_VERSION@|$(VERSION)|' \ -e 's|@CMAKE_INSTALL_LIBDIR@|$(LIBDIR:$(PREFIX)/%=%)|' \ @@ -72,9 +66,6 @@ $(LANGUAGE_NAME).pc: bindings/c/$(LANGUAGE_NAME).pc.in -e 's|@PROJECT_HOMEPAGE_URL@|$(HOMEPAGE_URL)|' \ -e 's|@CMAKE_INSTALL_PREFIX@|$(PREFIX)|' $< > $@ -$(SRC_DIR)/grammar.json: grammar.js - $(TS) generate --no-parser $^ - $(PARSER): $(SRC_DIR)/grammar.json $(TS) generate $^ @@ -84,15 +75,8 @@ install: all install -m644 $(LANGUAGE_NAME).pc '$(DESTDIR)$(PCLIBDIR)'/$(LANGUAGE_NAME).pc install -m644 lib$(LANGUAGE_NAME).a '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).a install -m755 lib$(LANGUAGE_NAME).$(SOEXT) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER) -ifneq ($(findstring mingw32,$(MACHINE)),) - install -d '$(DESTDIR)$(BINDIR)' - install -m755 lib$(LANGUAGE_NAME).dll '$(DESTDIR)$(BINDIR)'/lib$(LANGUAGE_NAME).dll - install -m755 lib$(LANGUAGE_NAME).dll.a '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).dll.a -else - install -m755 lib$(LANGUAGE_NAME).$(SOEXT) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER) - cd '$(DESTDIR)$(LIBDIR)' && ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER) lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) - cd '$(DESTDIR)$(LIBDIR)' && ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) lib$(LANGUAGE_NAME).$(SOEXT) -endif + ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) + ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXT) ifneq ($(wildcard queries/*.scm),) install -m644 queries/*.scm '$(DESTDIR)$(DATADIR)'/tree-sitter/queries/KEBAB_PARSER_NAME endif @@ -107,7 +91,7 @@ uninstall: $(RM) -r '$(DESTDIR)$(DATADIR)'/tree-sitter/queries/KEBAB_PARSER_NAME clean: - $(RM) $(OBJS) $(LANGUAGE_NAME).pc lib$(LANGUAGE_NAME).a lib$(LANGUAGE_NAME).$(SOEXT) lib$(LANGUAGE_NAME).dll.a + $(RM) $(OBJS) $(LANGUAGE_NAME).pc lib$(LANGUAGE_NAME).a lib$(LANGUAGE_NAME).$(SOEXT) test: $(TS) test diff --git a/crates/cli/src/templates/package.json b/cli/src/templates/package.json similarity index 87% rename from crates/cli/src/templates/package.json rename to cli/src/templates/package.json index d9dbbd33..a2ee2e59 100644 --- a/crates/cli/src/templates/package.json +++ b/cli/src/templates/package.json @@ -2,11 +2,7 @@ "name": "tree-sitter-PARSER_NAME", "version": "PARSER_VERSION", "description": "PARSER_DESCRIPTION", - "type": "module", - "repository": { - "type": "git", - "url": "git+PARSER_URL.git" - }, + "repository": "PARSER_URL", "funding": "FUNDING_URL", "license": "PARSER_LICENSE", "author": { @@ -38,11 +34,11 @@ }, "devDependencies": { "prebuildify": "^6.0.1", - "tree-sitter": "^0.25.0", + "tree-sitter": "^0.22.4", "tree-sitter-cli": "^CLI_VERSION" }, "peerDependencies": { - "tree-sitter": "^0.25.0" + "tree-sitter": "^0.22.4" }, "peerDependenciesMeta": { "tree-sitter": { diff --git a/crates/cli/src/templates/package.swift b/cli/src/templates/package.swift similarity index 100% rename from crates/cli/src/templates/package.swift rename to cli/src/templates/package.swift diff --git a/crates/cli/src/templates/py-binding.c b/cli/src/templates/py-binding.c similarity index 100% rename from crates/cli/src/templates/py-binding.c rename to cli/src/templates/py-binding.c diff --git a/crates/cli/src/templates/pyproject.toml b/cli/src/templates/pyproject.toml similarity index 100% rename from crates/cli/src/templates/pyproject.toml rename to cli/src/templates/pyproject.toml diff --git a/crates/cli/src/templates/root.zig b/cli/src/templates/root.zig similarity index 100% rename from crates/cli/src/templates/root.zig rename to cli/src/templates/root.zig diff --git a/crates/cli/src/templates/setup.py b/cli/src/templates/setup.py similarity index 96% rename from crates/cli/src/templates/setup.py rename to cli/src/templates/setup.py index bcf184b7..7f92eaee 100644 --- a/crates/cli/src/templates/setup.py +++ b/cli/src/templates/setup.py @@ -32,7 +32,7 @@ class BuildExt(build_ext): class BdistWheel(bdist_wheel): def get_tag(self): python, abi, platform = super().get_tag() - if python.startswith("cp") and not get_config_var("Py_GIL_DISABLED"): + if python.startswith("cp"): python, abi = "cp310", "abi3" return python, abi, platform diff --git a/crates/cli/src/templates/test.zig b/cli/src/templates/test.zig similarity index 85% rename from crates/cli/src/templates/test.zig rename to cli/src/templates/test.zig index af84322f..7baec557 100644 --- a/crates/cli/src/templates/test.zig +++ b/cli/src/templates/test.zig @@ -9,7 +9,7 @@ test "can load grammar" { const parser = Parser.create(); defer parser.destroy(); - const lang: *const ts.Language = Language.fromRaw(root.language()); + const lang: *const ts.Language = @ptrCast(root.language()); defer lang.destroy(); try testing.expectEqual(void{}, parser.setLanguage(lang)); diff --git a/crates/cli/src/templates/test_binding.py b/cli/src/templates/test_binding.py similarity index 100% rename from crates/cli/src/templates/test_binding.py rename to cli/src/templates/test_binding.py diff --git a/crates/cli/src/templates/tests.swift b/cli/src/templates/tests.swift similarity index 100% rename from crates/cli/src/templates/tests.swift rename to cli/src/templates/tests.swift diff --git a/cli/src/test.rs b/cli/src/test.rs new file mode 100644 index 00000000..e128bc80 --- /dev/null +++ b/cli/src/test.rs @@ -0,0 +1,1584 @@ +use std::{ + collections::BTreeMap, + ffi::OsStr, + fmt::Write as _, + fs, + io::{self, Write}, + path::{Path, PathBuf}, + str, + sync::LazyLock, + time::Duration, +}; + +use anstyle::{AnsiColor, Color, Style}; +use anyhow::{anyhow, Context, Result}; +use clap::ValueEnum; +use indoc::indoc; +use regex::{ + bytes::{Regex as ByteRegex, RegexBuilder as ByteRegexBuilder}, + Regex, +}; +use similar::{ChangeTag, TextDiff}; +use tree_sitter::{format_sexp, Language, LogType, Parser, Query, Tree}; +use walkdir::WalkDir; + +use super::util; +use crate::parse::Stats; + +static HEADER_REGEX: LazyLock = LazyLock::new(|| { + ByteRegexBuilder::new( + r"^(?x) + (?P(?:=+){3,}) + (?P[^=\r\n][^\r\n]*)? + \r?\n + (?P(?:([^=\r\n]|\s+:)[^\r\n]*\r?\n)+) + ===+ + (?P[^=\r\n][^\r\n]*)?\r?\n", + ) + .multi_line(true) + .build() + .unwrap() +}); + +static DIVIDER_REGEX: LazyLock = LazyLock::new(|| { + ByteRegexBuilder::new(r"^(?P(?:-+){3,})(?P[^-\r\n][^\r\n]*)?\r?\n") + .multi_line(true) + .build() + .unwrap() +}); + +static COMMENT_REGEX: LazyLock = LazyLock::new(|| Regex::new(r"(?m)^\s*;.*$").unwrap()); + +static WHITESPACE_REGEX: LazyLock = LazyLock::new(|| Regex::new(r"\s+").unwrap()); + +static SEXP_FIELD_REGEX: LazyLock = LazyLock::new(|| Regex::new(r" \w+: \(").unwrap()); + +static POINT_REGEX: LazyLock = + LazyLock::new(|| Regex::new(r"\s*\[\s*\d+\s*,\s*\d+\s*\]\s*").unwrap()); + +#[derive(Debug, PartialEq, Eq)] +pub enum TestEntry { + Group { + name: String, + children: Vec, + file_path: Option, + }, + Example { + name: String, + input: Vec, + output: String, + header_delim_len: usize, + divider_delim_len: usize, + has_fields: bool, + attributes_str: String, + attributes: TestAttributes, + file_name: Option, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TestAttributes { + pub skip: bool, + pub platform: bool, + pub fail_fast: bool, + pub error: bool, + pub languages: Vec>, +} + +impl Default for TestEntry { + fn default() -> Self { + Self::Group { + name: String::new(), + children: Vec::new(), + file_path: None, + } + } +} + +impl Default for TestAttributes { + fn default() -> Self { + Self { + skip: false, + platform: true, + fail_fast: false, + error: false, + languages: vec!["".into()], + } + } +} + +#[derive(ValueEnum, Default, Copy, Clone, PartialEq, Eq)] +pub enum TestStats { + All, + #[default] + OutliersAndTotal, + TotalOnly, +} + +pub struct TestOptions<'a> { + pub output: &'a mut String, + pub path: PathBuf, + pub debug: bool, + pub debug_graph: bool, + pub include: Option, + pub exclude: Option, + pub file_name: Option, + pub update: bool, + pub open_log: bool, + pub languages: BTreeMap<&'a str, &'a Language>, + pub color: bool, + pub test_num: usize, + /// Whether a test ran for the nth line in `output`, the true parse rate, and the adjusted + /// parse rate + pub parse_rates: &'a mut Vec<(bool, Option<(f64, f64)>)>, + pub stat_display: TestStats, + pub stats: &'a mut Stats, + pub show_fields: bool, + pub overview_only: bool, +} + +pub fn run_tests_at_path(parser: &mut Parser, opts: &mut TestOptions) -> Result<()> { + let test_entry = parse_tests(&opts.path)?; + let mut _log_session = None; + + if opts.debug_graph { + _log_session = Some(util::log_graphs(parser, "log.html", opts.open_log)?); + } else if opts.debug { + parser.set_logger(Some(Box::new(|log_type, message| { + if log_type == LogType::Lex { + io::stderr().write_all(b" ").unwrap(); + } + writeln!(&mut io::stderr(), "{message}").unwrap(); + }))); + } + + let mut failures = Vec::new(); + let mut corrected_entries = Vec::new(); + let mut has_parse_errors = false; + run_tests( + parser, + test_entry, + opts, + 0, + &mut failures, + &mut corrected_entries, + &mut has_parse_errors, + )?; + + let (count, total_adj_parse_time) = opts + .parse_rates + .iter() + .flat_map(|(_, rates)| rates) + .fold((0usize, 0.0f64), |(count, rate_accum), (_, adj_rate)| { + (count + 1, rate_accum + adj_rate) + }); + + let avg = total_adj_parse_time / count as f64; + let std_dev = { + let variance = opts + .parse_rates + .iter() + .flat_map(|(_, rates)| rates) + .map(|(_, rate_i)| (rate_i - avg).powi(2)) + .sum::() + / count as f64; + variance.sqrt() + }; + + for ((is_test, rates), out_line) in opts.parse_rates.iter().zip(opts.output.lines()) { + let stat_display = if !is_test { + // Test group, no actual parsing took place + String::new() + } else { + match (opts.stat_display, rates) { + (TestStats::TotalOnly, _) | (_, None) => String::new(), + (display, Some((true_rate, adj_rate))) => { + let mut stats = if display == TestStats::All { + format!(" ({true_rate:.3} bytes/ms)") + } else { + String::new() + }; + // 3 standard deviations below the mean, aka the "Empirical Rule" + if *adj_rate < 3.0f64.mul_add(-std_dev, avg) { + stats += &paint( + opts.color.then_some(AnsiColor::Red), + &format!(" -- Warning: Slow parse rate ({true_rate:.3} bytes/ms)"), + ); + } + stats + } + } + }; + println!("{out_line}{stat_display}"); + } + + parser.stop_printing_dot_graphs(); + + if failures.is_empty() { + Ok(()) + } else { + println!(); + + if opts.update && !has_parse_errors { + if failures.len() == 1 { + println!("1 update:\n"); + } else { + println!("{} updates:\n", failures.len()); + } + + for (i, (name, ..)) in failures.iter().enumerate() { + println!(" {}. {name}", i + 1); + } + + Ok(()) + } else { + has_parse_errors = opts.update && has_parse_errors; + + if !opts.overview_only { + if !has_parse_errors { + if failures.len() == 1 { + println!("1 failure:"); + } else { + println!("{} failures:", failures.len()); + } + } + + if opts.color { + print_diff_key(); + } + for (i, (name, actual, expected)) in failures.iter().enumerate() { + if expected == "NO ERROR" { + println!("\n {}. {name}:\n", i + 1); + println!(" Expected an ERROR node, but got:"); + println!( + " {}", + paint( + opts.color.then_some(AnsiColor::Red), + &format_sexp(actual, 2) + ) + ); + } else { + println!("\n {}. {name}:", i + 1); + let actual = format_sexp(actual, 2); + let expected = format_sexp(expected, 2); + print_diff(&actual, &expected, opts.color); + } + } + } + + if has_parse_errors { + Err(anyhow!(indoc! {" + Some tests failed to parse with unexpected `ERROR` or `MISSING` nodes, as shown above, and cannot be updated automatically. + Either fix the grammar or manually update the tests if this is expected."})) + } else { + Err(anyhow!("")) + } + } + } +} + +pub fn check_queries_at_path(language: &Language, path: &Path) -> Result<()> { + if path.exists() { + for entry in WalkDir::new(path) + .into_iter() + .filter_map(std::result::Result::ok) + .filter(|e| { + e.file_type().is_file() + && e.path().extension().and_then(OsStr::to_str) == Some("scm") + && !e.path().starts_with(".") + }) + { + let filepath = entry.file_name().to_str().unwrap_or(""); + let content = fs::read_to_string(entry.path()) + .with_context(|| format!("Error reading query file {filepath:?}"))?; + Query::new(language, &content) + .with_context(|| format!("Error in query file {filepath:?}"))?; + } + } + Ok(()) +} + +pub fn print_diff_key() { + println!( + "\ncorrect / {} / {}", + paint(Some(AnsiColor::Green), "expected"), + paint(Some(AnsiColor::Red), "unexpected") + ); +} + +pub fn print_diff(actual: &str, expected: &str, use_color: bool) { + let diff = TextDiff::from_lines(actual, expected); + for diff in diff.iter_all_changes() { + match diff.tag() { + ChangeTag::Equal => { + if use_color { + print!("{diff}"); + } else { + print!(" {diff}"); + } + } + ChangeTag::Insert => { + if use_color { + print!("{}", paint(Some(AnsiColor::Green), diff.as_str().unwrap())); + } else { + print!("+{diff}"); + } + if diff.missing_newline() { + println!(); + } + } + ChangeTag::Delete => { + if use_color { + print!("{}", paint(Some(AnsiColor::Red), diff.as_str().unwrap())); + } else { + print!("-{diff}"); + } + if diff.missing_newline() { + println!(); + } + } + } + } + + println!(); +} + +pub fn paint(color: Option>, text: &str) -> String { + let style = Style::new().fg_color(color.map(Into::into)); + format!("{style}{text}{style:#}") +} + +/// This will return false if we want to "fail fast". It will bail and not parse any more tests. +#[allow(clippy::too_many_arguments)] +fn run_tests( + parser: &mut Parser, + test_entry: TestEntry, + opts: &mut TestOptions, + mut indent_level: u32, + failures: &mut Vec<(String, String, String)>, + corrected_entries: &mut Vec<(String, String, String, String, usize, usize)>, + has_parse_errors: &mut bool, +) -> Result { + match test_entry { + TestEntry::Example { + name, + input, + output, + header_delim_len, + divider_delim_len, + has_fields, + attributes_str, + attributes, + .. + } => { + write!(opts.output, "{}", " ".repeat(indent_level as usize))?; + + if attributes.skip { + writeln!( + opts.output, + "{:>3}. ⌀ {}", + opts.test_num, + paint(opts.color.then_some(AnsiColor::Yellow), &name), + )?; + opts.parse_rates.push((true, None)); + opts.test_num += 1; + return Ok(true); + } + + if !attributes.platform { + writeln!( + opts.output, + "{:>3}. ⌀ {}", + opts.test_num, + paint(opts.color.then_some(AnsiColor::Magenta), &name), + )?; + opts.parse_rates.push((true, None)); + opts.test_num += 1; + return Ok(true); + } + + for (i, language_name) in attributes.languages.iter().enumerate() { + if !language_name.is_empty() { + let language = opts + .languages + .get(language_name.as_ref()) + .ok_or_else(|| anyhow!("Language not found: {language_name}"))?; + parser.set_language(language)?; + } + let start = std::time::Instant::now(); + let tree = parser.parse(&input, None).unwrap(); + { + let parse_time = start.elapsed(); + let true_parse_rate = tree.root_node().byte_range().len() as f64 + / (parse_time.as_nanos() as f64 / 1_000_000.0); + let adj_parse_rate = adjusted_parse_rate(&tree, parse_time); + + opts.parse_rates + .push((true, Some((true_parse_rate, adj_parse_rate)))); + opts.stats.total_parses += 1; + opts.stats.total_duration += parse_time; + opts.stats.total_bytes += tree.root_node().byte_range().len(); + } + + if attributes.error { + if tree.root_node().has_error() { + writeln!( + opts.output, + "{:>3}. ✓ {}", + opts.test_num, + paint(opts.color.then_some(AnsiColor::Green), &name), + )?; + opts.stats.successful_parses += 1; + if opts.update { + let input = String::from_utf8(input.clone()).unwrap(); + let output = format_sexp(&output, 0); + corrected_entries.push(( + name.clone(), + input, + output, + attributes_str.clone(), + header_delim_len, + divider_delim_len, + )); + } + } else { + if opts.update { + let input = String::from_utf8(input.clone()).unwrap(); + // Keep the original `expected` output if the actual output has no error + let output = format_sexp(&output, 0); + corrected_entries.push(( + name.clone(), + input, + output, + attributes_str.clone(), + header_delim_len, + divider_delim_len, + )); + } + writeln!( + opts.output, + "{:>3}. ✗ {}", + opts.test_num, + paint(opts.color.then_some(AnsiColor::Red), &name), + )?; + failures.push(( + name.clone(), + tree.root_node().to_sexp(), + "NO ERROR".to_string(), + )); + } + + if attributes.fail_fast { + return Ok(false); + } + } else { + let mut actual = tree.root_node().to_sexp(); + if !(opts.show_fields || has_fields) { + actual = strip_sexp_fields(&actual); + } + + if actual == output { + writeln!( + opts.output, + "{:>3}. ✓ {}", + opts.test_num, + paint(opts.color.then_some(AnsiColor::Green), &name), + )?; + opts.stats.successful_parses += 1; + if opts.update { + let input = String::from_utf8(input.clone()).unwrap(); + let output = format_sexp(&output, 0); + corrected_entries.push(( + name.clone(), + input, + output, + attributes_str.clone(), + header_delim_len, + divider_delim_len, + )); + } + } else { + if opts.update { + let input = String::from_utf8(input.clone()).unwrap(); + let expected_output = format_sexp(&output, 0); + let actual_output = format_sexp(&actual, 0); + + // Only bail early before updating if the actual is not the output, + // sometimes users want to test cases that + // are intended to have errors, hence why this + // check isn't shown above + if actual.contains("ERROR") || actual.contains("MISSING") { + *has_parse_errors = true; + + // keep the original `expected` output if the actual output has an + // error + corrected_entries.push(( + name.clone(), + input, + expected_output, + attributes_str.clone(), + header_delim_len, + divider_delim_len, + )); + } else { + corrected_entries.push(( + name.clone(), + input, + actual_output, + attributes_str.clone(), + header_delim_len, + divider_delim_len, + )); + writeln!( + opts.output, + "{:>3}. ✓ {}", + opts.test_num, + paint(opts.color.then_some(AnsiColor::Blue), &name), + )?; + } + } else { + writeln!( + opts.output, + "{:>3}. ✗ {}", + opts.test_num, + paint(opts.color.then_some(AnsiColor::Red), &name), + )?; + } + failures.push((name.clone(), actual, output.clone())); + + if attributes.fail_fast { + return Ok(false); + } + } + } + + if i == attributes.languages.len() - 1 { + // reset to the first language + parser.set_language(opts.languages.values().next().unwrap())?; + } + } + opts.test_num += 1; + } + TestEntry::Group { + name, + children, + file_path, + } => { + if children.is_empty() { + return Ok(true); + } + + indent_level += 1; + let failure_count = failures.len(); + let mut has_printed = false; + + let matches_filter = |name: &str, file_name: &Option, opts: &TestOptions| { + if let (Some(test_file_path), Some(filter_file_name)) = (file_name, &opts.file_name) + { + if !filter_file_name.eq(test_file_path) { + return false; + } + } + if let Some(include) = &opts.include { + include.is_match(name) + } else if let Some(exclude) = &opts.exclude { + !exclude.is_match(name) + } else { + true + } + }; + + let should_skip = |entry: &TestEntry, opts: &TestOptions| match entry { + TestEntry::Example { + name, file_name, .. + } => !matches_filter(name, file_name, opts), + TestEntry::Group { .. } => false, + }; + + for child in children { + if let TestEntry::Example { + ref name, + ref input, + ref output, + ref attributes_str, + header_delim_len, + divider_delim_len, + .. + } = child + { + if should_skip(&child, opts) { + let input = String::from_utf8(input.clone()).unwrap(); + let output = format_sexp(output, 0); + corrected_entries.push(( + name.clone(), + input, + output, + attributes_str.clone(), + header_delim_len, + divider_delim_len, + )); + + opts.test_num += 1; + + continue; + } + } + if !has_printed && indent_level > 1 { + has_printed = true; + writeln!( + opts.output, + "{}{name}:", + " ".repeat((indent_level - 1) as usize) + )?; + opts.parse_rates.push((false, None)); + } + if !run_tests( + parser, + child, + opts, + indent_level, + failures, + corrected_entries, + has_parse_errors, + )? { + // fail fast + return Ok(false); + } + } + + if let Some(file_path) = file_path { + if opts.update && failures.len() - failure_count > 0 { + write_tests(&file_path, corrected_entries)?; + } + corrected_entries.clear(); + } + } + } + Ok(true) +} + +// Parse time is interpreted in ns before converting to ms to avoid truncation issues +// Parse rates often have several outliers, leading to a large standard deviation. Taking +// the log of these rates serves to "flatten" out the distribution, yielding a more +// usable standard deviation for finding statistically significant slow parse rates +// NOTE: This is just a heuristic +#[must_use] +pub fn adjusted_parse_rate(tree: &Tree, parse_time: Duration) -> f64 { + f64::ln( + tree.root_node().byte_range().len() as f64 / (parse_time.as_nanos() as f64 / 1_000_000.0), + ) +} + +fn write_tests( + file_path: &Path, + corrected_entries: &[(String, String, String, String, usize, usize)], +) -> Result<()> { + let mut buffer = fs::File::create(file_path)?; + write_tests_to_buffer(&mut buffer, corrected_entries) +} + +fn write_tests_to_buffer( + buffer: &mut impl Write, + corrected_entries: &[(String, String, String, String, usize, usize)], +) -> Result<()> { + for (i, (name, input, output, attributes_str, header_delim_len, divider_delim_len)) in + corrected_entries.iter().enumerate() + { + if i > 0 { + writeln!(buffer)?; + } + writeln!( + buffer, + "{}\n{name}\n{}{}\n{input}\n{}\n\n{}", + "=".repeat(*header_delim_len), + if attributes_str.is_empty() { + attributes_str.clone() + } else { + format!("{attributes_str}\n") + }, + "=".repeat(*header_delim_len), + "-".repeat(*divider_delim_len), + output.trim() + )?; + } + Ok(()) +} + +pub fn parse_tests(path: &Path) -> io::Result { + let name = path + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("") + .to_string(); + if path.is_dir() { + let mut children = Vec::new(); + for entry in fs::read_dir(path)? { + let entry = entry?; + let hidden = entry.file_name().to_str().unwrap_or("").starts_with('.'); + if !hidden { + children.push(entry.path()); + } + } + children.sort_by(|a, b| { + a.file_name() + .unwrap_or_default() + .cmp(b.file_name().unwrap_or_default()) + }); + let children = children + .iter() + .map(|path| parse_tests(path)) + .collect::>>()?; + Ok(TestEntry::Group { + name, + children, + file_path: None, + }) + } else { + let content = fs::read_to_string(path)?; + Ok(parse_test_content(name, &content, Some(path.to_path_buf()))) + } +} + +#[must_use] +pub fn strip_sexp_fields(sexp: &str) -> String { + SEXP_FIELD_REGEX.replace_all(sexp, " (").to_string() +} + +#[must_use] +pub fn strip_points(sexp: &str) -> String { + POINT_REGEX.replace_all(sexp, "").to_string() +} + +fn parse_test_content(name: String, content: &str, file_path: Option) -> TestEntry { + let mut children = Vec::new(); + let bytes = content.as_bytes(); + let mut prev_name = String::new(); + let mut prev_attributes_str = String::new(); + let mut prev_header_end = 0; + + // Find the first test header in the file, and determine if it has a + // custom suffix. If so, then this suffix will be used to identify + // all subsequent headers and divider lines in the file. + let first_suffix = HEADER_REGEX + .captures(bytes) + .and_then(|c| c.name("suffix1")) + .map(|m| String::from_utf8_lossy(m.as_bytes())); + + // Find all of the `===` test headers, which contain the test names. + // Ignore any matches whose suffix does not match the first header + // suffix in the file. + let header_matches = HEADER_REGEX.captures_iter(bytes).filter_map(|c| { + let header_delim_len = c.name("equals").map_or(80, |m| m.as_bytes().len()); + let suffix1 = c + .name("suffix1") + .map(|m| String::from_utf8_lossy(m.as_bytes())); + let suffix2 = c + .name("suffix2") + .map(|m| String::from_utf8_lossy(m.as_bytes())); + + let (mut skip, mut platform, mut fail_fast, mut error, mut languages) = + (false, None, false, false, vec![]); + + let test_name_and_markers = c + .name("test_name_and_markers") + .map_or("".as_bytes(), |m| m.as_bytes()); + + let mut test_name = String::new(); + let mut attributes_str = String::new(); + + let mut seen_marker = false; + + let test_name_and_markers = str::from_utf8(test_name_and_markers).unwrap(); + for line in test_name_and_markers + .split_inclusive('\n') + .filter(|s| !s.is_empty()) + { + let trimmed_line = line.trim(); + match trimmed_line.split('(').next().unwrap() { + ":skip" => (seen_marker, skip) = (true, true), + ":platform" => { + if let Some(platforms) = trimmed_line.strip_prefix(':').and_then(|s| { + s.strip_prefix("platform(") + .and_then(|s| s.strip_suffix(')')) + }) { + seen_marker = true; + platform = Some( + platform.unwrap_or(false) || platforms.trim() == std::env::consts::OS, + ); + } + } + ":fail-fast" => (seen_marker, fail_fast) = (true, true), + ":error" => (seen_marker, error) = (true, true), + ":language" => { + if let Some(lang) = trimmed_line.strip_prefix(':').and_then(|s| { + s.strip_prefix("language(") + .and_then(|s| s.strip_suffix(')')) + }) { + seen_marker = true; + languages.push(lang.into()); + } + } + _ if !seen_marker => { + test_name.push_str(line); + } + _ => {} + } + } + attributes_str.push_str(test_name_and_markers.strip_prefix(&test_name).unwrap()); + + // prefer skip over error, both shouldn't be set + if skip { + error = false; + } + + // add a default language if none are specified, will defer to the first language + if languages.is_empty() { + languages.push("".into()); + } + + if suffix1 == first_suffix && suffix2 == first_suffix { + let header_range = c.get(0).unwrap().range(); + let test_name = if test_name.is_empty() { + None + } else { + Some(test_name.trim_end().to_string()) + }; + let attributes_str = if attributes_str.is_empty() { + None + } else { + Some(attributes_str.trim_end().to_string()) + }; + Some(( + header_delim_len, + header_range, + test_name, + attributes_str, + TestAttributes { + skip, + platform: platform.unwrap_or(true), + fail_fast, + error, + languages, + }, + )) + } else { + None + } + }); + + let (mut prev_header_len, mut prev_attributes) = (80, TestAttributes::default()); + for (header_delim_len, header_range, test_name, attributes_str, attributes) in header_matches + .chain(Some(( + 80, + bytes.len()..bytes.len(), + None, + None, + TestAttributes::default(), + ))) + { + // Find the longest line of dashes following each test description. That line + // separates the input from the expected output. Ignore any matches whose suffix + // does not match the first suffix in the file. + if prev_header_end > 0 { + let divider_range = DIVIDER_REGEX + .captures_iter(&bytes[prev_header_end..header_range.start]) + .filter_map(|m| { + let divider_delim_len = m.name("hyphens").map_or(80, |m| m.as_bytes().len()); + let suffix = m + .name("suffix") + .map(|m| String::from_utf8_lossy(m.as_bytes())); + if suffix == first_suffix { + let range = m.get(0).unwrap().range(); + Some(( + divider_delim_len, + (prev_header_end + range.start)..(prev_header_end + range.end), + )) + } else { + None + } + }) + .max_by_key(|(_, range)| range.len()); + + if let Some((divider_delim_len, divider_range)) = divider_range { + if let Ok(output) = str::from_utf8(&bytes[divider_range.end..header_range.start]) { + let mut input = bytes[prev_header_end..divider_range.start].to_vec(); + + // Remove trailing newline from the input. + input.pop(); + #[cfg(target_os = "windows")] + if input.last() == Some(&b'\r') { + input.pop(); + } + + // Remove all comments + let output = COMMENT_REGEX.replace_all(output, "").to_string(); + + // Normalize the whitespace in the expected output. + let output = WHITESPACE_REGEX.replace_all(output.trim(), " "); + let output = output.replace(" )", ")"); + + // Identify if the expected output has fields indicated. If not, then + // fields will not be checked. + let has_fields = SEXP_FIELD_REGEX.is_match(&output); + + let file_name = if let Some(ref path) = file_path { + path.file_name().map(|n| n.to_string_lossy().to_string()) + } else { + None + }; + + let t = TestEntry::Example { + name: prev_name, + input, + output, + header_delim_len: prev_header_len, + divider_delim_len, + has_fields, + attributes_str: prev_attributes_str, + attributes: prev_attributes, + file_name, + }; + + children.push(t); + } + } + } + prev_attributes = attributes; + prev_name = test_name.unwrap_or_default(); + prev_attributes_str = attributes_str.unwrap_or_default(); + prev_header_len = header_delim_len; + prev_header_end = header_range.end; + } + TestEntry::Group { + name, + children, + file_path, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_test_content_simple() { + let entry = parse_test_content( + "the-filename".to_string(), + r" +=============== +The first test +=============== + +a b c + +--- + +(a + (b c)) + +================ +The second test +================ +d +--- +(d) + " + .trim(), + None, + ); + + assert_eq!( + entry, + TestEntry::Group { + name: "the-filename".to_string(), + children: vec![ + TestEntry::Example { + name: "The first test".to_string(), + input: b"\na b c\n".to_vec(), + output: "(a (b c))".to_string(), + header_delim_len: 15, + divider_delim_len: 3, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + }, + TestEntry::Example { + name: "The second test".to_string(), + input: b"d".to_vec(), + output: "(d)".to_string(), + header_delim_len: 16, + divider_delim_len: 3, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + }, + ], + file_path: None, + } + ); + } + + #[test] + fn test_parse_test_content_with_dashes_in_source_code() { + let entry = parse_test_content( + "the-filename".to_string(), + r" +================== +Code with dashes +================== +abc +--- +defg +---- +hijkl +------- + +(a (b)) + +========================= +Code ending with dashes +========================= +abc +----------- +------------------- + +(c (d)) + " + .trim(), + None, + ); + + assert_eq!( + entry, + TestEntry::Group { + name: "the-filename".to_string(), + children: vec![ + TestEntry::Example { + name: "Code with dashes".to_string(), + input: b"abc\n---\ndefg\n----\nhijkl".to_vec(), + output: "(a (b))".to_string(), + header_delim_len: 18, + divider_delim_len: 7, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + }, + TestEntry::Example { + name: "Code ending with dashes".to_string(), + input: b"abc\n-----------".to_vec(), + output: "(c (d))".to_string(), + header_delim_len: 25, + divider_delim_len: 19, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + }, + ], + file_path: None, + } + ); + } + + #[test] + fn test_format_sexp() { + assert_eq!(format_sexp("", 0), ""); + assert_eq!( + format_sexp("(a b: (c) (d) e: (f (g (h (MISSING i)))))", 0), + r" +(a + b: (c) + (d) + e: (f + (g + (h + (MISSING i))))) +" + .trim() + ); + assert_eq!( + format_sexp("(program (ERROR (UNEXPECTED ' ')) (identifier))", 0), + r" +(program + (ERROR + (UNEXPECTED ' ')) + (identifier)) +" + .trim() + ); + assert_eq!( + format_sexp(r#"(source_file (MISSING ")"))"#, 0), + r#" +(source_file + (MISSING ")")) + "# + .trim() + ); + assert_eq!( + format_sexp( + r"(source_file (ERROR (UNEXPECTED 'f') (UNEXPECTED '+')))", + 0 + ), + r" +(source_file + (ERROR + (UNEXPECTED 'f') + (UNEXPECTED '+'))) +" + .trim() + ); + } + + #[test] + fn test_write_tests_to_buffer() { + let mut buffer = Vec::new(); + let corrected_entries = vec![ + ( + "title 1".to_string(), + "input 1".to_string(), + "output 1".to_string(), + String::new(), + 80, + 80, + ), + ( + "title 2".to_string(), + "input 2".to_string(), + "output 2".to_string(), + String::new(), + 80, + 80, + ), + ]; + write_tests_to_buffer(&mut buffer, &corrected_entries).unwrap(); + assert_eq!( + String::from_utf8(buffer).unwrap(), + r" +================================================================================ +title 1 +================================================================================ +input 1 +-------------------------------------------------------------------------------- + +output 1 + +================================================================================ +title 2 +================================================================================ +input 2 +-------------------------------------------------------------------------------- + +output 2 +" + .trim_start() + .to_string() + ); + } + + #[test] + fn test_parse_test_content_with_comments_in_sexp() { + let entry = parse_test_content( + "the-filename".to_string(), + r#" +================== +sexp with comment +================== +code +--- + +; Line start comment +(a (b)) + +================== +sexp with comment between +================== +code +--- + +; Line start comment +(a +; ignore this + (b) + ; also ignore this +) + +========================= +sexp with ';' +========================= +code +--- + +(MISSING ";") + "# + .trim(), + None, + ); + + assert_eq!( + entry, + TestEntry::Group { + name: "the-filename".to_string(), + children: vec![ + TestEntry::Example { + name: "sexp with comment".to_string(), + input: b"code".to_vec(), + output: "(a (b))".to_string(), + header_delim_len: 18, + divider_delim_len: 3, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + }, + TestEntry::Example { + name: "sexp with comment between".to_string(), + input: b"code".to_vec(), + output: "(a (b))".to_string(), + header_delim_len: 18, + divider_delim_len: 3, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + }, + TestEntry::Example { + name: "sexp with ';'".to_string(), + input: b"code".to_vec(), + output: "(MISSING \";\")".to_string(), + header_delim_len: 25, + divider_delim_len: 3, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + } + ], + file_path: None, + } + ); + } + + #[test] + fn test_parse_test_content_with_suffixes() { + let entry = parse_test_content( + "the-filename".to_string(), + r" +==================asdf\()[]|{}*+?^$.- +First test +==================asdf\()[]|{}*+?^$.- + +========================= +NOT A TEST HEADER +========================= +------------------------- + +---asdf\()[]|{}*+?^$.- + +(a) + +==================asdf\()[]|{}*+?^$.- +Second test +==================asdf\()[]|{}*+?^$.- + +========================= +NOT A TEST HEADER +========================= +------------------------- + +---asdf\()[]|{}*+?^$.- + +(a) + +=========================asdf\()[]|{}*+?^$.- +Test name with = symbol +=========================asdf\()[]|{}*+?^$.- + +========================= +NOT A TEST HEADER +========================= +------------------------- + +---asdf\()[]|{}*+?^$.- + +(a) + +==============================asdf\()[]|{}*+?^$.- +Test containing equals +==============================asdf\()[]|{}*+?^$.- + +=== + +------------------------------asdf\()[]|{}*+?^$.- + +(a) + +==============================asdf\()[]|{}*+?^$.- +Subsequent test containing equals +==============================asdf\()[]|{}*+?^$.- + +=== + +------------------------------asdf\()[]|{}*+?^$.- + +(a) +" + .trim(), + None, + ); + + let expected_input = b"\n=========================\n\ + NOT A TEST HEADER\n\ + =========================\n\ + -------------------------\n" + .to_vec(); + pretty_assertions::assert_eq!( + entry, + TestEntry::Group { + name: "the-filename".to_string(), + children: vec![ + TestEntry::Example { + name: "First test".to_string(), + input: expected_input.clone(), + output: "(a)".to_string(), + header_delim_len: 18, + divider_delim_len: 3, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + }, + TestEntry::Example { + name: "Second test".to_string(), + input: expected_input.clone(), + output: "(a)".to_string(), + header_delim_len: 18, + divider_delim_len: 3, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + }, + TestEntry::Example { + name: "Test name with = symbol".to_string(), + input: expected_input, + output: "(a)".to_string(), + header_delim_len: 25, + divider_delim_len: 3, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + }, + TestEntry::Example { + name: "Test containing equals".to_string(), + input: "\n===\n".into(), + output: "(a)".into(), + header_delim_len: 30, + divider_delim_len: 30, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + }, + TestEntry::Example { + name: "Subsequent test containing equals".to_string(), + input: "\n===\n".into(), + output: "(a)".into(), + header_delim_len: 30, + divider_delim_len: 30, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + } + ], + file_path: None, + } + ); + } + + #[test] + fn test_parse_test_content_with_newlines_in_test_names() { + let entry = parse_test_content( + "the-filename".to_string(), + r" +=============== +name +with +newlines +=============== +a +--- +(b) + +==================== +name with === signs +==================== +code with ---- +--- +(d) +", + None, + ); + + assert_eq!( + entry, + TestEntry::Group { + name: "the-filename".to_string(), + file_path: None, + children: vec![ + TestEntry::Example { + name: "name\nwith\nnewlines".to_string(), + input: b"a".to_vec(), + output: "(b)".to_string(), + header_delim_len: 15, + divider_delim_len: 3, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + }, + TestEntry::Example { + name: "name with === signs".to_string(), + input: b"code with ----".to_vec(), + output: "(d)".to_string(), + header_delim_len: 20, + divider_delim_len: 3, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + } + ] + } + ); + } + + #[test] + fn test_parse_test_with_markers() { + // do one with :skip, we should not see it in the entry output + + let entry = parse_test_content( + "the-filename".to_string(), + r" +===================== +Test with skip marker +:skip +===================== +a +--- +(b) +", + None, + ); + + assert_eq!( + entry, + TestEntry::Group { + name: "the-filename".to_string(), + file_path: None, + children: vec![TestEntry::Example { + name: "Test with skip marker".to_string(), + input: b"a".to_vec(), + output: "(b)".to_string(), + header_delim_len: 21, + divider_delim_len: 3, + has_fields: false, + attributes_str: ":skip".to_string(), + attributes: TestAttributes { + skip: true, + platform: true, + fail_fast: false, + error: false, + languages: vec!["".into()] + }, + file_name: None, + }] + } + ); + + let entry = parse_test_content( + "the-filename".to_string(), + &format!( + r" +========================= +Test with platform marker +:platform({}) +:fail-fast +========================= +a +--- +(b) + +============================= +Test with bad platform marker +:platform({}) + +:language(foo) +============================= +a +--- +(b) +", + std::env::consts::OS, + if std::env::consts::OS == "linux" { + "macos" + } else { + "linux" + } + ), + None, + ); + + assert_eq!( + entry, + TestEntry::Group { + name: "the-filename".to_string(), + file_path: None, + children: vec![ + TestEntry::Example { + name: "Test with platform marker".to_string(), + input: b"a".to_vec(), + output: "(b)".to_string(), + header_delim_len: 25, + divider_delim_len: 3, + has_fields: false, + attributes_str: format!(":platform({})\n:fail-fast", std::env::consts::OS), + attributes: TestAttributes { + skip: false, + platform: true, + fail_fast: true, + error: false, + languages: vec!["".into()] + }, + file_name: None, + }, + TestEntry::Example { + name: "Test with bad platform marker".to_string(), + input: b"a".to_vec(), + output: "(b)".to_string(), + header_delim_len: 29, + divider_delim_len: 3, + has_fields: false, + attributes_str: if std::env::consts::OS == "linux" { + ":platform(macos)\n\n:language(foo)".to_string() + } else { + ":platform(linux)\n\n:language(foo)".to_string() + }, + attributes: TestAttributes { + skip: false, + platform: false, + fail_fast: false, + error: false, + languages: vec!["foo".into()] + }, + file_name: None, + } + ] + } + ); + } +} diff --git a/crates/cli/src/test_highlight.rs b/cli/src/test_highlight.rs similarity index 83% rename from crates/cli/src/test_highlight.rs rename to cli/src/test_highlight.rs index d96f90c2..d2f37e2f 100644 --- a/crates/cli/src/test_highlight.rs +++ b/cli/src/test_highlight.rs @@ -1,13 +1,14 @@ use std::{fs, path::Path}; +use anstyle::AnsiColor; use anyhow::{anyhow, Result}; use tree_sitter::Point; use tree_sitter_highlight::{Highlight, HighlightConfiguration, HighlightEvent, Highlighter}; use tree_sitter_loader::{Config, Loader}; -use crate::{ +use super::{ query_testing::{parse_position_comments, to_utf8_point, Assertion, Utf8Point}, - test::{TestInfo, TestOutcome, TestResult, TestSummary}, + test::paint, util, }; @@ -47,7 +48,19 @@ pub fn test_highlights( loader_config: &Config, highlighter: &mut Highlighter, directory: &Path, - test_summary: &mut TestSummary, + use_color: bool, +) -> Result<()> { + println!("syntax highlighting:"); + test_highlights_indented(loader, loader_config, highlighter, directory, use_color, 2) +} + +fn test_highlights_indented( + loader: &Loader, + loader_config: &Config, + highlighter: &mut Highlighter, + directory: &Path, + use_color: bool, + indent_level: usize, ) -> Result<()> { let mut failed = false; @@ -55,22 +68,25 @@ pub fn test_highlights( let highlight_test_file = highlight_test_file?; let test_file_path = highlight_test_file.path(); let test_file_name = highlight_test_file.file_name(); + print!( + "{indent:indent_level$}", + indent = "", + indent_level = indent_level * 2 + ); if test_file_path.is_dir() && test_file_path.read_dir()?.next().is_some() { - test_summary - .highlight_results - .add_group(test_file_name.to_string_lossy().as_ref()); - if test_highlights( + println!("{}:", test_file_name.to_string_lossy()); + if test_highlights_indented( loader, loader_config, highlighter, &test_file_path, - test_summary, + use_color, + indent_level + 1, ) .is_err() { failed = true; } - test_summary.highlight_results.pop_traversal(); } else { let (language, language_config) = loader .language_configuration_for_file_name(&test_file_path)? @@ -82,12 +98,7 @@ pub fn test_highlights( })?; let highlight_config = language_config .highlight_config(language, None)? - .ok_or_else(|| { - anyhow!( - "No highlighting config found for {}", - test_file_path.display() - ) - })?; + .ok_or_else(|| anyhow!("No highlighting config found for {test_file_path:?}"))?; match test_highlight( loader, highlighter, @@ -95,28 +106,30 @@ pub fn test_highlights( fs::read(&test_file_path)?.as_slice(), ) { Ok(assertion_count) => { - test_summary.highlight_results.add_case(TestResult { - name: test_file_name.to_string_lossy().to_string(), - info: TestInfo::AssertionTest { - outcome: TestOutcome::AssertionPassed { assertion_count }, - test_num: test_summary.test_num, - }, - }); + println!( + "✓ {} ({assertion_count} assertions)", + paint( + use_color.then_some(AnsiColor::Green), + test_file_name.to_string_lossy().as_ref() + ), + ); } Err(e) => { - test_summary.highlight_results.add_case(TestResult { - name: test_file_name.to_string_lossy().to_string(), - info: TestInfo::AssertionTest { - outcome: TestOutcome::AssertionFailed { - error: e.to_string(), - }, - test_num: test_summary.test_num, - }, - }); + println!( + "✗ {}", + paint( + use_color.then_some(AnsiColor::Red), + test_file_name.to_string_lossy().as_ref() + ) + ); + println!( + "{indent:indent_level$} {e}", + indent = "", + indent_level = indent_level * 2 + ); failed = true; } } - test_summary.test_num += 1; } } diff --git a/crates/cli/src/test_tags.rs b/cli/src/test_tags.rs similarity index 78% rename from crates/cli/src/test_tags.rs rename to cli/src/test_tags.rs index 882718e5..a0e6e3e9 100644 --- a/crates/cli/src/test_tags.rs +++ b/cli/src/test_tags.rs @@ -1,12 +1,13 @@ use std::{fs, path::Path}; +use anstyle::AnsiColor; use anyhow::{anyhow, Result}; use tree_sitter_loader::{Config, Loader}; use tree_sitter_tags::{TagsConfiguration, TagsContext}; -use crate::{ +use super::{ query_testing::{parse_position_comments, to_utf8_point, Assertion, Utf8Point}, - test::{TestInfo, TestOutcome, TestResult, TestSummary}, + test::paint, util, }; @@ -46,7 +47,19 @@ pub fn test_tags( loader_config: &Config, tags_context: &mut TagsContext, directory: &Path, - test_summary: &mut TestSummary, + use_color: bool, +) -> Result<()> { + println!("tags:"); + test_tags_indented(loader, loader_config, tags_context, directory, use_color, 2) +} + +pub fn test_tags_indented( + loader: &Loader, + loader_config: &Config, + tags_context: &mut TagsContext, + directory: &Path, + use_color: bool, + indent_level: usize, ) -> Result<()> { let mut failed = false; @@ -54,22 +67,25 @@ pub fn test_tags( let tag_test_file = tag_test_file?; let test_file_path = tag_test_file.path(); let test_file_name = tag_test_file.file_name(); + print!( + "{indent:indent_level$}", + indent = "", + indent_level = indent_level * 2 + ); if test_file_path.is_dir() && test_file_path.read_dir()?.next().is_some() { - test_summary - .tag_results - .add_group(test_file_name.to_string_lossy().as_ref()); - if test_tags( + println!("{}:", test_file_name.to_string_lossy()); + if test_tags_indented( loader, loader_config, tags_context, &test_file_path, - test_summary, + use_color, + indent_level + 1, ) .is_err() { failed = true; } - test_summary.tag_results.pop_traversal(); } else { let (language, language_config) = loader .language_configuration_for_file_name(&test_file_path)? @@ -81,35 +97,37 @@ pub fn test_tags( })?; let tags_config = language_config .tags_config(language)? - .ok_or_else(|| anyhow!("No tags config found for {}", test_file_path.display()))?; + .ok_or_else(|| anyhow!("No tags config found for {test_file_path:?}"))?; match test_tag( tags_context, tags_config, fs::read(&test_file_path)?.as_slice(), ) { Ok(assertion_count) => { - test_summary.tag_results.add_case(TestResult { - name: test_file_name.to_string_lossy().to_string(), - info: TestInfo::AssertionTest { - outcome: TestOutcome::AssertionPassed { assertion_count }, - test_num: test_summary.test_num, - }, - }); + println!( + "✓ {} ({assertion_count} assertions)", + paint( + use_color.then_some(AnsiColor::Green), + test_file_name.to_string_lossy().as_ref() + ), + ); } Err(e) => { - test_summary.tag_results.add_case(TestResult { - name: test_file_name.to_string_lossy().to_string(), - info: TestInfo::AssertionTest { - outcome: TestOutcome::AssertionFailed { - error: e.to_string(), - }, - test_num: test_summary.test_num, - }, - }); + println!( + "✗ {}", + paint( + use_color.then_some(AnsiColor::Red), + test_file_name.to_string_lossy().as_ref() + ) + ); + println!( + "{indent:indent_level$} {e}", + indent = "", + indent_level = indent_level * 2 + ); failed = true; } } - test_summary.test_num += 1; } } diff --git a/cli/src/tests/async_context_test.rs b/cli/src/tests/async_context_test.rs new file mode 100644 index 00000000..fbcc5c30 --- /dev/null +++ b/cli/src/tests/async_context_test.rs @@ -0,0 +1,278 @@ +use std::{ + future::Future, + pin::{pin, Pin}, + ptr, + task::{self, Context, Poll, RawWaker, RawWakerVTable, Waker}, +}; + +use tree_sitter::Parser; + +use super::helpers::fixtures::get_language; + +#[test] +fn test_node_in_fut() { + let (ret, pended) = tokio_like_spawn(async { + let mut parser = Parser::new(); + let language = get_language("bash"); + parser.set_language(&language).unwrap(); + + let tree = parser.parse("#", None).unwrap(); + + let root = tree.root_node(); + let root_ref = &root; + + let fut_val_fn = || async { + yield_now().await; + root.child(0).unwrap().kind() + }; + + yield_now().await; + + let fut_ref_fn = || async { + yield_now().await; + root_ref.child(0).unwrap().kind() + }; + + let f1 = fut_val_fn().await; + let f2 = fut_ref_fn().await; + assert_eq!(f1, f2); + + let fut_val = async { + yield_now().await; + root.child(0).unwrap().kind() + }; + + let fut_ref = async { + yield_now().await; + root_ref.child(0).unwrap().kind() + }; + + let f1 = fut_val.await; + let f2 = fut_ref.await; + assert_eq!(f1, f2); + + f1 + }) + .join(); + assert_eq!(ret, "comment"); + assert_eq!(pended, 5); +} + +#[test] +fn test_node_and_cursor_ref_in_fut() { + let ((), pended) = tokio_like_spawn(async { + let mut parser = Parser::new(); + let language = get_language("c"); + parser.set_language(&language).unwrap(); + + let tree = parser.parse("#", None).unwrap(); + + let root = tree.root_node(); + let root_ref = &root; + + let mut cursor = tree.walk(); + let cursor_ref = &mut cursor; + + cursor_ref.goto_first_child(); + + let fut_val = async { + yield_now().await; + let _ = root.to_sexp(); + }; + + yield_now().await; + + let fut_ref = async { + yield_now().await; + let _ = root_ref.to_sexp(); + cursor_ref.goto_first_child(); + }; + + fut_val.await; + fut_ref.await; + + cursor_ref.goto_first_child(); + }) + .join(); + assert_eq!(pended, 3); +} + +#[test] +fn test_node_and_cursor_ref_in_fut_with_fut_fabrics() { + let ((), pended) = tokio_like_spawn(async { + let mut parser = Parser::new(); + let language = get_language("javascript"); + parser.set_language(&language).unwrap(); + + let tree = parser.parse("#", None).unwrap(); + + let root = tree.root_node(); + let root_ref = &root; + + let mut cursor = tree.walk(); + let cursor_ref = &mut cursor; + + cursor_ref.goto_first_child(); + + let fut_val = || async { + yield_now().await; + let _ = root.to_sexp(); + }; + + yield_now().await; + + let fut_ref = || async move { + yield_now().await; + let _ = root_ref.to_sexp(); + cursor_ref.goto_first_child(); + }; + + fut_val().await; + fut_val().await; + fut_ref().await; + }) + .join(); + assert_eq!(pended, 4); +} + +#[test] +fn test_node_and_cursor_ref_in_fut_with_inner_spawns() { + let (ret, pended) = tokio_like_spawn(async { + let mut parser = Parser::new(); + let language = get_language("rust"); + parser.set_language(&language).unwrap(); + + let tree = parser.parse("#", None).unwrap(); + + let mut cursor = tree.walk(); + let cursor_ref = &mut cursor; + + cursor_ref.goto_first_child(); + + let fut_val = || { + let tree = tree.clone(); + async move { + let root = tree.root_node(); + let mut cursor = tree.walk(); + let cursor_ref = &mut cursor; + yield_now().await; + let _ = root.to_sexp(); + cursor_ref.goto_first_child(); + } + }; + + yield_now().await; + + let fut_ref = || { + let tree = tree.clone(); + async move { + let root = tree.root_node(); + let root_ref = &root; + let mut cursor = tree.walk(); + let cursor_ref = &mut cursor; + yield_now().await; + let _ = root_ref.to_sexp(); + cursor_ref.goto_first_child(); + } + }; + + let ((), p1) = tokio_like_spawn(fut_val()).await.unwrap(); + let ((), p2) = tokio_like_spawn(fut_ref()).await.unwrap(); + + cursor_ref.goto_first_child(); + + fut_val().await; + fut_val().await; + fut_ref().await; + + cursor_ref.goto_first_child(); + + p1 + p2 + }) + .join(); + assert_eq!(pended, 4); + assert_eq!(ret, 2); +} + +fn tokio_like_spawn(future: T) -> JoinHandle<(T::Output, usize)> +where + T: Future + Send + 'static, + T::Output: Send + 'static, +{ + // No runtime, just noop waker + + let waker = noop_waker(); + let mut cx = task::Context::from_waker(&waker); + + let mut pending = 0; + let mut future = pin!(future); + let ret = loop { + match future.as_mut().poll(&mut cx) { + Poll::Pending => pending += 1, + Poll::Ready(r) => { + break r; + } + } + }; + JoinHandle::new((ret, pending)) +} + +async fn yield_now() { + struct SimpleYieldNow { + yielded: bool, + } + + impl Future for SimpleYieldNow { + type Output = (); + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { + cx.waker().wake_by_ref(); + if self.yielded { + return Poll::Ready(()); + } + self.yielded = true; + Poll::Pending + } + } + + SimpleYieldNow { yielded: false }.await; +} + +pub const fn noop_waker() -> Waker { + const VTABLE: RawWakerVTable = RawWakerVTable::new( + // Cloning just returns a new no-op raw waker + |_| RAW, + // `wake` does nothing + |_| {}, + // `wake_by_ref` does nothing + |_| {}, + // Dropping does nothing as we don't allocate anything + |_| {}, + ); + const RAW: RawWaker = RawWaker::new(ptr::null(), &VTABLE); + unsafe { Waker::from_raw(RAW) } +} + +struct JoinHandle { + data: Option, +} + +impl JoinHandle { + #[must_use] + const fn new(data: T) -> Self { + Self { data: Some(data) } + } + + fn join(&mut self) -> T { + self.data.take().unwrap() + } +} + +impl Future for JoinHandle { + type Output = std::result::Result; + + fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll { + let data = self.get_mut().data.take().unwrap(); + Poll::Ready(Ok(data)) + } +} diff --git a/crates/cli/src/tests/corpus_test.rs b/cli/src/tests/corpus_test.rs similarity index 95% rename from crates/cli/src/tests/corpus_test.rs rename to cli/src/tests/corpus_test.rs index ba3fd68e..6d272f51 100644 --- a/crates/cli/src/tests/corpus_test.rs +++ b/cli/src/tests/corpus_test.rs @@ -1,6 +1,5 @@ use std::{collections::HashMap, env, fs}; -use anyhow::Context; use tree_sitter::Parser; use tree_sitter_proc_macro::test_with_seed; @@ -16,7 +15,7 @@ use crate::{ LOG_GRAPH_ENABLED, START_SEED, }, parse::perform_edit, - test::{parse_tests, strip_sexp_fields, DiffKey, TestDiff}, + test::{parse_tests, print_diff, print_diff_key, strip_sexp_fields}, tests::{ allocations, helpers::fixtures::{fixtures_dir, get_language, get_test_language, SCRATCH_BASE_DIR}, @@ -209,14 +208,15 @@ pub fn test_language_corpus( if actual_output != test.output { println!("Incorrect initial parse for {test_name}"); - DiffKey::print(); - println!("{}", TestDiff::new(&actual_output, &test.output)); + print_diff_key(); + print_diff(&actual_output, &test.output, true); println!(); return false; } true - }); + }) + .unwrap(); if !passed { failure_count += 1; @@ -297,8 +297,8 @@ pub fn test_language_corpus( if actual_output != test.output { println!("Incorrect parse for {test_name} - seed {seed}"); - DiffKey::print(); - println!("{}", TestDiff::new(&actual_output, &test.output)); + print_diff_key(); + print_diff(&actual_output, &test.output, true); println!(); return false; } @@ -311,7 +311,7 @@ pub fn test_language_corpus( } true - }); + }).unwrap(); if !passed { failure_count += 1; @@ -363,14 +363,7 @@ fn test_feature_corpus_files() { grammar_path = test_path.join("grammar.json"); } let error_message_path = test_path.join("expected_error.txt"); - let grammar_json = tree_sitter_generate::load_grammar_file(&grammar_path, None) - .with_context(|| { - format!( - "Could not load grammar file for test language '{language_name}' at {}", - grammar_path.display() - ) - }) - .unwrap(); + let grammar_json = tree_sitter_generate::load_grammar_file(&grammar_path, None).unwrap(); let generate_result = tree_sitter_generate::generate_parser_for_grammar(&grammar_json, Some((0, 0, 0))); @@ -428,12 +421,13 @@ fn test_feature_corpus_files() { if actual_output == test.output { true } else { - DiffKey::print(); - print!("{}", TestDiff::new(&actual_output, &test.output)); + print_diff_key(); + print_diff(&actual_output, &test.output, true); println!(); false } - }); + }) + .unwrap(); if !passed { failure_count += 1; diff --git a/crates/cli/src/tests/detect_language.rs b/cli/src/tests/detect_language.rs similarity index 98% rename from crates/cli/src/tests/detect_language.rs rename to cli/src/tests/detect_language.rs index c543c31e..50bb891e 100644 --- a/crates/cli/src/tests/detect_language.rs +++ b/cli/src/tests/detect_language.rs @@ -90,7 +90,7 @@ fn detect_language_by_first_line_regex() { } #[test] -fn detect_language_by_double_barrel_file_extension() { +fn detect_langauge_by_double_barrel_file_extension() { let blade_dir = tree_sitter_dir( r#"{ "grammars": [ @@ -229,7 +229,7 @@ fn tree_sitter_dir(tree_sitter_json: &str, name: &str) -> tempfile::TempDir { .unwrap(); fs::write( temp_dir.path().join("src/tree_sitter/parser.h"), - include_str!("../../../../lib/src/parser.h"), + include_str!("../../../lib/src/parser.h"), ) .unwrap(); temp_dir diff --git a/cli/src/tests/helpers/allocations.rs b/cli/src/tests/helpers/allocations.rs new file mode 100644 index 00000000..dec67b11 --- /dev/null +++ b/cli/src/tests/helpers/allocations.rs @@ -0,0 +1,121 @@ +use std::{ + collections::HashMap, + os::raw::c_void, + sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering::SeqCst}, + Mutex, + }, +}; + +#[ctor::ctor] +unsafe fn initialize_allocation_recording() { + tree_sitter::set_allocator( + Some(ts_record_malloc), + Some(ts_record_calloc), + Some(ts_record_realloc), + Some(ts_record_free), + ); +} + +#[derive(Debug, PartialEq, Eq, Hash)] +struct Allocation(*const c_void); +unsafe impl Send for Allocation {} +unsafe impl Sync for Allocation {} + +#[derive(Default)] +struct AllocationRecorder { + enabled: AtomicBool, + allocation_count: AtomicUsize, + outstanding_allocations: Mutex>, +} + +thread_local! { + static RECORDER: AllocationRecorder = AllocationRecorder::default(); +} + +extern "C" { + fn malloc(size: usize) -> *mut c_void; + fn calloc(count: usize, size: usize) -> *mut c_void; + fn realloc(ptr: *mut c_void, size: usize) -> *mut c_void; + fn free(ptr: *mut c_void); +} + +pub fn record(f: impl FnOnce() -> T) -> T { + RECORDER.with(|recorder| { + recorder.enabled.store(true, SeqCst); + recorder.allocation_count.store(0, SeqCst); + recorder.outstanding_allocations.lock().unwrap().clear(); + }); + + let value = f(); + + let outstanding_allocation_indices = RECORDER.with(|recorder| { + recorder.enabled.store(false, SeqCst); + recorder.allocation_count.store(0, SeqCst); + recorder + .outstanding_allocations + .lock() + .unwrap() + .drain() + .map(|e| e.1) + .collect::>() + }); + assert!( + outstanding_allocation_indices.is_empty(), + "Leaked allocation indices: {outstanding_allocation_indices:?}" + ); + value +} + +fn record_alloc(ptr: *mut c_void) { + RECORDER.with(|recorder| { + if recorder.enabled.load(SeqCst) { + let count = recorder.allocation_count.fetch_add(1, SeqCst); + recorder + .outstanding_allocations + .lock() + .unwrap() + .insert(Allocation(ptr), count); + } + }); +} + +fn record_dealloc(ptr: *mut c_void) { + RECORDER.with(|recorder| { + if recorder.enabled.load(SeqCst) { + recorder + .outstanding_allocations + .lock() + .unwrap() + .remove(&Allocation(ptr)); + } + }); +} + +unsafe extern "C" fn ts_record_malloc(size: usize) -> *mut c_void { + let result = malloc(size); + record_alloc(result); + result +} + +unsafe extern "C" fn ts_record_calloc(count: usize, size: usize) -> *mut c_void { + let result = calloc(count, size); + record_alloc(result); + result +} + +unsafe extern "C" fn ts_record_realloc(ptr: *mut c_void, size: usize) -> *mut c_void { + let result = realloc(ptr, size); + if ptr.is_null() { + record_alloc(result); + } else if !core::ptr::eq(ptr, result) { + record_dealloc(ptr); + record_alloc(result); + } + result +} + +unsafe extern "C" fn ts_record_free(ptr: *mut c_void) { + record_dealloc(ptr); + free(ptr); +} diff --git a/crates/cli/src/tests/helpers/dirs.rs b/cli/src/tests/helpers/dirs.rs similarity index 98% rename from crates/cli/src/tests/helpers/dirs.rs rename to cli/src/tests/helpers/dirs.rs index d63790cd..22e99588 100644 --- a/crates/cli/src/tests/helpers/dirs.rs +++ b/cli/src/tests/helpers/dirs.rs @@ -1,7 +1,5 @@ pub static ROOT_DIR: LazyLock = LazyLock::new(|| { PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() .parent() .unwrap() .to_owned() diff --git a/crates/cli/src/tests/helpers/edits.rs b/cli/src/tests/helpers/edits.rs similarity index 100% rename from crates/cli/src/tests/helpers/edits.rs rename to cli/src/tests/helpers/edits.rs diff --git a/crates/cli/src/tests/helpers/fixtures.rs b/cli/src/tests/helpers/fixtures.rs similarity index 78% rename from crates/cli/src/tests/helpers/fixtures.rs rename to cli/src/tests/helpers/fixtures.rs index 0e0ff69d..0b046bcc 100644 --- a/crates/cli/src/tests/helpers/fixtures.rs +++ b/cli/src/tests/helpers/fixtures.rs @@ -23,9 +23,6 @@ static TEST_LOADER: LazyLock = LazyLock::new(|| { loader }); -#[cfg(feature = "wasm")] -pub static ENGINE: LazyLock = LazyLock::new(Default::default); - pub fn test_loader() -> &'static Loader { &TEST_LOADER } @@ -46,19 +43,10 @@ pub fn get_language(name: &str) -> Language { } pub fn get_test_fixture_language(name: &str) -> Language { - get_test_fixture_language_internal(name, false) -} - -#[cfg(feature = "wasm")] -pub fn get_test_fixture_language_wasm(name: &str) -> Language { - get_test_fixture_language_internal(name, true) -} - -fn get_test_fixture_language_internal(name: &str, wasm: bool) -> Language { let grammar_dir_path = fixtures_dir().join("test_grammars").join(name); let grammar_json = load_grammar_file(&grammar_dir_path.join("grammar.js"), None).unwrap(); let (parser_name, parser_code) = generate_parser(&grammar_json).unwrap(); - get_test_language_internal(&parser_name, &parser_code, Some(&grammar_dir_path), wasm) + get_test_language(&parser_name, &parser_code, Some(&grammar_dir_path)) } pub fn get_language_queries_path(language_name: &str) -> PathBuf { @@ -99,15 +87,6 @@ pub fn get_tags_config(language_name: &str) -> TagsConfiguration { } pub fn get_test_language(name: &str, parser_code: &str, path: Option<&Path>) -> Language { - get_test_language_internal(name, parser_code, path, false) -} - -fn get_test_language_internal( - name: &str, - parser_code: &str, - path: Option<&Path>, - wasm: bool, -) -> Language { let src_dir = scratch_dir().join("src").join(name); fs::create_dir_all(&src_dir).unwrap(); @@ -157,21 +136,5 @@ fn get_test_language_internal( config.header_paths = vec![&HEADER_DIR]; config.name = name.to_string(); - if wasm { - #[cfg(feature = "wasm")] - { - let mut loader = Loader::with_parser_lib_path(SCRATCH_DIR.clone()); - loader.use_wasm(&ENGINE); - if env::var("TREE_SITTER_GRAMMAR_DEBUG").is_ok() { - loader.debug_build(true); - } - loader.load_language_at_path_with_name(config).unwrap() - } - #[cfg(not(feature = "wasm"))] - { - unimplemented!("Wasm feature is not enabled") - } - } else { - TEST_LOADER.load_language_at_path_with_name(config).unwrap() - } + TEST_LOADER.load_language_at_path_with_name(config).unwrap() } diff --git a/crates/cli/src/tests/helpers.rs b/cli/src/tests/helpers/mod.rs similarity index 67% rename from crates/cli/src/tests/helpers.rs rename to cli/src/tests/helpers/mod.rs index 4d2e6128..298179c7 100644 --- a/crates/cli/src/tests/helpers.rs +++ b/cli/src/tests/helpers/mod.rs @@ -1,4 +1,4 @@ -pub use crate::fuzz::allocations; +pub mod allocations; pub mod edits; pub(super) mod fixtures; pub(super) mod query_helpers; diff --git a/crates/cli/src/tests/helpers/query_helpers.rs b/cli/src/tests/helpers/query_helpers.rs similarity index 97% rename from crates/cli/src/tests/helpers/query_helpers.rs rename to cli/src/tests/helpers/query_helpers.rs index e2c68f17..976a4f46 100644 --- a/crates/cli/src/tests/helpers/query_helpers.rs +++ b/cli/src/tests/helpers/query_helpers.rs @@ -12,7 +12,7 @@ pub struct Pattern { named: bool, field: Option<&'static str>, capture: Option, - children: Vec, + children: Vec, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -131,12 +131,11 @@ impl Pattern { if self.named { string.push('('); - let mut has_contents = if let Some(kind) = &self.kind { + let mut has_contents = false; + if let Some(kind) = &self.kind { write!(string, "{kind}").unwrap(); - true - } else { - false - }; + has_contents = true; + } for child in &self.children { let indent = indent + 2; if has_contents { @@ -182,9 +181,7 @@ impl Pattern { } matches.sort_unstable(); - for m in &mut matches { - m.last_node = None; - } + matches.iter_mut().for_each(|m| m.last_node = None); matches.dedup(); matches } @@ -225,7 +222,7 @@ impl Pattern { } // Find every matching combination of child patterns and child nodes. - let mut finished_matches = Vec::>::new(); + let mut finished_matches = Vec::::new(); if cursor.goto_first_child() { let mut match_states = vec![(0, mat)]; loop { diff --git a/crates/cli/src/tests/highlight_test.rs b/cli/src/tests/highlight_test.rs similarity index 98% rename from crates/cli/src/tests/highlight_test.rs rename to cli/src/tests/highlight_test.rs index 88209bfe..76c43f43 100644 --- a/crates/cli/src/tests/highlight_test.rs +++ b/cli/src/tests/highlight_test.rs @@ -481,7 +481,7 @@ fn test_highlighting_cancellation() { // The initial `highlight` call, which eagerly parses the outer document, should not fail. let mut highlighter = Highlighter::new(); - let mut events = highlighter + let events = highlighter .highlight( &HTML_HIGHLIGHT, source.as_bytes(), @@ -492,18 +492,14 @@ fn test_highlighting_cancellation() { // Iterating the scopes should not panic. It should return an error once the // cancellation is detected. - let found_cancellation_error = events.any(|event| match event { - Ok(_) => false, - Err(Error::Cancelled) => true, - Err(Error::InvalidLanguage | Error::Unknown) => { - unreachable!("Unexpected error type while iterating events") + for event in events { + if let Err(e) = event { + assert_eq!(e, Error::Cancelled); + return; } - }); + } - assert!( - found_cancellation_error, - "Expected a cancellation error while iterating events" - ); + panic!("Expected an error while iterating highlighter"); } #[test] diff --git a/crates/cli/src/tests/language_test.rs b/cli/src/tests/language_test.rs similarity index 100% rename from crates/cli/src/tests/language_test.rs rename to cli/src/tests/language_test.rs diff --git a/crates/cli/src/tests.rs b/cli/src/tests/mod.rs similarity index 92% rename from crates/cli/src/tests.rs rename to cli/src/tests/mod.rs index 2439be38..c64744f8 100644 --- a/crates/cli/src/tests.rs +++ b/cli/src/tests/mod.rs @@ -1,10 +1,11 @@ -mod async_boundary_test; +mod async_context_test; mod corpus_test; mod detect_language; mod helpers; mod highlight_test; mod language_test; mod node_test; +mod parser_hang_test; mod parser_test; mod pathological_test; mod query_test; @@ -26,8 +27,6 @@ pub use crate::fuzz::{ ITERATION_COUNT, }; -pub use helpers::fixtures::get_language; - /// This is a simple wrapper around [`tree_sitter_generate::generate_parser_for_grammar`], because /// our tests do not need to pass in a version number, only the grammar JSON. fn generate_parser(grammar_json: &str) -> GenerateResult<(String, String)> { diff --git a/crates/cli/src/tests/node_test.rs b/cli/src/tests/node_test.rs similarity index 94% rename from crates/cli/src/tests/node_test.rs rename to cli/src/tests/node_test.rs index 614bfdb9..515d73aa 100644 --- a/crates/cli/src/tests/node_test.rs +++ b/cli/src/tests/node_test.rs @@ -1,4 +1,4 @@ -use tree_sitter::{InputEdit, Node, Parser, Point, Tree}; +use tree_sitter::{Node, Parser, Point, Tree}; use tree_sitter_generate::load_grammar_file; use super::{ @@ -843,92 +843,6 @@ fn test_node_is_error() { assert!(child.is_error()); } -#[test] -fn test_edit_point() { - let edit = InputEdit { - start_byte: 5, - old_end_byte: 5, - new_end_byte: 10, - start_position: Point::new(0, 5), - old_end_position: Point::new(0, 5), - new_end_position: Point::new(0, 10), - }; - - // Point after edit - let mut point = Point::new(0, 8); - let mut byte = 8; - edit.edit_point(&mut point, &mut byte); - assert_eq!(point, Point::new(0, 13)); - assert_eq!(byte, 13); - - // Point before edit - let mut point = Point::new(0, 2); - let mut byte = 2; - edit.edit_point(&mut point, &mut byte); - assert_eq!(point, Point::new(0, 2)); - assert_eq!(byte, 2); - - // Point at edit start - let mut point = Point::new(0, 5); - let mut byte = 5; - edit.edit_point(&mut point, &mut byte); - assert_eq!(point, Point::new(0, 10)); - assert_eq!(byte, 10); -} - -#[test] -fn test_edit_range() { - use tree_sitter::{InputEdit, Point, Range}; - - let edit = InputEdit { - start_byte: 10, - old_end_byte: 15, - new_end_byte: 20, - start_position: Point::new(1, 0), - old_end_position: Point::new(1, 5), - new_end_position: Point::new(2, 0), - }; - - // Range after edit - let mut range = Range { - start_byte: 20, - end_byte: 25, - start_point: Point::new(2, 0), - end_point: Point::new(2, 5), - }; - edit.edit_range(&mut range); - assert_eq!(range.start_byte, 25); - assert_eq!(range.end_byte, 30); - assert_eq!(range.start_point, Point::new(3, 0)); - assert_eq!(range.end_point, Point::new(3, 5)); - - // Range before edit - let mut range = Range { - start_byte: 5, - end_byte: 8, - start_point: Point::new(0, 5), - end_point: Point::new(0, 8), - }; - edit.edit_range(&mut range); - assert_eq!(range.start_byte, 5); - assert_eq!(range.end_byte, 8); - assert_eq!(range.start_point, Point::new(0, 5)); - assert_eq!(range.end_point, Point::new(0, 8)); - - // Range overlapping edit - let mut range = Range { - start_byte: 8, - end_byte: 12, - start_point: Point::new(0, 8), - end_point: Point::new(1, 2), - }; - edit.edit_range(&mut range); - assert_eq!(range.start_byte, 8); - assert_eq!(range.end_byte, 10); - assert_eq!(range.start_point, Point::new(0, 8)); - assert_eq!(range.end_point, Point::new(1, 0)); -} - #[test] fn test_node_sexp() { let mut parser = Parser::new(); diff --git a/cli/src/tests/parser_hang_test.rs b/cli/src/tests/parser_hang_test.rs new file mode 100644 index 00000000..1ff9d17e --- /dev/null +++ b/cli/src/tests/parser_hang_test.rs @@ -0,0 +1,104 @@ +// For some reasons `Command::spawn` doesn't work in CI env for many exotic arches. +#![cfg(all(any(target_arch = "x86_64", target_arch = "x86"), not(sanitizing)))] + +use std::{ + env::VarError, + process::{Command, Stdio}, +}; + +use tree_sitter::Parser; +use tree_sitter_generate::load_grammar_file; + +use super::generate_parser; +use crate::tests::helpers::fixtures::{fixtures_dir, get_test_language}; + +// The `sanitizing` cfg is required to don't run tests under specific sunitizer +// because they don't work well with subprocesses _(it's an assumption)_. +// +// Below are two alternative examples of how to disable tests for some arches +// if a way with excluding the whole mod from compilation wouldn't work well. +// +// XXX: Also may be it makes sense to keep such tests as ignored by default +// to omit surprises and enable them on CI by passing an extra option explicitly: +// +// > cargo test -- --include-ignored +// +// #[cfg(all(any(target_arch = "x86_64", target_arch = "x86"), not(sanitizing)))] +// #[cfg_attr(not(all(any(target_arch = "x86_64", target_arch = "x86"), not(sanitizing))), ignore)] +// +#[test] +fn test_grammar_that_should_hang_and_not_segfault() { + let parent_sleep_millis = 1000; + let test_name = "test_grammar_that_should_hang_and_not_segfault"; + let test_var = "CARGO_HANG_TEST"; + + eprintln!(" {test_name}"); + + let tests_exec_path = std::env::args() + .next() + .expect("Failed to get tests executable path"); + + match std::env::var(test_var) { + Ok(v) if v == test_name => { + eprintln!(" child process id {}", std::process::id()); + hang_test(); + } + + Err(VarError::NotPresent) => { + eprintln!(" parent process id {}", std::process::id()); + let mut command = Command::new(tests_exec_path); + command.arg(test_name).env(test_var, test_name); + + if std::env::args().any(|x| x == "--nocapture") { + command.arg("--nocapture"); + } else { + command.stdout(Stdio::null()).stderr(Stdio::null()); + } + + match command.spawn() { + Ok(mut child) => { + std::thread::sleep(std::time::Duration::from_millis(parent_sleep_millis)); + match child.try_wait() { + Ok(Some(status)) if status.success() => { + panic!("Child didn't hang and exited successfully") + } + Ok(Some(status)) => panic!( + "Child didn't hang and exited with status code: {:?}", + status.code() + ), + _ => (), + } + if let Err(e) = child.kill() { + eprintln!( + "Failed to kill hang test's process id: {}, error: {e}", + child.id() + ); + } + } + Err(e) => panic!("{e}"), + } + } + + Err(e) => panic!("Env var error: {e}"), + + _ => unreachable!(), + } +} + +fn hang_test() { + let test_grammar_dir = fixtures_dir() + .join("test_grammars") + .join("get_col_should_hang_not_crash"); + + let grammar_json = load_grammar_file(&test_grammar_dir.join("grammar.js"), None).unwrap(); + let (parser_name, parser_code) = generate_parser(grammar_json.as_str()).unwrap(); + + let language = get_test_language(&parser_name, &parser_code, Some(test_grammar_dir.as_path())); + + let mut parser = Parser::new(); + parser.set_language(&language).unwrap(); + + let code_that_should_hang = "\nHello"; + + parser.parse(code_that_should_hang, None).unwrap(); +} diff --git a/crates/cli/src/tests/parser_test.rs b/cli/src/tests/parser_test.rs similarity index 92% rename from crates/cli/src/tests/parser_test.rs rename to cli/src/tests/parser_test.rs index f1d50319..d8b9767d 100644 --- a/crates/cli/src/tests/parser_test.rs +++ b/cli/src/tests/parser_test.rs @@ -1,17 +1,11 @@ use std::{ - ops::ControlFlow, - sync::{ - atomic::{AtomicUsize, Ordering}, - mpsc, - }, - thread, - time::{self, Duration}, + sync::atomic::{AtomicUsize, Ordering}, + thread, time, }; use tree_sitter::{ Decode, IncludedRangesError, InputEdit, LogType, ParseOptions, ParseState, Parser, Point, Range, }; -use tree_sitter_generate::load_grammar_file; use tree_sitter_proc_macro::retry; use super::helpers::{ @@ -22,11 +16,7 @@ use super::helpers::{ use crate::{ fuzz::edits::Edit, parse::perform_edit, - tests::{ - generate_parser, - helpers::fixtures::{fixtures_dir, get_test_fixture_language}, - invert_edit, - }, + tests::{generate_parser, helpers::fixtures::get_test_fixture_language, invert_edit}, }; #[test] @@ -97,6 +87,7 @@ fn test_parsing_with_logging() { } #[test] +#[cfg(unix)] fn test_parsing_with_debug_graph_enabled() { use std::io::{BufRead, BufReader, Seek}; @@ -708,13 +699,7 @@ fn test_parsing_on_multiple_threads() { fn test_parsing_cancelled_by_another_thread() { let cancellation_flag = std::sync::Arc::new(AtomicUsize::new(0)); let flag = cancellation_flag.clone(); - let callback = &mut |_: &ParseState| { - if cancellation_flag.load(Ordering::SeqCst) != 0 { - ControlFlow::Break(()) - } else { - ControlFlow::Continue(()) - } - }; + let callback = &mut |_: &ParseState| cancellation_flag.load(Ordering::SeqCst) != 0; let mut parser = Parser::new(); parser.set_language(&get_language("javascript")).unwrap(); @@ -779,13 +764,9 @@ fn test_parsing_with_a_timeout() { } }, None, - Some(ParseOptions::new().progress_callback(&mut |_| { - if start_time.elapsed().as_micros() > 1000 { - ControlFlow::Break(()) - } else { - ControlFlow::Continue(()) - } - })), + Some( + ParseOptions::new().progress_callback(&mut |_| start_time.elapsed().as_micros() > 1000), + ), ); assert!(tree.is_none()); assert!(start_time.elapsed().as_micros() < 2000); @@ -801,13 +782,9 @@ fn test_parsing_with_a_timeout() { } }, None, - Some(ParseOptions::new().progress_callback(&mut |_| { - if start_time.elapsed().as_micros() > 5000 { - ControlFlow::Break(()) - } else { - ControlFlow::Continue(()) - } - })), + Some( + ParseOptions::new().progress_callback(&mut |_| start_time.elapsed().as_micros() > 5000), + ), ); assert!(tree.is_none()); assert!(start_time.elapsed().as_micros() > 100); @@ -845,13 +822,7 @@ fn test_parsing_with_a_timeout_and_a_reset() { } }, None, - Some(ParseOptions::new().progress_callback(&mut |_| { - if start_time.elapsed().as_micros() > 5 { - ControlFlow::Break(()) - } else { - ControlFlow::Continue(()) - } - })), + Some(ParseOptions::new().progress_callback(&mut |_| start_time.elapsed().as_micros() > 5)), ); assert!(tree.is_none()); @@ -882,13 +853,7 @@ fn test_parsing_with_a_timeout_and_a_reset() { } }, None, - Some(ParseOptions::new().progress_callback(&mut |_| { - if start_time.elapsed().as_micros() > 5 { - ControlFlow::Break(()) - } else { - ControlFlow::Continue(()) - } - })), + Some(ParseOptions::new().progress_callback(&mut |_| start_time.elapsed().as_micros() > 5)), ); assert!(tree.is_none()); @@ -928,13 +893,10 @@ fn test_parsing_with_a_timeout_and_implicit_reset() { } }, None, - Some(ParseOptions::new().progress_callback(&mut |_| { - if start_time.elapsed().as_micros() > 5 { - ControlFlow::Break(()) - } else { - ControlFlow::Continue(()) - } - })), + Some( + ParseOptions::new() + .progress_callback(&mut |_| start_time.elapsed().as_micros() > 5), + ), ); assert!(tree.is_none()); @@ -975,13 +937,10 @@ fn test_parsing_with_timeout_and_no_completion() { } }, None, - Some(ParseOptions::new().progress_callback(&mut |_| { - if start_time.elapsed().as_micros() > 5 { - ControlFlow::Break(()) - } else { - ControlFlow::Continue(()) - } - })), + Some( + ParseOptions::new() + .progress_callback(&mut |_| start_time.elapsed().as_micros() > 5), + ), ); assert!(tree.is_none()); @@ -1020,10 +979,10 @@ fn test_parsing_with_timeout_during_balancing() { // are in the balancing phase. if state.current_byte_offset() != current_byte_offset { current_byte_offset = state.current_byte_offset(); - ControlFlow::Continue(()) + false } else { in_balancing = true; - ControlFlow::Break(()) + true } })), ); @@ -1045,10 +1004,10 @@ fn test_parsing_with_timeout_during_balancing() { Some(ParseOptions::new().progress_callback(&mut |state| { if state.current_byte_offset() != current_byte_offset { current_byte_offset = state.current_byte_offset(); - ControlFlow::Continue(()) + false } else { in_balancing = true; - ControlFlow::Break(()) + true } })), ); @@ -1072,7 +1031,7 @@ fn test_parsing_with_timeout_during_balancing() { // Because we've already finished parsing, we should only be resuming the // balancing phase. assert!(state.current_byte_offset() == current_byte_offset); - ControlFlow::Continue(()) + false })), ) .unwrap(); @@ -1098,11 +1057,7 @@ fn test_parsing_with_timeout_when_error_detected() { None, Some(ParseOptions::new().progress_callback(&mut |state| { offset = state.current_byte_offset(); - if state.has_error() { - ControlFlow::Break(()) - } else { - ControlFlow::Continue(()) - } + state.has_error() })), ); @@ -1782,7 +1737,7 @@ fn test_parsing_by_halting_at_offset() { None, Some(ParseOptions::new().progress_callback(&mut |p| { seen_byte_offsets.push(p.current_byte_offset()); - ControlFlow::Continue(()) + false })), ) .unwrap(); @@ -2083,97 +2038,3 @@ const fn simple_range(start: usize, end: usize) -> Range { fn chunked_input<'a>(text: &'a str, size: usize) -> impl FnMut(usize, Point) -> &'a [u8] { move |offset, _| &text.as_bytes()[offset..text.len().min(offset + size)] } - -#[test] -fn test_parse_options_reborrow() { - let mut parser = Parser::new(); - parser.set_language(&get_language("rust")).unwrap(); - - let parse_count = AtomicUsize::new(0); - - let mut callback = |_: &ParseState| { - parse_count.fetch_add(1, Ordering::SeqCst); - ControlFlow::Continue(()) - }; - let mut options = ParseOptions::new().progress_callback(&mut callback); - - let text1 = "fn first() {}".repeat(20); - let text2 = "fn second() {}".repeat(20); - - let tree1 = parser - .parse_with_options( - &mut |offset, _| { - if offset >= text1.len() { - &[] - } else { - &text1.as_bytes()[offset..] - } - }, - None, - Some(options.reborrow()), - ) - .unwrap(); - - assert_eq!(tree1.root_node().child(0).unwrap().kind(), "function_item"); - - let tree2 = parser - .parse_with_options( - &mut |offset, _| { - if offset >= text2.len() { - &[] - } else { - &text2.as_bytes()[offset..] - } - }, - None, - Some(options.reborrow()), - ) - .unwrap(); - - assert_eq!(tree2.root_node().child(0).unwrap().kind(), "function_item"); - - assert!(parse_count.load(Ordering::SeqCst) > 0); -} - -#[test] -fn test_grammar_that_should_hang_and_not_segfault() { - fn hang_test() { - let test_grammar_dir = fixtures_dir() - .join("test_grammars") - .join("get_col_should_hang_not_crash"); - - let grammar_json = load_grammar_file(&test_grammar_dir.join("grammar.js"), None) - .expect("Failed to load grammar file"); - - let (parser_name, parser_code) = - generate_parser(grammar_json.as_str()).expect("Failed to generate parser"); - - let language = - get_test_language(&parser_name, &parser_code, Some(test_grammar_dir.as_path())); - - let mut parser = Parser::new(); - parser - .set_language(&language) - .expect("Failed to set parser language"); - - let code_that_should_hang = "\nHello"; - - parser - .parse(code_that_should_hang, None) - .expect("Parse operation completed unexpectedly"); - } - - let timeout = Duration::from_millis(500); - let (tx, rx) = mpsc::channel(); - - thread::spawn(move || tx.send(std::panic::catch_unwind(hang_test))); - - match rx.recv_timeout(timeout) { - Ok(Ok(())) => panic!("The test completed rather than hanging"), - Ok(Err(panic_info)) => panic!("The test panicked unexpectedly: {panic_info:?}"), - Err(mpsc::RecvTimeoutError::Timeout) => {} // Expected - Err(mpsc::RecvTimeoutError::Disconnected) => { - panic!("The test thread disconnected unexpectedly") - } - } -} diff --git a/crates/cli/src/tests/pathological_test.rs b/cli/src/tests/pathological_test.rs similarity index 100% rename from crates/cli/src/tests/pathological_test.rs rename to cli/src/tests/pathological_test.rs diff --git a/crates/cli/src/tests/proc_macro/Cargo.toml b/cli/src/tests/proc_macro/Cargo.toml similarity index 95% rename from crates/cli/src/tests/proc_macro/Cargo.toml rename to cli/src/tests/proc_macro/Cargo.toml index 915bd172..e834383b 100644 --- a/crates/cli/src/tests/proc_macro/Cargo.toml +++ b/cli/src/tests/proc_macro/Cargo.toml @@ -14,4 +14,5 @@ proc-macro = true [dependencies] proc-macro2 = "1.0.93" quote = "1.0.38" +rand = "0.8.5" syn = { version = "2.0.96", features = ["full"] } diff --git a/crates/cli/src/tests/proc_macro/src/lib.rs b/cli/src/tests/proc_macro/src/lib.rs similarity index 100% rename from crates/cli/src/tests/proc_macro/src/lib.rs rename to cli/src/tests/proc_macro/src/lib.rs diff --git a/crates/cli/src/tests/query_test.rs b/cli/src/tests/query_test.rs similarity index 95% rename from crates/cli/src/tests/query_test.rs rename to cli/src/tests/query_test.rs index 3f1467e5..b8773acb 100644 --- a/crates/cli/src/tests/query_test.rs +++ b/cli/src/tests/query_test.rs @@ -1,4 +1,4 @@ -use std::{env, fmt::Write, ops::ControlFlow, sync::LazyLock}; +use std::{env, fmt::Write, sync::LazyLock}; use indoc::indoc; use rand::{prelude::StdRng, SeedableRng}; @@ -8,7 +8,6 @@ use tree_sitter::{ QueryCursorOptions, QueryError, QueryErrorKind, QueryPredicate, QueryPredicateArg, QueryProperty, Range, }; -use tree_sitter_generate::load_grammar_file; use unindent::Unindent; use super::helpers::{ @@ -121,24 +120,12 @@ fn test_query_errors_on_invalid_syntax() { // Unclosed sibling expression with predicate assert_eq!( - Query::new(&language, r"((identifier) (#a?)") + Query::new(&language, r"((identifier) (#a)") .unwrap_err() .message, [ - "((identifier) (#a?)", // - " ^", - ] - .join("\n") - ); - - // Predicate not ending in `?` or `!` - assert_eq!( - Query::new(&language, r"((identifier) (#a))") - .unwrap_err() - .message, - [ - "((identifier) (#a))", // - " ^", + "((identifier) (#a)", // + " ^", ] .join("\n") ); @@ -238,20 +225,6 @@ fn test_query_errors_on_invalid_syntax() { ] .join("\n") ); - assert_eq!( - Query::new(&language, "(statement / export_statement)").unwrap_err(), - QueryError { - row: 0, - offset: 11, - column: 11, - kind: QueryErrorKind::Syntax, - message: [ - "(statement / export_statement)", // - " ^" - ] - .join("\n") - } - ); }); } @@ -267,7 +240,7 @@ fn test_query_errors_on_invalid_symbols() { offset: 1, column: 1, kind: QueryErrorKind::NodeType, - message: "\">>>>\"".to_string() + message: ">>>>".to_string() } ); assert_eq!( @@ -277,7 +250,7 @@ fn test_query_errors_on_invalid_symbols() { offset: 1, column: 1, kind: QueryErrorKind::NodeType, - message: "\"te\\\"st\"".to_string() + message: "te\\\"st".to_string() } ); assert_eq!( @@ -287,7 +260,7 @@ fn test_query_errors_on_invalid_symbols() { offset: 1, column: 1, kind: QueryErrorKind::NodeType, - message: "\"\\\\\"".to_string() + message: "\\\\".to_string() } ); assert_eq!( @@ -297,7 +270,7 @@ fn test_query_errors_on_invalid_symbols() { offset: 1, column: 1, kind: QueryErrorKind::NodeType, - message: "\"clas\"".to_string() + message: "clas".to_string() } ); assert_eq!( @@ -307,7 +280,7 @@ fn test_query_errors_on_invalid_symbols() { offset: 15, column: 15, kind: QueryErrorKind::NodeType, - message: "\"arrayyyyy\"".to_string() + message: "arrayyyyy".to_string() }, ); assert_eq!( @@ -317,7 +290,7 @@ fn test_query_errors_on_invalid_symbols() { offset: 26, column: 26, kind: QueryErrorKind::NodeType, - message: "\"non_existent3\"".to_string() + message: "non_existent3".to_string() }, ); assert_eq!( @@ -327,7 +300,7 @@ fn test_query_errors_on_invalid_symbols() { offset: 14, column: 14, kind: QueryErrorKind::Field, - message: "\"condit\"".to_string() + message: "condit".to_string() }, ); assert_eq!( @@ -337,7 +310,7 @@ fn test_query_errors_on_invalid_symbols() { offset: 14, column: 14, kind: QueryErrorKind::Field, - message: "\"conditioning\"".to_string() + message: "conditioning".to_string() } ); assert_eq!( @@ -347,7 +320,7 @@ fn test_query_errors_on_invalid_symbols() { offset: 15, column: 15, kind: QueryErrorKind::Field, - message: "\"alternativ\"".to_string() + message: "alternativ".to_string() } ); assert_eq!( @@ -357,7 +330,7 @@ fn test_query_errors_on_invalid_symbols() { offset: 15, column: 15, kind: QueryErrorKind::Field, - message: "\"alternatives\"".to_string() + message: "alternatives".to_string() } ); assert_eq!( @@ -367,7 +340,7 @@ fn test_query_errors_on_invalid_symbols() { offset: 0, column: 0, kind: QueryErrorKind::Field, - message: "\"fakefield\"".to_string() + message: "fakefield".to_string() } ); }); @@ -410,7 +383,7 @@ fn test_query_errors_on_invalid_predicates() { row: 0, column: 29, offset: 29, - message: "\"ok\"".to_string(), + message: "ok".to_string(), } ); }); @@ -430,11 +403,11 @@ fn test_query_errors_on_impossible_patterns() { Err(QueryError { kind: QueryErrorKind::Structure, row: 0, - offset: 37, - column: 37, + offset: 51, + column: 51, message: [ "(binary_expression left: (expression (identifier)) left: (expression (identifier)))", - " ^", + " ^", ] .join("\n"), }) @@ -839,7 +812,8 @@ fn test_query_matches_with_many_overlapping_results() { // .foo(bar(BAZ)) // .foo(bar(BAZ)) // ... - let source = format!("a{}", "\n .foo(bar(BAZ))".repeat(count)); + let mut source = "a".to_string(); + source += &"\n .foo(bar(BAZ))".repeat(count); assert_query_matches( &language, @@ -2669,64 +2643,6 @@ fn test_query_matches_within_range_of_long_repetition() { }); } -#[test] -fn test_query_matches_contained_within_range() { - allocations::record(|| { - let language = get_language("json"); - let query = Query::new( - &language, - r#" - ("[" @l_bracket "]" @r_bracket) - ("{" @l_brace "}" @r_brace) - "#, - ) - .unwrap(); - - let source = r#" - [ - {"key1": "value1"}, - {"key2": "value2"}, - {"key3": "value3"}, - {"key4": "value4"}, - {"key5": "value5"}, - {"key6": "value6"}, - {"key7": "value7"}, - {"key8": "value8"}, - {"key9": "value9"}, - {"key10": "value10"}, - {"key11": "value11"}, - {"key12": "value12"}, - ] - "# - .unindent(); - - let mut parser = Parser::new(); - parser.set_language(&language).unwrap(); - let tree = parser.parse(&source, None).unwrap(); - - let expected_matches = [ - (1, vec![("l_brace", "{"), ("r_brace", "}")]), - (1, vec![("l_brace", "{"), ("r_brace", "}")]), - ]; - { - let mut cursor = QueryCursor::new(); - let matches = cursor - .set_containing_point_range(Point::new(5, 0)..Point::new(7, 0)) - .matches(&query, tree.root_node(), source.as_bytes()); - assert_eq!(collect_matches(matches, &query, &source), &expected_matches); - } - { - let mut cursor = QueryCursor::new(); - let matches = cursor.set_containing_byte_range(78..120).matches( - &query, - tree.root_node(), - source.as_bytes(), - ); - assert_eq!(collect_matches(matches, &query, &source), &expected_matches); - } - }); -} - #[test] fn test_query_matches_different_queries_same_cursor() { allocations::record(|| { @@ -4259,9 +4175,12 @@ fn test_query_random() { let pattern = pattern_ast.to_string(); let expected_matches = pattern_ast.matches_in_tree(&test_tree); - let query = Query::new(&language, &pattern).unwrap_or_else(|e| { - panic!("failed to build query for pattern {pattern}. seed: {seed}\n{e}") - }); + let query = match Query::new(&language, &pattern) { + Ok(query) => query, + Err(e) => { + panic!("failed to build query for pattern {pattern} - {e}. seed: {seed}"); + } + }; let mut actual_matches = Vec::new(); let mut match_iter = cursor.matches( &query, @@ -5090,26 +5009,6 @@ fn test_query_quantified_captures() { ("comment.documentation", "// quuz"), ], }, - Row { - description: "multiple quantifiers should not hang query parsing", - language: get_language("c"), - code: indoc! {" - // foo - // bar - // baz - "}, - pattern: r" - ((comment) ?+ @comment) - ", - // This should be identical to the `*` quantifier. - captures: &[ - ("comment", "// foo"), - ("comment", "// foo"), - ("comment", "// foo"), - ("comment", "// bar"), - ("comment", "// baz"), - ], - }, ]; allocations::record(|| { @@ -5249,7 +5148,7 @@ fn test_query_error_does_not_oob() { offset: 1, column: 1, kind: QueryErrorKind::NodeType, - message: "\"clas\"".to_string() + message: "clas".to_string() } ); } @@ -5536,13 +5435,8 @@ fn test_query_execution_with_timeout() { &query, tree.root_node(), source_code.as_bytes(), - QueryCursorOptions::new().progress_callback(&mut |_| { - if start_time.elapsed().as_micros() > 1000 { - ControlFlow::Break(()) - } else { - ControlFlow::Continue(()) - } - }), + QueryCursorOptions::new() + .progress_callback(&mut |_| start_time.elapsed().as_micros() > 1000), ) .count(); assert!(matches < 1000); @@ -5855,109 +5749,3 @@ fn test_query_allows_error_nodes_with_children() { assert_eq!(matches, &[(0, vec![("error", ".bar")])]); }); } - -#[test] -fn test_query_assertion_on_unreachable_node_with_child() { - // The `await_binding` rule is unreachable because it has a lower precedence than - // `identifier`, so we'll always reduce to an expression of type `identifier` - // instead whenever we see the token `await` followed by an identifier. - // - // A query that tries to capture the `await` token in the `await_binding` rule - // should not cause an assertion failure during query analysis. - let grammar = r#" -export default grammar({ - name: "query_assertion_crash", - - rules: { - source_file: $ => repeat($.expression), - - expression: $ => choice( - $.await_binding, - $.await_expr, - $.equal_expr, - prec(3, $.identifier), - ), - - await_binding: $ => prec(1, seq('await', $.identifier, '=', $.expression)), - - await_expr: $ => prec(1, seq('await', $.expression)), - - equal_expr: $ => prec.right(2, seq($.expression, '=', $.expression)), - - identifier: _ => /[a-z]+/, - } -}); - "#; - - let file = tempfile::NamedTempFile::with_suffix(".js").unwrap(); - std::fs::write(file.path(), grammar).unwrap(); - - let grammar_json = load_grammar_file(file.path(), None).unwrap(); - - let (parser_name, parser_code) = generate_parser(&grammar_json).unwrap(); - - let language = get_test_language(&parser_name, &parser_code, None); - - let query_result = Query::new(&language, r#"(await_binding "await")"#); - - assert!(query_result.is_err()); - assert_eq!( - query_result.unwrap_err(), - QueryError { - kind: QueryErrorKind::Structure, - row: 0, - offset: 0, - column: 0, - message: ["(await_binding \"await\")", "^"].join("\n"), - } - ); -} - -#[test] -fn test_query_supertype_with_anonymous_node() { - let grammar = r#" -export default grammar({ - name: "supertype_anonymous_test", - - extras: $ => [/\s/, $.comment], - - supertypes: $ => [$.expression], - - word: $ => $.identifier, - - rules: { - source_file: $ => repeat($.expression), - - expression: $ => choice( - $.function_call, - '()' // an empty tuple, which should be queryable with the supertype syntax - ), - - function_call: $ => seq($.identifier, '()'), - - identifier: _ => /[a-zA-Z_][a-zA-Z0-9_]*/, - - comment: _ => token(seq('//', /.*/)), - } -}); - "#; - - let file = tempfile::NamedTempFile::with_suffix(".js").unwrap(); - std::fs::write(file.path(), grammar).unwrap(); - - let grammar_json = load_grammar_file(file.path(), None).unwrap(); - - let (parser_name, parser_code) = generate_parser(&grammar_json).unwrap(); - - let language = get_test_language(&parser_name, &parser_code, None); - - let query_result = Query::new(&language, r#"(expression/"()") @tuple"#); - - assert!(query_result.is_ok()); - - let query = query_result.unwrap(); - - let source = "foo()\n()"; - - assert_query_matches(&language, &query, source, &[(0, vec![("tuple", "()")])]); -} diff --git a/crates/cli/src/tests/tags_test.rs b/cli/src/tests/tags_test.rs similarity index 95% rename from crates/cli/src/tests/tags_test.rs rename to cli/src/tests/tags_test.rs index 0c9f7111..232a01dc 100644 --- a/crates/cli/src/tests/tags_test.rs +++ b/cli/src/tests/tags_test.rs @@ -1,7 +1,6 @@ use std::{ ffi::{CStr, CString}, fs, ptr, slice, str, - sync::atomic::{AtomicUsize, Ordering}, }; use tree_sitter::Point; @@ -263,34 +262,34 @@ fn test_tags_ruby() { #[test] fn test_tags_cancellation() { + use std::sync::atomic::{AtomicUsize, Ordering}; + allocations::record(|| { // Large javascript document - let source = "/* hi */ class A { /* ok */ b() {} }\n".repeat(500); + let source = (0..500) + .map(|_| "/* hi */ class A { /* ok */ b() {} }\n") + .collect::(); + let cancellation_flag = AtomicUsize::new(0); let language = get_language("javascript"); let tags_config = TagsConfiguration::new(language, JS_TAG_QUERY, "").unwrap(); + let mut tag_context = TagsContext::new(); let tags = tag_context .generate_tags(&tags_config, source.as_bytes(), Some(&cancellation_flag)) .unwrap(); - let found_cancellation_error = tags.0.enumerate().any(|(i, tag)| { + for (i, tag) in tags.0.enumerate() { if i == 150 { cancellation_flag.store(1, Ordering::SeqCst); } - match tag { - Ok(_) => false, - Err(Error::Cancelled) => true, - Err(e) => { - unreachable!("Unexpected error type while iterating tags: {e}") - } + if let Err(e) = tag { + assert_eq!(e, Error::Cancelled); + return; } - }); + } - assert!( - found_cancellation_error, - "Expected to halt tagging with a cancellation error" - ); + panic!("Expected to halt tagging with an error"); }); } diff --git a/crates/cli/src/tests/test_highlight_test.rs b/cli/src/tests/test_highlight_test.rs similarity index 100% rename from crates/cli/src/tests/test_highlight_test.rs rename to cli/src/tests/test_highlight_test.rs diff --git a/crates/cli/src/tests/test_tags_test.rs b/cli/src/tests/test_tags_test.rs similarity index 100% rename from crates/cli/src/tests/test_tags_test.rs rename to cli/src/tests/test_tags_test.rs diff --git a/crates/cli/src/tests/text_provider_test.rs b/cli/src/tests/text_provider_test.rs similarity index 98% rename from crates/cli/src/tests/text_provider_test.rs rename to cli/src/tests/text_provider_test.rs index 6c4e8939..d9ed454e 100644 --- a/crates/cli/src/tests/text_provider_test.rs +++ b/cli/src/tests/text_provider_test.rs @@ -21,6 +21,7 @@ where let mut parser = Parser::new(); parser.set_language(&language).unwrap(); let tree = parser.parse_with_options(callback, None, None).unwrap(); + // eprintln!("{}", tree.clone().root_node().to_sexp()); assert_eq!("comment", tree.root_node().child(0).unwrap().kind()); (tree, language) } diff --git a/crates/cli/src/tests/tree_test.rs b/cli/src/tests/tree_test.rs similarity index 100% rename from crates/cli/src/tests/tree_test.rs rename to cli/src/tests/tree_test.rs diff --git a/crates/cli/src/tests/wasm_language_test.rs b/cli/src/tests/wasm_language_test.rs similarity index 87% rename from crates/cli/src/tests/wasm_language_test.rs rename to cli/src/tests/wasm_language_test.rs index d62095c0..dcf8c193 100644 --- a/crates/cli/src/tests/wasm_language_test.rs +++ b/cli/src/tests/wasm_language_test.rs @@ -1,13 +1,14 @@ -use std::fs; +use std::{fs, sync::LazyLock}; use streaming_iterator::StreamingIterator; -use tree_sitter::{Parser, Query, QueryCursor, WasmError, WasmErrorKind, WasmStore}; - -use crate::tests::helpers::{ - allocations, - fixtures::{get_test_fixture_language_wasm, ENGINE, WASM_DIR}, +use tree_sitter::{ + wasmtime::Engine, Parser, Query, QueryCursor, WasmError, WasmErrorKind, WasmStore, }; +use crate::tests::helpers::{allocations, fixtures::WASM_DIR}; + +static ENGINE: LazyLock = LazyLock::new(Engine::default); + #[test] fn test_wasm_stdlib_symbols() { let symbols = tree_sitter::wasm_stdlib_symbols().collect::>(); @@ -91,33 +92,6 @@ fn test_load_wasm_javascript_language() { }); } -#[test] -fn test_load_wasm_python_language() { - allocations::record(|| { - let mut store = WasmStore::new(&ENGINE).unwrap(); - let mut parser = Parser::new(); - let wasm = fs::read(WASM_DIR.join("tree-sitter-python.wasm")).unwrap(); - let language = store.load_language("python", &wasm).unwrap(); - parser.set_wasm_store(store).unwrap(); - parser.set_language(&language).unwrap(); - let tree = parser.parse("a = b\nc = d", None).unwrap(); - assert_eq!(tree.root_node().to_sexp(), "(module (expression_statement (assignment left: (identifier) right: (identifier))) (expression_statement (assignment left: (identifier) right: (identifier))))"); - }); -} - -#[test] -fn test_load_fixture_language_wasm() { - allocations::record(|| { - let store = WasmStore::new(&ENGINE).unwrap(); - let mut parser = Parser::new(); - let language = get_test_fixture_language_wasm("epsilon_external_tokens"); - parser.set_wasm_store(store).unwrap(); - parser.set_language(&language).unwrap(); - let tree = parser.parse("hello", None).unwrap(); - assert_eq!(tree.root_node().to_sexp(), "(document (zero_width))"); - }); -} - #[test] fn test_load_multiple_wasm_languages() { allocations::record(|| { @@ -142,7 +116,7 @@ fn test_load_multiple_wasm_languages() { let mut query_cursor = QueryCursor::new(); // First, parse with the store that originally loaded the languages. - // Then parse with a new parser and Wasm store, so that the languages + // Then parse with a new parser and wasm store, so that the languages // are added one-by-one, in between parses. for mut parser in [parser, parser2] { for _ in 0..2 { @@ -252,7 +226,7 @@ fn test_load_wasm_errors() { store.load_language("rust", bad_wasm).unwrap_err(), WasmError { kind: WasmErrorKind::Parse, - message: "failed to parse dylink section of Wasm module".into(), + message: "failed to parse dylink section of wasm module".into(), } ); diff --git a/crates/cli/src/util.rs b/cli/src/util.rs similarity index 99% rename from crates/cli/src/util.rs rename to cli/src/util.rs index 72968db8..fd4f4699 100644 --- a/crates/cli/src/util.rs +++ b/cli/src/util.rs @@ -9,7 +9,6 @@ use std::{ use anyhow::{anyhow, Context, Result}; use indoc::indoc; -use log::error; use tree_sitter::{Parser, Tree}; use tree_sitter_config::Config; use tree_sitter_loader::Config as LoaderConfig; @@ -121,7 +120,7 @@ impl Drop for LogSession { webbrowser::open(&self.path.to_string_lossy()).unwrap(); } } else { - error!( + eprintln!( "Dot failed: {} {}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) diff --git a/cli/src/version.rs b/cli/src/version.rs new file mode 100644 index 00000000..11871265 --- /dev/null +++ b/cli/src/version.rs @@ -0,0 +1,264 @@ +use std::{fs, path::PathBuf, process::Command}; + +use anyhow::{anyhow, Context, Result}; +use regex::Regex; +use tree_sitter_loader::TreeSitterJSON; + +pub struct Version { + pub version: String, + pub current_dir: PathBuf, +} + +impl Version { + #[must_use] + pub const fn new(version: String, current_dir: PathBuf) -> Self { + Self { + version, + current_dir, + } + } + + pub fn run(self) -> Result<()> { + let tree_sitter_json = self.current_dir.join("tree-sitter.json"); + + let tree_sitter_json = + serde_json::from_str::(&fs::read_to_string(tree_sitter_json)?)?; + + let is_multigrammar = tree_sitter_json.grammars.len() > 1; + + self.update_treesitter_json().with_context(|| { + format!( + "Failed to update tree-sitter.json at {}", + self.current_dir.display() + ) + })?; + self.update_cargo_toml().with_context(|| { + format!( + "Failed to update Cargo.toml at {}", + self.current_dir.display() + ) + })?; + self.update_package_json().with_context(|| { + format!( + "Failed to update package.json at {}", + self.current_dir.display() + ) + })?; + self.update_makefile(is_multigrammar).with_context(|| { + format!( + "Failed to update Makefile at {}", + self.current_dir.display() + ) + })?; + self.update_cmakelists_txt().with_context(|| { + format!( + "Failed to update CMakeLists.txt at {}", + self.current_dir.display() + ) + })?; + self.update_pyproject_toml().with_context(|| { + format!( + "Failed to update pyproject.toml at {}", + self.current_dir.display() + ) + })?; + + Ok(()) + } + + fn update_treesitter_json(&self) -> Result<()> { + let tree_sitter_json = &fs::read_to_string(self.current_dir.join("tree-sitter.json"))?; + + let tree_sitter_json = tree_sitter_json + .lines() + .map(|line| { + if line.contains("\"version\":") { + let prefix_index = line.find("\"version\":").unwrap() + "\"version\":".len(); + let start_quote = line[prefix_index..].find('"').unwrap() + prefix_index + 1; + let end_quote = line[start_quote + 1..].find('"').unwrap() + start_quote + 1; + + format!( + "{}{}{}", + &line[..start_quote], + self.version, + &line[end_quote..] + ) + } else { + line.to_string() + } + }) + .collect::>() + .join("\n") + + "\n"; + + fs::write(self.current_dir.join("tree-sitter.json"), tree_sitter_json)?; + + Ok(()) + } + + fn update_cargo_toml(&self) -> Result<()> { + if !self.current_dir.join("Cargo.toml").exists() { + return Ok(()); + } + + let cargo_toml = fs::read_to_string(self.current_dir.join("Cargo.toml"))?; + + let cargo_toml = cargo_toml + .lines() + .map(|line| { + if line.starts_with("version =") { + format!("version = \"{}\"", self.version) + } else { + line.to_string() + } + }) + .collect::>() + .join("\n") + + "\n"; + + fs::write(self.current_dir.join("Cargo.toml"), cargo_toml)?; + + if self.current_dir.join("Cargo.lock").exists() { + let Ok(cmd) = Command::new("cargo") + .arg("generate-lockfile") + .arg("--offline") + .current_dir(&self.current_dir) + .output() + else { + return Ok(()); // cargo is not `executable`, ignore + }; + + if !cmd.status.success() { + let stderr = String::from_utf8_lossy(&cmd.stderr); + return Err(anyhow!( + "Failed to run `cargo generate-lockfile`:\n{stderr}" + )); + } + } + + Ok(()) + } + + fn update_package_json(&self) -> Result<()> { + if !self.current_dir.join("package.json").exists() { + return Ok(()); + } + + let package_json = &fs::read_to_string(self.current_dir.join("package.json"))?; + + let package_json = package_json + .lines() + .map(|line| { + if line.contains("\"version\":") { + let prefix_index = line.find("\"version\":").unwrap() + "\"version\":".len(); + let start_quote = line[prefix_index..].find('"').unwrap() + prefix_index + 1; + let end_quote = line[start_quote + 1..].find('"').unwrap() + start_quote + 1; + + format!( + "{}{}{}", + &line[..start_quote], + self.version, + &line[end_quote..] + ) + } else { + line.to_string() + } + }) + .collect::>() + .join("\n") + + "\n"; + + fs::write(self.current_dir.join("package.json"), package_json)?; + + if self.current_dir.join("package-lock.json").exists() { + let Ok(cmd) = Command::new("npm") + .arg("install") + .arg("--package-lock-only") + .current_dir(&self.current_dir) + .output() + else { + return Ok(()); // npm is not `executable`, ignore + }; + + if !cmd.status.success() { + let stderr = String::from_utf8_lossy(&cmd.stderr); + return Err(anyhow!("Failed to run `npm install`:\n{stderr}")); + } + } + + Ok(()) + } + + fn update_makefile(&self, is_multigrammar: bool) -> Result<()> { + let makefile = if is_multigrammar { + if !self.current_dir.join("common").join("common.mak").exists() { + return Ok(()); + } + + fs::read_to_string(self.current_dir.join("Makefile"))? + } else { + if !self.current_dir.join("Makefile").exists() { + return Ok(()); + } + + fs::read_to_string(self.current_dir.join("Makefile"))? + }; + + let makefile = makefile + .lines() + .map(|line| { + if line.starts_with("VERSION") { + format!("VERSION := {}", self.version) + } else { + line.to_string() + } + }) + .collect::>() + .join("\n") + + "\n"; + + fs::write(self.current_dir.join("Makefile"), makefile)?; + + Ok(()) + } + + fn update_cmakelists_txt(&self) -> Result<()> { + if !self.current_dir.join("CMakeLists.txt").exists() { + return Ok(()); + } + + let cmake = fs::read_to_string(self.current_dir.join("CMakeLists.txt"))?; + + let re = Regex::new(r#"(\s*VERSION\s+)"[0-9]+\.[0-9]+\.[0-9]+""#)?; + let cmake = re.replace(&cmake, format!(r#"$1"{}""#, self.version)); + + fs::write(self.current_dir.join("CMakeLists.txt"), cmake.as_bytes())?; + + Ok(()) + } + + fn update_pyproject_toml(&self) -> Result<()> { + if !self.current_dir.join("pyproject.toml").exists() { + return Ok(()); + } + + let pyproject_toml = fs::read_to_string(self.current_dir.join("pyproject.toml"))?; + + let pyproject_toml = pyproject_toml + .lines() + .map(|line| { + if line.starts_with("version =") { + format!("version = \"{}\"", self.version) + } else { + line.to_string() + } + }) + .collect::>() + .join("\n") + + "\n"; + + fs::write(self.current_dir.join("pyproject.toml"), pyproject_toml)?; + + Ok(()) + } +} diff --git a/crates/cli/src/wasm.rs b/cli/src/wasm.rs similarity index 90% rename from crates/cli/src/wasm.rs rename to cli/src/wasm.rs index 09fb459b..eef6d08b 100644 --- a/crates/cli/src/wasm.rs +++ b/cli/src/wasm.rs @@ -5,13 +5,13 @@ use std::{ use anyhow::{anyhow, Context, Result}; use tree_sitter::wasm_stdlib_symbols; -use tree_sitter_generate::{load_grammar_file, parse_grammar::GrammarJSON}; +use tree_sitter_generate::parse_grammar::GrammarJSON; use tree_sitter_loader::Loader; use wasmparser::Parser; pub fn load_language_wasm_file(language_dir: &Path) -> Result<(String, Vec)> { let grammar_name = get_grammar_name(language_dir) - .with_context(|| "Failed to get Wasm filename") + .with_context(|| "Failed to get wasm filename") .unwrap(); let wasm_filename = format!("tree-sitter-{grammar_name}.wasm"); let contents = fs::read(language_dir.join(&wasm_filename)).with_context(|| { @@ -40,27 +40,30 @@ pub fn get_grammar_name(language_dir: &Path) -> Result { pub fn compile_language_to_wasm( loader: &Loader, + root_dir: Option<&Path>, language_dir: &Path, output_dir: &Path, output_file: Option, + force_docker: bool, ) -> Result<()> { - let grammar_name = get_grammar_name(language_dir) - .or_else(|_| load_grammar_file(&language_dir.join("grammar.js"), None))?; + let grammar_name = get_grammar_name(language_dir)?; let output_filename = output_file.unwrap_or_else(|| output_dir.join(format!("tree-sitter-{grammar_name}.wasm"))); let src_path = language_dir.join("src"); let scanner_path = loader.get_scanner_path(&src_path); loader.compile_parser_to_wasm( &grammar_name, + root_dir, &src_path, scanner_path .as_ref() .and_then(|p| Some(Path::new(p.file_name()?))), &output_filename, + force_docker, )?; // Exit with an error if the external scanner uses symbols from the - // C or C++ standard libraries that aren't available to Wasm parsers. + // C or C++ standard libraries that aren't available to wasm parsers. let stdlib_symbols = wasm_stdlib_symbols().collect::>(); let dylink_symbols = [ "__indirect_function_table", @@ -99,7 +102,7 @@ pub fn compile_language_to_wasm( if !missing_symbols.is_empty() { Err(anyhow!( concat!( - "This external scanner uses a symbol that isn't available to Wasm parsers.\n", + "This external scanner uses a symbol that isn't available to wasm parsers.\n", "\n", "Missing symbols:\n", " {}\n", diff --git a/crates/cli/LICENSE b/crates/cli/LICENSE deleted file mode 100644 index 971b81f9..00000000 --- a/crates/cli/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2018 Max Brunsfeld - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/crates/cli/eslint/.gitignore b/crates/cli/eslint/.gitignore deleted file mode 100644 index 6b1d0bfa..00000000 --- a/crates/cli/eslint/.gitignore +++ /dev/null @@ -1 +0,0 @@ -LICENSE diff --git a/crates/cli/npm/package-lock.json b/crates/cli/npm/package-lock.json deleted file mode 100644 index 739e69f1..00000000 --- a/crates/cli/npm/package-lock.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "name": "tree-sitter-cli", - "version": "0.27.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "tree-sitter-cli", - "version": "0.27.0", - "hasInstallScript": true, - "license": "MIT", - "bin": { - "tree-sitter": "cli.js" - }, - "engines": { - "node": ">=12.0.0" - } - } - } -} diff --git a/crates/cli/package.nix b/crates/cli/package.nix deleted file mode 100644 index eea05e12..00000000 --- a/crates/cli/package.nix +++ /dev/null @@ -1,69 +0,0 @@ -{ - lib, - src, - rustPlatform, - version, - clang, - libclang, - cmake, - pkg-config, - nodejs_22, - test-grammars, - stdenv, - installShellFiles, -}: -let - isCross = stdenv.targetPlatform == stdenv.buildPlatform; -in -rustPlatform.buildRustPackage { - pname = "tree-sitter-cli"; - - inherit src version; - - cargoBuildFlags = [ "--all-features" ]; - - nativeBuildInputs = [ - clang - cmake - pkg-config - nodejs_22 - ] - ++ lib.optionals (!isCross) [ installShellFiles ]; - - cargoLock.lockFile = ../../Cargo.lock; - - env.LIBCLANG_PATH = "${libclang.lib}/lib"; - - preBuild = '' - rm -rf test/fixtures - mkdir -p test/fixtures - cp -r ${test-grammars}/fixtures/* test/fixtures/ - chmod -R u+w test/fixtures - ''; - - preCheck = "export HOME=$TMPDIR"; - doCheck = !isCross; - - postInstall = lib.optionalString (!isCross) '' - installShellCompletion --cmd tree-sitter \ - --bash <($out/bin/tree-sitter complete --shell bash) \ - --zsh <($out/bin/tree-sitter complete --shell zsh) \ - --fish <($out/bin/tree-sitter complete --shell fish) - ''; - - meta = { - description = "Tree-sitter CLI - A tool for developing, testing, and using Tree-sitter parsers"; - longDescription = '' - Tree-sitter is a parser generator tool and an incremental parsing library. - It can build a concrete syntax tree for a source file and efficiently update - the syntax tree as the source file is edited. This package provides the CLI - tool for developing, testing, and using Tree-sitter parsers. - ''; - homepage = "https://tree-sitter.github.io/tree-sitter"; - changelog = "https://github.com/tree-sitter/tree-sitter/releases/tag/v${version}"; - license = lib.licenses.mit; - maintainers = with lib.maintainers; [ amaanq ]; - platforms = lib.platforms.all; - mainProgram = "tree-sitter"; - }; -} diff --git a/crates/cli/src/logger.rs b/crates/cli/src/logger.rs deleted file mode 100644 index 41b11906..00000000 --- a/crates/cli/src/logger.rs +++ /dev/null @@ -1,55 +0,0 @@ -use std::io::Write; - -use anstyle::{AnsiColor, Color, Style}; -use log::{Level, LevelFilter, Log, Metadata, Record}; - -pub fn paint(color: Option>, text: &str) -> String { - let style = Style::new().fg_color(color.map(Into::into)); - format!("{style}{text}{style:#}") -} - -struct Logger; - -impl Log for Logger { - fn enabled(&self, _: &Metadata) -> bool { - true - } - - fn log(&self, record: &Record) { - match record.level() { - Level::Error => eprintln!( - "{} {}", - paint(Some(AnsiColor::Red), "Error:"), - record.args() - ), - Level::Warn => eprintln!( - "{} {}", - paint(Some(AnsiColor::Yellow), "Warning:"), - record.args() - ), - Level::Info | Level::Debug => eprintln!("{}", record.args()), - Level::Trace => eprintln!( - "[{}] {}", - record - .module_path() - .unwrap_or_default() - .trim_start_matches("rust_tree_sitter_cli::"), - record.args() - ), - } - } - - fn flush(&self) { - let mut stderr = std::io::stderr().lock(); - let _ = stderr.flush(); - } -} - -pub fn init() { - log::set_boxed_logger(Box::new(Logger {})).unwrap(); - log::set_max_level(LevelFilter::Info); -} - -pub fn enable_debug() { - log::set_max_level(LevelFilter::Debug); -} diff --git a/crates/cli/src/playground.html b/crates/cli/src/playground.html deleted file mode 100644 index 147516ac..00000000 --- a/crates/cli/src/playground.html +++ /dev/null @@ -1,481 +0,0 @@ - - - - - tree-sitter THE_LANGUAGE_NAME - - - - - - - - - - - - - - - - - - diff --git a/crates/cli/src/templates/__init__.py b/crates/cli/src/templates/__init__.py deleted file mode 100644 index 784887a7..00000000 --- a/crates/cli/src/templates/__init__.py +++ /dev/null @@ -1,43 +0,0 @@ -"""PARSER_DESCRIPTION""" - -from importlib.resources import files as _files - -from ._binding import language - - -def _get_query(name, file): - try: - query = _files(f"{__package__}") / file - globals()[name] = query.read_text() - except FileNotFoundError: - globals()[name] = None - return globals()[name] - - -def __getattr__(name): - if name == "HIGHLIGHTS_QUERY": - return _get_query("HIGHLIGHTS_QUERY", "HIGHLIGHTS_QUERY_PATH") - if name == "INJECTIONS_QUERY": - return _get_query("INJECTIONS_QUERY", "INJECTIONS_QUERY_PATH") - if name == "LOCALS_QUERY": - return _get_query("LOCALS_QUERY", "LOCALS_QUERY_PATH") - if name == "TAGS_QUERY": - return _get_query("TAGS_QUERY", "TAGS_QUERY_PATH") - - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - - -__all__ = [ - "language", - "HIGHLIGHTS_QUERY", - "INJECTIONS_QUERY", - "LOCALS_QUERY", - "TAGS_QUERY", -] - - -def __dir__(): - return sorted(__all__ + [ - "__all__", "__builtins__", "__cached__", "__doc__", "__file__", - "__loader__", "__name__", "__package__", "__path__", "__spec__", - ]) diff --git a/crates/cli/src/templates/__init__.pyi b/crates/cli/src/templates/__init__.pyi deleted file mode 100644 index 5c88ff6c..00000000 --- a/crates/cli/src/templates/__init__.pyi +++ /dev/null @@ -1,17 +0,0 @@ -from typing import Final -from typing_extensions import CapsuleType - -HIGHLIGHTS_QUERY: Final[str] | None -"""The syntax highlighting query for this grammar.""" - -INJECTIONS_QUERY: Final[str] | None -"""The language injection query for this grammar.""" - -LOCALS_QUERY: Final[str] | None -"""The local variable query for this grammar.""" - -TAGS_QUERY: Final[str] | None -"""The symbol tagging query for this grammar.""" - -def language() -> CapsuleType: - """The tree-sitter language function for this grammar.""" diff --git a/crates/cli/src/templates/binding.java b/crates/cli/src/templates/binding.java deleted file mode 100644 index 704064a0..00000000 --- a/crates/cli/src/templates/binding.java +++ /dev/null @@ -1,65 +0,0 @@ -package PARSER_NS_CLEANED.jtreesitter.LOWER_PARSER_NAME; - -import java.lang.foreign.*; - -public final class PARSER_CLASS_NAME { - private static final ValueLayout VOID_PTR = - ValueLayout.ADDRESS.withTargetLayout(MemoryLayout.sequenceLayout(Long.MAX_VALUE, ValueLayout.JAVA_BYTE)); - private static final FunctionDescriptor FUNC_DESC = FunctionDescriptor.of(VOID_PTR); - private static final Linker LINKER = Linker.nativeLinker(); - private static final PARSER_CLASS_NAME INSTANCE = new PARSER_CLASS_NAME(); - - private final Arena arena = Arena.ofAuto(); - private volatile SymbolLookup lookup = null; - - private PARSER_CLASS_NAME() {} - - /** - * Get the tree-sitter language for this grammar. - */ - public static MemorySegment language() { - if (INSTANCE.lookup == null) - INSTANCE.lookup = INSTANCE.findLibrary(); - return language(INSTANCE.lookup); - } - - /** - * Get the tree-sitter language for this grammar. - * - * The {@linkplain Arena} used in the {@code lookup} - * must not be closed while the language is being used. - */ - public static MemorySegment language(SymbolLookup lookup) { - return call(lookup, "tree_sitter_PARSER_NAME"); - } - - private SymbolLookup findLibrary() { - try { - var library = System.mapLibraryName("tree-sitter-KEBAB_PARSER_NAME"); - return SymbolLookup.libraryLookup(library, arena); - } catch (IllegalArgumentException ex1) { - try { - System.loadLibrary("tree-sitter-KEBAB_PARSER_NAME"); - return SymbolLookup.loaderLookup(); - } catch (UnsatisfiedLinkError ex2) { - ex1.addSuppressed(ex2); - throw ex1; - } - } - } - - private static UnsatisfiedLinkError unresolved(String name) { - return new UnsatisfiedLinkError("Unresolved symbol: %s".formatted(name)); - } - - @SuppressWarnings("SameParameterValue") - private static MemorySegment call(SymbolLookup lookup, String name) throws UnsatisfiedLinkError { - var address = lookup.find(name).orElseThrow(() -> unresolved(name)); - try { - var function = LINKER.downcallHandle(address, FUNC_DESC); - return (MemorySegment) function.invokeExact(); - } catch (Throwable e) { - throw new RuntimeException("Call to %s failed".formatted(name), e); - } - } -} diff --git a/crates/cli/src/templates/binding_test.js b/crates/cli/src/templates/binding_test.js deleted file mode 100644 index 7a91a84d..00000000 --- a/crates/cli/src/templates/binding_test.js +++ /dev/null @@ -1,11 +0,0 @@ -import assert from "node:assert"; -import { test } from "node:test"; -import Parser from "tree-sitter"; - -test("can load grammar", () => { - const parser = new Parser(); - assert.doesNotReject(async () => { - const { default: language } = await import("./index.js"); - parser.setLanguage(language); - }); -}); diff --git a/crates/cli/src/templates/build.rs b/crates/cli/src/templates/build.rs deleted file mode 100644 index e3fffe4b..00000000 --- a/crates/cli/src/templates/build.rs +++ /dev/null @@ -1,56 +0,0 @@ -fn main() { - let src_dir = std::path::Path::new("src"); - - let mut c_config = cc::Build::new(); - c_config.std("c11").include(src_dir); - - #[cfg(target_env = "msvc")] - c_config.flag("-utf-8"); - - if std::env::var("TARGET").unwrap() == "wasm32-unknown-unknown" { - let Ok(wasm_headers) = std::env::var("DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS") else { - panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS must be set by the language crate"); - }; - let Ok(wasm_src) = - std::env::var("DEP_TREE_SITTER_LANGUAGE_WASM_SRC").map(std::path::PathBuf::from) - else { - panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_SRC must be set by the language crate"); - }; - - c_config.include(&wasm_headers); - c_config.files([ - wasm_src.join("stdio.c"), - wasm_src.join("stdlib.c"), - wasm_src.join("string.c"), - ]); - } - - let parser_path = src_dir.join("parser.c"); - c_config.file(&parser_path); - println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap()); - - let scanner_path = src_dir.join("scanner.c"); - if scanner_path.exists() { - c_config.file(&scanner_path); - println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap()); - } - - c_config.compile("tree-sitter-KEBAB_PARSER_NAME"); - - println!("cargo:rustc-check-cfg=cfg(with_highlights_query)"); - if !"HIGHLIGHTS_QUERY_PATH".is_empty() && std::path::Path::new("HIGHLIGHTS_QUERY_PATH").exists() { - println!("cargo:rustc-cfg=with_highlights_query"); - } - println!("cargo:rustc-check-cfg=cfg(with_injections_query)"); - if !"INJECTIONS_QUERY_PATH".is_empty() && std::path::Path::new("INJECTIONS_QUERY_PATH").exists() { - println!("cargo:rustc-cfg=with_injections_query"); - } - println!("cargo:rustc-check-cfg=cfg(with_locals_query)"); - if !"LOCALS_QUERY_PATH".is_empty() && std::path::Path::new("LOCALS_QUERY_PATH").exists() { - println!("cargo:rustc-cfg=with_locals_query"); - } - println!("cargo:rustc-check-cfg=cfg(with_tags_query)"); - if !"TAGS_QUERY_PATH".is_empty() && std::path::Path::new("TAGS_QUERY_PATH").exists() { - println!("cargo:rustc-cfg=with_tags_query"); - } -} diff --git a/crates/cli/src/templates/index.d.ts b/crates/cli/src/templates/index.d.ts deleted file mode 100644 index 24576d32..00000000 --- a/crates/cli/src/templates/index.d.ts +++ /dev/null @@ -1,60 +0,0 @@ -type BaseNode = { - type: string; - named: boolean; -}; - -type ChildNode = { - multiple: boolean; - required: boolean; - types: BaseNode[]; -}; - -type NodeInfo = - | (BaseNode & { - subtypes: BaseNode[]; - }) - | (BaseNode & { - fields: { [name: string]: ChildNode }; - children: ChildNode[]; - }); - -/** - * The tree-sitter language object for this grammar. - * - * @see {@linkcode https://tree-sitter.github.io/node-tree-sitter/interfaces/Parser.Language.html Parser.Language} - * - * @example - * import Parser from "tree-sitter"; - * import CAMEL_PARSER_NAME from "tree-sitter-KEBAB_PARSER_NAME"; - * - * const parser = new Parser(); - * parser.setLanguage(CAMEL_PARSER_NAME); - */ -declare const binding: { - /** - * The inner language object. - * @private - */ - language: unknown; - - /** - * The content of the `node-types.json` file for this grammar. - * - * @see {@linkplain https://tree-sitter.github.io/tree-sitter/using-parsers/6-static-node-types Static Node Types} - */ - nodeTypeInfo: NodeInfo[]; - - /** The syntax highlighting query for this grammar. */ - HIGHLIGHTS_QUERY?: string; - - /** The language injection query for this grammar. */ - INJECTIONS_QUERY?: string; - - /** The local variable query for this grammar. */ - LOCALS_QUERY?: string; - - /** The symbol tagging query for this grammar. */ - TAGS_QUERY?: string; -}; - -export default binding; diff --git a/crates/cli/src/templates/index.js b/crates/cli/src/templates/index.js deleted file mode 100644 index b3edc2e3..00000000 --- a/crates/cli/src/templates/index.js +++ /dev/null @@ -1,37 +0,0 @@ -import { readFileSync } from "node:fs"; -import { fileURLToPath } from "node:url"; - -const root = fileURLToPath(new URL("../..", import.meta.url)); - -const binding = typeof process.versions.bun === "string" - // Support `bun build --compile` by being statically analyzable enough to find the .node file at build-time - ? await import(`${root}/prebuilds/${process.platform}-${process.arch}/tree-sitter-KEBAB_PARSER_NAME.node`) - : (await import("node-gyp-build")).default(root); - -try { - const nodeTypes = await import(`${root}/src/node-types.json`, { with: { type: "json" } }); - binding.nodeTypeInfo = nodeTypes.default; -} catch { } - -const queries = [ - ["HIGHLIGHTS_QUERY", `${root}/HIGHLIGHTS_QUERY_PATH`], - ["INJECTIONS_QUERY", `${root}/INJECTIONS_QUERY_PATH`], - ["LOCALS_QUERY", `${root}/LOCALS_QUERY_PATH`], - ["TAGS_QUERY", `${root}/TAGS_QUERY_PATH`], -]; - -for (const [prop, path] of queries) { - Object.defineProperty(binding, prop, { - configurable: true, - enumerable: true, - get() { - delete binding[prop]; - try { - binding[prop] = readFileSync(path, "utf8"); - } catch { } - return binding[prop]; - } - }); -} - -export default binding; diff --git a/crates/cli/src/templates/pom.xml b/crates/cli/src/templates/pom.xml deleted file mode 100644 index 661fe42b..00000000 --- a/crates/cli/src/templates/pom.xml +++ /dev/null @@ -1,154 +0,0 @@ - - - 4.0.0 - PARSER_NS - jtreesitter-KEBAB_PARSER_NAME - JTreeSitter CAMEL_PARSER_NAME - PARSER_VERSION - PARSER_DESCRIPTION - PARSER_URL - - - PARSER_LICENSE - https://spdx.org/licenses/PARSER_LICENSE.html - - - - - PARSER_AUTHOR_NAME - PARSER_AUTHOR_EMAIL - PARSER_AUTHOR_URL - - - - PARSER_URL - scm:git:git://PARSER_URL_STRIPPED.git - scm:git:ssh://PARSER_URL_STRIPPED.git - - - 23 - UTF-8 - true - true - false - true - - - - io.github.tree-sitter - jtreesitter - 0.26.0 - true - - - org.junit.jupiter - junit-jupiter-api - 6.0.1 - test - - - - bindings/java/main - bindings/java/test - - - maven-surefire-plugin - 3.5.4 - - - ${project.build.directory}/reports/surefire - - --enable-native-access=ALL-UNNAMED - - - - maven-javadoc-plugin - 3.12.0 - - - - jar - - - - - public - true - true - all,-missing - - - - maven-source-plugin - 3.3.1 - - - - jar-no-fork - - - - - - maven-gpg-plugin - 3.2.8 - - - verify - - sign - - - true - - --no-tty - --pinentry-mode - loopback - - - - - - - io.github.mavenplugins - central-publishing-maven-plugin - 1.1.1 - - - deploy - - publish - - - validated - ${publish.auto} - ${publish.skip} - ${project.artifactId}-${project.version}.zip - ${project.artifactId}-${project.version}.zip - - - - true - - - - - - ci - - - env.CI - true - - - - false - true - false - - - - diff --git a/crates/cli/src/templates/test.java b/crates/cli/src/templates/test.java deleted file mode 100644 index 8bf81ea0..00000000 --- a/crates/cli/src/templates/test.java +++ /dev/null @@ -1,12 +0,0 @@ -import io.github.treesitter.jtreesitter.Language; -import PARSER_NS_CLEANED.jtreesitter.LOWER_PARSER_NAME.PARSER_CLASS_NAME; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; - -public class PARSER_CLASS_NAMETest { - @Test - public void testCanLoadLanguage() { - assertDoesNotThrow(() -> new Language(PARSER_CLASS_NAME.language())); - } -} diff --git a/crates/cli/src/test.rs b/crates/cli/src/test.rs deleted file mode 100644 index b568e4a3..00000000 --- a/crates/cli/src/test.rs +++ /dev/null @@ -1,2427 +0,0 @@ -use std::{ - collections::BTreeMap, - ffi::OsStr, - fmt::Display as _, - fs, - io::{self, Write}, - path::{Path, PathBuf}, - str, - sync::LazyLock, - time::Duration, -}; - -use anstyle::AnsiColor; -use anyhow::{anyhow, Context, Result}; -use clap::ValueEnum; -use indoc::indoc; -use regex::{ - bytes::{Regex as ByteRegex, RegexBuilder as ByteRegexBuilder}, - Regex, -}; -use schemars::{JsonSchema, Schema, SchemaGenerator}; -use serde::Serialize; -use similar::{ChangeTag, TextDiff}; -use tree_sitter::{format_sexp, Language, LogType, Parser, Query, Tree}; -use walkdir::WalkDir; - -use super::util; -use crate::{ - logger::paint, - parse::{ - render_cst, ParseDebugType, ParseFileOptions, ParseOutput, ParseStats, ParseTheme, Stats, - }, -}; - -static HEADER_REGEX: LazyLock = LazyLock::new(|| { - ByteRegexBuilder::new( - r"^(?x) - (?P(?:=+){3,}) - (?P[^=\r\n][^\r\n]*)? - \r?\n - (?P(?:([^=\r\n]|\s+:)[^\r\n]*\r?\n)+) - ===+ - (?P[^=\r\n][^\r\n]*)?\r?\n", - ) - .multi_line(true) - .build() - .unwrap() -}); - -static DIVIDER_REGEX: LazyLock = LazyLock::new(|| { - ByteRegexBuilder::new(r"^(?P(?:-+){3,})(?P[^-\r\n][^\r\n]*)?\r?\n") - .multi_line(true) - .build() - .unwrap() -}); - -static COMMENT_REGEX: LazyLock = LazyLock::new(|| Regex::new(r"(?m)^\s*;.*$").unwrap()); - -static WHITESPACE_REGEX: LazyLock = LazyLock::new(|| Regex::new(r"\s+").unwrap()); - -static SEXP_FIELD_REGEX: LazyLock = LazyLock::new(|| Regex::new(r" \w+: \(").unwrap()); - -static POINT_REGEX: LazyLock = - LazyLock::new(|| Regex::new(r"\s*\[\s*\d+\s*,\s*\d+\s*\]\s*").unwrap()); - -#[derive(Debug, PartialEq, Eq)] -pub enum TestEntry { - Group { - name: String, - children: Vec, - file_path: Option, - }, - Example { - name: String, - input: Vec, - output: String, - header_delim_len: usize, - divider_delim_len: usize, - has_fields: bool, - attributes_str: String, - attributes: TestAttributes, - file_name: Option, - }, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct TestAttributes { - pub skip: bool, - pub platform: bool, - pub fail_fast: bool, - pub error: bool, - pub cst: bool, - pub languages: Vec>, -} - -impl Default for TestEntry { - fn default() -> Self { - Self::Group { - name: String::new(), - children: Vec::new(), - file_path: None, - } - } -} - -impl Default for TestAttributes { - fn default() -> Self { - Self { - skip: false, - platform: true, - fail_fast: false, - error: false, - cst: false, - languages: vec!["".into()], - } - } -} - -#[derive(ValueEnum, Default, Debug, Copy, Clone, PartialEq, Eq, Serialize)] -pub enum TestStats { - All, - #[default] - OutliersAndTotal, - TotalOnly, -} - -pub struct TestOptions<'a> { - pub path: PathBuf, - pub debug: bool, - pub debug_graph: bool, - pub include: Option, - pub exclude: Option, - pub file_name: Option, - pub update: bool, - pub open_log: bool, - pub languages: BTreeMap<&'a str, &'a Language>, - pub color: bool, - pub show_fields: bool, - pub overview_only: bool, -} - -/// A stateful object used to collect results from running a grammar's test suite -#[derive(Debug, Default, Serialize, JsonSchema)] -pub struct TestSummary { - // Parse test results and associated data - #[schemars(schema_with = "schema_as_array")] - #[serde(serialize_with = "serialize_as_array")] - pub parse_results: TestResultHierarchy, - pub parse_failures: Vec, - pub parse_stats: Stats, - #[schemars(skip)] - #[serde(skip)] - pub has_parse_errors: bool, - #[schemars(skip)] - #[serde(skip)] - pub parse_stat_display: TestStats, - - // Other test results - #[schemars(schema_with = "schema_as_array")] - #[serde(serialize_with = "serialize_as_array")] - pub highlight_results: TestResultHierarchy, - #[schemars(schema_with = "schema_as_array")] - #[serde(serialize_with = "serialize_as_array")] - pub tag_results: TestResultHierarchy, - #[schemars(schema_with = "schema_as_array")] - #[serde(serialize_with = "serialize_as_array")] - pub query_results: TestResultHierarchy, - - // Data used during construction - #[schemars(skip)] - #[serde(skip)] - pub test_num: usize, - // Options passed in from the CLI which control how the summary is displayed - #[schemars(skip)] - #[serde(skip)] - pub color: bool, - #[schemars(skip)] - #[serde(skip)] - pub overview_only: bool, - #[schemars(skip)] - #[serde(skip)] - pub update: bool, - #[schemars(skip)] - #[serde(skip)] - pub json: bool, -} - -impl TestSummary { - #[must_use] - pub fn new( - color: bool, - stat_display: TestStats, - parse_update: bool, - overview_only: bool, - json_summary: bool, - ) -> Self { - Self { - color, - parse_stat_display: stat_display, - update: parse_update, - overview_only, - json: json_summary, - test_num: 1, - ..Default::default() - } - } -} - -#[derive(Debug, Default, JsonSchema)] -pub struct TestResultHierarchy { - root_group: Vec, - traversal_idxs: Vec, -} - -fn serialize_as_array(results: &TestResultHierarchy, serializer: S) -> Result -where - S: serde::Serializer, -{ - results.root_group.serialize(serializer) -} - -fn schema_as_array(gen: &mut SchemaGenerator) -> Schema { - gen.subschema_for::>() -} - -/// Stores arbitrarily nested parent test groups and child cases. Supports creation -/// in DFS traversal order -impl TestResultHierarchy { - /// Signifies the start of a new group's traversal during construction. - fn push_traversal(&mut self, idx: usize) { - self.traversal_idxs.push(idx); - } - - /// Signifies the end of the current group's traversal during construction. - /// Must be paired with a prior call to [`TestResultHierarchy::add_group`]. - pub fn pop_traversal(&mut self) { - self.traversal_idxs.pop(); - } - - /// Adds a new group as a child of the current group. Caller is responsible - /// for calling [`TestResultHierarchy::pop_traversal`] once the group is done - /// being traversed. - pub fn add_group(&mut self, group_name: &str) { - let new_group_idx = self.curr_group_len(); - self.push(TestResult { - name: group_name.to_string(), - info: TestInfo::Group { - children: Vec::new(), - }, - }); - self.push_traversal(new_group_idx); - } - - /// Adds a new test example as a child of the current group. - /// Asserts that `test_case.info` is not [`TestInfo::Group`]. - pub fn add_case(&mut self, test_case: TestResult) { - assert!(!matches!(test_case.info, TestInfo::Group { .. })); - self.push(test_case); - } - - /// Adds a new `TestResult` to the current group. - fn push(&mut self, result: TestResult) { - // If there are no traversal steps, we're adding to the root - if self.traversal_idxs.is_empty() { - self.root_group.push(result); - return; - } - - #[allow(clippy::manual_let_else)] - let mut curr_group = match self.root_group[self.traversal_idxs[0]].info { - TestInfo::Group { ref mut children } => children, - _ => unreachable!(), - }; - for idx in self.traversal_idxs.iter().skip(1) { - curr_group = match curr_group[*idx].info { - TestInfo::Group { ref mut children } => children, - _ => unreachable!(), - }; - } - - curr_group.push(result); - } - - fn curr_group_len(&self) -> usize { - if self.traversal_idxs.is_empty() { - return self.root_group.len(); - } - - #[allow(clippy::manual_let_else)] - let mut curr_group = match self.root_group[self.traversal_idxs[0]].info { - TestInfo::Group { ref children } => children, - _ => unreachable!(), - }; - for idx in self.traversal_idxs.iter().skip(1) { - curr_group = match curr_group[*idx].info { - TestInfo::Group { ref children } => children, - _ => unreachable!(), - }; - } - curr_group.len() - } - - #[allow(clippy::iter_without_into_iter)] - #[must_use] - pub fn iter(&self) -> TestResultIterWithDepth<'_> { - let mut stack = Vec::with_capacity(self.root_group.len()); - for child in self.root_group.iter().rev() { - stack.push((0, child)); - } - TestResultIterWithDepth { stack } - } -} - -pub struct TestResultIterWithDepth<'a> { - stack: Vec<(usize, &'a TestResult)>, -} - -impl<'a> Iterator for TestResultIterWithDepth<'a> { - type Item = (usize, &'a TestResult); - - fn next(&mut self) -> Option { - self.stack.pop().inspect(|(depth, result)| { - if let TestInfo::Group { children } = &result.info { - for child in children.iter().rev() { - self.stack.push((depth + 1, child)); - } - } - }) - } -} - -#[derive(Debug, Serialize, JsonSchema)] -pub struct TestResult { - pub name: String, - #[schemars(flatten)] - #[serde(flatten)] - pub info: TestInfo, -} - -#[derive(Debug, Serialize, JsonSchema)] -#[schemars(untagged)] -#[serde(untagged)] -pub enum TestInfo { - Group { - children: Vec, - }, - ParseTest { - outcome: TestOutcome, - // True parse rate, adjusted parse rate - #[schemars(schema_with = "parse_rate_schema")] - #[serde(serialize_with = "serialize_parse_rates")] - parse_rate: Option<(f64, f64)>, - test_num: usize, - }, - AssertionTest { - outcome: TestOutcome, - test_num: usize, - }, -} - -fn serialize_parse_rates( - parse_rate: &Option<(f64, f64)>, - serializer: S, -) -> Result -where - S: serde::Serializer, -{ - match parse_rate { - None => serializer.serialize_none(), - Some((first, _)) => serializer.serialize_some(first), - } -} - -fn parse_rate_schema(gen: &mut SchemaGenerator) -> Schema { - gen.subschema_for::>() -} - -#[derive(Debug, Clone, Eq, PartialEq, Serialize, JsonSchema)] -pub enum TestOutcome { - // Parse outcomes - Passed, - Failed, - Updated, - Skipped, - Platform, - - // Highlight/Tag/Query outcomes - AssertionPassed { assertion_count: usize }, - AssertionFailed { error: String }, -} - -impl TestSummary { - fn fmt_parse_results(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let (count, total_adj_parse_time) = self - .parse_results - .iter() - .filter_map(|(_, result)| match result.info { - TestInfo::Group { .. } => None, - TestInfo::ParseTest { parse_rate, .. } => parse_rate, - _ => unreachable!(), - }) - .fold((0usize, 0.0f64), |(count, rate_accum), (_, adj_rate)| { - (count + 1, rate_accum + adj_rate) - }); - - let avg = total_adj_parse_time / count as f64; - let std_dev = { - let variance = self - .parse_results - .iter() - .filter_map(|(_, result)| match result.info { - TestInfo::Group { .. } => None, - TestInfo::ParseTest { parse_rate, .. } => parse_rate, - _ => unreachable!(), - }) - .map(|(_, rate_i)| (rate_i - avg).powi(2)) - .sum::() - / count as f64; - variance.sqrt() - }; - - for (depth, entry) in self.parse_results.iter() { - write!(f, "{}", " ".repeat(depth + 1))?; - match &entry.info { - TestInfo::Group { .. } => writeln!(f, "{}:", entry.name)?, - TestInfo::ParseTest { - outcome, - parse_rate, - test_num, - } => { - let (color, result_char) = match outcome { - TestOutcome::Passed => (AnsiColor::Green, "✓"), - TestOutcome::Failed => (AnsiColor::Red, "✗"), - TestOutcome::Updated => (AnsiColor::Blue, "✓"), - TestOutcome::Skipped => (AnsiColor::Yellow, "⌀"), - TestOutcome::Platform => (AnsiColor::Magenta, "⌀"), - _ => unreachable!(), - }; - let stat_display = match (self.parse_stat_display, parse_rate) { - (TestStats::TotalOnly, _) | (_, None) => String::new(), - (display, Some((true_rate, adj_rate))) => { - let mut stats = if display == TestStats::All { - format!(" ({true_rate:.3} bytes/ms)") - } else { - String::new() - }; - // 3 standard deviations below the mean, aka the "Empirical Rule" - if *adj_rate < 3.0f64.mul_add(-std_dev, avg) { - stats += &paint( - self.color.then_some(AnsiColor::Yellow), - &format!( - " -- Warning: Slow parse rate ({true_rate:.3} bytes/ms)" - ), - ); - } - stats - } - }; - writeln!( - f, - "{test_num:>3}. {result_char} {}{stat_display}", - paint(self.color.then_some(color), &entry.name), - )?; - } - TestInfo::AssertionTest { .. } => unreachable!(), - } - } - - // Parse failure info - if !self.parse_failures.is_empty() && self.update && !self.has_parse_errors { - writeln!( - f, - "\n{} update{}:\n", - self.parse_failures.len(), - if self.parse_failures.len() == 1 { - "" - } else { - "s" - } - )?; - - for (i, TestFailure { name, .. }) in self.parse_failures.iter().enumerate() { - writeln!(f, " {}. {name}", i + 1)?; - } - } else if !self.parse_failures.is_empty() && !self.overview_only { - if !self.has_parse_errors { - writeln!( - f, - "\n{} failure{}:", - self.parse_failures.len(), - if self.parse_failures.len() == 1 { - "" - } else { - "s" - } - )?; - } - - if self.color { - DiffKey.fmt(f)?; - } - for ( - i, - TestFailure { - name, - actual, - expected, - is_cst, - }, - ) in self.parse_failures.iter().enumerate() - { - if expected == "NO ERROR" { - writeln!(f, "\n {}. {name}:\n", i + 1)?; - writeln!(f, " Expected an ERROR node, but got:")?; - let actual = if *is_cst { - actual - } else { - &format_sexp(actual, 2) - }; - writeln!( - f, - " {}", - paint(self.color.then_some(AnsiColor::Red), actual) - )?; - } else { - writeln!(f, "\n {}. {name}:", i + 1)?; - if *is_cst { - writeln!( - f, - "{}", - TestDiff::new(actual, expected).with_color(self.color) - )?; - } else { - writeln!( - f, - "{}", - TestDiff::new(&format_sexp(actual, 2), &format_sexp(expected, 2)) - .with_color(self.color,) - )?; - } - } - } - } else { - writeln!(f)?; - } - - Ok(()) - } -} - -impl std::fmt::Display for TestSummary { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.fmt_parse_results(f)?; - - let mut render_assertion_results = - |name: &str, results: &TestResultHierarchy| -> std::fmt::Result { - writeln!(f, "{name}:")?; - for (depth, entry) in results.iter() { - write!(f, "{}", " ".repeat(depth + 2))?; - match &entry.info { - TestInfo::Group { .. } => writeln!(f, "{}", entry.name)?, - TestInfo::AssertionTest { outcome, test_num } => match outcome { - TestOutcome::AssertionPassed { assertion_count } => writeln!( - f, - "{:>3}. ✓ {} ({assertion_count} assertions)", - test_num, - paint(self.color.then_some(AnsiColor::Green), &entry.name) - )?, - TestOutcome::AssertionFailed { error } => { - writeln!( - f, - "{:>3}. ✗ {}", - test_num, - paint(self.color.then_some(AnsiColor::Red), &entry.name) - )?; - writeln!(f, "{} {error}", " ".repeat(depth + 1))?; - } - _ => unreachable!(), - }, - TestInfo::ParseTest { .. } => unreachable!(), - } - } - Ok(()) - }; - - if !self.highlight_results.root_group.is_empty() { - render_assertion_results("syntax highlighting", &self.highlight_results)?; - } - - if !self.tag_results.root_group.is_empty() { - render_assertion_results("tags", &self.tag_results)?; - } - - if !self.query_results.root_group.is_empty() { - render_assertion_results("queries", &self.query_results)?; - } - - write!(f, "{}", self.parse_stats)?; - - Ok(()) - } -} - -pub fn run_tests_at_path( - parser: &mut Parser, - opts: &TestOptions, - test_summary: &mut TestSummary, -) -> Result<()> { - let test_entry = parse_tests(&opts.path)?; - let mut _log_session = None; - - if opts.debug_graph { - _log_session = Some(util::log_graphs(parser, "log.html", opts.open_log)?); - } else if opts.debug { - parser.set_logger(Some(Box::new(|log_type, message| { - if log_type == LogType::Lex { - io::stderr().write_all(b" ").unwrap(); - } - writeln!(&mut io::stderr(), "{message}").unwrap(); - }))); - } - - let mut corrected_entries = Vec::new(); - run_tests( - parser, - test_entry, - opts, - test_summary, - &mut corrected_entries, - true, - )?; - - parser.stop_printing_dot_graphs(); - - if test_summary.parse_failures.is_empty() || (opts.update && !test_summary.has_parse_errors) { - Ok(()) - } else if opts.update && test_summary.has_parse_errors { - Err(anyhow!(indoc! {" - Some tests failed to parse with unexpected `ERROR` or `MISSING` nodes, as shown above, and cannot be updated automatically. - Either fix the grammar or manually update the tests if this is expected."})) - } else { - Err(anyhow!("")) - } -} - -pub fn check_queries_at_path(language: &Language, path: &Path) -> Result<()> { - if path.exists() { - for entry in WalkDir::new(path) - .into_iter() - .filter_map(std::result::Result::ok) - .filter(|e| { - e.file_type().is_file() - && e.path().extension().and_then(OsStr::to_str) == Some("scm") - && !e.path().starts_with(".") - }) - { - let filepath = entry.file_name().to_str().unwrap_or(""); - let content = fs::read_to_string(entry.path()) - .with_context(|| format!("Error reading query file {filepath:?}"))?; - Query::new(language, &content) - .with_context(|| format!("Error in query file {filepath:?}"))?; - } - } - Ok(()) -} - -pub struct DiffKey; - -impl std::fmt::Display for DiffKey { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "\ncorrect / {} / {}", - paint(Some(AnsiColor::Green), "expected"), - paint(Some(AnsiColor::Red), "unexpected") - )?; - Ok(()) - } -} - -impl DiffKey { - /// Writes [`DiffKey`] to stdout - pub fn print() { - println!("{Self}"); - } -} - -pub struct TestDiff<'a> { - pub actual: &'a str, - pub expected: &'a str, - pub color: bool, -} - -impl<'a> TestDiff<'a> { - #[must_use] - pub const fn new(actual: &'a str, expected: &'a str) -> Self { - Self { - actual, - expected, - color: true, - } - } - - #[must_use] - pub const fn with_color(mut self, color: bool) -> Self { - self.color = color; - self - } -} - -impl std::fmt::Display for TestDiff<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let diff = TextDiff::from_lines(self.actual, self.expected); - for diff in diff.iter_all_changes() { - match diff.tag() { - ChangeTag::Equal => { - if self.color { - write!(f, "{diff}")?; - } else { - write!(f, " {diff}")?; - } - } - ChangeTag::Insert => { - if self.color { - write!( - f, - "{}", - paint(Some(AnsiColor::Green), diff.as_str().unwrap()) - )?; - } else { - write!(f, "+{diff}")?; - } - if diff.missing_newline() { - writeln!(f)?; - } - } - ChangeTag::Delete => { - if self.color { - write!(f, "{}", paint(Some(AnsiColor::Red), diff.as_str().unwrap()))?; - } else { - write!(f, "-{diff}")?; - } - if diff.missing_newline() { - writeln!(f)?; - } - } - } - } - - Ok(()) - } -} - -#[derive(Debug, Serialize, JsonSchema)] -pub struct TestFailure { - name: String, - actual: String, - expected: String, - is_cst: bool, -} - -impl TestFailure { - fn new(name: T, actual: U, expected: V, is_cst: bool) -> Self - where - T: Into, - U: Into, - V: Into, - { - Self { - name: name.into(), - actual: actual.into(), - expected: expected.into(), - is_cst, - } - } -} - -struct TestCorrection { - name: String, - input: String, - output: String, - attributes_str: String, - header_delim_len: usize, - divider_delim_len: usize, -} - -impl TestCorrection { - fn new( - name: T, - input: U, - output: V, - attributes_str: W, - header_delim_len: usize, - divider_delim_len: usize, - ) -> Self - where - T: Into, - U: Into, - V: Into, - W: Into, - { - Self { - name: name.into(), - input: input.into(), - output: output.into(), - attributes_str: attributes_str.into(), - header_delim_len, - divider_delim_len, - } - } -} - -/// This will return false if we want to "fail fast". It will bail and not parse any more tests. -fn run_tests( - parser: &mut Parser, - test_entry: TestEntry, - opts: &TestOptions, - test_summary: &mut TestSummary, - corrected_entries: &mut Vec, - is_root: bool, -) -> Result { - match test_entry { - TestEntry::Example { - name, - input, - output, - header_delim_len, - divider_delim_len, - has_fields, - attributes_str, - attributes, - .. - } => { - if attributes.skip { - test_summary.parse_results.add_case(TestResult { - name: name.clone(), - info: TestInfo::ParseTest { - outcome: TestOutcome::Skipped, - parse_rate: None, - test_num: test_summary.test_num, - }, - }); - test_summary.test_num += 1; - return Ok(true); - } - - if !attributes.platform { - test_summary.parse_results.add_case(TestResult { - name: name.clone(), - info: TestInfo::ParseTest { - outcome: TestOutcome::Platform, - parse_rate: None, - test_num: test_summary.test_num, - }, - }); - test_summary.test_num += 1; - return Ok(true); - } - - for (i, language_name) in attributes.languages.iter().enumerate() { - if !language_name.is_empty() { - let language = opts - .languages - .get(language_name.as_ref()) - .ok_or_else(|| anyhow!("Language not found: {language_name}"))?; - parser.set_language(language)?; - } - let start = std::time::Instant::now(); - let tree = parser.parse(&input, None).unwrap(); - let parse_rate = { - let parse_time = start.elapsed(); - let true_parse_rate = tree.root_node().byte_range().len() as f64 - / (parse_time.as_nanos() as f64 / 1_000_000.0); - let adj_parse_rate = adjusted_parse_rate(&tree, parse_time); - - test_summary.parse_stats.total_parses += 1; - test_summary.parse_stats.total_duration += parse_time; - test_summary.parse_stats.total_bytes += tree.root_node().byte_range().len(); - - Some((true_parse_rate, adj_parse_rate)) - }; - - if attributes.error { - if tree.root_node().has_error() { - test_summary.parse_results.add_case(TestResult { - name: name.clone(), - info: TestInfo::ParseTest { - outcome: TestOutcome::Passed, - parse_rate, - test_num: test_summary.test_num, - }, - }); - test_summary.parse_stats.successful_parses += 1; - if opts.update { - let input = String::from_utf8(input.clone()).unwrap(); - let output = if attributes.cst { - output.clone() - } else { - format_sexp(&output, 0) - }; - corrected_entries.push(TestCorrection::new( - &name, - input, - output, - &attributes_str, - header_delim_len, - divider_delim_len, - )); - } - } else { - if opts.update { - let input = String::from_utf8(input.clone()).unwrap(); - // Keep the original `expected` output if the actual output has no error - let output = if attributes.cst { - output.clone() - } else { - format_sexp(&output, 0) - }; - corrected_entries.push(TestCorrection::new( - &name, - input, - output, - &attributes_str, - header_delim_len, - divider_delim_len, - )); - } - test_summary.parse_results.add_case(TestResult { - name: name.clone(), - info: TestInfo::ParseTest { - outcome: TestOutcome::Failed, - parse_rate, - test_num: test_summary.test_num, - }, - }); - let actual = if attributes.cst { - render_test_cst(&input, &tree)? - } else { - tree.root_node().to_sexp() - }; - test_summary.parse_failures.push(TestFailure::new( - &name, - actual, - "NO ERROR", - attributes.cst, - )); - } - - if attributes.fail_fast { - return Ok(false); - } - } else { - let mut actual = if attributes.cst { - render_test_cst(&input, &tree)? - } else { - tree.root_node().to_sexp() - }; - if !(attributes.cst || opts.show_fields || has_fields) { - actual = strip_sexp_fields(&actual); - } - - if actual == output { - test_summary.parse_results.add_case(TestResult { - name: name.clone(), - info: TestInfo::ParseTest { - outcome: TestOutcome::Passed, - parse_rate, - test_num: test_summary.test_num, - }, - }); - test_summary.parse_stats.successful_parses += 1; - if opts.update { - let input = String::from_utf8(input.clone()).unwrap(); - let output = if attributes.cst { - actual - } else { - format_sexp(&output, 0) - }; - corrected_entries.push(TestCorrection::new( - &name, - input, - output, - &attributes_str, - header_delim_len, - divider_delim_len, - )); - } - } else { - if opts.update { - let input = String::from_utf8(input.clone()).unwrap(); - let (expected_output, actual_output) = if attributes.cst { - (output.clone(), actual.clone()) - } else { - (format_sexp(&output, 0), format_sexp(&actual, 0)) - }; - - // Only bail early before updating if the actual is not the output, - // sometimes users want to test cases that - // are intended to have errors, hence why this - // check isn't shown above - if actual.contains("ERROR") || actual.contains("MISSING") { - test_summary.has_parse_errors = true; - - // keep the original `expected` output if the actual output has an - // error - corrected_entries.push(TestCorrection::new( - &name, - input, - expected_output, - &attributes_str, - header_delim_len, - divider_delim_len, - )); - } else { - corrected_entries.push(TestCorrection::new( - &name, - input, - actual_output, - &attributes_str, - header_delim_len, - divider_delim_len, - )); - test_summary.parse_results.add_case(TestResult { - name: name.clone(), - info: TestInfo::ParseTest { - outcome: TestOutcome::Updated, - parse_rate, - test_num: test_summary.test_num, - }, - }); - } - } else { - test_summary.parse_results.add_case(TestResult { - name: name.clone(), - info: TestInfo::ParseTest { - outcome: TestOutcome::Failed, - parse_rate, - test_num: test_summary.test_num, - }, - }); - } - test_summary.parse_failures.push(TestFailure::new( - &name, - actual, - &output, - attributes.cst, - )); - - if attributes.fail_fast { - return Ok(false); - } - } - } - - if i == attributes.languages.len() - 1 { - // reset to the first language - parser.set_language(opts.languages.values().next().unwrap())?; - } - } - test_summary.test_num += 1; - } - TestEntry::Group { - name, - children, - file_path, - } => { - if children.is_empty() { - return Ok(true); - } - - let mut ran_test_in_group = false; - - let matches_filter = |name: &str, file_name: &Option, opts: &TestOptions| { - if let (Some(test_file_path), Some(filter_file_name)) = (file_name, &opts.file_name) - { - if !filter_file_name.eq(test_file_path) { - return false; - } - } - if let Some(include) = &opts.include { - include.is_match(name) - } else if let Some(exclude) = &opts.exclude { - !exclude.is_match(name) - } else { - true - } - }; - - for child in children { - if let TestEntry::Example { - ref name, - ref file_name, - ref input, - ref output, - ref attributes_str, - header_delim_len, - divider_delim_len, - .. - } = child - { - if !matches_filter(name, file_name, opts) { - if opts.update { - let input = String::from_utf8(input.clone()).unwrap(); - let output = format_sexp(output, 0); - corrected_entries.push(TestCorrection::new( - name, - input, - output, - attributes_str, - header_delim_len, - divider_delim_len, - )); - } - - test_summary.test_num += 1; - continue; - } - } - - if !ran_test_in_group && !is_root { - test_summary.parse_results.add_group(&name); - ran_test_in_group = true; - } - if !run_tests(parser, child, opts, test_summary, corrected_entries, false)? { - // fail fast - return Ok(false); - } - } - // Now that we're done traversing the children of the current group, pop - // the index - test_summary.parse_results.pop_traversal(); - - if let Some(file_path) = file_path { - if opts.update { - write_tests(&file_path, corrected_entries)?; - } - corrected_entries.clear(); - } - } - } - Ok(true) -} - -/// Convenience wrapper to render a CST for a test entry. -fn render_test_cst(input: &[u8], tree: &Tree) -> Result { - let mut rendered_cst: Vec = Vec::new(); - let mut cursor = tree.walk(); - let opts = ParseFileOptions { - edits: &[], - output: ParseOutput::Cst, - stats: &mut ParseStats::default(), - print_time: false, - timeout: 0, - debug: ParseDebugType::Quiet, - debug_graph: false, - cancellation_flag: None, - encoding: None, - open_log: false, - no_ranges: false, - parse_theme: &ParseTheme::empty(), - }; - render_cst(input, tree, &mut cursor, &opts, &mut rendered_cst)?; - Ok(String::from_utf8_lossy(&rendered_cst).trim().to_string()) -} - -// Parse time is interpreted in ns before converting to ms to avoid truncation issues -// Parse rates often have several outliers, leading to a large standard deviation. Taking -// the log of these rates serves to "flatten" out the distribution, yielding a more -// usable standard deviation for finding statistically significant slow parse rates -// NOTE: This is just a heuristic -#[must_use] -pub fn adjusted_parse_rate(tree: &Tree, parse_time: Duration) -> f64 { - f64::ln( - tree.root_node().byte_range().len() as f64 / (parse_time.as_nanos() as f64 / 1_000_000.0), - ) -} - -fn write_tests(file_path: &Path, corrected_entries: &[TestCorrection]) -> Result<()> { - let mut buffer = fs::File::create(file_path)?; - write_tests_to_buffer(&mut buffer, corrected_entries) -} - -fn write_tests_to_buffer( - buffer: &mut impl Write, - corrected_entries: &[TestCorrection], -) -> Result<()> { - for ( - i, - TestCorrection { - name, - input, - output, - attributes_str, - header_delim_len, - divider_delim_len, - }, - ) in corrected_entries.iter().enumerate() - { - if i > 0 { - writeln!(buffer)?; - } - writeln!( - buffer, - "{}\n{name}\n{}{}\n{input}\n{}\n\n{}", - "=".repeat(*header_delim_len), - if attributes_str.is_empty() { - attributes_str.clone() - } else { - format!("{attributes_str}\n") - }, - "=".repeat(*header_delim_len), - "-".repeat(*divider_delim_len), - output.trim() - )?; - } - Ok(()) -} - -pub fn parse_tests(path: &Path) -> io::Result { - let name = path - .file_stem() - .and_then(|s| s.to_str()) - .unwrap_or("") - .to_string(); - if path.is_dir() { - let mut children = Vec::new(); - for entry in fs::read_dir(path)? { - let entry = entry?; - let hidden = entry.file_name().to_str().unwrap_or("").starts_with('.'); - if !hidden { - children.push(entry.path()); - } - } - children.sort_by(|a, b| { - a.file_name() - .unwrap_or_default() - .cmp(b.file_name().unwrap_or_default()) - }); - let children = children - .iter() - .map(|path| parse_tests(path)) - .collect::>>()?; - Ok(TestEntry::Group { - name, - children, - file_path: None, - }) - } else { - let content = fs::read_to_string(path)?; - Ok(parse_test_content(name, &content, Some(path.to_path_buf()))) - } -} - -#[must_use] -pub fn strip_sexp_fields(sexp: &str) -> String { - SEXP_FIELD_REGEX.replace_all(sexp, " (").to_string() -} - -#[must_use] -pub fn strip_points(sexp: &str) -> String { - POINT_REGEX.replace_all(sexp, "").to_string() -} - -fn parse_test_content(name: String, content: &str, file_path: Option) -> TestEntry { - let mut children = Vec::new(); - let bytes = content.as_bytes(); - let mut prev_name = String::new(); - let mut prev_attributes_str = String::new(); - let mut prev_header_end = 0; - - // Find the first test header in the file, and determine if it has a - // custom suffix. If so, then this suffix will be used to identify - // all subsequent headers and divider lines in the file. - let first_suffix = HEADER_REGEX - .captures(bytes) - .and_then(|c| c.name("suffix1")) - .map(|m| String::from_utf8_lossy(m.as_bytes())); - - // Find all of the `===` test headers, which contain the test names. - // Ignore any matches whose suffix does not match the first header - // suffix in the file. - let header_matches = HEADER_REGEX.captures_iter(bytes).filter_map(|c| { - let header_delim_len = c.name("equals").map_or(80, |m| m.as_bytes().len()); - let suffix1 = c - .name("suffix1") - .map(|m| String::from_utf8_lossy(m.as_bytes())); - let suffix2 = c - .name("suffix2") - .map(|m| String::from_utf8_lossy(m.as_bytes())); - - let (mut skip, mut platform, mut fail_fast, mut error, mut cst, mut languages) = - (false, None, false, false, false, vec![]); - - let test_name_and_markers = c - .name("test_name_and_markers") - .map_or("".as_bytes(), |m| m.as_bytes()); - - let mut test_name = String::new(); - let mut attributes_str = String::new(); - - let mut seen_marker = false; - - let test_name_and_markers = str::from_utf8(test_name_and_markers).unwrap(); - for line in test_name_and_markers - .split_inclusive('\n') - .filter(|s| !s.is_empty()) - { - let trimmed_line = line.trim(); - match trimmed_line.split('(').next().unwrap() { - ":skip" => (seen_marker, skip) = (true, true), - ":platform" => { - if let Some(platforms) = trimmed_line.strip_prefix(':').and_then(|s| { - s.strip_prefix("platform(") - .and_then(|s| s.strip_suffix(')')) - }) { - seen_marker = true; - platform = Some( - platform.unwrap_or(false) || platforms.trim() == std::env::consts::OS, - ); - } - } - ":fail-fast" => (seen_marker, fail_fast) = (true, true), - ":error" => (seen_marker, error) = (true, true), - ":language" => { - if let Some(lang) = trimmed_line.strip_prefix(':').and_then(|s| { - s.strip_prefix("language(") - .and_then(|s| s.strip_suffix(')')) - }) { - seen_marker = true; - languages.push(lang.into()); - } - } - ":cst" => (seen_marker, cst) = (true, true), - _ if !seen_marker => { - test_name.push_str(line); - } - _ => {} - } - } - attributes_str.push_str(test_name_and_markers.strip_prefix(&test_name).unwrap()); - - // prefer skip over error, both shouldn't be set - if skip { - error = false; - } - - // add a default language if none are specified, will defer to the first language - if languages.is_empty() { - languages.push("".into()); - } - - if suffix1 == first_suffix && suffix2 == first_suffix { - let header_range = c.get(0).unwrap().range(); - let test_name = if test_name.is_empty() { - None - } else { - Some(test_name.trim_end().to_string()) - }; - let attributes_str = if attributes_str.is_empty() { - None - } else { - Some(attributes_str.trim_end().to_string()) - }; - Some(( - header_delim_len, - header_range, - test_name, - attributes_str, - TestAttributes { - skip, - platform: platform.unwrap_or(true), - fail_fast, - error, - cst, - languages, - }, - )) - } else { - None - } - }); - - let (mut prev_header_len, mut prev_attributes) = (80, TestAttributes::default()); - for (header_delim_len, header_range, test_name, attributes_str, attributes) in header_matches - .chain(Some(( - 80, - bytes.len()..bytes.len(), - None, - None, - TestAttributes::default(), - ))) - { - // Find the longest line of dashes following each test description. That line - // separates the input from the expected output. Ignore any matches whose suffix - // does not match the first suffix in the file. - if prev_header_end > 0 { - let divider_range = DIVIDER_REGEX - .captures_iter(&bytes[prev_header_end..header_range.start]) - .filter_map(|m| { - let divider_delim_len = m.name("hyphens").map_or(80, |m| m.as_bytes().len()); - let suffix = m - .name("suffix") - .map(|m| String::from_utf8_lossy(m.as_bytes())); - if suffix == first_suffix { - let range = m.get(0).unwrap().range(); - Some(( - divider_delim_len, - (prev_header_end + range.start)..(prev_header_end + range.end), - )) - } else { - None - } - }) - .max_by_key(|(_, range)| range.len()); - - if let Some((divider_delim_len, divider_range)) = divider_range { - if let Ok(output) = str::from_utf8(&bytes[divider_range.end..header_range.start]) { - let mut input = bytes[prev_header_end..divider_range.start].to_vec(); - - // Remove trailing newline from the input. - input.pop(); - if input.last() == Some(&b'\r') { - input.pop(); - } - - let (output, has_fields) = if prev_attributes.cst { - (output.trim().to_string(), false) - } else { - // Remove all comments - let output = COMMENT_REGEX.replace_all(output, "").to_string(); - - // Normalize the whitespace in the expected output. - let output = WHITESPACE_REGEX.replace_all(output.trim(), " "); - let output = output.replace(" )", ")"); - - // Identify if the expected output has fields indicated. If not, then - // fields will not be checked. - let has_fields = SEXP_FIELD_REGEX.is_match(&output); - - (output, has_fields) - }; - - let file_name = if let Some(ref path) = file_path { - path.file_name().map(|n| n.to_string_lossy().to_string()) - } else { - None - }; - - let t = TestEntry::Example { - name: prev_name, - input, - output, - header_delim_len: prev_header_len, - divider_delim_len, - has_fields, - attributes_str: prev_attributes_str, - attributes: prev_attributes, - file_name, - }; - - children.push(t); - } - } - } - prev_attributes = attributes; - prev_name = test_name.unwrap_or_default(); - prev_attributes_str = attributes_str.unwrap_or_default(); - prev_header_len = header_delim_len; - prev_header_end = header_range.end; - } - TestEntry::Group { - name, - children, - file_path, - } -} - -#[cfg(test)] -mod tests { - use serde_json::json; - - use crate::tests::get_language; - - use super::*; - - #[test] - fn test_parse_test_content_simple() { - let entry = parse_test_content( - "the-filename".to_string(), - r" -=============== -The first test -=============== - -a b c - ---- - -(a - (b c)) - -================ -The second test -================ -d ---- -(d) - " - .trim(), - None, - ); - - assert_eq!( - entry, - TestEntry::Group { - name: "the-filename".to_string(), - children: vec![ - TestEntry::Example { - name: "The first test".to_string(), - input: b"\na b c\n".to_vec(), - output: "(a (b c))".to_string(), - header_delim_len: 15, - divider_delim_len: 3, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - file_name: None, - }, - TestEntry::Example { - name: "The second test".to_string(), - input: b"d".to_vec(), - output: "(d)".to_string(), - header_delim_len: 16, - divider_delim_len: 3, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - file_name: None, - }, - ], - file_path: None, - } - ); - } - - #[test] - fn test_parse_test_content_with_dashes_in_source_code() { - let entry = parse_test_content( - "the-filename".to_string(), - r" -================== -Code with dashes -================== -abc ---- -defg ----- -hijkl -------- - -(a (b)) - -========================= -Code ending with dashes -========================= -abc ------------ -------------------- - -(c (d)) - " - .trim(), - None, - ); - - assert_eq!( - entry, - TestEntry::Group { - name: "the-filename".to_string(), - children: vec![ - TestEntry::Example { - name: "Code with dashes".to_string(), - input: b"abc\n---\ndefg\n----\nhijkl".to_vec(), - output: "(a (b))".to_string(), - header_delim_len: 18, - divider_delim_len: 7, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - file_name: None, - }, - TestEntry::Example { - name: "Code ending with dashes".to_string(), - input: b"abc\n-----------".to_vec(), - output: "(c (d))".to_string(), - header_delim_len: 25, - divider_delim_len: 19, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - file_name: None, - }, - ], - file_path: None, - } - ); - } - - #[test] - fn test_format_sexp() { - assert_eq!(format_sexp("", 0), ""); - assert_eq!( - format_sexp("(a b: (c) (d) e: (f (g (h (MISSING i)))))", 0), - r" -(a - b: (c) - (d) - e: (f - (g - (h - (MISSING i))))) -" - .trim() - ); - assert_eq!( - format_sexp("(program (ERROR (UNEXPECTED ' ')) (identifier))", 0), - r" -(program - (ERROR - (UNEXPECTED ' ')) - (identifier)) -" - .trim() - ); - assert_eq!( - format_sexp(r#"(source_file (MISSING ")"))"#, 0), - r#" -(source_file - (MISSING ")")) - "# - .trim() - ); - assert_eq!( - format_sexp( - r"(source_file (ERROR (UNEXPECTED 'f') (UNEXPECTED '+')))", - 0 - ), - r" -(source_file - (ERROR - (UNEXPECTED 'f') - (UNEXPECTED '+'))) -" - .trim() - ); - } - - #[test] - fn test_write_tests_to_buffer() { - let mut buffer = Vec::new(); - let corrected_entries = vec![ - TestCorrection::new( - "title 1".to_string(), - "input 1".to_string(), - "output 1".to_string(), - String::new(), - 80, - 80, - ), - TestCorrection::new( - "title 2".to_string(), - "input 2".to_string(), - "output 2".to_string(), - String::new(), - 80, - 80, - ), - ]; - write_tests_to_buffer(&mut buffer, &corrected_entries).unwrap(); - assert_eq!( - String::from_utf8(buffer).unwrap(), - r" -================================================================================ -title 1 -================================================================================ -input 1 --------------------------------------------------------------------------------- - -output 1 - -================================================================================ -title 2 -================================================================================ -input 2 --------------------------------------------------------------------------------- - -output 2 -" - .trim_start() - .to_string() - ); - } - - #[test] - fn test_parse_test_content_with_comments_in_sexp() { - let entry = parse_test_content( - "the-filename".to_string(), - r#" -================== -sexp with comment -================== -code ---- - -; Line start comment -(a (b)) - -================== -sexp with comment between -================== -code ---- - -; Line start comment -(a -; ignore this - (b) - ; also ignore this -) - -========================= -sexp with ';' -========================= -code ---- - -(MISSING ";") - "# - .trim(), - None, - ); - - assert_eq!( - entry, - TestEntry::Group { - name: "the-filename".to_string(), - children: vec![ - TestEntry::Example { - name: "sexp with comment".to_string(), - input: b"code".to_vec(), - output: "(a (b))".to_string(), - header_delim_len: 18, - divider_delim_len: 3, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - file_name: None, - }, - TestEntry::Example { - name: "sexp with comment between".to_string(), - input: b"code".to_vec(), - output: "(a (b))".to_string(), - header_delim_len: 18, - divider_delim_len: 3, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - file_name: None, - }, - TestEntry::Example { - name: "sexp with ';'".to_string(), - input: b"code".to_vec(), - output: "(MISSING \";\")".to_string(), - header_delim_len: 25, - divider_delim_len: 3, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - file_name: None, - } - ], - file_path: None, - } - ); - } - - #[test] - fn test_parse_test_content_with_suffixes() { - let entry = parse_test_content( - "the-filename".to_string(), - r" -==================asdf\()[]|{}*+?^$.- -First test -==================asdf\()[]|{}*+?^$.- - -========================= -NOT A TEST HEADER -========================= -------------------------- - ----asdf\()[]|{}*+?^$.- - -(a) - -==================asdf\()[]|{}*+?^$.- -Second test -==================asdf\()[]|{}*+?^$.- - -========================= -NOT A TEST HEADER -========================= -------------------------- - ----asdf\()[]|{}*+?^$.- - -(a) - -=========================asdf\()[]|{}*+?^$.- -Test name with = symbol -=========================asdf\()[]|{}*+?^$.- - -========================= -NOT A TEST HEADER -========================= -------------------------- - ----asdf\()[]|{}*+?^$.- - -(a) - -==============================asdf\()[]|{}*+?^$.- -Test containing equals -==============================asdf\()[]|{}*+?^$.- - -=== - -------------------------------asdf\()[]|{}*+?^$.- - -(a) - -==============================asdf\()[]|{}*+?^$.- -Subsequent test containing equals -==============================asdf\()[]|{}*+?^$.- - -=== - -------------------------------asdf\()[]|{}*+?^$.- - -(a) -" - .trim(), - None, - ); - - let expected_input = b"\n=========================\n\ - NOT A TEST HEADER\n\ - =========================\n\ - -------------------------\n" - .to_vec(); - pretty_assertions::assert_eq!( - entry, - TestEntry::Group { - name: "the-filename".to_string(), - children: vec![ - TestEntry::Example { - name: "First test".to_string(), - input: expected_input.clone(), - output: "(a)".to_string(), - header_delim_len: 18, - divider_delim_len: 3, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - file_name: None, - }, - TestEntry::Example { - name: "Second test".to_string(), - input: expected_input.clone(), - output: "(a)".to_string(), - header_delim_len: 18, - divider_delim_len: 3, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - file_name: None, - }, - TestEntry::Example { - name: "Test name with = symbol".to_string(), - input: expected_input, - output: "(a)".to_string(), - header_delim_len: 25, - divider_delim_len: 3, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - file_name: None, - }, - TestEntry::Example { - name: "Test containing equals".to_string(), - input: "\n===\n".into(), - output: "(a)".into(), - header_delim_len: 30, - divider_delim_len: 30, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - file_name: None, - }, - TestEntry::Example { - name: "Subsequent test containing equals".to_string(), - input: "\n===\n".into(), - output: "(a)".into(), - header_delim_len: 30, - divider_delim_len: 30, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - file_name: None, - } - ], - file_path: None, - } - ); - } - - #[test] - fn test_parse_test_content_with_newlines_in_test_names() { - let entry = parse_test_content( - "the-filename".to_string(), - r" -=============== -name -with -newlines -=============== -a ---- -(b) - -==================== -name with === signs -==================== -code with ---- ---- -(d) -", - None, - ); - - assert_eq!( - entry, - TestEntry::Group { - name: "the-filename".to_string(), - file_path: None, - children: vec![ - TestEntry::Example { - name: "name\nwith\nnewlines".to_string(), - input: b"a".to_vec(), - output: "(b)".to_string(), - header_delim_len: 15, - divider_delim_len: 3, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - file_name: None, - }, - TestEntry::Example { - name: "name with === signs".to_string(), - input: b"code with ----".to_vec(), - output: "(d)".to_string(), - header_delim_len: 20, - divider_delim_len: 3, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - file_name: None, - } - ] - } - ); - } - - #[test] - fn test_parse_test_with_markers() { - // do one with :skip, we should not see it in the entry output - - let entry = parse_test_content( - "the-filename".to_string(), - r" -===================== -Test with skip marker -:skip -===================== -a ---- -(b) -", - None, - ); - - assert_eq!( - entry, - TestEntry::Group { - name: "the-filename".to_string(), - file_path: None, - children: vec![TestEntry::Example { - name: "Test with skip marker".to_string(), - input: b"a".to_vec(), - output: "(b)".to_string(), - header_delim_len: 21, - divider_delim_len: 3, - has_fields: false, - attributes_str: ":skip".to_string(), - attributes: TestAttributes { - skip: true, - platform: true, - fail_fast: false, - error: false, - cst: false, - languages: vec!["".into()] - }, - file_name: None, - }] - } - ); - - let entry = parse_test_content( - "the-filename".to_string(), - &format!( - r" -========================= -Test with platform marker -:platform({}) -:fail-fast -========================= -a ---- -(b) - -============================= -Test with bad platform marker -:platform({}) - -:language(foo) -============================= -a ---- -(b) - -==================== -Test with cst marker -:cst -==================== -1 ---- -0:0 - 1:0 source_file -0:0 - 0:1 expression -0:0 - 0:1 number_literal `1` -", - std::env::consts::OS, - if std::env::consts::OS == "linux" { - "macos" - } else { - "linux" - } - ), - None, - ); - - assert_eq!( - entry, - TestEntry::Group { - name: "the-filename".to_string(), - file_path: None, - children: vec![ - TestEntry::Example { - name: "Test with platform marker".to_string(), - input: b"a".to_vec(), - output: "(b)".to_string(), - header_delim_len: 25, - divider_delim_len: 3, - has_fields: false, - attributes_str: format!(":platform({})\n:fail-fast", std::env::consts::OS), - attributes: TestAttributes { - skip: false, - platform: true, - fail_fast: true, - error: false, - cst: false, - languages: vec!["".into()] - }, - file_name: None, - }, - TestEntry::Example { - name: "Test with bad platform marker".to_string(), - input: b"a".to_vec(), - output: "(b)".to_string(), - header_delim_len: 29, - divider_delim_len: 3, - has_fields: false, - attributes_str: if std::env::consts::OS == "linux" { - ":platform(macos)\n\n:language(foo)".to_string() - } else { - ":platform(linux)\n\n:language(foo)".to_string() - }, - attributes: TestAttributes { - skip: false, - platform: false, - fail_fast: false, - error: false, - cst: false, - languages: vec!["foo".into()] - }, - file_name: None, - }, - TestEntry::Example { - name: "Test with cst marker".to_string(), - input: b"1".to_vec(), - output: "0:0 - 1:0 source_file -0:0 - 0:1 expression -0:0 - 0:1 number_literal `1`" - .to_string(), - header_delim_len: 20, - divider_delim_len: 3, - has_fields: false, - attributes_str: ":cst".to_string(), - attributes: TestAttributes { - skip: false, - platform: true, - fail_fast: false, - error: false, - cst: true, - languages: vec!["".into()] - }, - file_name: None, - } - ] - } - ); - } - - fn clear_parse_rate(result: &mut TestResult) { - let test_case_info = &mut result.info; - match test_case_info { - TestInfo::ParseTest { - ref mut parse_rate, .. - } => { - assert!(parse_rate.is_some()); - *parse_rate = None; - } - TestInfo::Group { .. } | TestInfo::AssertionTest { .. } => { - panic!("Unexpected test result") - } - } - } - - #[test] - fn run_tests_simple() { - let mut parser = Parser::new(); - let language = get_language("c"); - parser - .set_language(&language) - .expect("Failed to set language"); - let mut languages = BTreeMap::new(); - languages.insert("c", &language); - let opts = TestOptions { - path: PathBuf::from("foo"), - debug: true, - debug_graph: false, - include: None, - exclude: None, - file_name: None, - update: false, - open_log: false, - languages, - color: true, - show_fields: false, - overview_only: false, - }; - - // NOTE: The following test cases are combined to work around a race condition - // in the loader - { - let test_entry = TestEntry::Group { - name: "foo".to_string(), - file_path: None, - children: vec![TestEntry::Example { - name: "C Test 1".to_string(), - input: b"1;\n".to_vec(), - output: "(translation_unit (expression_statement (number_literal)))" - .to_string(), - header_delim_len: 25, - divider_delim_len: 3, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - file_name: None, - }], - }; - - let mut test_summary = TestSummary::new(true, TestStats::All, false, false, false); - let mut corrected_entries = Vec::new(); - run_tests( - &mut parser, - test_entry, - &opts, - &mut test_summary, - &mut corrected_entries, - true, - ) - .expect("Failed to run tests"); - - // parse rates will always be different, so we need to clear out these - // fields to reliably assert equality below - clear_parse_rate(&mut test_summary.parse_results.root_group[0]); - test_summary.parse_stats.total_duration = Duration::from_secs(0); - - let json_results = serde_json::to_string(&test_summary).unwrap(); - - assert_eq!( - json_results, - json!({ - "parse_results": [ - { - "name": "C Test 1", - "outcome": "Passed", - "parse_rate": null, - "test_num": 1 - } - ], - "parse_failures": [], - "parse_stats": { - "successful_parses": 1, - "total_parses": 1, - "total_bytes": 3, - "total_duration": { - "secs": 0, - "nanos": 0, - } - }, - "highlight_results": [], - "tag_results": [], - "query_results": [] - }) - .to_string() - ); - } - { - let test_entry = TestEntry::Group { - name: "corpus".to_string(), - file_path: None, - children: vec![ - TestEntry::Group { - name: "group1".to_string(), - // This test passes - children: vec![TestEntry::Example { - name: "C Test 1".to_string(), - input: b"1;\n".to_vec(), - output: "(translation_unit (expression_statement (number_literal)))" - .to_string(), - header_delim_len: 25, - divider_delim_len: 3, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - file_name: None, - }], - file_path: None, - }, - TestEntry::Group { - name: "group2".to_string(), - children: vec![ - // This test passes - TestEntry::Example { - name: "C Test 2".to_string(), - input: b"1;\n".to_vec(), - output: - "(translation_unit (expression_statement (number_literal)))" - .to_string(), - header_delim_len: 25, - divider_delim_len: 3, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - file_name: None, - }, - // This test fails, and is marked with fail-fast - TestEntry::Example { - name: "C Test 3".to_string(), - input: b"1;\n".to_vec(), - output: - "(translation_unit (expression_statement (string_literal)))" - .to_string(), - header_delim_len: 25, - divider_delim_len: 3, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes { - fail_fast: true, - ..Default::default() - }, - file_name: None, - }, - ], - file_path: None, - }, - // This group never runs because of the previous failure - TestEntry::Group { - name: "group3".to_string(), - // This test fails, and is marked with fail-fast - children: vec![TestEntry::Example { - name: "C Test 4".to_string(), - input: b"1;\n".to_vec(), - output: "(translation_unit (expression_statement (number_literal)))" - .to_string(), - header_delim_len: 25, - divider_delim_len: 3, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - file_name: None, - }], - file_path: None, - }, - ], - }; - - let mut test_summary = TestSummary::new(true, TestStats::All, false, false, false); - let mut corrected_entries = Vec::new(); - run_tests( - &mut parser, - test_entry, - &opts, - &mut test_summary, - &mut corrected_entries, - true, - ) - .expect("Failed to run tests"); - - // parse rates will always be different, so we need to clear out these - // fields to reliably assert equality below - { - let test_group_1_info = &mut test_summary.parse_results.root_group[0].info; - match test_group_1_info { - TestInfo::Group { - ref mut children, .. - } => clear_parse_rate(&mut children[0]), - TestInfo::ParseTest { .. } | TestInfo::AssertionTest { .. } => { - panic!("Unexpected test result"); - } - } - let test_group_2_info = &mut test_summary.parse_results.root_group[1].info; - match test_group_2_info { - TestInfo::Group { - ref mut children, .. - } => { - clear_parse_rate(&mut children[0]); - clear_parse_rate(&mut children[1]); - } - TestInfo::ParseTest { .. } | TestInfo::AssertionTest { .. } => { - panic!("Unexpected test result"); - } - } - test_summary.parse_stats.total_duration = Duration::from_secs(0); - } - - let json_results = serde_json::to_string(&test_summary).unwrap(); - - assert_eq!( - json_results, - json!({ - "parse_results": [ - { - "name": "group1", - "children": [ - { - "name": "C Test 1", - "outcome": "Passed", - "parse_rate": null, - "test_num": 1 - } - ] - }, - { - "name": "group2", - "children": [ - { - "name": "C Test 2", - "outcome": "Passed", - "parse_rate": null, - "test_num": 2 - }, - { - "name": "C Test 3", - "outcome": "Failed", - "parse_rate": null, - "test_num": 3 - } - ] - } - ], - "parse_failures": [ - { - "name": "C Test 3", - "actual": "(translation_unit (expression_statement (number_literal)))", - "expected": "(translation_unit (expression_statement (string_literal)))", - "is_cst": false, - } - ], - "parse_stats": { - "successful_parses": 2, - "total_parses": 3, - "total_bytes": 9, - "total_duration": { - "secs": 0, - "nanos": 0, - } - }, - "highlight_results": [], - "tag_results": [], - "query_results": [] - }) - .to_string() - ); - } - } -} diff --git a/crates/cli/src/tests/async_boundary_test.rs b/crates/cli/src/tests/async_boundary_test.rs deleted file mode 100644 index 254ed931..00000000 --- a/crates/cli/src/tests/async_boundary_test.rs +++ /dev/null @@ -1,150 +0,0 @@ -use std::{ - future::Future, - pin::Pin, - ptr, - task::{Context, Poll, RawWaker, RawWakerVTable, Waker}, -}; - -use tree_sitter::Parser; - -use super::helpers::fixtures::get_language; - -#[test] -fn test_node_across_async_boundaries() { - let mut parser = Parser::new(); - let language = get_language("bash"); - parser.set_language(&language).unwrap(); - let tree = parser.parse("#", None).unwrap(); - let root = tree.root_node(); - - let (result, yields) = simple_async_executor(async { - let root_ref = &root; - - // Test node captured by value - let fut_by_value = async { - yield_once().await; - root.child(0).unwrap().kind() - }; - - // Test node captured by reference - let fut_by_ref = async { - yield_once().await; - root_ref.child(0).unwrap().kind() - }; - - let result1 = fut_by_value.await; - let result2 = fut_by_ref.await; - - assert_eq!(result1, result2); - result1 - }); - - assert_eq!(result, "comment"); - assert_eq!(yields, 2); -} - -#[test] -fn test_cursor_across_async_boundaries() { - let mut parser = Parser::new(); - let language = get_language("c"); - parser.set_language(&language).unwrap(); - let tree = parser.parse("#", None).unwrap(); - let mut cursor = tree.walk(); - - let ((), yields) = simple_async_executor(async { - cursor.goto_first_child(); - - // Test cursor usage across yield point - yield_once().await; - cursor.goto_first_child(); - - // Test cursor in async block - let cursor_ref = &mut cursor; - let fut = async { - yield_once().await; - cursor_ref.goto_first_child(); - }; - fut.await; - }); - - assert_eq!(yields, 2); -} - -#[test] -fn test_node_and_cursor_together() { - let mut parser = Parser::new(); - let language = get_language("javascript"); - parser.set_language(&language).unwrap(); - let tree = parser.parse("#", None).unwrap(); - let root = tree.root_node(); - let mut cursor = tree.walk(); - - let ((), yields) = simple_async_executor(async { - cursor.goto_first_child(); - - let fut = async { - yield_once().await; - let _ = root.to_sexp(); - cursor.goto_first_child(); - }; - - yield_once().await; - fut.await; - }); - - assert_eq!(yields, 2); -} - -fn simple_async_executor(future: F) -> (F::Output, u32) -where - F: Future, -{ - let waker = noop_waker(); - let mut cx = Context::from_waker(&waker); - let mut yields = 0; - let mut future = Box::pin(future); - - loop { - match future.as_mut().poll(&mut cx) { - Poll::Ready(result) => return (result, yields), - Poll::Pending => yields += 1, - } - } -} - -async fn yield_once() { - struct YieldOnce { - yielded: bool, - } - - impl Future for YieldOnce { - type Output = (); - - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { - if self.yielded { - Poll::Ready(()) - } else { - self.yielded = true; - cx.waker().wake_by_ref(); - Poll::Pending - } - } - } - - YieldOnce { yielded: false }.await; -} - -const fn noop_waker() -> Waker { - const VTABLE: RawWakerVTable = RawWakerVTable::new( - // Cloning just returns a new no-op raw waker - |_| RAW, - // `wake` does nothing - |_| {}, - // `wake_by_ref` does nothing - |_| {}, - // Dropping does nothing as we don't allocate anything - |_| {}, - ); - const RAW: RawWaker = RawWaker::new(ptr::null(), &VTABLE); - unsafe { Waker::from_raw(RAW) } -} diff --git a/crates/cli/src/version.rs b/crates/cli/src/version.rs deleted file mode 100644 index 757306c7..00000000 --- a/crates/cli/src/version.rs +++ /dev/null @@ -1,396 +0,0 @@ -use std::{fs, path::PathBuf, process::Command}; - -use clap::ValueEnum; -use log::{info, warn}; -use regex::Regex; -use semver::Version as SemverVersion; -use std::cmp::Ordering; -use tree_sitter_loader::TreeSitterJSON; - -#[derive(Clone, Copy, Default, ValueEnum)] -pub enum BumpLevel { - #[default] - Patch, - Minor, - Major, -} - -pub struct Version { - pub version: Option, - pub current_dir: PathBuf, - pub bump: Option, -} - -#[derive(thiserror::Error, Debug)] -pub enum VersionError { - #[error(transparent)] - Json(#[from] serde_json::Error), - #[error(transparent)] - Io(#[from] std::io::Error), - #[error("Failed to update one or more files:\n\n{0}")] - Update(UpdateErrors), -} - -#[derive(thiserror::Error, Debug)] -pub struct UpdateErrors(Vec); - -impl std::fmt::Display for UpdateErrors { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - for error in &self.0 { - writeln!(f, "{error}\n")?; - } - Ok(()) - } -} - -#[derive(thiserror::Error, Debug)] -pub enum UpdateError { - #[error("Failed to update {1}:\n{0}")] - Io(std::io::Error, PathBuf), - #[error("Failed to run `{0}`:\n{1}")] - Command(&'static str, String), -} - -impl Version { - #[must_use] - pub const fn new( - version: Option, - current_dir: PathBuf, - bump: Option, - ) -> Self { - Self { - version, - current_dir, - bump, - } - } - - pub fn run(mut self) -> Result<(), VersionError> { - let tree_sitter_json = self.current_dir.join("tree-sitter.json"); - - let tree_sitter_json = - serde_json::from_str::(&fs::read_to_string(tree_sitter_json)?)?; - - let current_version = tree_sitter_json.metadata.version; - self.version = match (self.version.is_some(), self.bump) { - (false, None) => { - info!("Current version: {current_version}"); - return Ok(()); - } - (true, None) => self.version, - (false, Some(bump)) => { - let mut v = current_version.clone(); - match bump { - BumpLevel::Patch => v.patch += 1, - BumpLevel::Minor => { - v.minor += 1; - v.patch = 0; - } - BumpLevel::Major => { - v.major += 1; - v.minor = 0; - v.patch = 0; - } - } - Some(v) - } - (true, Some(_)) => unreachable!(), - }; - - let new_version = self.version.as_ref().unwrap(); - match new_version.cmp(¤t_version) { - Ordering::Less => { - warn!("New version is lower than current!"); - warn!("Reverting version {current_version} to {new_version}"); - } - Ordering::Greater => { - info!("Bumping version {current_version} to {new_version}"); - } - Ordering::Equal => { - info!("Keeping version {current_version}"); - } - } - - let is_multigrammar = tree_sitter_json.grammars.len() > 1; - - let mut errors = Vec::new(); - - // Helper to push errors into the errors vector, returns true if an error was pushed - let mut push_err = |result: Result<(), UpdateError>| -> bool { - if let Err(e) = result { - errors.push(e); - return true; - } - false - }; - - push_err(self.update_treesitter_json()); - - // Only update Cargo.lock if Cargo.toml was updated - push_err(self.update_cargo_toml()).then(|| push_err(self.update_cargo_lock())); - - // Only update package-lock.json if package.json was updated - push_err(self.update_package_json()).then(|| push_err(self.update_package_lock_json())); - - push_err(self.update_makefile(is_multigrammar)); - push_err(self.update_cmakelists_txt()); - push_err(self.update_pyproject_toml()); - push_err(self.update_zig_zon()); - - if errors.is_empty() { - Ok(()) - } else { - Err(VersionError::Update(UpdateErrors(errors))) - } - } - - fn update_file_with(&self, path: &PathBuf, update_fn: F) -> Result<(), UpdateError> - where - F: Fn(&str) -> String, - { - let content = fs::read_to_string(path).map_err(|e| UpdateError::Io(e, path.clone()))?; - let updated_content = update_fn(&content); - fs::write(path, updated_content).map_err(|e| UpdateError::Io(e, path.clone())) - } - - fn update_treesitter_json(&self) -> Result<(), UpdateError> { - let json_path = self.current_dir.join("tree-sitter.json"); - self.update_file_with(&json_path, |content| { - content - .lines() - .map(|line| { - if line.contains("\"version\":") { - let prefix_index = - line.find("\"version\":").unwrap() + "\"version\":".len(); - let start_quote = - line[prefix_index..].find('"').unwrap() + prefix_index + 1; - let end_quote = - line[start_quote + 1..].find('"').unwrap() + start_quote + 1; - - format!( - "{}{}{}", - &line[..start_quote], - self.version.as_ref().unwrap(), - &line[end_quote..] - ) - } else { - line.to_string() - } - }) - .collect::>() - .join("\n") - + "\n" - }) - } - - fn update_cargo_toml(&self) -> Result<(), UpdateError> { - let cargo_toml_path = self.current_dir.join("Cargo.toml"); - if !cargo_toml_path.exists() { - return Ok(()); - } - - self.update_file_with(&cargo_toml_path, |content| { - content - .lines() - .map(|line| { - if line.starts_with("version =") { - format!("version = \"{}\"", self.version.as_ref().unwrap()) - } else { - line.to_string() - } - }) - .collect::>() - .join("\n") - + "\n" - })?; - - Ok(()) - } - - fn update_cargo_lock(&self) -> Result<(), UpdateError> { - if self.current_dir.join("Cargo.lock").exists() { - let Ok(cmd) = Command::new("cargo") - .arg("generate-lockfile") - .arg("--offline") - .current_dir(&self.current_dir) - .output() - else { - return Ok(()); // cargo is not `executable`, ignore - }; - - if !cmd.status.success() { - let stderr = String::from_utf8_lossy(&cmd.stderr); - return Err(UpdateError::Command( - "cargo generate-lockfile", - stderr.to_string(), - )); - } - } - - Ok(()) - } - - fn update_package_json(&self) -> Result<(), UpdateError> { - let package_json_path = self.current_dir.join("package.json"); - if !package_json_path.exists() { - return Ok(()); - } - - self.update_file_with(&package_json_path, |content| { - content - .lines() - .map(|line| { - if line.contains("\"version\":") { - let prefix_index = - line.find("\"version\":").unwrap() + "\"version\":".len(); - let start_quote = - line[prefix_index..].find('"').unwrap() + prefix_index + 1; - let end_quote = - line[start_quote + 1..].find('"').unwrap() + start_quote + 1; - - format!( - "{}{}{}", - &line[..start_quote], - self.version.as_ref().unwrap(), - &line[end_quote..] - ) - } else { - line.to_string() - } - }) - .collect::>() - .join("\n") - + "\n" - })?; - - Ok(()) - } - - fn update_package_lock_json(&self) -> Result<(), UpdateError> { - if self.current_dir.join("package-lock.json").exists() { - let Ok(cmd) = Command::new("npm") - .arg("install") - .arg("--package-lock-only") - .current_dir(&self.current_dir) - .output() - else { - return Ok(()); // npm is not `executable`, ignore - }; - - if !cmd.status.success() { - let stderr = String::from_utf8_lossy(&cmd.stderr); - return Err(UpdateError::Command("npm install", stderr.to_string())); - } - } - - Ok(()) - } - - fn update_makefile(&self, is_multigrammar: bool) -> Result<(), UpdateError> { - let makefile_path = if is_multigrammar { - self.current_dir.join("common").join("common.mak") - } else { - self.current_dir.join("Makefile") - }; - - self.update_file_with(&makefile_path, |content| { - content - .lines() - .map(|line| { - if line.starts_with("VERSION") { - format!("VERSION := {}", self.version.as_ref().unwrap()) - } else { - line.to_string() - } - }) - .collect::>() - .join("\n") - + "\n" - })?; - - Ok(()) - } - - fn update_cmakelists_txt(&self) -> Result<(), UpdateError> { - let cmake_lists_path = self.current_dir.join("CMakeLists.txt"); - if !cmake_lists_path.exists() { - return Ok(()); - } - - self.update_file_with(&cmake_lists_path, |content| { - let re = Regex::new(r#"(\s*VERSION\s+)"[0-9]+\.[0-9]+\.[0-9]+""#) - .expect("Failed to compile regex"); - re.replace( - content, - format!(r#"$1"{}""#, self.version.as_ref().unwrap()), - ) - .to_string() - })?; - - Ok(()) - } - - fn update_pyproject_toml(&self) -> Result<(), UpdateError> { - let pyproject_toml_path = self.current_dir.join("pyproject.toml"); - if !pyproject_toml_path.exists() { - return Ok(()); - } - - self.update_file_with(&pyproject_toml_path, |content| { - content - .lines() - .map(|line| { - if line.starts_with("version =") { - format!("version = \"{}\"", self.version.as_ref().unwrap()) - } else { - line.to_string() - } - }) - .collect::>() - .join("\n") - + "\n" - })?; - - Ok(()) - } - - fn update_zig_zon(&self) -> Result<(), UpdateError> { - let zig_zon_path = self.current_dir.join("build.zig.zon"); - if !zig_zon_path.exists() { - return Ok(()); - } - - self.update_file_with(&zig_zon_path, |content| { - let zig_version_prefix = ".version ="; - content - .lines() - .map(|line| { - if line - .trim_start_matches(|c: char| c.is_ascii_whitespace()) - .starts_with(zig_version_prefix) - { - let prefix_index = - line.find(zig_version_prefix).unwrap() + zig_version_prefix.len(); - let start_quote = - line[prefix_index..].find('"').unwrap() + prefix_index + 1; - let end_quote = - line[start_quote + 1..].find('"').unwrap() + start_quote + 1; - - format!( - "{}{}{}", - &line[..start_quote], - self.version.as_ref().unwrap(), - &line[end_quote..] - ) - } else { - line.to_string() - } - }) - .collect::>() - .join("\n") - + "\n" - })?; - - Ok(()) - } -} diff --git a/crates/config/LICENSE b/crates/config/LICENSE deleted file mode 100644 index 971b81f9..00000000 --- a/crates/config/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2018 Max Brunsfeld - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/crates/generate/LICENSE b/crates/generate/LICENSE deleted file mode 100644 index 971b81f9..00000000 --- a/crates/generate/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2018 Max Brunsfeld - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/crates/generate/src/parser.h.inc b/crates/generate/src/parser.h.inc deleted file mode 100644 index 858107de..00000000 --- a/crates/generate/src/parser.h.inc +++ /dev/null @@ -1,286 +0,0 @@ -#ifndef TREE_SITTER_PARSER_H_ -#define TREE_SITTER_PARSER_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -#define ts_builtin_sym_error ((TSSymbol)-1) -#define ts_builtin_sym_end 0 -#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024 - -#ifndef TREE_SITTER_API_H_ -typedef uint16_t TSStateId; -typedef uint16_t TSSymbol; -typedef uint16_t TSFieldId; -typedef struct TSLanguage TSLanguage; -typedef struct TSLanguageMetadata { - uint8_t major_version; - uint8_t minor_version; - uint8_t patch_version; -} TSLanguageMetadata; -#endif - -typedef struct { - TSFieldId field_id; - uint8_t child_index; - bool inherited; -} TSFieldMapEntry; - -// Used to index the field and supertype maps. -typedef struct { - uint16_t index; - uint16_t length; -} TSMapSlice; - -typedef struct { - bool visible; - bool named; - bool supertype; -} TSSymbolMetadata; - -typedef struct TSLexer TSLexer; - -struct TSLexer { - int32_t lookahead; - TSSymbol result_symbol; - void (*advance)(TSLexer *, bool); - void (*mark_end)(TSLexer *); - uint32_t (*get_column)(TSLexer *); - bool (*is_at_included_range_start)(const TSLexer *); - bool (*eof)(const TSLexer *); - void (*log)(const TSLexer *, const char *, ...); -}; - -typedef enum { - TSParseActionTypeShift, - TSParseActionTypeReduce, - TSParseActionTypeAccept, - TSParseActionTypeRecover, -} TSParseActionType; - -typedef union { - struct { - uint8_t type; - TSStateId state; - bool extra; - bool repetition; - } shift; - struct { - uint8_t type; - uint8_t child_count; - TSSymbol symbol; - int16_t dynamic_precedence; - uint16_t production_id; - } reduce; - uint8_t type; -} TSParseAction; - -typedef struct { - uint16_t lex_state; - uint16_t external_lex_state; -} TSLexMode; - -typedef struct { - uint16_t lex_state; - uint16_t external_lex_state; - uint16_t reserved_word_set_id; -} TSLexerMode; - -typedef union { - TSParseAction action; - struct { - uint8_t count; - bool reusable; - } entry; -} TSParseActionEntry; - -typedef struct { - int32_t start; - int32_t end; -} TSCharacterRange; - -struct TSLanguage { - uint32_t abi_version; - uint32_t symbol_count; - uint32_t alias_count; - uint32_t token_count; - uint32_t external_token_count; - uint32_t state_count; - uint32_t large_state_count; - uint32_t production_id_count; - uint32_t field_count; - uint16_t max_alias_sequence_length; - const uint16_t *parse_table; - const uint16_t *small_parse_table; - const uint32_t *small_parse_table_map; - const TSParseActionEntry *parse_actions; - const char * const *symbol_names; - const char * const *field_names; - const TSMapSlice *field_map_slices; - const TSFieldMapEntry *field_map_entries; - const TSSymbolMetadata *symbol_metadata; - const TSSymbol *public_symbol_map; - const uint16_t *alias_map; - const TSSymbol *alias_sequences; - const TSLexerMode *lex_modes; - bool (*lex_fn)(TSLexer *, TSStateId); - bool (*keyword_lex_fn)(TSLexer *, TSStateId); - TSSymbol keyword_capture_token; - struct { - const bool *states; - const TSSymbol *symbol_map; - void *(*create)(void); - void (*destroy)(void *); - bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist); - unsigned (*serialize)(void *, char *); - void (*deserialize)(void *, const char *, unsigned); - } external_scanner; - const TSStateId *primary_state_ids; - const char *name; - const TSSymbol *reserved_words; - uint16_t max_reserved_word_set_size; - uint32_t supertype_count; - const TSSymbol *supertype_symbols; - const TSMapSlice *supertype_map_slices; - const TSSymbol *supertype_map_entries; - TSLanguageMetadata metadata; -}; - -static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, int32_t lookahead) { - uint32_t index = 0; - uint32_t size = len - index; - while (size > 1) { - uint32_t half_size = size / 2; - uint32_t mid_index = index + half_size; - const TSCharacterRange *range = &ranges[mid_index]; - if (lookahead >= range->start && lookahead <= range->end) { - return true; - } else if (lookahead > range->end) { - index = mid_index; - } - size -= half_size; - } - const TSCharacterRange *range = &ranges[index]; - return (lookahead >= range->start && lookahead <= range->end); -} - -/* - * Lexer Macros - */ - -#ifdef _MSC_VER -#define UNUSED __pragma(warning(suppress : 4101)) -#else -#define UNUSED __attribute__((unused)) -#endif - -#define START_LEXER() \ - bool result = false; \ - bool skip = false; \ - UNUSED \ - bool eof = false; \ - int32_t lookahead; \ - goto start; \ - next_state: \ - lexer->advance(lexer, skip); \ - start: \ - skip = false; \ - lookahead = lexer->lookahead; - -#define ADVANCE(state_value) \ - { \ - state = state_value; \ - goto next_state; \ - } - -#define ADVANCE_MAP(...) \ - { \ - static const uint16_t map[] = { __VA_ARGS__ }; \ - for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) { \ - if (map[i] == lookahead) { \ - state = map[i + 1]; \ - goto next_state; \ - } \ - } \ - } - -#define SKIP(state_value) \ - { \ - skip = true; \ - state = state_value; \ - goto next_state; \ - } - -#define ACCEPT_TOKEN(symbol_value) \ - result = true; \ - lexer->result_symbol = symbol_value; \ - lexer->mark_end(lexer); - -#define END_STATE() return result; - -/* - * Parse Table Macros - */ - -#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT) - -#define STATE(id) id - -#define ACTIONS(id) id - -#define SHIFT(state_value) \ - {{ \ - .shift = { \ - .type = TSParseActionTypeShift, \ - .state = (state_value) \ - } \ - }} - -#define SHIFT_REPEAT(state_value) \ - {{ \ - .shift = { \ - .type = TSParseActionTypeShift, \ - .state = (state_value), \ - .repetition = true \ - } \ - }} - -#define SHIFT_EXTRA() \ - {{ \ - .shift = { \ - .type = TSParseActionTypeShift, \ - .extra = true \ - } \ - }} - -#define REDUCE(symbol_name, children, precedence, prod_id) \ - {{ \ - .reduce = { \ - .type = TSParseActionTypeReduce, \ - .symbol = symbol_name, \ - .child_count = children, \ - .dynamic_precedence = precedence, \ - .production_id = prod_id \ - }, \ - }} - -#define RECOVER() \ - {{ \ - .type = TSParseActionTypeRecover \ - }} - -#define ACCEPT_INPUT() \ - {{ \ - .type = TSParseActionTypeAccept \ - }} - -#ifdef __cplusplus -} -#endif - -#endif // TREE_SITTER_PARSER_H_ diff --git a/crates/generate/src/quickjs.rs b/crates/generate/src/quickjs.rs deleted file mode 100644 index d8c71cfe..00000000 --- a/crates/generate/src/quickjs.rs +++ /dev/null @@ -1,477 +0,0 @@ -use std::{ - collections::HashMap, - path::{Path, PathBuf}, - sync::{LazyLock, Mutex}, -}; - -use log::{error, info, warn}; -use rquickjs::{ - loader::{FileResolver, ScriptLoader}, - Context, Ctx, Function, Module, Object, Runtime, Type, Value, -}; - -use super::{IoError, JSError, JSResult}; - -const DSL: &[u8] = include_bytes!("dsl.js"); - -trait JSResultExt { - fn or_js_error(self, ctx: &Ctx) -> JSResult; -} - -impl JSResultExt for Result { - fn or_js_error(self, ctx: &Ctx) -> JSResult { - match self { - Ok(v) => Ok(v), - Err(rquickjs::Error::Exception) => Err(format_js_exception(ctx.catch())), - Err(e) => Err(JSError::QuickJS(e.to_string())), - } - } -} - -fn format_js_exception(v: Value) -> JSError { - let Some(exception) = v.into_exception() else { - return JSError::QuickJS("Expected a JS exception".to_string()); - }; - - let error_obj = exception.as_object(); - let mut parts = Vec::new(); - - for (key, label) in [("message", "Message"), ("stack", "Stack"), ("name", "Type")] { - if let Ok(value) = error_obj.get::<_, String>(key) { - parts.push(format!("{label}: {value}")); - } - } - - if parts.is_empty() { - JSError::QuickJS(exception.to_string()) - } else { - JSError::QuickJS(parts.join("\n")) - } -} - -static FILE_CACHE: LazyLock>> = - LazyLock::new(|| Mutex::new(HashMap::new())); - -#[rquickjs::function] -fn load_file(path: String) -> rquickjs::Result { - { - let cache = FILE_CACHE.lock().unwrap(); - if let Some(cached) = cache.get(&path) { - return Ok(cached.clone()); - } - } - - let content = std::fs::read_to_string(&path).map_err(|e| { - rquickjs::Error::new_from_js_message("IOError", "FileReadError", e.to_string()) - })?; - - { - let mut cache = FILE_CACHE.lock().unwrap(); - cache.insert(path, content.clone()); - } - - Ok(content) -} - -#[rquickjs::class] -#[derive(rquickjs::class::Trace, rquickjs::JsLifetime, Default)] -pub struct Console {} - -impl Console { - fn format_args(args: &[Value<'_>]) -> String { - args.iter() - .map(|v| match v.type_of() { - Type::Bool => v.as_bool().unwrap().to_string(), - Type::Int => v.as_int().unwrap().to_string(), - Type::Float => v.as_float().unwrap().to_string(), - Type::String => v - .as_string() - .unwrap() - .to_string() - .unwrap_or_else(|_| String::new()), - Type::Null => "null".to_string(), - Type::Undefined => "undefined".to_string(), - Type::Uninitialized => "uninitialized".to_string(), - Type::Module => "module".to_string(), - Type::BigInt => v.get::().unwrap_or_else(|_| "BigInt".to_string()), - Type::Unknown => "unknown".to_string(), - Type::Array => { - let js_vals = v - .as_array() - .unwrap() - .iter::>() - .filter_map(|x| x.ok()) - .map(|x| { - if x.is_string() { - format!("'{}'", Self::format_args(&[x])) - } else { - Self::format_args(&[x]) - } - }) - .collect::>() - .join(", "); - - format!("[ {js_vals} ]") - } - Type::Symbol - | Type::Object - | Type::Proxy - | Type::Function - | Type::Constructor - | Type::Promise - | Type::Exception => "[object Object]".to_string(), - }) - .collect::>() - .join(" ") - } -} - -#[rquickjs::methods] -impl Console { - #[qjs(constructor)] - pub const fn new() -> Self { - Console {} - } - - #[allow(clippy::needless_pass_by_value)] - pub fn log(&self, args: rquickjs::function::Rest>) -> rquickjs::Result<()> { - info!("{}", Self::format_args(&args)); - Ok(()) - } - - #[allow(clippy::needless_pass_by_value)] - pub fn warn(&self, args: rquickjs::function::Rest>) -> rquickjs::Result<()> { - warn!("{}", Self::format_args(&args)); - Ok(()) - } - - #[allow(clippy::needless_pass_by_value)] - pub fn error(&self, args: rquickjs::function::Rest>) -> rquickjs::Result<()> { - error!("Error: {}", Self::format_args(&args)); - Ok(()) - } -} - -fn resolve_module_path(base_path: &Path, module_path: &str) -> rquickjs::Result { - let candidates = if module_path.starts_with("./") || module_path.starts_with("../") { - let target = base_path.join(module_path); - vec![ - target.with_extension("js"), - target.with_extension("json"), - target.clone(), - ] - } else { - let local_target = base_path.join(module_path); - let node_modules_target = Path::new("node_modules").join(module_path); - - vec![ - local_target.with_extension("js"), - local_target.with_extension("json"), - local_target.clone(), - node_modules_target.with_extension("js"), - node_modules_target.with_extension("json"), - node_modules_target, - ] - }; - - for candidate in candidates { - if let Ok(resolved) = try_resolve_path(&candidate) { - return Ok(resolved); - } - } - - Err(rquickjs::Error::new_from_js_message( - "Error", - "ModuleNotFound", - format!("Module not found: {module_path}"), - )) -} - -fn try_resolve_path(path: &Path) -> rquickjs::Result { - let metadata = std::fs::metadata(path).map_err(|_| { - rquickjs::Error::new_from_js_message( - "Error", - "FileNotFound", - format!("Path not found: {}", path.display()), - ) - })?; - - if metadata.is_file() { - return Ok(path.to_path_buf()); - } - - if metadata.is_dir() { - let index_path = path.join("index.js"); - if index_path.exists() { - return Ok(index_path); - } - } - - Err(rquickjs::Error::new_from_js_message( - "Error", - "ResolutionFailed", - format!("Cannot resolve: {}", path.display()), - )) -} - -#[allow(clippy::needless_pass_by_value)] -fn require_from_module<'js>( - ctx: Ctx<'js>, - module_path: String, - from_module: &str, -) -> rquickjs::Result> { - let current_module = PathBuf::from(from_module); - let current_dir = if current_module.is_file() { - current_module.parent().unwrap_or(Path::new(".")) - } else { - current_module.as_path() - }; - - let resolved_path = resolve_module_path(current_dir, &module_path)?; - - let contents = load_file(resolved_path.to_string_lossy().to_string())?; - - load_module_from_content(&ctx, &resolved_path, &contents) -} - -fn load_module_from_content<'js>( - ctx: &Ctx<'js>, - path: &Path, - contents: &str, -) -> rquickjs::Result> { - if path.extension().is_some_and(|ext| ext == "json") { - return ctx.eval::, _>(format!("JSON.parse({contents:?})")); - } - - let exports = Object::new(ctx.clone())?; - let module_obj = Object::new(ctx.clone())?; - module_obj.set("exports", exports.clone())?; - - let filename = path.to_string_lossy().to_string(); - let dirname = path - .parent() - .map_or_else(|| ".".to_string(), |p| p.to_string_lossy().to_string()); - - // Require function specific to *this* module - let module_path = filename.clone(); - let require = Function::new( - ctx.clone(), - move |ctx_inner: Ctx<'js>, target_path: String| -> rquickjs::Result> { - require_from_module(ctx_inner, target_path, &module_path) - }, - )?; - - let wrapper = - format!("(function(exports, require, module, __filename, __dirname) {{ {contents} }})"); - - let module_func = ctx.eval::, _>(wrapper)?; - module_func.call::<_, Value<'js>>((exports, require, module_obj.clone(), filename, dirname))?; - - module_obj.get("exports") -} - -pub fn execute_native_runtime(grammar_path: &Path) -> JSResult { - let runtime = Runtime::new()?; - - runtime.set_memory_limit(64 * 1024 * 1024); // 64MB - runtime.set_max_stack_size(256 * 1024); // 256KB - - let context = Context::full(&runtime)?; - - let resolver = FileResolver::default() - .with_path("./node_modules") - .with_path("./") - .with_pattern("{}.mjs"); - let loader = ScriptLoader::default().with_extension("mjs"); - runtime.set_loader(resolver, loader); - - let cwd = std::env::current_dir().map_err(|e| JSError::IO(IoError::new(&e, None)))?; - let relative_path = pathdiff::diff_paths(grammar_path, &cwd) - .map(|p| p.to_string_lossy().to_string()) - .ok_or(JSError::RelativePath)?; - - context.with(|ctx| -> JSResult { - let globals = ctx.globals(); - - globals.set("native", true).or_js_error(&ctx)?; - globals - .set("__ts_grammar_path", relative_path) - .or_js_error(&ctx)?; - - let console = rquickjs::Class::instance(ctx.clone(), Console::new()).or_js_error(&ctx)?; - globals.set("console", console).or_js_error(&ctx)?; - - let process = Object::new(ctx.clone()).or_js_error(&ctx)?; - let env = Object::new(ctx.clone()).or_js_error(&ctx)?; - for (key, value) in std::env::vars() { - env.set(key, value).or_js_error(&ctx)?; - } - process.set("env", env).or_js_error(&ctx)?; - globals.set("process", process).or_js_error(&ctx)?; - - let module = Object::new(ctx.clone()).or_js_error(&ctx)?; - module - .set("exports", Object::new(ctx.clone()).or_js_error(&ctx)?) - .or_js_error(&ctx)?; - globals.set("module", module).or_js_error(&ctx)?; - - let grammar_path_string = grammar_path.to_string_lossy().to_string(); - let main_require = Function::new( - ctx.clone(), - move |ctx_inner, target_path: String| -> rquickjs::Result { - require_from_module(ctx_inner, target_path, &grammar_path_string) - }, - )?; - globals.set("require", main_require).or_js_error(&ctx)?; - - let promise = Module::evaluate(ctx.clone(), "dsl", DSL).or_js_error(&ctx)?; - promise.finish::<()>().or_js_error(&ctx)?; - - let grammar_json = ctx - .eval::("globalThis.output") - .map(|s| s.to_string()) - .or_js_error(&ctx)? - .or_js_error(&ctx)?; - - let parsed = serde_json::from_str::(&grammar_json)?; - Ok(serde_json::to_string_pretty(&parsed)?) - }) -} - -#[cfg(test)] -mod tests { - use std::{ - fs, - sync::{Arc, Mutex, OnceLock}, - }; - use tempfile::TempDir; - - use super::*; - - static TEST_MUTEX: OnceLock>> = OnceLock::new(); - - fn with_test_lock(test: F) -> R - where - F: FnOnce() -> R, - { - let _guard = TEST_MUTEX.get_or_init(|| Arc::new(Mutex::new(()))).lock(); - let result = test(); - cleanup_runtime_state(); - result - } - - fn cleanup_runtime_state() { - FILE_CACHE.lock().unwrap().clear(); - } - - #[test] - fn test_basic_grammar_execution() { - with_test_lock(|| { - let temp_dir = TempDir::new().unwrap(); - std::env::set_current_dir(temp_dir.path()).unwrap(); - - let grammar_path = temp_dir.path().join("grammar.js"); - fs::write( - &grammar_path, - r" - module.exports = grammar({ - name: 'test', - rules: { source_file: $ => 'hello' } - }); - ", - ) - .unwrap(); - - let json = execute_native_runtime(&grammar_path).expect("Failed to execute grammar"); - assert!(json.contains("\"name\": \"test\"")); - assert!(json.contains("\"hello\"")); - }); - } - - #[test] - fn test_module_imports() { - with_test_lock(|| { - let temp_dir = TempDir::new().unwrap(); - std::env::set_current_dir(temp_dir.path()).unwrap(); - - fs::write( - temp_dir.path().join("common.js"), - r" - module.exports = { identifier: $ => /[a-zA-Z_][a-zA-Z0-9_]*/ }; - ", - ) - .unwrap(); - - fs::write( - temp_dir.path().join("grammar.js"), - r" - const common = require('./common'); - module.exports = grammar({ - name: 'test_import', - rules: { source_file: common.identifier } - }); - ", - ) - .unwrap(); - - let json = execute_native_runtime(&temp_dir.path().join("grammar.js")) - .expect("Failed to execute grammar with imports"); - assert!(json.contains("\"name\": \"test_import\"")); - }); - } - - #[test] - fn test_json_module_loading() { - with_test_lock(|| { - let temp_dir = TempDir::new().unwrap(); - std::env::set_current_dir(temp_dir.path()).unwrap(); - - fs::write( - temp_dir.path().join("package.json"), - r#"{"version": "1.0.0"}"#, - ) - .unwrap(); - fs::write( - temp_dir.path().join("grammar.js"), - r" - const pkg = require('./package.json'); - module.exports = grammar({ - name: 'json_test', - rules: { - source_file: $ => 'version_' + pkg.version.replace(/\./g, '_') - } - }); - ", - ) - .unwrap(); - - let json = execute_native_runtime(&temp_dir.path().join("grammar.js")) - .expect("Failed to execute grammar with JSON import"); - assert!(json.contains("version_1_0_0")); - }); - } - - #[test] - fn test_resource_limits() { - with_test_lock(|| { - let temp_dir = TempDir::new().unwrap(); - std::env::set_current_dir(temp_dir.path()).unwrap(); - - fs::write( - temp_dir.path().join("grammar.js"), - r" - const huge = new Array(10000000).fill('x'.repeat(1000)); - module.exports = grammar({ - name: 'resource_test', - rules: { source_file: $ => 'test' } - }); - ", - ) - .unwrap(); - - let result = execute_native_runtime(&temp_dir.path().join("grammar.js")); - assert!(result.is_err()); - assert!(matches!(result.unwrap_err(), JSError::QuickJS(_))); - }); - } -} diff --git a/crates/highlight/LICENSE b/crates/highlight/LICENSE deleted file mode 100644 index 971b81f9..00000000 --- a/crates/highlight/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2018 Max Brunsfeld - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/crates/language/LICENSE b/crates/language/LICENSE deleted file mode 100644 index 971b81f9..00000000 --- a/crates/language/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2018 Max Brunsfeld - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/crates/language/build.rs b/crates/language/build.rs deleted file mode 100644 index 930b3dff..00000000 --- a/crates/language/build.rs +++ /dev/null @@ -1,13 +0,0 @@ -fn main() { - if std::env::var("TARGET") - .unwrap_or_default() - .starts_with("wasm32-unknown") - { - let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap(); - let wasm_headers = std::path::Path::new(&manifest_dir).join("wasm/include"); - let wasm_src = std::path::Path::new(&manifest_dir).join("wasm/src"); - - println!("cargo::metadata=wasm-headers={}", wasm_headers.display()); - println!("cargo::metadata=wasm-src={}", wasm_src.display()); - } -} diff --git a/crates/language/wasm/include/assert.h b/crates/language/wasm/include/assert.h deleted file mode 100644 index e981a20e..00000000 --- a/crates/language/wasm/include/assert.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef TREE_SITTER_WASM_ASSERT_H_ -#define TREE_SITTER_WASM_ASSERT_H_ - -#ifdef NDEBUG -#define assert(e) ((void)0) -#else -__attribute__((noreturn)) void __assert_fail(const char *assertion, const char *file, unsigned line, const char *function) { - __builtin_trap(); -} -#define assert(expression) \ - ((expression) ? (void)0 : __assert_fail(#expression, __FILE__, __LINE__, __func__)) -#endif - -#endif // TREE_SITTER_WASM_ASSERT_H_ diff --git a/crates/language/wasm/include/ctype.h b/crates/language/wasm/include/ctype.h deleted file mode 100644 index cea32970..00000000 --- a/crates/language/wasm/include/ctype.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef TREE_SITTER_WASM_CTYPE_H_ -#define TREE_SITTER_WASM_CTYPE_H_ - -static inline int isprint(int c) { - return c >= 0x20 && c <= 0x7E; -} - -#endif // TREE_SITTER_WASM_CTYPE_H_ diff --git a/crates/language/wasm/include/endian.h b/crates/language/wasm/include/endian.h deleted file mode 100644 index f35a5962..00000000 --- a/crates/language/wasm/include/endian.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef TREE_SITTER_WASM_ENDIAN_H_ -#define TREE_SITTER_WASM_ENDIAN_H_ - -#define be16toh(x) __builtin_bswap16(x) -#define be32toh(x) __builtin_bswap32(x) -#define be64toh(x) __builtin_bswap64(x) -#define le16toh(x) (x) -#define le32toh(x) (x) -#define le64toh(x) (x) - - -#endif // TREE_SITTER_WASM_ENDIAN_H_ diff --git a/crates/language/wasm/include/inttypes.h b/crates/language/wasm/include/inttypes.h deleted file mode 100644 index f5cccd07..00000000 --- a/crates/language/wasm/include/inttypes.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef TREE_SITTER_WASM_INTTYPES_H_ -#define TREE_SITTER_WASM_INTTYPES_H_ - -// https://github.com/llvm/llvm-project/blob/0c3cf200f5b918fb5c1114e9f1764c2d54d1779b/libc/include/llvm-libc-macros/inttypes-macros.h#L209 - -#define PRId32 "d" - -#endif // TREE_SITTER_WASM_INTTYPES_H_ diff --git a/crates/language/wasm/include/stdint.h b/crates/language/wasm/include/stdint.h deleted file mode 100644 index 10cc35dc..00000000 --- a/crates/language/wasm/include/stdint.h +++ /dev/null @@ -1,46 +0,0 @@ -#ifndef TREE_SITTER_WASM_STDINT_H_ -#define TREE_SITTER_WASM_STDINT_H_ - -// https://github.com/llvm/llvm-project/blob/0c3cf200f5b918fb5c1114e9f1764c2d54d1779b/clang/test/Preprocessor/init.c#L1672 - -typedef signed char int8_t; - -typedef short int16_t; - -typedef int int32_t; - -typedef long long int int64_t; - -typedef unsigned char uint8_t; - -typedef unsigned short uint16_t; - -typedef unsigned int uint32_t; - -typedef long long unsigned int uint64_t; - -typedef long unsigned int size_t; - -typedef long unsigned int uintptr_t; - -#define INT8_MAX 127 -#define INT16_MAX 32767 -#define INT32_MAX 2147483647L -#define INT64_MAX 9223372036854775807LL - -#define UINT8_MAX 255 -#define UINT16_MAX 65535 -#define UINT32_MAX 4294967295U -#define UINT64_MAX 18446744073709551615ULL - -#if defined(__wasm32__) - -#define SIZE_MAX 4294967295UL - -#elif defined(__wasm64__) - -#define SIZE_MAX 18446744073709551615UL - -#endif - -#endif // TREE_SITTER_WASM_STDINT_H_ diff --git a/crates/language/wasm/include/stdio.h b/crates/language/wasm/include/stdio.h deleted file mode 100644 index 4089cccc..00000000 --- a/crates/language/wasm/include/stdio.h +++ /dev/null @@ -1,36 +0,0 @@ -#ifndef TREE_SITTER_WASM_STDIO_H_ -#define TREE_SITTER_WASM_STDIO_H_ - -#include -#include - -typedef struct FILE FILE; - -typedef __builtin_va_list va_list; -#define va_start(ap, last) __builtin_va_start(ap, last) -#define va_end(ap) __builtin_va_end(ap) -#define va_arg(ap, type) __builtin_va_arg(ap, type) - -#define stdout ((FILE *)0) - -#define stderr ((FILE *)1) - -#define stdin ((FILE *)2) - -int fclose(FILE *stream); - -FILE *fdopen(int fd, const char *mode); - -int fputc(int c, FILE *stream); - -int fputs(const char *restrict s, FILE *restrict stream); - -size_t fwrite(const void *restrict buffer, size_t size, size_t nmemb, FILE *restrict stream); - -int fprintf(FILE *restrict stream, const char *restrict format, ...); - -int snprintf(char *restrict buffer, size_t buffsz, const char *restrict format, ...); - -int vsnprintf(char *restrict buffer, size_t buffsz, const char *restrict format, va_list vlist); - -#endif // TREE_SITTER_WASM_STDIO_H_ diff --git a/crates/language/wasm/include/stdlib.h b/crates/language/wasm/include/stdlib.h deleted file mode 100644 index 2da313ab..00000000 --- a/crates/language/wasm/include/stdlib.h +++ /dev/null @@ -1,15 +0,0 @@ -#ifndef TREE_SITTER_WASM_STDLIB_H_ -#define TREE_SITTER_WASM_STDLIB_H_ - -#include - -#define NULL ((void*)0) - -void* malloc(size_t); -void* calloc(size_t, size_t); -void free(void*); -void* realloc(void*, size_t); - -__attribute__((noreturn)) void abort(void); - -#endif // TREE_SITTER_WASM_STDLIB_H_ diff --git a/crates/language/wasm/include/string.h b/crates/language/wasm/include/string.h deleted file mode 100644 index 10f11958..00000000 --- a/crates/language/wasm/include/string.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef TREE_SITTER_WASM_STRING_H_ -#define TREE_SITTER_WASM_STRING_H_ - -#include - -int memcmp(const void *lhs, const void *rhs, size_t count); - -void *memcpy(void *restrict dst, const void *restrict src, size_t size); - -void *memmove(void *dst, const void *src, size_t count); - -void *memset(void *dst, int value, size_t count); - -int strncmp(const char *left, const char *right, size_t n); - -size_t strlen(const char *str); - -#endif // TREE_SITTER_WASM_STRING_H_ diff --git a/crates/language/wasm/include/wctype.h b/crates/language/wasm/include/wctype.h deleted file mode 100644 index 8d1e8c82..00000000 --- a/crates/language/wasm/include/wctype.h +++ /dev/null @@ -1,168 +0,0 @@ -#ifndef TREE_SITTER_WASM_WCTYPE_H_ -#define TREE_SITTER_WASM_WCTYPE_H_ - -typedef int wint_t; - -static inline bool iswalpha(wint_t wch) { - switch (wch) { - case L'a': - case L'b': - case L'c': - case L'd': - case L'e': - case L'f': - case L'g': - case L'h': - case L'i': - case L'j': - case L'k': - case L'l': - case L'm': - case L'n': - case L'o': - case L'p': - case L'q': - case L'r': - case L's': - case L't': - case L'u': - case L'v': - case L'w': - case L'x': - case L'y': - case L'z': - case L'A': - case L'B': - case L'C': - case L'D': - case L'E': - case L'F': - case L'G': - case L'H': - case L'I': - case L'J': - case L'K': - case L'L': - case L'M': - case L'N': - case L'O': - case L'P': - case L'Q': - case L'R': - case L'S': - case L'T': - case L'U': - case L'V': - case L'W': - case L'X': - case L'Y': - case L'Z': - return true; - default: - return false; - } -} - -static inline bool iswdigit(wint_t wch) { - switch (wch) { - case L'0': - case L'1': - case L'2': - case L'3': - case L'4': - case L'5': - case L'6': - case L'7': - case L'8': - case L'9': - return true; - default: - return false; - } -} - -static inline bool iswalnum(wint_t wch) { - switch (wch) { - case L'a': - case L'b': - case L'c': - case L'd': - case L'e': - case L'f': - case L'g': - case L'h': - case L'i': - case L'j': - case L'k': - case L'l': - case L'm': - case L'n': - case L'o': - case L'p': - case L'q': - case L'r': - case L's': - case L't': - case L'u': - case L'v': - case L'w': - case L'x': - case L'y': - case L'z': - case L'A': - case L'B': - case L'C': - case L'D': - case L'E': - case L'F': - case L'G': - case L'H': - case L'I': - case L'J': - case L'K': - case L'L': - case L'M': - case L'N': - case L'O': - case L'P': - case L'Q': - case L'R': - case L'S': - case L'T': - case L'U': - case L'V': - case L'W': - case L'X': - case L'Y': - case L'Z': - case L'0': - case L'1': - case L'2': - case L'3': - case L'4': - case L'5': - case L'6': - case L'7': - case L'8': - case L'9': - return true; - default: - return false; - } -} - -static inline bool iswspace(wint_t wch) { - switch (wch) { - case L' ': - case L'\t': - case L'\n': - case L'\v': - case L'\f': - case L'\r': - return true; - default: - return false; - } -} - -#endif // TREE_SITTER_WASM_WCTYPE_H_ diff --git a/crates/language/wasm/src/stdio.c b/crates/language/wasm/src/stdio.c deleted file mode 100644 index 470c1ecc..00000000 --- a/crates/language/wasm/src/stdio.c +++ /dev/null @@ -1,299 +0,0 @@ -#include -#include - -typedef struct { - bool left_justify; // - - bool zero_pad; // 0 - bool show_sign; // + - bool space_prefix; // ' ' - bool alternate_form; // # -} format_flags_t; - -static const char* parse_format_spec( - const char *format, - int *width, - int *precision, - format_flags_t *flags -) { - *width = 0; - *precision = -1; - flags->left_justify = false; - flags->zero_pad = false; - flags->show_sign = false; - flags->space_prefix = false; - flags->alternate_form = false; - - const char *p = format; - - // Parse flags - while (*p == '-' || *p == '+' || *p == ' ' || *p == '#' || *p == '0') { - switch (*p) { - case '-': flags->left_justify = true; break; - case '0': flags->zero_pad = true; break; - case '+': flags->show_sign = true; break; - case ' ': flags->space_prefix = true; break; - case '#': flags->alternate_form = true; break; - } - p++; - } - - // width - while (*p >= '0' && *p <= '9') { - *width = (*width * 10) + (*p - '0'); - p++; - } - - // precision - if (*p == '.') { - p++; - *precision = 0; - while (*p >= '0' && *p <= '9') { - *precision = (*precision * 10) + (*p - '0'); - p++; - } - } - - return p; -} - -static int int_to_str( - long long value, - char *buffer, - int base, - bool is_signed, - bool uppercase -) { - if (base < 2 || base > 16) return 0; - - const char *digits = uppercase ? "0123456789ABCDEF" : "0123456789abcdef"; - char temp[32]; - int i = 0, len = 0; - bool is_negative = false; - - if (value == 0) { - buffer[0] = '0'; - buffer[1] = '\0'; - return 1; - } - - if (is_signed && value < 0 && base == 10) { - is_negative = true; - value = -value; - } - - unsigned long long uval = (unsigned long long)value; - while (uval > 0) { - temp[i++] = digits[uval % base]; - uval /= base; - } - - if (is_negative) { - buffer[len++] = '-'; - } - - while (i > 0) { - buffer[len++] = temp[--i]; - } - - buffer[len] = '\0'; - return len; -} - -static int ptr_to_str(void *ptr, char *buffer) { - buffer[0] = '0'; - buffer[1] = 'x'; - int len = int_to_str((uintptr_t)ptr, buffer + 2, 16, 0, 0); - return 2 + len; -} - -char *strncpy(char *dest, const char *src, size_t n) { - char *d = dest; - const char *s = src; - while (n-- && (*d++ = *s++)); - if (n == (size_t)-1) *d = '\0'; - return dest; -} - -static int write_formatted_to_buffer( - char *buffer, - size_t buffer_size, - size_t *pos, - const char *str, - int width, - const format_flags_t *flags -) { - int len = strlen(str); - int written = 0; - int pad_len = (width > len) ? (width - len) : 0; - int zero_pad = flags->zero_pad && !flags->left_justify; - - if (!flags->left_justify && pad_len > 0) { - char pad_char = zero_pad ? '0' : ' '; - for (int i = 0; i < pad_len && *pos < buffer_size - 1; i++) { - buffer[(*pos)++] = pad_char; - written++; - } - } - - for (int i = 0; i < len && *pos < buffer_size - 1; i++) { - buffer[(*pos)++] = str[i]; - written++; - } - - if (flags->left_justify && pad_len > 0) { - for (int i = 0; i < pad_len && *pos < buffer_size - 1; i++) { - buffer[(*pos)++] = ' '; - written++; - } - } - - return written; -} - -static int vsnprintf_impl(char *buffer, size_t buffsz, const char *format, va_list args) { - if (!buffer || buffsz == 0 || !format) return -1; - - size_t pos = 0; - int total_chars = 0; - const char *p = format; - - while (*p) { - if (*p == '%') { - p++; - if (*p == '%') { - if (pos < buffsz - 1) buffer[pos++] = '%'; - total_chars++; - p++; - continue; - } - - int width, precision; - format_flags_t flags; - p = parse_format_spec(p, &width, &precision, &flags); - - char temp_buf[64]; - const char *output_str = temp_buf; - - switch (*p) { - case 's': { - const char *str = va_arg(args, const char*); - if (!str) str = "(null)"; - - int str_len = strlen(str); - if (precision >= 0 && str_len > precision) { - strncpy(temp_buf, str, precision); - temp_buf[precision] = '\0'; - output_str = temp_buf; - } else { - output_str = str; - } - break; - } - case 'd': - case 'i': { - int value = va_arg(args, int); - int_to_str(value, temp_buf, 10, true, false); - break; - } - case 'u': { - unsigned int value = va_arg(args, unsigned int); - int_to_str(value, temp_buf, 10, false, false); - break; - } - case 'x': { - unsigned int value = va_arg(args, unsigned int); - int_to_str(value, temp_buf, 16, false, false); - break; - } - case 'X': { - unsigned int value = va_arg(args, unsigned int); - int_to_str(value, temp_buf, 16, false, true); - break; - } - case 'p': { - void *ptr = va_arg(args, void*); - ptr_to_str(ptr, temp_buf); - break; - } - case 'c': { - int c = va_arg(args, int); - temp_buf[0] = (char)c; - temp_buf[1] = '\0'; - break; - } - case 'z': { - if (*(p + 1) == 'u') { - size_t value = va_arg(args, size_t); - int_to_str(value, temp_buf, 10, false, false); - p++; - } else { - temp_buf[0] = 'z'; - temp_buf[1] = '\0'; - } - break; - } - default: - temp_buf[0] = '%'; - temp_buf[1] = *p; - temp_buf[2] = '\0'; - break; - } - - int str_len = strlen(output_str); - int formatted_len = (width > str_len) ? width : str_len; - total_chars += formatted_len; - - if (pos < buffsz - 1) { - write_formatted_to_buffer(buffer, buffsz, &pos, output_str, width, &flags); - } - - } else { - if (pos < buffsz - 1) buffer[pos++] = *p; - total_chars++; - } - p++; - } - - if (buffsz > 0) buffer[pos < buffsz ? pos : buffsz - 1] = '\0'; - - return total_chars; -} - -int snprintf(char *restrict buffer, size_t buffsz, const char *restrict format, ...) { - if (!buffer || buffsz == 0 || !format) return -1; - - va_list args; - va_start(args, format); - int result = vsnprintf_impl(buffer, buffsz, format, args); - va_end(args); - - return result; -} - -int vsnprintf(char *restrict buffer, size_t buffsz, const char *restrict format, va_list vlist) { - return vsnprintf_impl(buffer, buffsz, format, vlist); -} - -int fclose(FILE *stream) { - return 0; -} - -FILE* fdopen(int fd, const char *mode) { - return 0; -} - -int fputc(int c, FILE *stream) { - return c; -} - -int fputs(const char *restrict str, FILE *restrict stream) { - return 0; -} - -size_t fwrite(const void *restrict buffer, size_t size, size_t nmemb, FILE *restrict stream) { - return size * nmemb; -} - -int fprintf(FILE *restrict stream, const char *restrict format, ...) { - return 0; -} diff --git a/crates/language/wasm/src/string.c b/crates/language/wasm/src/string.c deleted file mode 100644 index 2d0d9096..00000000 --- a/crates/language/wasm/src/string.c +++ /dev/null @@ -1,66 +0,0 @@ -#include - -int memcmp(const void *lhs, const void *rhs, size_t count) { - const unsigned char *l = lhs; - const unsigned char *r = rhs; - while (count--) { - if (*l != *r) { - return *l - *r; - } - l++; - r++; - } - return 0; -} - -void *memcpy(void *restrict dst, const void *restrict src, size_t size) { - unsigned char *d = dst; - const unsigned char *s = src; - while (size--) { - *d++ = *s++; - } - return dst; -} - -void *memmove(void *dst, const void *src, size_t count) { - unsigned char *d = dst; - const unsigned char *s = src; - if (d < s) { - while (count--) { - *d++ = *s++; - } - } else if (d > s) { - d += count; - s += count; - while (count--) { - *(--d) = *(--s); - } - } - return dst; -} - -void *memset(void *dst, int value, size_t count) { - unsigned char *p = dst; - while (count--) { - *p++ = (unsigned char)value; - } - return dst; -} - -int strncmp(const char *left, const char *right, size_t n) { - while (n-- > 0) { - if (*left != *right) { - return *(unsigned char *)left - *(unsigned char *)right; - } - if (*left == '\0') break; - left++; - right++; - } - return 0; -} - -size_t strlen(const char *str) { - const char *s = str; - while (*s) s++; - return s - str; -} diff --git a/crates/loader/LICENSE b/crates/loader/LICENSE deleted file mode 100644 index 971b81f9..00000000 --- a/crates/loader/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2018 Max Brunsfeld - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/crates/loader/emscripten-version b/crates/loader/emscripten-version deleted file mode 100644 index af253c16..00000000 --- a/crates/loader/emscripten-version +++ /dev/null @@ -1 +0,0 @@ -4.0.15 diff --git a/crates/loader/wasi-sdk-version b/crates/loader/wasi-sdk-version deleted file mode 100644 index 231f5c77..00000000 --- a/crates/loader/wasi-sdk-version +++ /dev/null @@ -1 +0,0 @@ -29.0 diff --git a/crates/tags/LICENSE b/crates/tags/LICENSE deleted file mode 100644 index 971b81f9..00000000 --- a/crates/tags/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2018 Max Brunsfeld - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/crates/xtask/src/build_wasm.rs b/crates/xtask/src/build_wasm.rs deleted file mode 100644 index fbb231ce..00000000 --- a/crates/xtask/src/build_wasm.rs +++ /dev/null @@ -1,513 +0,0 @@ -use std::{ - collections::HashSet, - ffi::{OsStr, OsString}, - fmt::Write, - fs, - path::{Path, PathBuf}, - process::Command, - time::Duration, -}; - -use anyhow::{anyhow, Result}; -use etcetera::BaseStrategy as _; -use indoc::indoc; -use notify::{ - event::{AccessKind, AccessMode}, - EventKind, RecursiveMode, -}; -use notify_debouncer_full::new_debouncer; -use tree_sitter_loader::{IoError, LoaderError, WasiSDKClangError}; - -use crate::{ - bail_on_err, embed_sources::embed_sources_in_map, watch_wasm, BuildWasm, EMSCRIPTEN_TAG, -}; - -#[derive(PartialEq, Eq)] -enum EmccSource { - Native, - Docker, - Podman, -} - -const EXPORTED_RUNTIME_METHODS: [&str; 20] = [ - "AsciiToString", - "stringToUTF8", - "UTF8ToString", - "lengthBytesUTF8", - "stringToUTF16", - "loadWebAssemblyModule", - "getValue", - "setValue", - "HEAPF32", - "HEAPF64", - "HEAP_DATA_VIEW", - "HEAP8", - "HEAPU8", - "HEAP16", - "HEAPU16", - "HEAP32", - "HEAPU32", - "HEAP64", - "HEAPU64", - "LE_HEAP_STORE_I64", -]; - -const WASI_SDK_VERSION: &str = include_str!("../../loader/wasi-sdk-version").trim_ascii(); - -pub fn run_wasm(args: &BuildWasm) -> Result<()> { - let mut emscripten_flags = if args.debug { - vec!["-O0", "--minify", "0"] - } else { - vec!["-O3", "--minify", "0"] - }; - - if args.debug { - emscripten_flags.extend(["-s", "ASSERTIONS=1", "-s", "SAFE_HEAP=1", "-g"]); - } - - if args.verbose { - emscripten_flags.extend(["-s", "VERBOSE=1", "-v"]); - } - - let emcc_name = if cfg!(windows) { "emcc.bat" } else { "emcc" }; - - // Order of preference: emscripten > docker > podman > error - let source = if !args.docker && Command::new(emcc_name).output().is_ok() { - EmccSource::Native - } else if Command::new("docker") - .output() - .is_ok_and(|out| out.status.success()) - { - EmccSource::Docker - } else if Command::new("podman") - .arg("--version") - .output() - .is_ok_and(|out| out.status.success()) - { - EmccSource::Podman - } else { - return Err(anyhow!( - "You must have either emcc, docker, or podman on your PATH to run this command" - )); - }; - - let mut command = match source { - EmccSource::Native => Command::new(emcc_name), - EmccSource::Docker | EmccSource::Podman => { - let mut command = match source { - EmccSource::Docker => Command::new("docker"), - EmccSource::Podman => Command::new("podman"), - _ => unreachable!(), - }; - command.args(["run", "--rm"]); - - // Mount the root directory as a volume, which is the repo root - let mut volume_string = OsString::from(std::env::current_dir().unwrap()); - volume_string.push(":/src:Z"); - command.args([OsStr::new("--volume"), &volume_string]); - - // In case `docker` is an alias to `podman`, ensure that podman - // mounts the current directory as writable by the container - // user which has the same uid as the host user. Setting the - // podman-specific variable is more reliable than attempting to - // detect whether `docker` is an alias for `podman`. - // see https://docs.podman.io/en/latest/markdown/podman-run.1.html#userns-mode - command.env("PODMAN_USERNS", "keep-id"); - - // Get the current user id so that files created in the docker container will have - // the same owner. - #[cfg(unix)] - { - #[link(name = "c")] - extern "C" { - fn getuid() -> u32; - } - // don't need to set user for podman since PODMAN_USERNS=keep-id is already set - if source == EmccSource::Docker { - let user_id = unsafe { getuid() }; - command.args(["--user", &user_id.to_string()]); - } - }; - - // Run `emcc` in a container using the `emscripten-slim` image - command.args([EMSCRIPTEN_TAG, "emcc"]); - command - } - }; - - fs::create_dir_all("target/scratch").unwrap(); - - let exported_functions = format!( - "{}{}", - fs::read_to_string("lib/src/wasm/stdlib-symbols.txt")?, - fs::read_to_string("lib/binding_web/lib/exports.txt")? - ) - .replace('"', "") - .lines() - .fold(String::new(), |mut output, line| { - let _ = write!(output, "_{line}"); - output - }) - .trim_end_matches(',') - .to_string(); - - let exported_functions = format!("EXPORTED_FUNCTIONS={exported_functions}"); - let exported_runtime_methods = format!( - "EXPORTED_RUNTIME_METHODS={}", - EXPORTED_RUNTIME_METHODS.join(",") - ); - - // Clean up old files from prior runs - for file in [ - "web-tree-sitter.mjs", - "web-tree-sitter.cjs", - "web-tree-sitter.wasm", - "web-tree-sitter.wasm.map", - ] { - fs::remove_file(PathBuf::from("lib/binding_web/lib").join(file)).ok(); - } - - if !args.cjs { - emscripten_flags.extend(["-s", "EXPORT_ES6=1"]); - } - - macro_rules! binding_file { - ($ext:literal) => { - concat!("lib/binding_web/lib/web-tree-sitter", $ext) - }; - } - - #[rustfmt::skip] - emscripten_flags.extend([ - "-gsource-map=inline", - "-fno-exceptions", - "-std=c11", - "-s", "WASM=1", - "-s", "MODULARIZE=1", - "-s", "INITIAL_MEMORY=33554432", - "-s", "ALLOW_MEMORY_GROWTH=1", - "-s", "SUPPORT_BIG_ENDIAN=1", - "-s", "WASM_BIGINT=1", - "-s", "MAIN_MODULE=2", - "-s", "FILESYSTEM=0", - "-s", "NODEJS_CATCH_EXIT=0", - "-s", "NODEJS_CATCH_REJECTION=0", - "-s", &exported_functions, - "-s", &exported_runtime_methods, - "-D", "fprintf(...)=", - "-D", "printf(...)=", - "-D", "NDEBUG=", - "-D", "_POSIX_C_SOURCE=200112L", - "-D", "_DEFAULT_SOURCE=", - "-D", "_BSD_SOURCE=", - "-D", "_DARWIN_C_SOURCE=", - "-I", "lib/src", - "-I", "lib/include", - "--js-library", "lib/binding_web/lib/imports.js", - "--pre-js", "lib/binding_web/lib/prefix.js", - "-o", if args.cjs { binding_file!(".cjs") } else { binding_file!(".mjs") }, - "lib/src/lib.c", - "lib/binding_web/lib/tree-sitter.c", - ]); - if args.emit_tsd { - emscripten_flags.extend(["--emit-tsd", "web-tree-sitter.d.ts"]); - } - - let command = command.args(&emscripten_flags); - - if args.watch { - watch_wasm!(|| build_wasm(command, args.emit_tsd)); - } else { - build_wasm(command, args.emit_tsd)?; - } - - Ok(()) -} - -fn build_wasm(cmd: &mut Command, edit_tsd: bool) -> Result<()> { - bail_on_err( - &cmd.spawn()?.wait_with_output()?, - "Failed to compile the Tree-sitter Wasm library", - )?; - - if edit_tsd { - let file = "lib/binding_web/lib/web-tree-sitter.d.ts"; - let content = fs::read_to_string(file)? - .replace("Automatically generated", "Automatically @generated") - .replace( - "AsciiToString(ptr: any): string", - "AsciiToString(ptr: number): string", - ) - .replace( - "stringToUTF8(str: any, outPtr: any, maxBytesToWrite: any): any", - "stringToUTF8(str: string, outPtr: number, maxBytesToWrite: number): number", - ) - .replace( - "UTF8ToString(ptr: number, maxBytesToRead?: number | undefined): string", - "UTF8ToString(ptr: number, maxBytesToRead?: number): string", - ) - .replace( - "lengthBytesUTF8(str: any): number", - "lengthBytesUTF8(str: string): number", - ) - .replace( - "stringToUTF16(str: any, outPtr: any, maxBytesToWrite: any): number", - "stringToUTF16(str: string, outPtr: number, maxBytesToWrite: number): number", - ) - .replace( - concat!( - "loadWebAssemblyModule(binary: any, flags: any, libName?: string | ", - "undefined, localScope?: any | undefined, handle?: number | undefined): any" - ), - concat!( - "loadWebAssemblyModule(binary: Uint8Array | WebAssembly.Module, flags: Record,", - " libName?: string, localScope?: Record, handle?: number):", - " Promise number>>" - ), - ) - .replace( - "getValue(ptr: number, type?: string): any", - "getValue(ptr: number, type?: string): number", - ) - .replace("HEAPF32: any", "HEAPF32: Float32Array") - .replace("HEAPF64: any", "HEAPF64: Float64Array") - .replace("HEAP_DATA_VIEW: any", "HEAP_DATA_VIEW: DataView") - .replace("HEAP8: any", "HEAP8: Int8Array") - .replace("HEAPU8: any", "HEAPU8: Uint8Array") - .replace("HEAP16: any", "HEAP16: Int16Array") - .replace("HEAPU16: any", "HEAPU16: Uint16Array") - .replace("HEAP32: any", "HEAP32: Int32Array") - .replace("HEAPU32: any", "HEAPU32: Uint32Array") - .replace("HEAP64: any", "HEAP64: BigInt64Array") - .replace("HEAPU64: any", "HEAPU64: BigUint64Array") - .replace("BigInt;", "bigint;") - .replace("BigInt)", "bigint)") - .replace( - "WasmModule & typeof RuntimeExports;", - indoc! {" - WasmModule & typeof RuntimeExports & { - currentParseCallback: ((index: number, position: {row: number, column: number}) => string | undefined) | null; - currentLogCallback: ((message: string, isLex: boolean) => void) | null; - currentProgressCallback: ((state: {currentOffset: number, hasError: boolean}) => void) | null; - currentQueryProgressCallback: ((state: {currentOffset: number}) => void) | null; - }; - "}, - ) - .replace( - "MainModuleFactory (options?: unknown): Promise", - "MainModuleFactory(options?: Partial): Promise", - ); - fs::write(file, content)?; - } - - // Post-process the source map to embed source content for optimized builds - let map_path = Path::new("lib") - .join("binding_web") - .join("lib") - .join("web-tree-sitter.wasm.map"); - if map_path.exists() { - if let Err(e) = embed_sources_in_map(&map_path) { - eprintln!("Warning: Failed to embed sources in source map: {e}"); - } - } - - Ok(()) -} - -/// This ensures that the wasi-sdk is available, downloading and extracting it if necessary, -/// and returns the path to the `clang` executable. -/// -/// If `TREE_SITTER_WASI_SDK_PATH` is set, it will use that path to look for the clang executable. -/// -/// Note that this is just a minimially modified version of -/// `tree_sitter_loader::ensure_wasi_sdk_exists`. In the loader, this functionality is implemented -/// as a private method of `Loader`. Rather than add this to the public API, we just -/// re-implement it. Any fixes and/or modifications made to the loader's copy should be reflected -/// here. -pub fn ensure_wasi_sdk_exists() -> Result { - let possible_executables = if cfg!(windows) { - vec![ - "clang.exe", - "wasm32-unknown-wasi-clang.exe", - "wasm32-wasi-clang.exe", - ] - } else { - vec!["clang", "wasm32-unknown-wasi-clang", "wasm32-wasi-clang"] - }; - - if let Ok(wasi_sdk_path) = std::env::var("TREE_SITTER_WASI_SDK_PATH") { - let wasi_sdk_dir = PathBuf::from(wasi_sdk_path); - - for exe in &possible_executables { - let clang_exe = wasi_sdk_dir.join("bin").join(exe); - if clang_exe.exists() { - return Ok(clang_exe); - } - } - - Err(LoaderError::WasiSDKClang(WasiSDKClangError { - wasi_sdk_dir: wasi_sdk_dir.to_string_lossy().to_string(), - possible_executables: possible_executables.clone(), - download: false, - }))?; - } - - let cache_dir = etcetera::choose_base_strategy()? - .cache_dir() - .join("tree-sitter"); - fs::create_dir_all(&cache_dir).map_err(|error| { - LoaderError::IO(IoError { - error, - path: Some(cache_dir.to_string_lossy().to_string()), - }) - })?; - - let wasi_sdk_dir = cache_dir.join("wasi-sdk"); - - for exe in &possible_executables { - let clang_exe = wasi_sdk_dir.join("bin").join(exe); - if clang_exe.exists() { - return Ok(clang_exe); - } - } - - fs::create_dir_all(&wasi_sdk_dir).map_err(|error| { - LoaderError::IO(IoError { - error, - path: Some(wasi_sdk_dir.to_string_lossy().to_string()), - }) - })?; - - let arch_os = if cfg!(target_os = "macos") { - if cfg!(target_arch = "aarch64") { - "arm64-macos" - } else { - "x86_64-macos" - } - } else if cfg!(target_os = "windows") { - if cfg!(target_arch = "aarch64") { - "arm64-windows" - } else { - "x86_64-windows" - } - } else if cfg!(target_os = "linux") { - if cfg!(target_arch = "aarch64") { - "arm64-linux" - } else { - "x86_64-linux" - } - } else { - Err(LoaderError::WasiSDKPlatform)? - }; - - let sdk_filename = format!("wasi-sdk-{WASI_SDK_VERSION}-{arch_os}.tar.gz"); - let wasi_sdk_major_version = WASI_SDK_VERSION - .trim_end_matches(char::is_numeric) // trim minor version... - .trim_end_matches('.'); // ...and '.' separator - let sdk_url = format!( - "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-{wasi_sdk_major_version}/{sdk_filename}", - ); - - eprintln!("Downloading wasi-sdk from {sdk_url}..."); - let temp_tar_path = cache_dir.join(sdk_filename); - - let status = Command::new("curl") - .arg("-f") - .arg("-L") - .arg("-o") - .arg(&temp_tar_path) - .arg(&sdk_url) - .status() - .map_err(|e| LoaderError::Curl(sdk_url.clone(), e))?; - - if !status.success() { - Err(LoaderError::WasiSDKDownload(sdk_url))?; - } - - eprintln!("Extracting wasi-sdk to {}...", wasi_sdk_dir.display()); - extract_tar_gz_with_strip(&temp_tar_path, &wasi_sdk_dir)?; - - fs::remove_file(temp_tar_path).ok(); - for exe in &possible_executables { - let clang_exe = wasi_sdk_dir.join("bin").join(exe); - if clang_exe.exists() { - return Ok(clang_exe); - } - } - - Err(LoaderError::WasiSDKClang(WasiSDKClangError { - wasi_sdk_dir: wasi_sdk_dir.to_string_lossy().to_string(), - possible_executables, - download: true, - }))? -} - -/// Extracts a tar.gz archive with `tar`, stripping the first path component. -fn extract_tar_gz_with_strip(archive_path: &Path, destination: &Path) -> Result<()> { - let status = Command::new("tar") - .arg("-xzf") - .arg(archive_path) - .arg("--strip-components=1") - .arg("-C") - .arg(destination) - .status() - .map_err(|e| LoaderError::Tar(archive_path.to_string_lossy().to_string(), e))?; - - if !status.success() { - Err(LoaderError::Extraction( - archive_path.to_string_lossy().to_string(), - destination.to_string_lossy().to_string(), - ))?; - } - - Ok(()) -} - -pub fn run_wasm_stdlib() -> Result<()> { - let export_flags = include_str!("../../../lib/src/wasm/stdlib-symbols.txt") - .lines() - .map(|line| format!("-Wl,--export={}", &line[1..line.len() - 2])) - .collect::>(); - - let clang_exe = ensure_wasi_sdk_exists()?; - - let output = Command::new(&clang_exe) - .args([ - "-o", - "stdlib.wasm", - "-Os", - "-fPIC", - "-DTREE_SITTER_FEATURE_WASM", - "-Wl,--no-entry", - "-Wl,--stack-first", - "-Wl,-z", - "-Wl,stack-size=65536", - "-Wl,--import-undefined", - "-Wl,--import-memory", - "-Wl,--import-table", - "-Wl,--strip-debug", - "-Wl,--export=__wasm_call_ctors", - "-Wl,--export=__stack_pointer", - "-Wl,--export=reset_heap", - ]) - .args(&export_flags) - .arg("crates/language/wasm/src/stdlib.c") - .output()?; - - bail_on_err(&output, "Failed to compile the Tree-sitter Wasm stdlib")?; - - let xxd = Command::new("xxd") - .args(["-C", "-i", "stdlib.wasm"]) - .output()?; - - bail_on_err( - &xxd, - "Failed to run xxd on the compiled Tree-sitter Wasm stdlib", - )?; - - fs::write("lib/src/wasm/wasm-stdlib.h", xxd.stdout)?; - - fs::rename("stdlib.wasm", "target/stdlib.wasm")?; - - Ok(()) -} diff --git a/crates/xtask/src/bump.rs b/crates/xtask/src/bump.rs deleted file mode 100644 index 02254274..00000000 --- a/crates/xtask/src/bump.rs +++ /dev/null @@ -1,295 +0,0 @@ -use std::{cmp::Ordering, path::Path}; - -use anyhow::{anyhow, Context, Result}; -use indoc::indoc; -use semver::{Prerelease, Version}; - -use crate::{create_commit, BumpVersion}; - -pub fn get_latest_tag() -> Result { - let output = std::process::Command::new("git") - .args(["tag", "-l"]) - .output()?; - if !output.status.success() { - anyhow::bail!( - "Failed to list tags: {}", - String::from_utf8_lossy(&output.stderr) - ); - } - - let mut tags = String::from_utf8(output.stdout)? - .lines() - .filter_map(|tag| Version::parse(tag.strip_prefix('v').unwrap_or(tag)).ok()) - .collect::>(); - - tags.sort_by( - |a, b| match (a.pre != Prerelease::EMPTY, b.pre != Prerelease::EMPTY) { - (true, true) | (false, false) => a.cmp(b), - (true, false) => Ordering::Less, - (false, true) => Ordering::Greater, - }, - ); - - tags.last() - .map(std::string::ToString::to_string) - .ok_or_else(|| anyhow!("No tags found")) -} - -pub fn run(args: BumpVersion) -> Result<()> { - let latest_tag = get_latest_tag()?; - let current_version = Version::parse(&latest_tag)?; - - let output = std::process::Command::new("git") - .args(["rev-parse", &format!("v{latest_tag}")]) - .output()?; - if !output.status.success() { - anyhow::bail!( - "Failed to get tag SHA: {}", - String::from_utf8_lossy(&output.stderr) - ); - } - - let workspace_toml_version = Version::parse(&fetch_workspace_version()?)?; - - if current_version.major != workspace_toml_version.major - && current_version.minor != workspace_toml_version.minor - { - eprintln!( - indoc! {" - Seems like the workspace Cargo.toml ({}) version does not match up with the latest git tag ({}). - Please ensure you don't change that yourself, this subcommand will handle this for you. - "}, - workspace_toml_version, latest_tag - ); - return Ok(()); - } - - let next_version = args.version; - - println!("Bumping from {current_version} to {next_version}"); - update_crates(¤t_version, &next_version)?; - update_makefile(&next_version)?; - update_cmake(&next_version)?; - update_nix(&next_version)?; - update_npm(&next_version)?; - update_zig(&next_version)?; - tag_next_version(&next_version)?; - - Ok(()) -} - -fn tag_next_version(next_version: &Version) -> Result<()> { - let commit_sha = create_commit( - &format!("{next_version}"), - &[ - "Cargo.lock", - "Cargo.toml", - "Makefile", - "build.zig.zon", - "flake.nix", - "crates/cli/Cargo.toml", - "crates/cli/npm/package.json", - "crates/cli/npm/package-lock.json", - "crates/config/Cargo.toml", - "crates/highlight/Cargo.toml", - "crates/loader/Cargo.toml", - "crates/tags/Cargo.toml", - "CMakeLists.txt", - "lib/Cargo.toml", - "lib/binding_web/package.json", - "lib/binding_web/package-lock.json", - ], - )?; - - // Create tag - let output = std::process::Command::new("git") - .args([ - "tag", - "-a", - &format!("v{next_version}"), - "-m", - &format!("v{next_version}"), - &commit_sha, - ]) - .output()?; - if !output.status.success() { - anyhow::bail!( - "Failed to create tag: {}", - String::from_utf8_lossy(&output.stderr) - ); - } - - println!("Tagged commit {commit_sha} with tag v{next_version}"); - - Ok(()) -} - -fn update_makefile(next_version: &Version) -> Result<()> { - let makefile = std::fs::read_to_string("Makefile")?; - let makefile = makefile - .lines() - .map(|line| { - if line.starts_with("VERSION") { - format!("VERSION := {next_version}") - } else { - line.to_string() - } - }) - .collect::>() - .join("\n") - + "\n"; - - std::fs::write("Makefile", makefile)?; - - Ok(()) -} - -fn update_cmake(next_version: &Version) -> Result<()> { - let cmake = std::fs::read_to_string("CMakeLists.txt")?; - let cmake = cmake - .lines() - .map(|line| { - if line.contains(" VERSION") { - let start_quote = line.find('"').unwrap(); - let end_quote = line.rfind('"').unwrap(); - format!( - "{}{next_version}{}", - &line[..=start_quote], - &line[end_quote..] - ) - } else { - line.to_string() - } - }) - .collect::>() - .join("\n") - + "\n"; - - std::fs::write("CMakeLists.txt", cmake)?; - - Ok(()) -} - -fn update_nix(next_version: &Version) -> Result<()> { - let nix = std::fs::read_to_string("flake.nix")?; - let nix = nix - .lines() - .map(|line| { - if line.trim_start().starts_with("version =") { - format!(" version = \"{next_version}\";") - } else { - line.to_string() - } - }) - .collect::>() - .join("\n") - + "\n"; - - std::fs::write("flake.nix", nix)?; - - Ok(()) -} - -fn update_crates(current_version: &Version, next_version: &Version) -> Result<()> { - let mut cmd = std::process::Command::new("cargo"); - cmd.arg("workspaces").arg("version"); - - if next_version.minor > current_version.minor { - cmd.arg("minor"); - } else { - cmd.arg("patch"); - } - - cmd.arg("--no-git-commit") - .arg("--yes") - .arg("--force") - .arg("tree-sitter{,-cli,-config,-generate,-loader,-highlight,-tags}") - .arg("--ignore-changes") - .arg("crates/language/*"); - - let status = cmd.status()?; - - if !status.success() { - return Err(anyhow!("Failed to update crates")); - } - - Ok(()) -} - -fn update_npm(next_version: &Version) -> Result<()> { - for npm_project in ["lib/binding_web", "crates/cli/npm"] { - let npm_path = Path::new(npm_project); - - let package_json_path = npm_path.join("package.json"); - - let package_json = serde_json::from_str::( - &std::fs::read_to_string(&package_json_path) - .with_context(|| format!("Failed to read {}", package_json_path.display()))?, - )?; - - let mut package_json = package_json - .as_object() - .ok_or_else(|| anyhow!("Invalid package.json"))? - .clone(); - package_json.insert( - "version".to_string(), - serde_json::Value::String(next_version.to_string()), - ); - - let package_json = serde_json::to_string_pretty(&package_json)? + "\n"; - - std::fs::write(package_json_path, package_json)?; - - let Ok(cmd) = std::process::Command::new("npm") - .arg("install") - .arg("--package-lock-only") - .arg("--ignore-scripts") - .current_dir(npm_path) - .output() - else { - return Ok(()); // npm is not `executable`, ignore - }; - - if !cmd.status.success() { - let stderr = String::from_utf8_lossy(&cmd.stderr); - return Err(anyhow!( - "Failed to run `npm install` in {}:\n{stderr}", - npm_path.display() - )); - } - } - - Ok(()) -} - -fn update_zig(next_version: &Version) -> Result<()> { - let zig = std::fs::read_to_string("build.zig.zon")? - .lines() - .map(|line| { - if line.starts_with(" .version") { - format!(" .version = \"{next_version}\",") - } else { - line.to_string() - } - }) - .collect::>() - .join("\n") - + "\n"; - - std::fs::write("build.zig.zon", zig)?; - - Ok(()) -} - -/// read Cargo.toml and get the version -fn fetch_workspace_version() -> Result { - std::fs::read_to_string("Cargo.toml")? - .lines() - .find(|line| line.starts_with("version = ")) - .and_then(|line| { - line.split_terminator('"') - .next_back() - .map(|s| s.to_string()) - }) - .ok_or_else(|| anyhow!("No version found in Cargo.toml")) -} diff --git a/crates/xtask/src/embed_sources.rs b/crates/xtask/src/embed_sources.rs deleted file mode 100644 index ce8ec403..00000000 --- a/crates/xtask/src/embed_sources.rs +++ /dev/null @@ -1,61 +0,0 @@ -use anyhow::Result; -use std::fs; -use std::path::Path; - -/// Restores sourcesContent if it was stripped by Binaryen. -/// -/// This is a workaround for Binaryen where `wasm-opt -O2` and higher -/// optimization levels strip the `sourcesContent` field from source maps, -/// even when the source map was generated with `--sources` flag. -/// -/// This is fixed upstream in Binaryen as of Apr 9, 2025, but there hasn't been a release with the fix yet. -/// See: -/// -/// This reads the original source files and embeds them in the -/// source map's `sourcesContent` field, making debugging possible even -/// with optimized builds. -/// -/// TODO: Once Binaryen releases a version with the fix, and emscripten updates to that -/// version, and we update our emscripten version, this function can be removed. -pub fn embed_sources_in_map(map_path: &Path) -> Result<()> { - let map_content = fs::read_to_string(map_path)?; - let mut map: serde_json::Value = serde_json::from_str(&map_content)?; - - if let Some(sources_content) = map.get("sourcesContent") { - if let Some(arr) = sources_content.as_array() { - if !arr.is_empty() && arr.iter().any(|v| !v.is_null()) { - return Ok(()); - } - } - } - - let sources = map["sources"] - .as_array() - .ok_or_else(|| anyhow::anyhow!("No sources array in source map"))?; - - let map_dir = map_path.parent().unwrap_or(Path::new(".")); - let mut sources_content = Vec::new(); - - for source in sources { - let source_path = source.as_str().unwrap_or(""); - let full_path = map_dir.join(source_path); - - let content = if full_path.exists() { - match fs::read_to_string(&full_path) { - Ok(content) => serde_json::Value::String(content), - Err(_) => serde_json::Value::Null, - } - } else { - serde_json::Value::Null - }; - - sources_content.push(content); - } - - map["sourcesContent"] = serde_json::Value::Array(sources_content); - - let output = serde_json::to_string(&map)?; - fs::write(map_path, output)?; - - Ok(()) -} diff --git a/crates/xtask/src/fetch.rs b/crates/xtask/src/fetch.rs deleted file mode 100644 index 6fa431c6..00000000 --- a/crates/xtask/src/fetch.rs +++ /dev/null @@ -1,140 +0,0 @@ -use crate::{bail_on_err, root_dir, FetchFixtures, EMSCRIPTEN_VERSION}; -use anyhow::Result; -use std::{fs, process::Command}; - -pub fn run_fixtures(args: &FetchFixtures) -> Result<()> { - let fixtures_dir = root_dir().join("test").join("fixtures"); - let grammars_dir = fixtures_dir.join("grammars"); - let fixtures_path = fixtures_dir.join("fixtures.json"); - - // grammar name, tag - let mut fixtures: Vec<(String, String)> = - serde_json::from_str(&fs::read_to_string(&fixtures_path)?)?; - - for (grammar, tag) in &mut fixtures { - let grammar_dir = grammars_dir.join(&grammar); - let grammar_url = format!("https://github.com/tree-sitter/tree-sitter-{grammar}"); - - println!("Fetching the {grammar} grammar..."); - - if !grammar_dir.exists() { - let mut command = Command::new("git"); - command.args([ - "clone", - "--depth", - "1", - "--branch", - tag, - &grammar_url, - &grammar_dir.to_string_lossy(), - ]); - bail_on_err( - &command.spawn()?.wait_with_output()?, - &format!("Failed to clone the {grammar} grammar"), - )?; - } else { - let mut describe_command = Command::new("git"); - describe_command.current_dir(&grammar_dir).args([ - "describe", - "--tags", - "--exact-match", - "HEAD", - ]); - - let output = describe_command.output()?; - let current_tag = String::from_utf8_lossy(&output.stdout); - let current_tag = current_tag.trim(); - - if current_tag != tag { - println!("Updating {grammar} grammar from {current_tag} to {tag}..."); - - let mut fetch_command = Command::new("git"); - fetch_command.current_dir(&grammar_dir).args([ - "fetch", - "origin", - &format!("refs/tags/{tag}:refs/tags/{tag}"), - ]); - bail_on_err( - &fetch_command.spawn()?.wait_with_output()?, - &format!("Failed to fetch tag {tag} for {grammar} grammar"), - )?; - - let mut reset_command = Command::new("git"); - reset_command - .current_dir(&grammar_dir) - .args(["reset", "--hard", "HEAD"]); - bail_on_err( - &reset_command.spawn()?.wait_with_output()?, - &format!("Failed to reset {grammar} grammar working tree"), - )?; - - let mut checkout_command = Command::new("git"); - checkout_command - .current_dir(&grammar_dir) - .args(["checkout", tag]); - bail_on_err( - &checkout_command.spawn()?.wait_with_output()?, - &format!("Failed to checkout tag {tag} for {grammar} grammar"), - )?; - } else { - println!("{grammar} grammar is already at tag {tag}"); - } - } - } - - if args.update { - println!("Updating the fixtures lock file"); - fs::write( - &fixtures_path, - // format the JSON without extra newlines - serde_json::to_string(&fixtures)? - .replace("[[", "[\n [") - .replace("],", "],\n ") - .replace("]]", "]\n]"), - )?; - } - - Ok(()) -} - -pub fn run_emscripten() -> Result<()> { - let emscripten_dir = root_dir().join("target").join("emsdk"); - if emscripten_dir.exists() { - println!("Emscripten SDK already exists"); - return Ok(()); - } - println!("Cloning the Emscripten SDK..."); - - let mut command = Command::new("git"); - command.args([ - "clone", - "https://github.com/emscripten-core/emsdk.git", - &emscripten_dir.to_string_lossy(), - ]); - bail_on_err( - &command.spawn()?.wait_with_output()?, - "Failed to clone the Emscripten SDK", - )?; - - std::env::set_current_dir(&emscripten_dir)?; - - let emsdk = if cfg!(windows) { - "emsdk.bat" - } else { - "./emsdk" - }; - - let mut command = Command::new(emsdk); - command.args(["install", EMSCRIPTEN_VERSION]); - bail_on_err( - &command.spawn()?.wait_with_output()?, - "Failed to install Emscripten", - )?; - - let mut command = Command::new(emsdk); - command.args(["activate", EMSCRIPTEN_VERSION]); - bail_on_err( - &command.spawn()?.wait_with_output()?, - "Failed to activate Emscripten", - ) -} diff --git a/crates/xtask/src/test_schema.rs b/crates/xtask/src/test_schema.rs deleted file mode 100644 index a2f65ed2..00000000 --- a/crates/xtask/src/test_schema.rs +++ /dev/null @@ -1,25 +0,0 @@ -use std::path::PathBuf; - -use anyhow::Result; -use serde_json::to_writer_pretty; - -use tree_sitter_cli::test::TestSummary; - -pub fn run_test_schema() -> Result<()> { - let schema = schemars::schema_for!(TestSummary); - - let xtask_path: PathBuf = env!("CARGO_MANIFEST_DIR").into(); - let schema_path = xtask_path - .parent() - .unwrap() - .parent() - .unwrap() - .join("docs") - .join("src") - .join("assets") - .join("schemas") - .join("test-summary.schema.json"); - let mut file = std::fs::File::create(schema_path)?; - - Ok(to_writer_pretty(&mut file, &schema)?) -} diff --git a/crates/xtask/src/upgrade_wasmtime.rs b/crates/xtask/src/upgrade_wasmtime.rs deleted file mode 100644 index 90e5b0f2..00000000 --- a/crates/xtask/src/upgrade_wasmtime.rs +++ /dev/null @@ -1,101 +0,0 @@ -use std::process::Command; - -use anyhow::{Context, Result}; -use semver::Version; - -use crate::{create_commit, UpgradeWasmtime}; - -const WASMTIME_RELEASE_URL: &str = "https://github.com/bytecodealliance/wasmtime/releases/download"; - -fn update_cargo(version: &Version) -> Result<()> { - let file = std::fs::read_to_string("lib/Cargo.toml")?; - let mut old_lines = file.lines(); - let mut new_lines = Vec::with_capacity(old_lines.size_hint().0); - - while let Some(line) = old_lines.next() { - new_lines.push(line.to_string()); - if line == "[dependencies.wasmtime-c-api]" { - let _ = old_lines.next(); - new_lines.push(format!("version = \"{version}\"")); - } - } - - std::fs::write("lib/Cargo.toml", new_lines.join("\n") + "\n")?; - - Command::new("cargo") - .arg("update") - .status() - .map(|_| ()) - .with_context(|| "Failed to execute cargo update") -} - -fn zig_fetch(lines: &mut Vec, version: &Version, url_suffix: &str) -> Result<()> { - let url = &format!("{WASMTIME_RELEASE_URL}/v{version}/wasmtime-v{version}-{url_suffix}"); - println!(" Fetching {url}"); - lines.push(format!(" .url = \"{url}\",")); - - let output = Command::new("zig") - .arg("fetch") - .arg(url) - .output() - .with_context(|| format!("Failed to execute zig fetch {url}"))?; - - let hash = String::from_utf8_lossy(&output.stdout); - lines.push(format!(" .hash = \"{}\",", hash.trim_end())); - - Ok(()) -} - -fn update_zig(version: &Version) -> Result<()> { - let file = std::fs::read_to_string("build.zig.zon")?; - let mut old_lines = file.lines(); - let new_lines = &mut Vec::with_capacity(old_lines.size_hint().0); - - macro_rules! match_wasmtime_zig_dep { - ($line:ident, {$($platform:literal => [$($arch:literal),*]),*,}) => { - match $line { - $($(concat!(" .wasmtime_c_api_", $arch, "_", $platform, " = .{") => { - let (_, _) = (old_lines.next(), old_lines.next()); - let suffix = if $platform == "windows" || $platform == "mingw" { - concat!($arch, "-", $platform, "-c-api.zip") - } else { - concat!($arch, "-", $platform, "-c-api.tar.xz") - }; - zig_fetch(new_lines, version, suffix)?; - })*)* - _ => {} - } - }; - } - - while let Some(line) = old_lines.next() { - new_lines.push(line.to_string()); - match_wasmtime_zig_dep!(line, { - "android" => ["aarch64", "x86_64"], - "linux" => ["aarch64", "armv7", "i686", "riscv64gc", "s390x", "x86_64"], - "macos" => ["aarch64", "x86_64"], - "mingw" => ["x86_64"], - "musl" => ["aarch64", "x86_64"], - "windows" => ["aarch64", "i686", "x86_64"], - }); - } - - std::fs::write("build.zig.zon", new_lines.join("\n") + "\n")?; - - Ok(()) -} - -pub fn run(args: &UpgradeWasmtime) -> Result<()> { - println!("Upgrading wasmtime for Rust"); - update_cargo(&args.version)?; - - println!("Upgrading wasmtime for Zig"); - update_zig(&args.version)?; - - create_commit( - &format!("build(deps): bump wasmtime-c-api to v{}", args.version), - &["lib/Cargo.toml", "Cargo.lock", "build.zig.zon"], - )?; - - Ok(()) -} diff --git a/docs/book.toml b/docs/book.toml index 0894c988..664a1f24 100644 --- a/docs/book.toml +++ b/docs/book.toml @@ -4,6 +4,7 @@ authors = [ "Amaan Qureshi ", ] language = "en" +multilingual = false src = "src" title = "Tree-sitter" diff --git a/docs/package.nix b/docs/package.nix deleted file mode 100644 index 1d07631f..00000000 --- a/docs/package.nix +++ /dev/null @@ -1,33 +0,0 @@ -{ - stdenv, - lib, - version, - mdbook, - mdbook-admonish, -}: -stdenv.mkDerivation { - inherit version; - - src = ./.; - pname = "tree-sitter-docs"; - - nativeBuildInputs = [ - mdbook - mdbook-admonish - ]; - - buildPhase = '' - mdbook build - ''; - - installPhase = '' - mkdir -p $out/share/doc - cp -r book $out/share/doc/tree-sitter - ''; - - meta = { - description = "Tree-sitter documentation"; - homepage = "https://tree-sitter.github.io/tree-sitter"; - license = lib.licenses.mit; - }; -} diff --git a/docs/src/3-syntax-highlighting.md b/docs/src/3-syntax-highlighting.md index c6356fbb..b11a2e86 100644 --- a/docs/src/3-syntax-highlighting.md +++ b/docs/src/3-syntax-highlighting.md @@ -73,8 +73,9 @@ The behaviors of these three files are described in the next section. ## Queries -Tree-sitter's syntax highlighting system is based on *tree queries*, which are a general system for pattern-matching on -Tree-sitter's syntax trees. See [this section][pattern matching] of the documentation for more information about tree queries. +Tree-sitter's syntax highlighting system is based on *tree queries*, which are a general system for pattern-matching on Tree-sitter's +syntax trees. See [this section][pattern matching] of the documentation for more information +about tree queries. Syntax highlighting is controlled by *three* different types of query files that are usually included in the `queries` folder. The default names for the query files use the `.scm` file. We chose this extension because it commonly used for files written @@ -179,16 +180,6 @@ The capture names are as follows: - `@local.reference` — indicates that a syntax node contains the *name*, which *may* refer to an earlier definition within some enclosing scope. -Additionally, to ignore certain nodes from being tagged, you can use the `@ignore` capture. This is useful if you want to -exclude a subset of nodes from being tagged. When writing a query leveraging this, you should ensure this pattern comes -before any other patterns that would be used for tagging, for example: - -```scheme -(expression (identifier) @ignore) - -(identifier) @local.reference -``` - When highlighting a file, Tree-sitter will keep track of the set of scopes that contains any given position, and the set of definitions within each scope. When processing a syntax node that is captured as a `local.reference`, Tree-sitter will try to find a definition for a name that matches the node's text. If it finds a match, Tree-sitter will ensure that the @@ -432,7 +423,7 @@ not the `keyword` class. ``` [erb]: https://en.wikipedia.org/wiki/ERuby -[highlight crate]: https://github.com/tree-sitter/tree-sitter/tree/master/crates/highlight +[highlight crate]: https://github.com/tree-sitter/tree-sitter/tree/master/highlight [init-config]: ./cli/init-config.md [init]: ./cli/init.md#structure-of-tree-sitterjson [js grammar]: https://github.com/tree-sitter/tree-sitter-javascript diff --git a/docs/src/4-code-navigation.md b/docs/src/4-code-navigation.md index 02a9fa4d..46d60307 100644 --- a/docs/src/4-code-navigation.md +++ b/docs/src/4-code-navigation.md @@ -3,8 +3,7 @@ Tree-sitter can be used in conjunction with its [query language][query language] as a part of code navigation systems. An example of such a system can be seen in the `tree-sitter tags` command, which emits a textual dump of the interesting syntactic nodes in its file argument. A notable application of this is GitHub's support for [search-based code navigation][gh search]. -This document exists to describe how to integrate with such systems, and how to extend this functionality to any language -with a Tree-sitter grammar. +This document exists to describe how to integrate with such systems, and how to extend this functionality to any language with a Tree-sitter grammar. ## Tagging and captures @@ -13,9 +12,9 @@ entities. Having found them, you use a syntax capture to label the entity and it The essence of a given tag lies in two pieces of data: the _role_ of the entity that is matched (i.e. whether it is a definition or a reference) and the _kind_ of that entity, which describes how the entity is used -(i.e. whether it's a class definition, function call, variable reference, and so on). Our convention is to use a syntax -capture following the `@role.kind` capture name format, and another inner capture, always called `@name`, that pulls out -the name of a given identifier. +(i.e. whether it's a class definition, function call, variable reference, and so on). Our convention is to use a syntax capture +following the `@role.kind` capture name format, and another inner capture, always called `@name`, that pulls out the name +of a given identifier. You may optionally include a capture named `@doc` to bind a docstring. For convenience purposes, the tagging system provides two built-in functions, `#select-adjacent!` and `#strip!` that are convenient for removing comment syntax from a docstring. diff --git a/docs/src/5-implementation.md b/docs/src/5-implementation.md index 36963d8e..987a91ff 100644 --- a/docs/src/5-implementation.md +++ b/docs/src/5-implementation.md @@ -51,10 +51,10 @@ WIP [crates]: https://crates.io [npm]: https://npmjs.com [gh]: https://github.com/tree-sitter/tree-sitter/releases/latest -[src]: https://github.com/tree-sitter/tree-sitter/tree/master/crates/cli/src +[src]: https://github.com/tree-sitter/tree-sitter/tree/master/cli/src [schema]: https://tree-sitter.github.io/tree-sitter/assets/schemas/grammar.schema.json -[parse grammar]: https://github.com/tree-sitter/tree-sitter/blob/master/crates/generate/src/parse_grammar.rs +[parse grammar]: https://github.com/tree-sitter/tree-sitter/blob/master/cli/generate/src/parse_grammar.rs [enum]: https://doc.rust-lang.org/book/ch06-01-defining-an-enum.html -[rules.rs]: https://github.com/tree-sitter/tree-sitter/blob/master/crates/generate/src/rules.rs -[prepare grammar]: https://github.com/tree-sitter/tree-sitter/tree/master/crates/generate/src/prepare_grammar +[rules.rs]: https://github.com/tree-sitter/tree-sitter/blob/master/cli/generate/src/rules.rs +[prepare grammar]: https://github.com/tree-sitter/tree-sitter/tree/master/cli/generate/src/prepare_grammar [symbols]: https://en.wikipedia.org/wiki/Terminal_and_nonterminal_symbols diff --git a/docs/src/6-contributing.md b/docs/src/6-contributing.md index 0fad6581..4584be4c 100644 --- a/docs/src/6-contributing.md +++ b/docs/src/6-contributing.md @@ -14,7 +14,7 @@ To make changes to Tree-sitter, you should have: 2. A [Rust toolchain][rust], for compiling the Rust bindings, the highlighting library, and the CLI. 3. Node.js and NPM, for generating parsers from `grammar.js` files. 4. Either [Emscripten][emscripten], [Docker][docker], or [podman][podman] for -compiling the library to Wasm. +compiling the library to WASM. ### Building @@ -25,7 +25,7 @@ git clone https://github.com/tree-sitter/tree-sitter cd tree-sitter ``` -Optionally, build the Wasm library. If you skip this step, then the `tree-sitter playground` command will require an internet +Optionally, build the WASM library. If you skip this step, then the `tree-sitter playground` command will require an internet connection. If you have Emscripten installed, this will use your `emcc` compiler. Otherwise, it will use Docker or Podman: ```sh @@ -45,15 +45,15 @@ This will create the `tree-sitter` CLI executable in the `target/release` folder If you want to automatically install the `tree-sitter` CLI in your system, you can run: ```sh -cargo install --path crates/cli +cargo install --path cli ``` If you're going to be in a fast iteration cycle and would like the CLI to build faster, you can use the `release-dev` profile: ```sh -cargo build --profile release-dev +cargo build --release --profile release-dev # or -cargo install --path crates/cli --profile release-dev +cargo install --path cli --profile release-dev ``` ### Testing @@ -76,26 +76,13 @@ Then you can run the tests: cargo xtask test ``` -Similarly, to test the Wasm binding, you need to compile these parsers to Wasm: +Similarly, to test the WASM binding, you need to compile these parsers to WASM: ```sh cargo xtask generate-fixtures --wasm cargo xtask test-wasm ``` -#### Wasm Stdlib - -The tree-sitter Wasm stdlib can be built via xtask: - -```sh -cargo xtask build-wasm-stdlib -``` - -This command looks for the [Wasi SDK][wasi_sdk] indicated by the `TREE_SITTER_WASI_SDK_PATH` -environment variable. If you don't have the binary, it can be downloaded from wasi-sdk's [releases][wasi-sdk-releases] -page. Note that any changes to `crates/language/wasm/**` requires rebuilding the tree-sitter Wasm stdlib via -`cargo xtask build-wasm-stdlib`. - ### Debugging The test script has a number of useful flags. You can list them all by running `cargo xtask test -h`. @@ -121,13 +108,6 @@ Additionally, if you want to run a particular _example_ from the corpus, you can cargo xtask test -l javascript -e Arrays ``` -If you are using `lldb` to debug the C library, tree-sitter provides custom pretty printers for several of its types. -You can enable these helpers by importing them: - -```sh -(lldb) command script import /path/to/tree-sitter/lib/lldb_pretty_printers/tree_sitter_types.py -``` - ## Published Packages The main [`tree-sitter/tree-sitter`][ts repo] repository contains the source code for @@ -139,7 +119,7 @@ several packages that are published to package registries for different language * [`tree-sitter-cli`][cli crate] — The command-line tool * JavaScript modules on [npmjs.com][npmjs]: - * [`web-tree-sitter`][web-ts] — A Wasm-based JavaScript binding to the core library + * [`web-tree-sitter`][web-ts] — A WASM-based JavaScript binding to the core library * [`tree-sitter-cli`][cli package] — The command-line tool There are also several other dependent repositories that contain other published packages: @@ -199,7 +179,7 @@ a short delay. Once you've made a change that you're happy with, you can submit The playground page is a little more complicated, but if you know some basic JavaScript and CSS you should be able to make changes. The playground code can be found in [`docs/src/assets/js/playground.js`][playground], and its corresponding css at [`docs/src/assets/css/playground.css`][playground css]. The editor of choice we use for the playground is [CodeMirror][codemirror], -and the tree-sitter module is fetched from [here][js url]. This, along with the Wasm module and Wasm parsers, live in the +and the tree-sitter module is fetched from [here][js url]. This, along with the wasm module and wasm parsers, live in the [.github.io repo][gh.io repo]. [admonish]: https://github.com/tommilligan/mdbook-admonish @@ -217,7 +197,7 @@ and the tree-sitter module is fetched from [here][js url]. This, along with the [go package]: https://pkg.go.dev/github.com/tree-sitter/go-tree-sitter [go ts]: https://github.com/tree-sitter/go-tree-sitter [highlight crate]: https://crates.io/crates/tree-sitter-highlight -[js url]: https://tree-sitter.github.io/web-tree-sitter.js +[js url]: https://tree-sitter.github.io/tree-sitter.js [lib crate]: https://crates.io/crates/tree-sitter [mdBook]: https://rust-lang.github.io/mdBook [mdbook cli]: https://rust-lang.github.io/mdBook/guide/installation.html @@ -233,6 +213,4 @@ and the tree-sitter module is fetched from [here][js url]. This, along with the [pypi]: https://pypi.org [rust]: https://rustup.rs [ts repo]: https://github.com/tree-sitter/tree-sitter -[wasi_sdk]: https://github.com/WebAssembly/wasi-sdk -[wasi-sdk-releases]: https://github.com/WebAssembly/wasi-sdk/releases [web-ts]: https://www.npmjs.com/package/web-tree-sitter diff --git a/docs/src/7-playground.md b/docs/src/7-playground.md index 81c61805..90f97685 100644 --- a/docs/src/7-playground.md +++ b/docs/src/7-playground.md @@ -7,52 +7,47 @@

Code

-
-
- -
-
Bash
-
C
-
C++
-
C#
-
Go
-
HTML
-
Java
-
JavaScript
-
PHP
-
Python
-
Ruby
-
Rust
-
TOML
-
TypeScript
-
YAML
-
- -
- -
+
+ +
+
Bash
+
C
+
C++
+
C#
+
Go
+
HTML
+
Java
+
JavaScript
+
PHP
+
Python
+
Ruby
+
Rust
+
TOML
+
TypeScript
+
YAML
+
@@ -75,16 +70,7 @@
-

- Tree - -

+

Tree


@@ -93,7 +79,7 @@
 

About

You can try out tree-sitter with a few pre-selected grammars on this page. You can also run playground locally (with your own grammar) using the -CLI's tree-sitter playground subcommand. +CLI's tree-sitter playground subcommand.

```admonish info @@ -116,7 +102,7 @@ you must use at least one capture, like (node_name) @capture-nameLANGUAGE_BASE_URL = "https://tree-sitter.github.io"; diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 231085ae..6e6eed94 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -15,7 +15,6 @@ - [Predicates and Directives](./using-parsers/queries/3-predicates-and-directives.md) - [API](./using-parsers/queries/4-api.md) - [Static Node Types](./using-parsers/6-static-node-types.md) - - [ABI versions](./using-parsers/7-abi-versions.md) - [Creating Parsers](./creating-parsers/index.md) - [Getting Started](./creating-parsers/1-getting-started.md) - [The Grammar DSL](./creating-parsers/2-the-grammar-dsl.md) diff --git a/docs/src/assets/css/playground.css b/docs/src/assets/css/playground.css index ab59ebec..45236937 100644 --- a/docs/src/assets/css/playground.css +++ b/docs/src/assets/css/playground.css @@ -54,23 +54,6 @@ display: inline-block; } -.language-container { - display: flex; - align-items: center; - gap: 16px; - margin-bottom: 16px; -} - -#language-version { - color: var(--light-text); - font-size: 14px; - font-weight: 500; - padding: 4px 8px; - background: var(--light-bg); - border-radius: 4px; - border: 1px solid var(--light-border); -} - #language-select { background-color: var(--light-bg); border: 1px solid var(--light-border); @@ -86,53 +69,6 @@ background-position: right 8px center; } -#copy-button { - background: none; - border: 1px solid var(--light-border); - border-radius: 4px; - padding: 6px; - cursor: pointer; - color: var(--light-text); - display: inline-flex; - align-items: center; - justify-content: center; - margin-left: 8px; -} - -#copy-button:hover { - background-color: var(--primary-color-alpha); - border-color: var(--light-hover-border); -} - -#copy-button:focus { - outline: none; - border-color: var(--primary-color); - box-shadow: 0 0 0 2px var(--primary-color-alpha); -} - -.toast { - position: fixed; - bottom: 20px; - right: 20px; - background-color: var(--lighbt-bg); - color: var(--light-text); - padding: 12px 16px; - border-radius: 6px; - box-shadow: 0 4px 12px rgba(0, 0, 0, 0.3); - font-size: 14px; - font-weight: 500; - opacity: 0; - transform: translateY(20px); - transition: all 0.3s ease; - z-index: 1000; - pointer-events: none; -} - -.toast.show { - opacity: 1; - transform: translateY(0); -} - .select-button { background-color: var(--light-bg); border: 1px solid var(--light-border); @@ -328,33 +264,13 @@ input[type="checkbox"]:focus { .coal, .navy { - & #language-version, & #language-select, - & #copy-button, & .select-button { background-color: var(--dark-bg); border-color: var(--dark-border); color: var(--dark-text); } - & #copy-button:hover, - & #language-select:hover, - & .select-button:hover { - border-color: var(--dark-border); - background-color: var(--primary-color-alpha-dark); - } - - & .toast { - background-color: var(--dark-bg); - color: var(--dark-text); - } - - #language-select:focus, - & .select-button:focus { - border-color: #79c0ff; - box-shadow: 0 0 0 2px var(--primary-color-alpha-dark); - } - & input[type="checkbox"] { border-color: var(--dark-border); background-color: var(--dark-bg); diff --git a/docs/src/assets/js/playground.js b/docs/src/assets/js/playground.js index ef65c371..babb2afb 100644 --- a/docs/src/assets/js/playground.js +++ b/docs/src/assets/js/playground.js @@ -61,7 +61,7 @@ function initializeCustomSelect({ initialValue = null, addListeners = false }) { } window.initializePlayground = async (opts) => { - const { Parser, Language, Query } = window.TreeSitter; + const { Parser, Language } = window.TreeSitter; const { local } = opts; if (local) { @@ -106,7 +106,6 @@ window.initializePlayground = async (opts) => { const codeInput = document.getElementById("code-input"); const languageSelect = document.getElementById("language-select"); - const languageVersion = document.getElementById('language-version'); const loggingCheckbox = document.getElementById("logging-checkbox"); const anonymousNodes = document.getElementById('anonymous-nodes-checkbox'); const outputContainer = document.getElementById("output-container"); @@ -118,7 +117,6 @@ window.initializePlayground = async (opts) => { const queryContainer = document.getElementById("query-container"); const queryInput = document.getElementById("query-input"); const accessibilityCheckbox = document.getElementById("accessibility-checkbox"); - const copyButton = document.getElementById("copy-button"); const updateTimeSpan = document.getElementById("update-time"); const languagesByName = {}; @@ -128,15 +126,16 @@ window.initializePlayground = async (opts) => { const parser = new Parser(); + console.log(parser, codeInput, queryInput); + const codeEditor = CodeMirror.fromTextArea(codeInput, { lineNumbers: true, showCursorWhenSelecting: true }); codeEditor.on('keydown', (_, event) => { - const key = event.key; - if (key === 'ArrowLeft' || key === 'ArrowRight' || key === '?') { - event.stopPropagation(); // Prevent mdBook from going back/forward, or showing help + if (event.key === 'ArrowLeft' || event.key === 'ArrowRight') { + event.stopPropagation(); // Prevent mdBook from going back/forward } }); @@ -146,9 +145,8 @@ window.initializePlayground = async (opts) => { }); queryEditor.on('keydown', (_, event) => { - const key = event.key; - if (key === 'ArrowLeft' || key === 'ArrowRight' || key === '?') { - event.stopPropagation(); // Prevent mdBook from going back/forward, or showing help + if (event.key === 'ArrowLeft' || event.key === 'ArrowRight') { + event.stopPropagation(); // Prevent mdBook from going back/forward } }); @@ -175,12 +173,11 @@ window.initializePlayground = async (opts) => { queryEditor.on("changes", debounce(handleQueryChange, 150)); loggingCheckbox.addEventListener("change", handleLoggingChange); - anonymousNodes.addEventListener("change", renderTree); + anonymousNodes.addEventListener('change', renderTree); queryCheckbox.addEventListener("change", handleQueryEnableChange); accessibilityCheckbox.addEventListener("change", handleQueryChange); languageSelect.addEventListener("change", handleLanguageChange); outputContainer.addEventListener("click", handleTreeClick); - copyButton?.addEventListener("click", handleCopy); handleQueryEnableChange(); await handleLanguageChange(); @@ -205,15 +202,6 @@ window.initializePlayground = async (opts) => { tree = null; languageName = newLanguageName; - - const metadata = languagesByName[languageName].metadata; - if (languageVersion && metadata) { - languageVersion.textContent = `v${metadata.major_version}.${metadata.minor_version}.${metadata.patch_version}`; - languageVersion.style.visibility = 'visible'; - } else if (languageVersion) { - languageVersion.style.visibility = 'hidden'; - } - parser.setLanguage(languagesByName[newLanguageName]); handleCodeChange(); handleQueryChange(); @@ -304,10 +292,10 @@ window.initializePlayground = async (opts) => { const nodeClass = displayName === 'ERROR' || displayName.startsWith('MISSING') - ? 'node-link error plain' + ? 'node-link error' : cursor.nodeIsNamed - ? 'node-link named plain' - : 'node-link anonymous plain'; + ? 'node-link named' + : 'node-link anonymous'; row = `
${" ".repeat(indentLevel)}${fieldName}` + ` { marks.forEach((m) => m.clear()); if (tree && query) { - const captures = query.captures(tree.rootNode, { - startPosition: { row: startRow, column: 0 }, - endPosition: { row: endRow, column: 0 }, - }); + const captures = query.captures( + tree.rootNode, + { row: startRow, column: 0 }, + { row: endRow, column: 0 }, + ); let lastNodeId; for (const { name, node } of captures) { if (node.id === lastNodeId) continue; @@ -409,7 +398,7 @@ window.initializePlayground = async (opts) => { const queryText = queryEditor.getValue(); try { - query = new Query(parser.language, queryText); + query = parser.language.query(queryText); let match; let row = 0; @@ -497,27 +486,16 @@ window.initializePlayground = async (opts) => { const containerHeight = outputContainerScroll.clientHeight; const offset = treeRowHighlightedIndex * lineHeight; if (scrollTop > offset - 20) { - outputContainerScroll.scrollTo({ top: offset - 20, behavior: 'smooth' }); + $(outputContainerScroll).animate({ scrollTop: offset - 20 }, 150); } else if (scrollTop < offset + lineHeight + 40 - containerHeight) { - outputContainerScroll.scrollTo({ - top: offset - containerHeight + 40, - behavior: 'smooth' - }); + $(outputContainerScroll).animate( + { scrollTop: offset - containerHeight + 40 }, + 150, + ); } } } - function handleCopy() { - const selection = window.getSelection(); - selection.removeAllRanges(); - const range = document.createRange(); - range.selectNodeContents(outputContainer); - selection.addRange(range); - navigator.clipboard.writeText(selection.toString()); - selection.removeRange(range); - showToast('Tree copied to clipboard!'); - } - function handleTreeClick(event) { if (event.target.tagName === "A") { event.preventDefault(); @@ -641,23 +619,4 @@ window.initializePlayground = async (opts) => { if (callNow) func.apply(context, args); }; } - - function showToast(message) { - const existingToast = document.querySelector('.toast'); - if (existingToast) { - existingToast.remove(); - } - - const toast = document.createElement('div'); - toast.className = 'toast'; - toast.textContent = message; - document.body.appendChild(toast); - - setTimeout(() => toast.classList.add('show'), 50); - - setTimeout(() => { - toast.classList.remove('show'); - setTimeout(() => toast.remove(), 200); - }, 1000); - } }; diff --git a/docs/src/assets/schemas/config.schema.json b/docs/src/assets/schemas/config.schema.json index 92453f37..f0fe92e6 100644 --- a/docs/src/assets/schemas/config.schema.json +++ b/docs/src/assets/schemas/config.schema.json @@ -188,6 +188,11 @@ "type": "string", "format": "uri", "description": "The project's funding link." + }, + "homepage": { + "type": "string", + "format": "uri", + "description": "The project's homepage." } }, "additionalProperties": false, diff --git a/docs/src/assets/schemas/node-types.schema.json b/docs/src/assets/schemas/node-types.schema.json deleted file mode 100644 index 7ea8a5af..00000000 --- a/docs/src/assets/schemas/node-types.schema.json +++ /dev/null @@ -1,108 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Tree-sitter node types specification", - "type": "array", - "items": { - "$ref": "#/definitions/NodeInfo" - }, - "definitions": { - "NodeInfo": { - "type": "object", - "required": [ - "type", - "named" - ], - "properties": { - "type": { - "type": "string" - }, - "named": { - "type": "boolean" - }, - "root": { - "type": "boolean", - "default": false - }, - "extra": { - "type": "boolean", - "default": false - }, - "fields": { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/FieldInfo" - } - }, - "children": { - "$ref": "#/definitions/FieldInfo" - }, - "subtypes": { - "type": "array", - "items": { - "$ref": "#/definitions/NodeType" - } - } - }, - "oneOf": [ - { - "description": "Regular node", - "properties": { - "subtypes": false - } - }, - { - "description": "Supertype node", - "required": [ - "subtypes" - ], - "properties": { - "children": false, - "fields": false - } - } - ] - }, - "NodeType": { - "type": "object", - "required": [ - "type", - "named" - ], - "properties": { - "type": { - "type": "string", - "description": "The kind of node type" - }, - "named": { - "type": "boolean", - "description": "Whether the node type is named" - } - } - }, - "FieldInfo": { - "type": "object", - "required": [ - "multiple", - "required", - "types" - ], - "properties": { - "multiple": { - "type": "boolean", - "default": false - }, - "required": { - "type": "boolean", - "default": true - }, - "types": { - "type": "array", - "default": [], - "items": { - "$ref": "#/definitions/NodeType" - } - } - } - } - } -} diff --git a/docs/src/assets/schemas/test-summary.schema.json b/docs/src/assets/schemas/test-summary.schema.json deleted file mode 100644 index 6211d60c..00000000 --- a/docs/src/assets/schemas/test-summary.schema.json +++ /dev/null @@ -1,247 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "TestSummary", - "description": "A stateful object used to collect results from running a grammar's test suite", - "type": "object", - "properties": { - "parse_results": { - "type": "array", - "items": { - "$ref": "#/$defs/TestResult" - } - }, - "parse_failures": { - "type": "array", - "items": { - "$ref": "#/$defs/TestFailure" - } - }, - "parse_stats": { - "$ref": "#/$defs/Stats" - }, - "highlight_results": { - "type": "array", - "items": { - "$ref": "#/$defs/TestResult" - } - }, - "tag_results": { - "type": "array", - "items": { - "$ref": "#/$defs/TestResult" - } - }, - "query_results": { - "type": "array", - "items": { - "$ref": "#/$defs/TestResult" - } - } - }, - "required": [ - "parse_results", - "parse_failures", - "parse_stats", - "highlight_results", - "tag_results", - "query_results" - ], - "$defs": { - "TestResult": { - "type": "object", - "properties": { - "name": { - "type": "string" - } - }, - "required": [ - "name" - ], - "anyOf": [ - { - "type": "object", - "properties": { - "children": { - "type": "array", - "items": { - "$ref": "#/$defs/TestResult" - } - } - }, - "required": [ - "children" - ] - }, - { - "type": "object", - "properties": { - "outcome": { - "$ref": "#/$defs/TestOutcome" - }, - "parse_rate": { - "type": [ - "number", - "null" - ], - "format": "double" - }, - "test_num": { - "type": "integer", - "format": "uint", - "minimum": 0 - } - }, - "required": [ - "outcome", - "parse_rate", - "test_num" - ] - }, - { - "type": "object", - "properties": { - "outcome": { - "$ref": "#/$defs/TestOutcome" - }, - "test_num": { - "type": "integer", - "format": "uint", - "minimum": 0 - } - }, - "required": [ - "outcome", - "test_num" - ] - } - ] - }, - "TestOutcome": { - "oneOf": [ - { - "type": "string", - "enum": [ - "Passed", - "Failed", - "Updated", - "Skipped", - "Platform" - ] - }, - { - "type": "object", - "properties": { - "AssertionPassed": { - "type": "object", - "properties": { - "assertion_count": { - "type": "integer", - "format": "uint", - "minimum": 0 - } - }, - "required": [ - "assertion_count" - ] - } - }, - "required": [ - "AssertionPassed" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "AssertionFailed": { - "type": "object", - "properties": { - "error": { - "type": "string" - } - }, - "required": [ - "error" - ] - } - }, - "required": [ - "AssertionFailed" - ], - "additionalProperties": false - } - ] - }, - "TestFailure": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "actual": { - "type": "string" - }, - "expected": { - "type": "string" - }, - "is_cst": { - "type": "boolean" - } - }, - "required": [ - "name", - "actual", - "expected", - "is_cst" - ] - }, - "Stats": { - "type": "object", - "properties": { - "successful_parses": { - "type": "integer", - "format": "uint", - "minimum": 0 - }, - "total_parses": { - "type": "integer", - "format": "uint", - "minimum": 0 - }, - "total_bytes": { - "type": "integer", - "format": "uint", - "minimum": 0 - }, - "total_duration": { - "$ref": "#/$defs/Duration" - } - }, - "required": [ - "successful_parses", - "total_parses", - "total_bytes", - "total_duration" - ] - }, - "Duration": { - "type": "object", - "properties": { - "secs": { - "type": "integer", - "format": "uint64", - "minimum": 0 - }, - "nanos": { - "type": "integer", - "format": "uint32", - "minimum": 0 - } - }, - "required": [ - "secs", - "nanos" - ] - } - } -} \ No newline at end of file diff --git a/docs/src/cli/build.md b/docs/src/cli/build.md index 44ee8271..180e7f92 100644 --- a/docs/src/cli/build.md +++ b/docs/src/cli/build.md @@ -1,7 +1,7 @@ # `tree-sitter build` The `build` command compiles your parser into a dynamically-loadable library, -either as a shared object (`.so`, `.dylib`, or `.dll`) or as a Wasm module. +either as a shared object (`.so`, `.dylib`, or `.dll`) or as a WASM module. ```bash tree-sitter build [OPTIONS] [PATH] # Aliases: b @@ -18,13 +18,16 @@ will attempt to build the parser in the current working directory. ### `-w/--wasm` -Compile the parser as a Wasm module. This command looks for the [Wasi SDK][wasi_sdk] indicated by the `TREE_SITTER_WASI_SDK_PATH` -environment variable. If you don't have the binary, the CLI will attempt to download it for you to `/tree-sitter/wasi-sdk/`, -where `` is resolved according to the [XDG base directory][XDG] or Window's [Known_Folder_Locations][Known_Folder]. +Compile the parser as a WASM module. + +### `-d/--docker` + +Use Docker or Podman to supply Emscripten. This removes the need to install Emscripten on your machine locally. +Note that this flag is only available when compiling to WASM. ### `-o/--output` -Specify where to output the shared object file (native or Wasm). This flag accepts either an absolute path or a relative +Specify where to output the shared object file (native or WASM). This flag accepts either an absolute path or a relative path. If you don't supply this flag, the CLI will attempt to figure out what the language name is based on the parent directory name to use for the output file. If the CLI can't figure it out, it will default to `parser`, thus generating `parser.so` or `parser.wasm` in the current working directory. @@ -37,9 +40,4 @@ in the external scanner does so using their allocator. ### `-0/--debug` -Compile the parser with debug flags enabled. This is useful when debugging issues that require a debugger like `gdb` or -`lldb`. - -[Known_Folder]: https://learn.microsoft.com/en-us/windows/win32/shell/knownfolderid -[wasi_sdk]: https://github.com/WebAssembly/wasi-sdk -[XDG]: https://specifications.freedesktop.org/basedir/latest/ +Compile the parser with debug flags enabled. This is useful when debugging issues that require a debugger like `gdb` or `lldb`. diff --git a/docs/src/cli/dump-languages.md b/docs/src/cli/dump-languages.md index f29daa57..1d1a6aaa 100644 --- a/docs/src/cli/dump-languages.md +++ b/docs/src/cli/dump-languages.md @@ -1,8 +1,6 @@ # `tree-sitter dump-languages` -The `dump-languages` command prints out a list of all the languages that the CLI knows about. This can be useful for debugging -purposes, or for scripting. The paths to search comes from the config file's [`parser-directories`][parser-directories] -object. +The `dump-languages` command prints out a list of all the languages that the CLI knows about. This can be useful for debugging purposes, or for scripting. The paths to search comes from the config file's [`parser-directories`][parser-directories] object. ```bash tree-sitter dump-languages [OPTIONS] # Aliases: langs @@ -12,7 +10,6 @@ tree-sitter dump-languages [OPTIONS] # Aliases: langs ### `--config-path` -The path to the configuration file. Ordinarily, the CLI will use the default location as explained in the [init-config](./init-config.md) -command. This flag allows you to explicitly override that default, and use a config defined elsewhere. +The path to the configuration file. Ordinarily, the CLI will use the default location as explained in the [init-config](./init-config.md) command. This flag allows you to explicitly override that default, and use a config defined elsewhere. [parser-directories]: ./init-config.md#parser-directories diff --git a/docs/src/cli/fuzz.md b/docs/src/cli/fuzz.md index 7f97f9ba..1f79bc00 100644 --- a/docs/src/cli/fuzz.md +++ b/docs/src/cli/fuzz.md @@ -17,18 +17,6 @@ A list of test names to skip fuzzing. The directory containing the parser. This is primarily useful in multi-language repositories. -### `-p/--grammar-path` - -The path to the directory containing the grammar. - -### `--lib-path` - -The path to the parser's dynamic library. This is used instead of the cached or automatically generated dynamic library. - -### `--lang-name` - -If `--lib-path` is used, the name of the language used to extract the library's language function - ### `--edits ` The maximum number of edits to perform. The default is 3. diff --git a/docs/src/cli/generate.md b/docs/src/cli/generate.md index df9111f0..3b2a94d3 100644 --- a/docs/src/cli/generate.md +++ b/docs/src/cli/generate.md @@ -1,47 +1,50 @@ # `tree-sitter generate` -The most important command for grammar development is `tree-sitter generate`, which reads the grammar in structured form -and outputs C files that can be compiled into a shared or static library (e.g., using the [`build`](./build.md) command). +The most important command you'll use is `tree-sitter generate`. This command reads the `grammar.js` file in your current +working directory and creates a file called `src/parser.c`, which implements the parser. After making changes to your grammar, +just run `tree-sitter generate` again. ```bash tree-sitter generate [OPTIONS] [GRAMMAR_PATH] # Aliases: gen, g ``` -The optional `GRAMMAR_PATH` argument should point to the structured grammar, in one of two forms: -- `grammar.js` a (ESM or CJS) JavaScript file; if the argument is omitted, it defaults to `./grammar.js`. -- `grammar.json` a structured representation of the grammar that is created as a byproduct of `generate`; this can be used -to regenerate a missing `parser.c` without requiring a JavaScript runtime (useful when distributing parsers to consumers). +The grammar path argument allows you to specify a path to a `grammar.js` JavaScript file, or `grammar.json` JSON file. +In case your `grammar.js` file is in a non-standard path, you can specify it yourself. But, if you are using a parser +where `grammar.json` was already generated, or it was hand-written, you can tell the CLI to generate the parser *based* +on this JSON file. This avoids relying on a JavaScript file and avoids the dependency on a JavaScript runtime. If there is an ambiguity or *local ambiguity* in your grammar, Tree-sitter will detect it during parser generation, and -it will exit with a `Unresolved conflict` error message. To learn more about conflicts and how to handle them, see +it will exit with a `Unresolved conflict` error message. To learn more about conflicts and how to handle them, check out the section on [`Structuring Rules Well`](../creating-parsers/3-writing-the-grammar.md#structuring-rules-well) in the user guide. -## Generated files - -- `src/parser.c` implements the parser logic specified in the grammar. -- `src/tree_sitter/parser.h` provides basic C definitions that are used in the generated `parser.c` file. -- `src/tree_sitter/alloc.h` provides memory allocation macros that can be used in an external scanner. -- `src/tree_sitter/array.h` provides array macros that can be used in an external scanner. -- `src/grammar.json` contains a structured representation of the grammar; can be used to regenerate the parser without having -to re-evaluate the `grammar.js`. -- `src/node-types.json` provides type information about individual syntax nodes; see the section on [`Static Node Types`](../using-parsers/6-static-node-types.md). - - ## Options ### `-l/--log` -Print the log of the parser generation process. This includes information such as what tokens are included in the error -recovery state, what keywords were extracted, what states were split and why, and the entry point state. +Print the log of the parser generation process. This is really only useful if you know what you're doing, or are investigating +a bug in the CLI itself. It logs info such as what tokens are included in the error recovery state, +what keywords were extracted, what states were split and why, and the entry point state. ### `--abi ` The ABI to use for parser generation. The default is ABI 15, with ABI 14 being a supported target. -### `--no-parser` +### `-b/--build` -Only generate `grammar.json` and `node-types.json` +Compile all defined languages in the current directory. The cli will automatically compile the parsers after generation, +and place them in the cache dir. + +### `-0/--debug-build` + +Compile the parser with debug flags enabled. This is useful when debugging issues that require a debugger like `gdb` or `lldb`. + +### `--libdir ` + +The directory to place the compiled parser(s) in. +On Unix systems, the default path is `$XDG_CACHE_HOME/tree-sitter` if `$XDG_CACHE_HOME` is set, +otherwise `$HOME/.config/tree-sitter` is used. On Windows, the default path is `%LOCALAPPDATA%\tree-sitter` if available, +otherwise `$HOME\AppData\Local\tree-sitter` is used. ### `-o/--output` @@ -53,19 +56,7 @@ Print the overview of states from the given rule. This is useful for debugging a item sets for all given states in a given rule. To solely view state count numbers for rules, pass in `-` for the rule argument. To view the overview of states for every rule, pass in `*` for the rule argument. -### `--json-summary` - -Report conflicts in a JSON format. - ### `--js-runtime ` The path to the JavaScript runtime executable to use when generating the parser. The default is `node`. -Note that you can also set this with `TREE_SITTER_JS_RUNTIME`. Starting from version 0.26, you can -also pass in `native` to use the experimental native QuickJS runtime that comes bundled with the CLI. -This avoids the dependency on a JavaScript runtime entirely. The native QuickJS runtime is compatible -with ESM as well as with CommonJS in strict mode. If your grammar depends on `npm` to install dependencies such as base -grammars, the native runtime can be used *after* running `npm install`. - -### `--disable-optimization` - -Disable optimizations when generating the parser. Currently, this only affects the merging of compatible parse states. +Note that you can also set this with `TREE_SITTER_JS_RUNTIME`. diff --git a/docs/src/cli/highlight.md b/docs/src/cli/highlight.md index 1a4ed1f6..c5b3e8a3 100644 --- a/docs/src/cli/highlight.md +++ b/docs/src/cli/highlight.md @@ -46,19 +46,10 @@ Suppress main output. The path to a file that contains paths to source files to highlight -### `-p/--grammar-path ` - -The path to the directory containing the grammar. - ### `--config-path ` -The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more -information. +The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more information. ### `-n/--test-number ` Highlight the contents of a specific test. - -### `-r/--rebuild` - -Force a rebuild of the parser before running the fuzzer. diff --git a/docs/src/cli/index.md b/docs/src/cli/index.md index 042c0196..7c982b40 100644 --- a/docs/src/cli/index.md +++ b/docs/src/cli/index.md @@ -1,8 +1,4 @@ # CLI Overview -The `tree-sitter` command-line interface is used to create, manage, test, and build tree-sitter parsers. It is controlled -by - -- a personal `tree-sitter/config.json` config file generated by [`tree-sitter init-config`](./init-config.md) -- a parser `tree-sitter.json` config file generated by [`tree-sitter init`](./init.md). - +Let's go over all of the functionality of the `tree-sitter` command line interface. +Once you feel that you have enough of a grasp on the CLI, you can move onto the grammar authoring section to learn more about writing your own parser. diff --git a/docs/src/cli/init-config.md b/docs/src/cli/init-config.md index 77aacd66..96cb96a7 100644 --- a/docs/src/cli/init-config.md +++ b/docs/src/cli/init-config.md @@ -30,8 +30,7 @@ on your filesystem. You can control this using the `"parser-directories"` key in { "parser-directories": [ "/Users/my-name/code", - "~/other-code", - "$HOME/another-code" + "/Users/my-name/other-code" ] } ``` diff --git a/docs/src/cli/init.md b/docs/src/cli/init.md index 568bc98f..2751b368 100644 --- a/docs/src/cli/init.md +++ b/docs/src/cli/init.md @@ -8,94 +8,26 @@ we recommend using git for version control of your grammar. tree-sitter init [OPTIONS] # Aliases: i ``` -## Generated files +## Options -### Required files +### `--update` -The following required files are always created if missing: - -- `tree-sitter.json` - The main configuration file that determines how `tree-sitter` interacts with the grammar. If missing, -the `init` command will prompt the user for the required fields. See [below](./init.md#structure-of-tree-sitterjson) for -the full documentation of the structure of this file. -- `package.json` - The `npm` manifest for the parser. This file is required for some `tree-sitter` subcommands, and if the -grammar has dependencies (e.g., another published base grammar that this grammar extends). -- `grammar.js` - An empty template for the main grammar file; see [the section on creating parsers](../2-creating-parser). - -### Language bindings - -Language bindings are files that allow your parser to be directly used by projects written in the respective language. -The following bindings are created if enabled in `tree-sitter.json`: - -#### C/C++ - -- `Makefile` — This file tells [`make`][make] how to compile your language. -- `CMakeLists.txt` — This file tells [`cmake`][cmake] how to compile your language. -- `bindings/c/tree_sitter/tree-sitter-language.h` — This file provides the C interface of your language. -- `bindings/c/tree-sitter-language.pc` — This file provides [pkg-config][pkg-config] metadata about your language's C library. - -#### Go - -- `go.mod` — This file is the manifest of the Go module. -- `bindings/go/binding.go` — This file wraps your language in a Go module. -- `bindings/go/binding_test.go` — This file contains a test for the Go package. - -#### Node - -- `binding.gyp` — This file tells Node.js how to compile your language. -- `bindings/node/binding.cc` — This file wraps your language in a JavaScript module for Node.js. -- `bindings/node/index.js` — This is the file that Node.js initially loads when using your language. -- `bindings/node/index.d.ts` — This file provides type hints for your parser when used in TypeScript. -- `bindings/node/binding_test.js` — This file contains a test for the Node.js package. - -#### Java - -- `pom.xml` - This file is the manifest of the Maven package. -- `bindings/java/main/namespace/language/TreeSitterLanguage.java` - This file wraps your language in a Java class. -- `bindings/java/test/TreeSitterLanguageTest.java` - This file contains a test for the Java package. - -#### Python - -- `pyproject.toml` — This file is the manifest of the Python package. -- `setup.py` — This file tells Python how to compile your language. -- `bindings/python/tree_sitter_language/binding.c` — This file wraps your language in a Python module. -- `bindings/python/tree_sitter_language/__init__.py` — This file tells Python how to load your language. -- `bindings/python/tree_sitter_language/__init__.pyi` — This file provides type hints for your parser when used in Python. -- `bindings/python/tree_sitter_language/py.typed` — This file provides type hints for your parser when used in Python. -- `bindings/python/tests/test_binding.py` — This file contains a test for the Python package. - -#### Rust - -- `Cargo.toml` — This file is the manifest of the Rust package. -- `bindings/rust/build.rs` — This file tells Rust how to compile your language. -- `bindings/rust/lib.rs` — This file wraps your language in a Rust crate when used in Rust. - -#### Swift - -- `Package.swift` — This file tells Swift how to compile your language. -- `bindings/swift/TreeSitterLanguage/language.h` — This file wraps your language in a Swift module when used in Swift. -- `bindings/swift/TreeSitterLanguageTests/TreeSitterLanguageTests.swift` — This file contains a test for the Swift package. - -#### Zig - -- `build.zig` - This file tells Zig how to compile your language. -- `build.zig.zon` - This file is the manifest of the Zig package. -- `bindings/zig/root.zig` - This file wraps your language in a Zig module. -- `bindings/zig/test.zig` - This file contains a test for the Zig package. - -### Additional files - -In addition, the following files are created that aim to improve the development experience: - -- `.editorconfig` — This file tells your editor how to format your code. More information about this file can be found [here][editorconfig]. -- `.gitattributes` — This file tells Git how to handle line endings and tells GitHub which files are generated. -- `.gitignore` — This file tells Git which files to ignore when committing changes. +Update outdated generated files, if needed. ## Structure of `tree-sitter.json` +The main file of interest for users to configure is `tree-sitter.json`, which tells the CLI information about your grammar, +such as the location of queries. + ### The `grammars` field This field is an array of objects, though you typically only need one object in this array unless your repo has -multiple grammars (for example, `Typescript` and `TSX`), e.g., +multiple grammars (for example, `Typescript` and `TSX`). + +### Example + +Typically, the objects in the `"tree-sitter"` array only needs to specify a few keys: + ```json { "tree-sitter": [ @@ -113,7 +45,7 @@ multiple grammars (for example, `Typescript` and `TSX`), e.g., } ``` -#### Basic fields +#### Basic Fields These keys specify basic information about the parser: @@ -129,12 +61,12 @@ parser to files that should be checked for modifications during recompilation. This is useful during development to have changes to other files besides scanner.c be picked up by the cli. -#### Language detection +#### Language Detection These keys help to decide whether the language applies to a given file: -- `file-types` — An array of filename suffix strings (not including the dot). The grammar will be used for files whose names -end with one of these suffixes. Note that the suffix may match an *entire* filename. +- `file-types` — An array of filename suffix strings. The grammar will be used for files whose names end with one of +these suffixes. Note that the suffix may match an *entire* filename. - `first-line-regex` — A regex pattern that will be tested against the first line of a file to determine whether this language applies to the file. If present, this regex will be used for any file whose @@ -149,14 +81,14 @@ no `content-regex` will be preferred over this one. should be used for a potential *language injection* site. Language injection is described in more detail in [the relevant section](../3-syntax-highlighting.md#language-injection). -#### Query paths +#### Query Paths These keys specify relative paths from the directory containing `tree-sitter.json` to the files that control syntax highlighting: - `highlights` — Path to a *highlight query*. Default: `queries/highlights.scm` - `locals` — Path to a *local variable query*. Default: `queries/locals.scm`. - `injections` — Path to an *injection query*. Default: `queries/injections.scm`. -- `tags` — Path to a *tag query*. Default: `queries/tags.scm`. +- `tags` — Path to an *tag query*. Default: `queries/tags.scm`. ### The `metadata` field @@ -168,7 +100,7 @@ Typically, this will all be set up when you run `tree-sitter init`, but you are - `description` — The brief description of your grammar - `authors` (required) — An array of objects that contain a `name` field, and optionally an `email` and `url` field. Each field is a string -- `links` — An object that contains a `repository` field, and optionally a `funding` field. Each field is a string +- `links` — An object that contains a `repository` field, and optionally a `homepage` field. Each field is a string - `namespace` — The namespace for the `Java` and `Kotlin` bindings, defaults to `io.github.tree-sitter` if not provided ### The `bindings` field @@ -179,25 +111,74 @@ Each key is a language name, and the value is a boolean. - `c` (default: `true`) - `go` (default: `true`) - `java` (default: `false`) +- `kotlin` (default: `false`) - `node` (default: `true`) - `python` (default: `true`) - `rust` (default: `true`) - `swift` (default: `false`) -- `zig` (default: `false`) -## Options +## Binding Files -### `-u/--update` +When you run `tree-sitter init`, the CLI will also generate a number of files in your repository that allow for your parser +to be used from different language. Here is a list of these bindings files that are generated, and what their purpose is: -Update outdated generated files, if possible. +### C/C++ -**Note:** Existing files that may have been edited manually are _not_ updated in general. To force an update to such files, -remove them and call `tree-sitter init -u` again. +- `Makefile` — This file tells [`make`][make] how to compile your language. +- `CMakeLists.txt` — This file tells [`cmake`][cmake] how to compile your language. +- `bindings/c/tree_sitter/tree-sitter-language.h` — This file provides the C interface of your language. +- `bindings/c/tree-sitter-language.pc` — This file provides [pkg-config][pkg-config] metadata about your language's C library. +- `src/tree_sitter/parser.h` — This file provides some basic C definitions that are used in your generated `parser.c` file. +- `src/tree_sitter/alloc.h` — This file provides some memory allocation macros that are to be used in your external scanner, +if you have one. +- `src/tree_sitter/array.h` — This file provides some array macros that are to be used in your external scanner, +if you have one. -### `-p/--grammar-path ` +### Go -The path to the directory containing the grammar. +- `go.mod` — This file is the manifest of the Go module. +- `bindings/go/binding.go` — This file wraps your language in a Go module. +- `bindings/go/binding_test.go` — This file contains a test for the Go package. +### Node + +- `binding.gyp` — This file tells Node.js how to compile your language. +- `package.json` — This file is the manifest of the Node.js package. +- `bindings/node/binding.cc` — This file wraps your language in a JavaScript module for Node.js. +- `bindings/node/index.js` — This is the file that Node.js initially loads when using your language. +- `bindings/node/index.d.ts` — This file provides type hints for your parser when used in TypeScript. +- `bindings/node/binding_test.js` — This file contains a test for the Node.js package. + +### Python + +- `pyproject.toml` — This file is the manifest of the Python package. +- `setup.py` — This file tells Python how to compile your language. +- `bindings/python/tree_sitter_language/binding.c` — This file wraps your language in a Python module. +- `bindings/python/tree_sitter_language/__init__.py` — This file tells Python how to load your language. + `bindings/python/tree_sitter_language/__init__.pyi` — This file provides type hints for your parser when used in Python. +- `bindings/python/tree_sitter_language/py.typed` — This file provides type hints for your parser when used in Python. +- `bindings/python/tests/test_binding.py` — This file contains a test for the Python package. + +### Rust + +- `Cargo.toml` — This file is the manifest of the Rust package. +- `bindings/rust/lib.rs` — This file wraps your language in a Rust crate when used in Rust. +- `bindings/rust/build.rs` — This file wraps the building process for the Rust crate. + +### Swift + +- `Package.swift` — This file tells Swift how to compile your language. +- `bindings/swift/TreeSitterLanguage/language.h` — This file wraps your language in a Swift module when used in Swift. +- `bindings/swift/TreeSitterLanguageTests/TreeSitterLanguageTests.swift` — This file contains a test for the Swift package. + +### Additional Files + +Additionally, there's a few other files that are generated when you run `tree-sitter init`, +that aim to improve the development experience: + +- `.editorconfig` — This file tells your editor how to format your code. More information about this file can be found [here][editorconfig] +- `.gitattributes` — This file tells Git how to handle line endings, and tells GitHub what files are generated. +- `.gitignore` — This file tells Git what files to ignore when committing changes. [cmake]: https://cmake.org/cmake/help/latest [editorconfig]: https://editorconfig.org diff --git a/docs/src/cli/parse.md b/docs/src/cli/parse.md index 2e9bc835..97c6422a 100644 --- a/docs/src/cli/parse.md +++ b/docs/src/cli/parse.md @@ -1,8 +1,8 @@ # `tree-sitter parse` The `parse` command parses source files using a Tree-sitter parser. You can pass any number of file paths and glob patterns -to `tree-sitter parse`, and it will parse all the given files. If no paths are provided, input will be parsed from stdin. -The command will exit with a non-zero status code if any parse errors occurred. +to `tree-sitter parse`, and it will parse all the given files. The command will exit with a non-zero status code if any +parse errors occurred. ```bash tree-sitter parse [OPTIONS] [PATHS]... # Aliases: p @@ -14,18 +14,6 @@ tree-sitter parse [OPTIONS] [PATHS]... # Aliases: p The path to a file that contains paths to source files to parse. -### `-p/--grammar-path ` - -The path to the directory containing the grammar. - -### `-l/--lib-path` - -The path to the parser's dynamic library. This is used instead of the cached or automatically generated dynamic library. - -### `--lang-name` - -If `--lib-path` is used, the name of the language used to extract the library's language function - ### `--scope ` The language scope to use for parsing. This is useful when the language is ambiguous. @@ -45,7 +33,7 @@ The graphs are constructed with [graphviz dot][dot], and the output is written t ### `--wasm` -Compile and run the parser as a Wasm module (only if the tree-sitter CLI was built with `--features=wasm`). +Compile and run the parser as a WASM module. ### `--dot` @@ -78,8 +66,7 @@ Suppress main output. ### `--edits ...` -Apply edits after parsing the file. Edits are in the form of `row,col|position delcount insert_text` where row and col, -or position are 0-indexed. +Apply edits after parsing the file. Edits are in the form of `row,col|position delcount insert_text` where row and col, or position are 0-indexed. ### `--encoding ` @@ -90,14 +77,9 @@ in `UTF-16BE` or `UTF-16LE`. If no `BOM` is present, `UTF-8` is the default. One When using the `--debug-graph` option, open the log file in the default browser. -### `-j/--json-summary` - -Output parsing results in a JSON format. - ### `--config-path ` -The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more -information. +The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more information. ### `-n/--test-number ` diff --git a/docs/src/cli/playground.md b/docs/src/cli/playground.md index c0bfb495..75ab4588 100644 --- a/docs/src/cli/playground.md +++ b/docs/src/cli/playground.md @@ -7,8 +7,8 @@ tree-sitter playground [OPTIONS] # Aliases: play, pg, web-ui ``` ```admonish note -For this to work, you must have already built the parser as a Wasm module. This can be done with the [`build`](./build.md) -subcommand (`tree-sitter build --wasm`). +For this to work, you must have already built the parser as a WASM module. This can be done with the [`build`](./build.md) subcommand +(`tree-sitter build --wasm`). ``` ## Options @@ -20,7 +20,3 @@ Don't automatically open the playground in the default browser. ### `--grammar-path ` The path to the directory containing the grammar and wasm files. - -### `-e/--export ` - -Export static playground files to the specified directory instead of serving them. diff --git a/docs/src/cli/query.md b/docs/src/cli/query.md index fbd6dafd..cbfb445a 100644 --- a/docs/src/cli/query.md +++ b/docs/src/cli/query.md @@ -8,18 +8,6 @@ tree-sitter query [OPTIONS] [PATHS]... # Aliases: q ## Options -### `-p/--grammar-path ` - -The path to the directory containing the grammar. - -### `--lib-path` - -The path to the parser's dynamic library. This is used instead of the cached or automatically generated dynamic library. - -### `--lang-name` - -If `--lib-path` is used, the name of the language used to extract the library's language function - ### `-t/--time` Print the time taken to execute the query on the file. @@ -36,20 +24,10 @@ The path to a file that contains paths to source files in which the query will b The range of byte offsets in which the query will be executed. The format is `start_byte:end_byte`. -### `--containing-byte-range ` - -The range of byte offsets in which the query will be executed. Only the matches that are fully contained within the provided -byte range will be returned. - ### `--row-range ` The range of rows in which the query will be executed. The format is `start_row:end_row`. -### `--containing-row-range ` - -The range of rows in which the query will be executed. Only the matches that are fully contained within the provided row -range will be returned. - ### `--scope ` The language scope to use for parsing and querying. This is useful when the language is ambiguous. @@ -64,13 +42,8 @@ Whether to run query tests or not. ### `--config-path ` -The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more -information. +The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more information. ### `-n/--test-number ` Query the contents of a specific test. - -### `-r/--rebuild` - -Force a rebuild of the parser before executing the query. diff --git a/docs/src/cli/tags.md b/docs/src/cli/tags.md index 8275237e..98e173c3 100644 --- a/docs/src/cli/tags.md +++ b/docs/src/cli/tags.md @@ -25,19 +25,10 @@ Suppress main output. The path to a file that contains paths to source files to tag. -### `-p/--grammar-path ` - -The path to the directory containing the grammar. - ### `--config-path ` -The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more -information. +The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more information. ### `-n/--test-number ` Generate tags from the contents of a specific test. - -### `-r/--rebuild` - -Force a rebuild of the parser before running the tags. diff --git a/docs/src/cli/test.md b/docs/src/cli/test.md index 7e662f60..ee82ee9b 100644 --- a/docs/src/cli/test.md +++ b/docs/src/cli/test.md @@ -16,22 +16,6 @@ Only run tests whose names match this regex. Skip tests whose names match this regex. -### `--file-name ` - -Only run tests from the given filename in the corpus. - -### `-p/--grammar-path ` - -The path to the directory containing the grammar. - -### `--lib-path` - -The path to the parser's dynamic library. This is used instead of the cached or automatically generated dynamic library. - -### `--lang-name` - -If `--lib-path` is used, the name of the language used to extract the library's language function - ### `-u/--update` Update the expected output of tests. @@ -55,7 +39,7 @@ The graphs are constructed with [graphviz dot][dot], and the output is written t ### `--wasm` -Compile and run the parser as a Wasm module (only if the tree-sitter CLI was built with `--features=wasm`). +Compile and run the parser as a WASM module. ### `--open-log` @@ -63,8 +47,7 @@ When using the `--debug-graph` option, open the log file in the default browser. ### `--config-path ` -The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more -information. +The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more information. ### `--show-fields` @@ -87,7 +70,3 @@ Force a rebuild of the parser before running tests. ### `--overview-only` Only show the overview of the test results, and not the diff. - -### `--json-summary` - -Output the test summary in a JSON format. diff --git a/docs/src/cli/version.md b/docs/src/cli/version.md index 39cda84a..464c98a4 100644 --- a/docs/src/cli/version.md +++ b/docs/src/cli/version.md @@ -17,39 +17,8 @@ This will update the version in several files, if they exist: * CMakeLists.txt * pyproject.toml -Alternative forms can use the version in `tree-sitter.json` to bump automatically: - -```bash -tree-sitter version --bump patch # patch bump -tree-sitter version --bump minor # minor bump -tree-sitter version --bump major # major bump -``` - -As a grammar author, you should keep the version of your grammar in sync across different bindings. However, doing so manually -is error-prone and tedious, so this command takes care of the burden. If you are using a version control system, it is recommended -to commit the changes made by this command, and to tag the commit with the new version. - -To print the current version without bumping it, use: - -```bash -tree-sitter version -``` - -Note that some of the binding updates require access to external tooling: - -* Updating Cargo.toml and Cargo.lock bindings requires that `cargo` is installed. -* Updating package-lock.json requires that `npm` is installed. - -## Options - -### `-p/--grammar-path ` - -The path to the directory containing the grammar. - -### `--bump` - -Automatically bump the version. Possible values are: - -- `patch`: Bump the patch version. -- `minor`: Bump the minor version. -- `major`: Bump the major version. +As a grammar author, you should keep the version of your grammar in sync across +different bindings. However, doing so manually is error-prone and tedious, so +this command takes care of the burden. If you are using a version control system, +it is recommended to commit the changes made by this command, and to tag the +commit with the new version. diff --git a/docs/src/creating-parsers/1-getting-started.md b/docs/src/creating-parsers/1-getting-started.md index 5095eb75..6cab1839 100644 --- a/docs/src/creating-parsers/1-getting-started.md +++ b/docs/src/creating-parsers/1-getting-started.md @@ -24,7 +24,7 @@ on any platform. See [the contributing docs](../6-contributing.md#developing-tre running the following command: `cargo install tree-sitter-cli --locked` - Install the `tree-sitter-cli` [Node.js module][node-module] using [`npm`][npm], the Node package manager. This approach -is fast, but it only works on certain platforms, because it relies on pre-built binaries. +is fast, but is only works on certain platforms, because it relies on pre-built binaries. - Download a binary for your platform from [the latest GitHub release][releases], and put it into a directory on your `PATH`. @@ -38,7 +38,7 @@ cd tree-sitter-${LOWER_PARSER_NAME} ``` ```admonish note -The `LOWER_` prefix here means the "lowercase" name of the language. +The `LOWER-` prefix here means the "lowercase" name of the language. ``` ### Init @@ -64,7 +64,7 @@ There should be a file called `grammar.js` with the following contents: /// // @ts-check -export default grammar({ +module.exports = grammar({ name: 'LOWER_PARSER_NAME', rules: { @@ -131,6 +131,6 @@ To learn more about this command, check the [reference page](../cli/generate.md) [npm]: https://docs.npmjs.com [path-env]: https://en.wikipedia.org/wiki/PATH_(variable) [releases]: https://github.com/tree-sitter/tree-sitter/releases/latest -[tree-sitter-cli]: https://github.com/tree-sitter/tree-sitter/tree/master/crates/cli +[tree-sitter-cli]: https://github.com/tree-sitter/tree-sitter/tree/master/cli [triple-slash]: https://www.typescriptlang.org/docs/handbook/triple-slash-directives.html [ts-check]: https://www.typescriptlang.org/docs/handbook/intro-to-js-ts.html diff --git a/docs/src/creating-parsers/2-the-grammar-dsl.md b/docs/src/creating-parsers/2-the-grammar-dsl.md index 0592c6bd..eff6c2eb 100644 --- a/docs/src/creating-parsers/2-the-grammar-dsl.md +++ b/docs/src/creating-parsers/2-the-grammar-dsl.md @@ -12,18 +12,18 @@ it generates its own regex-matching logic based on the Rust regex syntax as part used as a convenient way of writing regular expressions in your grammar. You can use Rust regular expressions in your grammar DSL through the `RustRegex` class. Simply pass your regex pattern as a string: - ```js - new RustRegex('(?i)[a-z_][a-z0-9_]*') // matches a simple identifier - ``` +```js +new RustRegex('(?i)[a-z_][a-z0-9_]*') // matches a simple identifier +``` - Unlike JavaScript's builtin `RegExp` class, which takes a pattern and flags as separate arguments, `RustRegex` only - accepts a single pattern string. While it doesn't support separate flags, you can use inline flags within the pattern - itself. For more details about Rust's regex syntax and capabilities, check out the [Rust regex documentation][rust regex]. +Unlike JavaScript's builtin `RegExp` class, which takes a pattern and flags as separate arguments, `RustRegex` only +accepts a single pattern string. While it doesn't support separate flags, you can use inline flags within the pattern itself. +For more details about Rust's regex syntax and capabilities, check out the [Rust regex documentation][rust regex]. - ```admonish note - Only a subset of the Regex engine is actually supported. This is due to certain features like lookahead and lookaround - assertions not feasible to use in an LR(1) grammar, as well as certain flags being unnecessary for tree-sitter. However, - plenty of features are supported by default: +- **Regex Limitations** — Only a subset of the Regex engine is actually +supported. This is due to certain features like lookahead and lookaround assertions +not feasible to use in an LR(1) grammar, as well as certain flags being unnecessary +for tree-sitter. However, plenty of features are supported by default: - Character classes - Character ranges @@ -33,7 +33,6 @@ DSL through the `RustRegex` class. Simply pass your regex pattern as a string: - Grouping - Unicode character escapes - Unicode property escapes - ``` - **Sequences : `seq(rule1, rule2, ...)`** — This function creates a rule that matches any number of other rules, one after another. It is analogous to simply writing multiple symbols next to each other in [EBNF notation][ebnf]. @@ -50,10 +49,10 @@ The previous `repeat` rule is implemented in `repeat1` but is included because i - **Options : `optional(rule)`** — This function creates a rule that matches *zero or one* occurrence of a given rule. It is analogous to the `[x]` (square bracket) syntax in EBNF notation. -- **Precedence : `prec(number, rule)`** — This function marks the given rule with a numerical precedence, which will be -used to resolve [*LR(1) Conflicts*][lr-conflict] at parser-generation time. When two rules overlap in a way that represents -either a true ambiguity or a *local* ambiguity given one token of lookahead, Tree-sitter will try to resolve the conflict -by matching the rule with the higher precedence. The default precedence of all rules is zero. This works similarly to the +- **Precedence : `prec(number, rule)`** — This function marks the given rule with a numerical precedence, which will be used +to resolve [*LR(1) Conflicts*][lr-conflict] at parser-generation time. When two rules overlap in a way that represents either +a true ambiguity or a *local* ambiguity given one token of lookahead, Tree-sitter will try to resolve the conflict by matching +the rule with the higher precedence. The default precedence of all rules is zero. This works similarly to the [precedence directives][yacc-prec] in Yacc grammars. This function can also be used to assign lexical precedence to a given @@ -107,7 +106,7 @@ grammar rules themselves. These fields are: - **`extras`** — an array of tokens that may appear *anywhere* in the language. This is often used for whitespace and comments. The default value of `extras` is to accept whitespace. To control whitespace explicitly, specify -`extras: $ => []` in your grammar. See the section on [using extras][extras] for more details. +`extras: $ => []` in your grammar. - **`inline`** — an array of rule names that should be automatically *removed* from the grammar by replacing all of their usages with a copy of their definition. This is useful for rules that are used in multiple places but for which you *don't* @@ -115,8 +114,8 @@ want to create syntax tree nodes at runtime. - **`conflicts`** — an array of arrays of rule names. Each inner array represents a set of rules that's involved in an *LR(1) conflict* that is *intended to exist* in the grammar. When these conflicts occur at runtime, Tree-sitter will use -the GLR algorithm to explore all the possible interpretations. If *multiple* parses end up succeeding, Tree-sitter will -pick the subtree whose corresponding rule has the highest total *dynamic precedence*. +the GLR algorithm to explore all the possible interpretations. If *multiple* parses end up succeeding, Tree-sitter will pick +the subtree whose corresponding rule has the highest total *dynamic precedence*. - **`externals`** — an array of token names which can be returned by an [*external scanner*][external-scanners]. External scanners allow you to write custom C code which runs during the lexing @@ -129,31 +128,24 @@ than globally. Can only be used with parse precedence, not lexical precedence. - **`word`** — the name of a token that will match keywords to the [keyword extraction][keyword-extraction] optimization. -- **`supertypes`** — an array of rule names which should be considered to be 'supertypes' in the generated -[*node types* file][static-node-types-supertypes]. Supertype rules are automatically hidden from the parse tree, regardless -of whether their names start with an underscore. The main use case for supertypes is to group together multiple different -kinds of nodes under a single abstract category, such as "expression" or "declaration". See the section on [`using supertypes`][supertypes] -for more details. +- **`supertypes`** — an array of hidden rule names which should be considered to be 'supertypes' in the generated +[*node types* file][static-node-types]. - **`reserved`** — similar in structure to the main `rules` property, an object of reserved word sets associated with an -array of reserved rules. The reserved rule in the array must be a terminal token meaning it must be a string, regex, token, -or terminal rule. The reserved rule must also exist and be used in the grammar, specifying arbitrary tokens will not work. -The *first* reserved word set in the object is the global word set, meaning it applies to every rule in every parse state. -However, certain keywords are contextual, depending on the rule. For example, in JavaScript, keywords are typically not -allowed as ordinary variables, however, they *can* be used as a property name. In this situation, the `reserved` function -would be used, and the word set to pass in would be the name of the word set that is declared in the `reserved` object that -corresponds to an empty array, signifying *no* keywords are reserved. +array of reserved rules. The reserved rule in the array must be a terminal token meaning it must be a string, regex, or token, +or a terminal rule. The *first* reserved word set in the object is the global word set, meaning it applies to every rule +in every parse state. However, certain keywords are contextual, depending on the rule. For example, in JavaScript, keywords +are typically not allowed as ordinary variables, however, they *can* be used as a property name. In this situation, the `reserved` +function would be used, and the word set to pass in would be the name of the word set that is declared in the `reserved` +object that coreesponds an empty array, signifying *no* keywords are reserved. [bison-dprec]: https://www.gnu.org/software/bison/manual/html_node/Generalized-LR-Parsing.html [ebnf]: https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form [external-scanners]: ./4-external-scanners.md -[extras]: ./3-writing-the-grammar.md#using-extras [keyword-extraction]: ./3-writing-the-grammar.md#keyword-extraction [lexical vs parse]: ./3-writing-the-grammar.md#lexical-precedence-vs-parse-precedence [lr-conflict]: https://en.wikipedia.org/wiki/LR_parser#Conflicts_in_the_constructed_tables [named-vs-anonymous-nodes]: ../using-parsers/2-basic-parsing.md#named-vs-anonymous-nodes [rust regex]: https://docs.rs/regex/1.1.8/regex/#grouping-and-flags [static-node-types]: ../using-parsers/6-static-node-types.md -[static-node-types-supertypes]: ../using-parsers/6-static-node-types.md#supertype-nodes -[supertypes]: ./3-writing-the-grammar.md#using-supertypes [yacc-prec]: https://docs.oracle.com/cd/E19504-01/802-5880/6i9k05dh3/index.html diff --git a/docs/src/creating-parsers/3-writing-the-grammar.md b/docs/src/creating-parsers/3-writing-the-grammar.md index 5048ff5b..047e6e92 100644 --- a/docs/src/creating-parsers/3-writing-the-grammar.md +++ b/docs/src/creating-parsers/3-writing-the-grammar.md @@ -1,7 +1,7 @@ # Writing the Grammar -Writing a grammar requires creativity. There are an infinite number of CFGs (context-free grammars) that can be used to -describe any given language. To produce a good Tree-sitter parser, you need to create a grammar with two important properties: +Writing a grammar requires creativity. There are an infinite number of CFGs (context-free grammars) that can be used to describe +any given language. To produce a good Tree-sitter parser, you need to create a grammar with two important properties: 1. **An intuitive structure** — Tree-sitter's output is a [concrete syntax tree][cst]; each node in the tree corresponds directly to a [terminal or non-terminal symbol][non-terminal] in the grammar. So to produce an easy-to-analyze tree, there @@ -74,11 +74,11 @@ you might start with something like this: return_statement: $ => seq( 'return', - $.expression, + $._expression, ';' ), - expression: $ => choice( + _expression: $ => choice( $.identifier, $.number // TODO: other kinds of expressions @@ -139,8 +139,8 @@ instead. It's often useful to check your progress by trying to parse some real c ## Structuring Rules Well Imagine that you were just starting work on the [Tree-sitter JavaScript parser][tree-sitter-javascript]. Naively, you might -try to directly mirror the structure of the [ECMAScript Language Spec][ecmascript-spec]. To illustrate the problem with -this approach, consider the following line of code: +try to directly mirror the structure of the [ECMAScript Language Spec][ecmascript-spec]. To illustrate the problem with this +approach, consider the following line of code: ```js return x + y; @@ -181,17 +181,16 @@ which are unrelated to the actual code. ## Standard Rule Names -Tree-sitter places no restrictions on how to name the rules of your grammar. It can be helpful, however, to follow certain -conventions used by many other established grammars in the ecosystem. Some of these well-established patterns are listed -below: +Tree-sitter places no restrictions on how to name the rules of your grammar. It can be helpful, however, to follow certain conventions +used by many other established grammars in the ecosystem. Some of these well-established patterns are listed below: - `source_file`: Represents an entire source file, this rule is commonly used as the root node for a grammar, -- `expression`/`statement`: Used to represent statements and expressions for a given language. Commonly defined as a choice -between several more specific sub-expression/sub-statement rules. +- `expression`/`statement`: Used to represent statements and expressions for a given language. Commonly defined as a choice between several +more specific sub-expression/sub-statement rules. - `block`: Used as the parent node for block scopes, with its children representing the block's contents. - `type`: Represents the types of a language such as `int`, `char`, and `void`. -- `identifier`: Used for constructs like variable names, function arguments, and object fields; this rule is commonly used -as the `word` token in grammars. +- `identifier`: Used for constructs like variable names, function arguments, and object fields; this rule is commonly used as the `word` +token in grammars. - `string`: Used to represent `"string literals"`. - `comment`: Used to represent comments, this rule is commonly used as an `extra`. @@ -203,7 +202,7 @@ To produce a readable syntax tree, we'd like to model JavaScript expressions usi { // ... - expression: $ => choice( + _expression: $ => choice( $.identifier, $.unary_expression, $.binary_expression, @@ -211,14 +210,14 @@ To produce a readable syntax tree, we'd like to model JavaScript expressions usi ), unary_expression: $ => choice( - seq('-', $.expression), - seq('!', $.expression), + seq('-', $._expression), + seq('!', $._expression), // ... ), binary_expression: $ => choice( - seq($.expression, '*', $.expression), - seq($.expression, '+', $.expression), + seq($._expression, '*', $._expression), + seq($._expression, '+', $._expression), // ... ), } @@ -253,7 +252,7 @@ ambiguity. For an expression like `-a * b`, it's not clear whether the `-` operator applies to the `a * b` or just to the `a`. This is where the `prec` function [described in the previous page][grammar dsl] comes into play. By wrapping a rule with `prec`, we can indicate that certain sequence of symbols should _bind to each other more tightly_ than others. For example, the -`'-', $.expression` sequence in `unary_expression` should bind more tightly than the `$.expression, '+', $.expression` +`'-', $._expression` sequence in `unary_expression` should bind more tightly than the `$._expression, '+', $._expression` sequence in `binary_expression`: ```js @@ -264,8 +263,8 @@ sequence in `binary_expression`: prec( 2, choice( - seq("-", $.expression), - seq("!", $.expression), + seq("-", $._expression), + seq("!", $._expression), // ... ), ); @@ -300,8 +299,8 @@ This is where `prec.left` and `prec.right` come into use. We want to select the // ... binary_expression: $ => choice( - prec.left(2, seq($.expression, '*', $.expression)), - prec.left(1, seq($.expression, '+', $.expression)), + prec.left(2, seq($._expression, '*', $._expression)), + prec.left(1, seq($._expression, '+', $._expression)), // ... ), } @@ -309,12 +308,12 @@ This is where `prec.left` and `prec.right` come into use. We want to select the ## Using Conflicts -Sometimes, conflicts are actually desirable. In our JavaScript grammar, expressions and patterns can create intentional -ambiguity. A construct like `[x, y]` could be legitimately parsed as both an array literal (like in `let a = [x, y]`) or -as a destructuring pattern (like in `let [x, y] = arr`). +Sometimes, conflicts are actually desirable. In our JavaScript grammar, expressions and patterns can create intentional ambiguity. +A construct like `[x, y]` could be legitimately parsed as both an array literal (like in `let a = [x, y]`) or as a destructuring +pattern (like in `let [x, y] = arr`). ```js -export default grammar({ +module.exports = grammar({ name: "javascript", rules: { @@ -396,131 +395,6 @@ function_definition: $ => Adding fields like this allows you to retrieve nodes using the [field APIs][field-names-section]. -## Using Extras - -Extras are tokens that can appear anywhere in the grammar, without being explicitly mentioned in a rule. This is useful -for things like whitespace and comments, which can appear between any two tokens in most programming languages. To define -an extra, you can use the `extras` function: - -```js -module.exports = grammar({ - name: "my_language", - - extras: ($) => [ - /\s/, // whitespace - $.comment, - ], - - rules: { - comment: ($) => - token( - choice(seq("//", /.*/), seq("/*", /[^*]*\*+([^/*][^*]*\*+)*/, "/")), - ), - }, -}); -``` - -```admonish warning -When adding more complicated tokens to `extras`, it's preferable to associate the pattern -with a rule. This way, you avoid the lexer inlining this pattern in a bunch of spots, -which can dramatically reduce the parser size. -``` - -For example, instead of defining the `comment` token inline in `extras`: - -```js -// ❌ Less preferable - -const comment = token( - choice(seq("//", /.*/), seq("/*", /[^*]*\*+([^/*][^*]*\*+)*/, "/")), -); - -module.exports = grammar({ - name: "my_language", - extras: ($) => [ - /\s/, // whitespace - comment, - ], - rules: { - // ... - }, -}); -``` - -We can define it as a rule and then reference it in `extras`: - -```js -// ✅ More preferable - -module.exports = grammar({ - name: "my_language", - - extras: ($) => [ - /\s/, // whitespace - $.comment, - ], - - rules: { - // ... - - comment: ($) => - token( - choice(seq("//", /.*/), seq("/*", /[^*]*\*+([^/*][^*]*\*+)*/, "/")), - ), - }, -}); -``` - -```admonish note -Tree-sitter intentionally simplifies the whitespace character class, `\s`, to `[ \t\n\r]` as a performance -optimization. This is because typically users do not require the full Unicode definition of whitespace. -``` - -## Using Supertypes - -Some rules in your grammar will represent abstract categories of syntax nodes, such as "expression", "type", or "declaration". -These rules are often defined as simple choices between several other rules. For example, in the JavaScript grammar, the -`_expression` rule is defined as a choice between many different kinds of expressions: - -```js -expression: $ => choice( - $.identifier, - $.unary_expression, - $.binary_expression, - $.call_expression, - $.member_expression, - // ... -), -``` - -By default, Tree-sitter will generate a visible node type for each of these abstract category rules, which can lead to -unnecessarily deep and complex syntax trees. To avoid this, you can add these abstract category rules to the grammar's `supertypes` -definition. Tree-sitter will then treat these rules as _supertypes_, and will not generate visible node types for them in -the syntax tree. - -```js -module.exports = grammar({ - name: "javascript", - - supertypes: $ => [ - $.expression, - ], - - rules: { - expression: $ => choice( - $.identifier, - // ... - ), - - // ... - }, -}); -_ -``` - -Although supertype rules are hidden from the syntax tree, they can still be used in queries. See the chapter on -[Query Syntax][query syntax] for more information. - # Lexical Analysis Tree-sitter's parsing process is divided into two phases: parsing (which is described above) and [lexing][lexing] — the @@ -565,8 +439,8 @@ as mentioned in the previous page, is `token(prec(N, ...))`. ## Keywords Many languages have a set of _keyword_ tokens (e.g. `if`, `for`, `return`), as well as a more general token (e.g. `identifier`) -that matches any word, including many of the keyword strings. For example, JavaScript has a keyword `instanceof`, which -is used as a binary operator, like this: +that matches any word, including many of the keyword strings. For example, JavaScript has a keyword `instanceof`, which is +used as a binary operator, like this: ```js if (a instanceof Something) b(); @@ -599,7 +473,7 @@ grammar({ word: $ => $.identifier, rules: { - expression: $ => + _expression: $ => choice( $.identifier, $.unary_expression, @@ -609,13 +483,13 @@ grammar({ binary_expression: $ => choice( - prec.left(1, seq($.expression, "instanceof", $.expression)), + prec.left(1, seq($._expression, "instanceof", $._expression)), // ... ), unary_expression: $ => choice( - prec.left(2, seq("typeof", $.expression)), + prec.left(2, seq("typeof", $._expression)), // ... ), @@ -652,6 +526,5 @@ rule that's called something else, you should just alias the word token instead, [field-names-section]: ../using-parsers/2-basic-parsing.md#node-field-names [non-terminal]: https://en.wikipedia.org/wiki/Terminal_and_nonterminal_symbols [peg]: https://en.wikipedia.org/wiki/Parsing_expression_grammar -[query syntax]: ../using-parsers/queries/1-syntax.md#supertype-nodes [tree-sitter-javascript]: https://github.com/tree-sitter/tree-sitter-javascript [yacc]: https://en.wikipedia.org/wiki/Yacc diff --git a/docs/src/creating-parsers/4-external-scanners.md b/docs/src/creating-parsers/4-external-scanners.md index 6c89e221..7d63d04b 100644 --- a/docs/src/creating-parsers/4-external-scanners.md +++ b/docs/src/creating-parsers/4-external-scanners.md @@ -23,7 +23,10 @@ grammar({ }); ``` -Then, add another C source file to your project. Its path must be src/scanner.c for the CLI to recognize it. +Then, add another C source file to your project. Its path must be src/scanner.c for the CLI to recognize it. Be sure to add +this file to the sources section of your `binding.gyp` file so that it will be included when your project is compiled by +Node.js and uncomment the appropriate block in your bindings/rust/build.rs file so that it will be included in your Rust +crate. In this new source file, define an [`enum`][enum] type containing the names of all of your external tokens. The ordering of this enum must match the order in your grammar's `externals` array; the actual names do not matter. @@ -143,10 +146,10 @@ the second argument, the current character will be treated as whitespace; whites associated with tokens emitted by the external scanner. - **`void (*mark_end)(TSLexer *)`** — A function for marking the end of the recognized token. This allows matching tokens -that require multiple characters of lookahead. By default, (if you don't call `mark_end`), any character that you moved -past using the `advance` function will be included in the size of the token. But once you call `mark_end`, then any later -calls to `advance` will _not_ increase the size of the returned token. You can call `mark_end` multiple times to increase -the size of the token. +that require multiple characters of lookahead. By default, (if you don't call `mark_end`), any character that you moved past +using the `advance` function will be included in the size of the token. But once you call `mark_end`, then any later calls +to `advance` will _not_ increase the size of the returned token. You can call `mark_end` multiple times to increase the size +of the token. - **`uint32_t (*get_column)(TSLexer *)`** — A function for querying the current column position of the lexer. It returns the number of codepoints since the start of the current line. The codepoint position is recalculated on every call to this @@ -185,9 +188,9 @@ if (valid_symbols[INDENT] || valid_symbols[DEDENT]) { ### Allocator -Instead of using libc's `malloc`, `calloc`, `realloc`, and `free`, you should use the versions prefixed with `ts_` from -`tree_sitter/alloc.h`. These macros can allow a potential consumer to override the default allocator with their own implementation, -but by default will use the libc functions. +Instead of using libc's `malloc`, `calloc`, `realloc`, and `free`, you should use the versions prefixed with `ts_` from `tree_sitter/alloc.h`. +These macros can allow a potential consumer to override the default allocator with their own implementation, but by default +will use the libc functions. As a consumer of the tree-sitter core library as well as any parser libraries that might use allocations, you can enable overriding the default allocator and have it use the same one as the library allocator, of which you can set with `ts_set_allocator`. @@ -195,8 +198,7 @@ To enable this overriding in scanners, you must compile them with the `TREE_SITT the library must be linked into your final app dynamically, since it needs to resolve the internal functions at runtime. If you are compiling an executable binary that uses the core library, but want to load parsers dynamically at runtime, then you will have to use a special linker flag on Unix. For non-Darwin systems, that would be `--dynamic-list` and for Darwin -systems, that would be `-exported_symbols_list`. The CLI does exactly this, so you can use it as a reference (check out -`cli/build.rs`). +systems, that would be `-exported_symbols_list`. The CLI does exactly this, so you can use it as a reference (check out `cli/build.rs`). For example, assuming you wanted to allocate 100 bytes for your scanner, you'd do so like the following example: @@ -294,10 +296,9 @@ bool tree_sitter_my_language_external_scanner_scan( ## Other External Scanner Details -External scanners have priority over Tree-sitter's normal lexing process. When a token listed in the externals array is -valid at a given position, the external scanner is called first. This makes external scanners a powerful way to override -Tree-sitter's default lexing behavior, especially for cases that can't be handled with regular lexical rules, parsing, or -dynamic precedence. +External scanners have priority over Tree-sitter's normal lexing process. When a token listed in the externals array is valid +at a given position, the external scanner is called first. This makes external scanners a powerful way to override Tree-sitter's +default lexing behavior, especially for cases that can't be handled with regular lexical rules, parsing, or dynamic precedence. During error recovery, Tree-sitter's first step is to call the external scanner's scan function with all tokens marked as valid. Your scanner should detect and handle this case appropriately. One simple approach is to add an unused "sentinel" diff --git a/docs/src/creating-parsers/5-writing-tests.md b/docs/src/creating-parsers/5-writing-tests.md index 33155cca..363c9c73 100644 --- a/docs/src/creating-parsers/5-writing-tests.md +++ b/docs/src/creating-parsers/5-writing-tests.md @@ -39,8 +39,8 @@ It only shows the *named* nodes, as described in [this section][named-vs-anonymo ``` The expected output section can also *optionally* show the [*field names*][node-field-names] associated with each child - node. To include field names in your tests, you write a node's field name followed by a colon, before the node itself - in the S-expression: + node. To include field names in your tests, you write a node's field name followed by a colon, before the node itself in + the S-expression: ```query (source_file @@ -77,21 +77,16 @@ These tests are important. They serve as the parser's API documentation, and the to verify that everything still parses correctly. By default, the `tree-sitter test` command runs all the tests in your `test/corpus/` folder. To run a particular test, you -can use the `-i` flag: +can use the `-f` flag: ```sh -tree-sitter test -i 'Return statements' +tree-sitter test -f 'Return statements' ``` The recommendation is to be comprehensive in adding tests. If it's a visible node, add it to a test file in your `test/corpus` directory. It's typically a good idea to test all the permutations of each language construct. This increases test coverage, but doubly acquaints readers with a way to examine expected outputs and understand the "edges" of a language. -```admonish tip -After modifying the grammar, you can run `tree-sitter test -u` -to update all syntax trees in corpus files with current parser output. -``` - ## Attributes Tests can be annotated with a few `attributes`. Attributes must be put in the header, below the test name, and start with @@ -104,18 +99,16 @@ you can repeat the attribute on a new line. The following attributes are available: -* `:cst` - This attribute specifies that the expected output should be in the form of a CST instead of the normal S-expression. -This CST matches the format given by `parse --cst`. +* `:skip` — This attribute will skip the test when running `tree-sitter test`. + This is useful when you want to temporarily disable running a test without deleting it. * `:error` — This attribute will assert that the parse tree contains an error. It's useful to just validate that a certain input is invalid without displaying the whole parse tree, as such you should omit the parse tree below the `---` line. -* `:fail-fast` — This attribute will stop the testing of additional cases if the test marked with this attribute fails. +* `:fail-fast` — This attribute will stop the testing additional tests if the test marked with this attribute fails. * `:language(LANG)` — This attribute will run the tests using the parser for the specified language. This is useful for multi-parser repos, such as XML and DTD, or Typescript and TSX. The default parser used will always be the first entry in the `grammars` field in the `tree-sitter.json` config file, so having a way to pick a second or even third parser is useful. * `:platform(PLATFORM)` — This attribute specifies the platform on which the test should run. It is useful to test platform-specific behavior (e.g. Windows newlines are different from Unix). This attribute must match up with Rust's [`std::env::consts::OS`][constants]. -* `:skip` — This attribute will skip the test when running `tree-sitter test`. -This is useful when you want to temporarily disable running a test without deleting it. Examples using attributes: @@ -172,4 +165,3 @@ file is changed. [external-scanners]: ./4-external-scanners.md [node-field-names]: ../using-parsers/2-basic-parsing.md#node-field-names [s-exp]: https://en.wikipedia.org/wiki/S-expression -[named-vs-anonymous-nodes]: ../using-parsers/2-basic-parsing.md#named-vs-anonymous-nodes diff --git a/docs/src/creating-parsers/index.md b/docs/src/creating-parsers/index.md index 4fb2c112..478cbeeb 100644 --- a/docs/src/creating-parsers/index.md +++ b/docs/src/creating-parsers/index.md @@ -1,4 +1,4 @@ # Creating parsers -Developing Tree-sitter grammars can have a difficult learning curve, but once you get the hang of it, it can be fun and -even zen-like. This document will help you to get started and to develop a useful mental model. +Developing Tree-sitter grammars can have a difficult learning curve, but once you get the hang of it, it can be fun and even +zen-like. This document will help you to get started and to develop a useful mental model. diff --git a/docs/src/index.md b/docs/src/index.md index ee92966a..5d5cb0ea 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -10,8 +10,7 @@ file and efficiently update the syntax tree as the source file is edited. Tree-s - **General** enough to parse any programming language - **Fast** enough to parse on every keystroke in a text editor - **Robust** enough to provide useful results even in the presence of syntax errors -- **Dependency-free** so that the runtime library (which is written in pure [C11](https://github.com/tree-sitter/tree-sitter/tree/master/lib)) -can be embedded in any application +- **Dependency-free** so that the runtime library (which is written in pure [C11](https://github.com/tree-sitter/tree-sitter/tree/master/lib)) can be embedded in any application ## Language Bindings @@ -22,40 +21,31 @@ There are bindings that allow Tree-sitter to be used from the following language - [C#](https://github.com/tree-sitter/csharp-tree-sitter) - [Go](https://github.com/tree-sitter/go-tree-sitter) - [Haskell](https://github.com/tree-sitter/haskell-tree-sitter) -- [Java (JDK 22+)](https://github.com/tree-sitter/java-tree-sitter) +- [Java (JDK 22)](https://github.com/tree-sitter/java-tree-sitter) - [JavaScript (Node.js)](https://github.com/tree-sitter/node-tree-sitter) - [JavaScript (Wasm)](https://github.com/tree-sitter/tree-sitter/tree/master/lib/binding_web) - [Kotlin](https://github.com/tree-sitter/kotlin-tree-sitter) - [Python](https://github.com/tree-sitter/py-tree-sitter) - [Rust](https://github.com/tree-sitter/tree-sitter/tree/master/lib/binding_rust) -- [Swift](https://github.com/tree-sitter/swift-tree-sitter) - [Zig](https://github.com/tree-sitter/zig-tree-sitter) ### Third-party -- [C# (.NET)](https://github.com/zabbius/dotnet-tree-sitter) -- [C++](https://github.com/nsumner/cpp-tree-sitter) -- [Crystal](https://github.com/crystal-lang-tools/crystal-tree-sitter) -- [D](https://github.com/aminya/d-tree-sitter) - [Delphi](https://github.com/modersohn/delphi-tree-sitter) - [ELisp](https://www.gnu.org/software/emacs/manual/html_node/elisp/Parsing-Program-Source.html) -- [Go](https://github.com/alexaandru/go-tree-sitter-bare) - [Guile](https://github.com/Z572/guile-ts) -- [Janet](https://github.com/sogaiu/janet-tree-sitter) - [Java (JDK 8+)](https://github.com/bonede/tree-sitter-ng) - [Java (JDK 11+)](https://github.com/seart-group/java-tree-sitter) - [Julia](https://github.com/MichaelHatherly/TreeSitter.jl) - [Lua](https://github.com/euclidianAce/ltreesitter) - [Lua](https://github.com/xcb-xwii/lua-tree-sitter) -- [OCaml](https://github.com/semgrep/ocaml-tree-sitter-core) +- [OCaml](https://github.com/returntocorp/ocaml-tree-sitter-core) - [Odin](https://github.com/laytan/odin-tree-sitter) - [Perl](https://metacpan.org/pod/Text::Treesitter) -- [Pharo](https://github.com/Evref-BL/Pharo-Tree-Sitter) -- [PHP](https://github.com/soulseekah/ext-treesitter) - [R](https://github.com/DavisVaughan/r-tree-sitter) - [Ruby](https://github.com/Faveod/ruby-tree-sitter) - -_Keep in mind that some of the bindings may be incomplete or out of date._ +- [Ruby](https://github.com/calicoday/ruby-tree-sitter-ffi) +- [Swift](https://github.com/ChimeHQ/SwiftTreeSitter) ## Parsers diff --git a/docs/src/using-parsers/2-basic-parsing.md b/docs/src/using-parsers/2-basic-parsing.md index 8c425d6b..f9b0fb3f 100644 --- a/docs/src/using-parsers/2-basic-parsing.md +++ b/docs/src/using-parsers/2-basic-parsing.md @@ -2,8 +2,7 @@ ## Providing the Code -In the example on the previous page, we parsed source code stored in a simple string using the `ts_parser_parse_string` -function: +In the example on the previous page, we parsed source code stored in a simple string using the `ts_parser_parse_string` function: ```c TSTree *ts_parser_parse_string( @@ -39,15 +38,15 @@ typedef struct { uint32_t *bytes_read ); TSInputEncoding encoding; - TSDecodeFunction decode; + DecodeFunction decode; } TSInput; ``` If you want to decode text that is not encoded in UTF-8 or UTF-16, you can set the `decode` field of the input to your function -that will decode text. The signature of the `TSDecodeFunction` is as follows: +that will decode text. The signature of the `DecodeFunction` is as follows: ```c -typedef uint32_t (*TSDecodeFunction)( +typedef uint32_t (*DecodeFunction)( const uint8_t *string, uint32_t length, int32_t *code_point @@ -87,10 +86,6 @@ TSPoint ts_node_start_point(TSNode); TSPoint ts_node_end_point(TSNode); ``` -```admonish note -A *newline* is considered to be a single line feed (`\n`) character. -``` - ## Retrieving Nodes Every tree has a _root node_: @@ -136,10 +131,10 @@ Consider a grammar rule like this: if_statement: $ => seq("if", "(", $._expression, ")", $._statement); ``` -A syntax node representing an `if_statement` in this language would have 5 children: the condition expression, the body -statement, as well as the `if`, `(`, and `)` tokens. The expression and the statement would be marked as _named_ nodes, -because they have been given explicit names in the grammar. But the `if`, `(`, and `)` nodes would _not_ be named nodes, -because they are represented in the grammar as simple strings. +A syntax node representing an `if_statement` in this language would have 5 children: the condition expression, the body statement, +as well as the `if`, `(`, and `)` tokens. The expression and the statement would be marked as _named_ nodes, because they +have been given explicit names in the grammar. But the `if`, `(`, and `)` nodes would _not_ be named nodes, because they +are represented in the grammar as simple strings. You can check whether any given node is named: diff --git a/docs/src/using-parsers/3-advanced-parsing.md b/docs/src/using-parsers/3-advanced-parsing.md index c1c92e24..df13fe30 100644 --- a/docs/src/using-parsers/3-advanced-parsing.md +++ b/docs/src/using-parsers/3-advanced-parsing.md @@ -19,11 +19,11 @@ typedef struct { void ts_tree_edit(TSTree *, const TSInputEdit *); ``` -Then, you can call `ts_parser_parse` again, passing in the old tree. This will create a new tree that internally shares -structure with the old tree. +Then, you can call `ts_parser_parse` again, passing in the old tree. This will create a new tree that internally shares structure +with the old tree. When you edit a syntax tree, the positions of its nodes will change. If you have stored any `TSNode` instances outside of -the `TSTree`, you must update their positions separately, using the same `TSInputEdit` value, in order to update their +the `TSTree`, you must update their positions separately, using the same `TSInput` value, in order to update their cached positions. ```c diff --git a/docs/src/using-parsers/6-static-node-types.md b/docs/src/using-parsers/6-static-node-types.md index 171f4314..5976d0bc 100644 --- a/docs/src/using-parsers/6-static-node-types.md +++ b/docs/src/using-parsers/6-static-node-types.md @@ -108,9 +108,9 @@ In Tree-sitter grammars, there are usually certain rules that represent abstract "type", "declaration"). In the `grammar.js` file, these are often written as [hidden rules][hidden rules] whose definition is a simple [`choice`][grammar dsl] where each member is just a single symbol. -Normally, hidden rules are not mentioned in the node types file, since they don't appear in the syntax tree. But if you -add a hidden rule to the grammar's [`supertypes` list][grammar dsl], then it _will_ show up in the node types file, with -the following special entry: +Normally, hidden rules are not mentioned in the node types file, since they don't appear in the syntax tree. But if you add +a hidden rule to the grammar's [`supertypes` list][grammar dsl], then it _will_ show up in the node +types file, with the following special entry: - `"subtypes"` — An array of objects that specify the _types_ of nodes that this 'supertype' node can wrap. diff --git a/docs/src/using-parsers/7-abi-versions.md b/docs/src/using-parsers/7-abi-versions.md deleted file mode 100644 index 1db42a5d..00000000 --- a/docs/src/using-parsers/7-abi-versions.md +++ /dev/null @@ -1,25 +0,0 @@ -# ABI versions - -Parsers generated with tree-sitter have an associated ABI version, which establishes hard compatibility boundaries -between the generated parser and the tree-sitter library. - -A given version of the tree-sitter library is only able to load parsers generated with supported ABI versions: - -| tree-sitter version | Min parser ABI version | Max parser ABI version | -|---------------------|------------------------|------------------------| -| 0.14 | 9 | 9 | -| >=0.15.0, <=0.15.7 | 9 | 10 | -| >=0.15.8, <=0.16 | 9 | 11 | -| 0.17, 0.18 | 9 | 12 | -| >=0.19, <=0.20.2 | 13 | 13 | -| >=0.20.3, <=0.24 | 13 | 14 | -| >=0.25 | 13 | 15 | - -By default, the tree-sitter CLI will generate parsers using the latest available ABI for that version, but an older ABI -(supported by the CLI) can be selected by passing the [`--abi` option][abi_option] to the `generate` command. - -Note that the ABI version range supported by the CLI can be smaller than for the library: When a new ABI version is released, -older versions will be phased out over a deprecation period, which starts with no longer being able to generate parsers -with the oldest ABI version. - -[abi_option]: ../cli/generate.md#--abi-version diff --git a/docs/src/using-parsers/index.md b/docs/src/using-parsers/index.md index 5b2c146d..48d61599 100644 --- a/docs/src/using-parsers/index.md +++ b/docs/src/using-parsers/index.md @@ -6,8 +6,8 @@ the core concepts remain the same. Tree-sitter's parsing functionality is implemented through its C API, with all functions documented in the [tree_sitter/api.h][api.h] header file, but if you're working in another language, you can use one of the following bindings found [here](../index.md#language-bindings), -each providing idiomatic access to Tree-sitter's functionality. Of these bindings, the official ones have their own API -doc hosted online at the following pages: +each providing idiomatic access to Tree-sitter's functionality. Of these bindings, the official ones have their own API docs +hosted online at the following pages: - [Go][go] - [Java] diff --git a/docs/src/using-parsers/queries/1-syntax.md b/docs/src/using-parsers/queries/1-syntax.md index 2e0a8853..5edd0047 100644 --- a/docs/src/using-parsers/queries/1-syntax.md +++ b/docs/src/using-parsers/queries/1-syntax.md @@ -1,9 +1,9 @@ # Query Syntax -A _query_ consists of one or more _patterns_, where each pattern is an [S-expression][s-exp] that matches a certain set -of nodes in a syntax tree. The expression to match a given node consists of a pair of parentheses containing two things: -the node's type, and optionally, a series of other S-expressions that match the node's children. For example, this pattern -would match any `binary_expression` node whose children are both `number_literal` nodes: +A _query_ consists of one or more _patterns_, where each pattern is an [S-expression][s-exp] that matches a certain set of +nodes in a syntax tree. The expression to match a given node consists of a pair of parentheses containing two things: the +node's type, and optionally, a series of other S-expressions that match the node's children. For example, this pattern would +match any `binary_expression` node whose children are both `number_literal` nodes: ```query (binary_expression (number_literal) (number_literal)) @@ -96,32 +96,6 @@ by `(ERROR)` queries. Specific missing node types can also be queried: (MISSING ";") @missing-semicolon ``` -### Supertype Nodes - -Some node types are marked as _supertypes_ in a grammar. A supertype is a node type that contains multiple -subtypes. For example, in the [JavaScript grammar example][grammar], `expression` is a supertype that can represent any -kind of expression, such as a `binary_expression`, `call_expression`, or `identifier`. You can use supertypes in queries -to match any of their subtypes, rather than having to list out each subtype individually. For example, this pattern would -match any kind of expression, even though it's not a visible node in the syntax tree: - -```query -(expression) @any-expression -``` - -To query specific subtypes of a supertype, you can use the syntax `supertype/subtype`. For example, this pattern would -match a `binary_expression` only if it is a child of `expression`: - -```query -(expression/binary_expression) @binary-expression -``` - -This also applies to anonymous nodes. For example, this pattern would match `"()"` only if it is a child of `expression`: - -```query -(expression/"()") @empty-expression -``` - -[grammar]: ../../creating-parsers/3-writing-the-grammar.md#structuring-rules-well [node-field-names]: ../2-basic-parsing.md#node-field-names [named-vs-anonymous-nodes]: ../2-basic-parsing.md#named-vs-anonymous-nodes [s-exp]: https://en.wikipedia.org/wiki/S-expression diff --git a/docs/src/using-parsers/queries/3-predicates-and-directives.md b/docs/src/using-parsers/queries/3-predicates-and-directives.md index 1f9aeada..a059a3ba 100644 --- a/docs/src/using-parsers/queries/3-predicates-and-directives.md +++ b/docs/src/using-parsers/queries/3-predicates-and-directives.md @@ -128,19 +128,19 @@ This pattern would match any builtin variable that is not a local variable, beca # Directives -Similar to predicates, directives are a way to associate arbitrary metadata with a pattern. The only difference between -predicates and directives is that directives end in a `!` character instead of `?` character. +Similar to predicates, directives are a way to associate arbitrary metadata with a pattern. The only difference between predicates +and directives is that directives end in a `!` character instead of `?` character. Tree-sitter's CLI supports the following directives by default: ## The `set!` directive -This directive allows you to associate key-value pairs with a pattern. The key and value can be any arbitrary text that -you see fit. +This directive allows you to associate key-value pairs with a pattern. The key and value can be any arbitrary text that you +see fit. ```query ((comment) @injection.content - (#match? @injection.content "/[*\/][!*\/] for the CLI," - echo " and .#lib- for the library (e.g. nix build .#cli-aarch64-linux)." - echo "" - echo " Available targets:" - echo " aarch64-linux - ARM64 Linux" - echo " armv7l-linux - ARMv7 Linux" - echo " x86_64-linux - x86_64 Linux" - echo " i686-linux - i686 Linux" - echo " loongarch64 - LoongArch64 Linux" - echo " mips - MIPS Linux" - echo " mips64 - MIPS64 Linux" - echo " musl64 - x86_64 MUSL Linux" - echo " powerpc64-linux - PowerPC64 Linux" - echo " riscv32 - RISC-V 32-bit Linux" - echo " riscv64 - RISC-V 64-bit Linux" - echo " s390x - s390x Linux" - echo " x86_64-windows - x86_64 Windows" - echo " x86_64-darwin - x86_64 macOS (Darwin only)" - echo " aarch64-darwin - ARM64 macOS (Darwin only)" - echo "" - echo "Apps:" - echo " nix run .#cli - Run tree-sitter CLI" - echo " nix run .#docs - Serve docs locally" - echo " nix run .#format - Format all code" - echo " nix run .#lint - Run all linting checks" - echo "" - echo "Tests & Checks:" - echo " nix flake check - Run all tests and checks" - echo "" - echo "Version: ${version}" - ''; - - env = { - RUST_BACKTRACE = 1; - LIBCLANG_PATH = "${pkgs.libclang.lib}/lib"; - LLVM_COV = "${pkgs.llvm}/bin/llvm-cov"; - LLVM_PROFDATA = "${pkgs.llvm}/bin/llvm-profdata"; - TREE_SITTER_WASI_SDK_PATH = "${pkgs.pkgsCross.wasi32.stdenv.cc}"; - }; - }; - } - ); - }; -} diff --git a/crates/highlight/Cargo.toml b/highlight/Cargo.toml similarity index 89% rename from crates/highlight/Cargo.toml rename to highlight/Cargo.toml index 502bb31f..f48789f1 100644 --- a/crates/highlight/Cargo.toml +++ b/highlight/Cargo.toml @@ -11,7 +11,6 @@ rust-version.workspace = true readme = "README.md" homepage.workspace = true repository.workspace = true -documentation = "https://docs.rs/tree-sitter-highlight" license.workspace = true keywords = ["incremental", "parsing", "syntax", "highlighting"] categories = ["parsing", "text-editors"] @@ -20,7 +19,6 @@ categories = ["parsing", "text-editors"] workspace = true [lib] -path = "src/highlight.rs" crate-type = ["lib", "staticlib"] [dependencies] diff --git a/crates/highlight/README.md b/highlight/README.md similarity index 94% rename from crates/highlight/README.md rename to highlight/README.md index edb89708..73e16b28 100644 --- a/crates/highlight/README.md +++ b/highlight/README.md @@ -12,8 +12,8 @@ to parse, to your `Cargo.toml`: ```toml [dependencies] -tree-sitter-highlight = "0.25.4" -tree-sitter-javascript = "0.23.1" +tree-sitter-highlight = "0.22.0" +tree-sitter-javascript = "0.21.3" ``` Define the list of highlight names that you will recognize: @@ -63,7 +63,7 @@ Load some highlighting queries from the `queries` directory of the language repo ```rust use tree_sitter_highlight::HighlightConfiguration; -let javascript_language = tree_sitter_javascript::LANGUAGE.into(); +let javascript_language = tree_sitter_javascript::language(); let mut javascript_config = HighlightConfiguration::new( javascript_language, diff --git a/crates/highlight/include/tree_sitter/highlight.h b/highlight/include/tree_sitter/highlight.h similarity index 100% rename from crates/highlight/include/tree_sitter/highlight.h rename to highlight/include/tree_sitter/highlight.h diff --git a/crates/highlight/src/c_lib.rs b/highlight/src/c_lib.rs similarity index 100% rename from crates/highlight/src/c_lib.rs rename to highlight/src/c_lib.rs diff --git a/crates/highlight/src/highlight.rs b/highlight/src/lib.rs similarity index 97% rename from crates/highlight/src/highlight.rs rename to highlight/src/lib.rs index d38351f6..9d296050 100644 --- a/crates/highlight/src/highlight.rs +++ b/highlight/src/lib.rs @@ -1,4 +1,4 @@ -#![cfg_attr(not(any(test, doctest)), doc = include_str!("../README.md"))] +#![doc = include_str!("../README.md")] pub mod c_lib; use core::slice; @@ -7,8 +7,7 @@ use std::{ iter, marker::PhantomData, mem::{self, MaybeUninit}, - ops::{self, ControlFlow}, - str, + ops, str, sync::{ atomic::{AtomicUsize, Ordering}, LazyLock, @@ -189,7 +188,7 @@ struct HighlightIterLayer<'a> { depth: usize, } -pub struct _QueryCaptures<'query, 'tree, T: TextProvider, I: AsRef<[u8]>> { +pub struct _QueryCaptures<'query, 'tree: 'query, T: TextProvider, I: AsRef<[u8]>> { ptr: *mut ffi::TSQueryCursor, query: &'query Query, text_provider: T, @@ -225,7 +224,7 @@ impl<'tree> _QueryMatch<'_, 'tree> { } } -impl<'query, 'tree, T: TextProvider, I: AsRef<[u8]>> Iterator +impl<'query, 'tree: 'query, T: TextProvider, I: AsRef<[u8]>> Iterator for _QueryCaptures<'query, 'tree, T, I> { type Item = (QueryMatch<'query, 'tree>, usize); @@ -276,7 +275,7 @@ impl Highlighter { } } - pub const fn parser(&mut self) -> &mut Parser { + pub fn parser(&mut self) -> &mut Parser { &mut self.parser } @@ -344,13 +343,11 @@ impl HighlightConfiguration { locals_query: &str, ) -> Result { // Concatenate the query strings, keeping track of the start offset of each section. - let mut query_source = String::with_capacity( - injection_query.len() + locals_query.len() + highlights_query.len(), - ); + let mut query_source = String::new(); query_source.push_str(injection_query); - let locals_query_offset = injection_query.len(); + let locals_query_offset = query_source.len(); query_source.push_str(locals_query); - let highlights_query_offset = injection_query.len() + locals_query.len(); + let highlights_query_offset = query_source.len(); query_source.push_str(highlights_query); // Construct a single query by concatenating the three query strings, but record the @@ -541,13 +538,9 @@ impl<'a> HighlightIterLayer<'a> { None, Some(ParseOptions::new().progress_callback(&mut |_| { if let Some(cancellation_flag) = cancellation_flag { - if cancellation_flag.load(Ordering::SeqCst) != 0 { - ControlFlow::Break(()) - } else { - ControlFlow::Continue(()) - } + cancellation_flag.load(Ordering::SeqCst) != 0 } else { - ControlFlow::Continue(()) + false } })), ) @@ -594,7 +587,6 @@ impl<'a> HighlightIterLayer<'a> { } } - // SAFETY: // The `captures` iterator borrows the `Tree` and the `QueryCursor`, which // prevents them from being moved. But both of these values are really just // pointers, so it's actually ok to move them. @@ -1106,7 +1098,7 @@ impl HtmlRenderer { result } - pub const fn set_carriage_return_highlight(&mut self, highlight: Option) { + pub fn set_carriage_return_highlight(&mut self, highlight: Option) { self.carriage_return_highlight = highlight; } @@ -1230,14 +1222,12 @@ impl HtmlRenderer { // At line boundaries, close and re-open all of the open tags. if c == b'\n' { - for _ in highlights { - self.end_highlight(); - } + highlights.iter().for_each(|_| self.end_highlight()); self.html.push(c); self.line_offsets.push(self.html.len() as u32); - for scope in highlights { - self.start_highlight(*scope, attribute_callback); - } + highlights + .iter() + .for_each(|scope| self.start_highlight(*scope, attribute_callback)); } else if let Some(escape) = html_escape(c) { self.html.extend_from_slice(escape); } else { diff --git a/CMakeLists.txt b/lib/CMakeLists.txt similarity index 87% rename from CMakeLists.txt rename to lib/CMakeLists.txt index f11895c0..42aa60d0 100644 --- a/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.13) project(tree-sitter - VERSION "0.27.0" + VERSION "0.25.9" DESCRIPTION "An incremental parsing system for programming tools" HOMEPAGE_URL "https://tree-sitter.github.io/tree-sitter/" LANGUAGES C) @@ -11,15 +11,15 @@ option(TREE_SITTER_FEATURE_WASM "Enable the Wasm feature" OFF) option(AMALGAMATED "Build using an amalgamated source" OFF) if(AMALGAMATED) - set(TS_SOURCE_FILES "${PROJECT_SOURCE_DIR}/lib/src/lib.c") + set(TS_SOURCE_FILES "${PROJECT_SOURCE_DIR}/src/lib.c") else() - file(GLOB TS_SOURCE_FILES lib/src/*.c) - list(REMOVE_ITEM TS_SOURCE_FILES "${PROJECT_SOURCE_DIR}/lib/src/lib.c") + file(GLOB TS_SOURCE_FILES src/*.c) + list(REMOVE_ITEM TS_SOURCE_FILES "${PROJECT_SOURCE_DIR}/src/lib.c") endif() add_library(tree-sitter ${TS_SOURCE_FILES}) -target_include_directories(tree-sitter PRIVATE lib/src lib/src/wasm PUBLIC lib/include) +target_include_directories(tree-sitter PRIVATE src src/wasm include) if(MSVC) target_compile_options(tree-sitter PRIVATE @@ -81,13 +81,13 @@ set_target_properties(tree-sitter SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}" DEFINE_SYMBOL "") -target_compile_definitions(tree-sitter PRIVATE _POSIX_C_SOURCE=200112L _DEFAULT_SOURCE _BSD_SOURCE _DARWIN_C_SOURCE) +target_compile_definitions(tree-sitter PRIVATE _POSIX_C_SOURCE=200112L _DEFAULT_SOURCE) include(GNUInstallDirs) -configure_file(lib/tree-sitter.pc.in "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter.pc" @ONLY) +configure_file(tree-sitter.pc.in "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter.pc" @ONLY) -install(FILES lib/include/tree_sitter/api.h +install(FILES include/tree_sitter/api.h DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/tree_sitter") install(FILES "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 64a9e8eb..9b4a3352 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -4,11 +4,10 @@ version.workspace = true description = "Rust bindings to the Tree-sitter parsing library" authors.workspace = true edition.workspace = true -rust-version = "1.77" +rust-version = "1.76" readme = "binding_rust/README.md" homepage.workspace = true repository.workspace = true -documentation = "https://docs.rs/tree-sitter" license.workspace = true keywords.workspace = true categories = [ @@ -30,7 +29,6 @@ include = [ "/src/unicode/*", "/src/wasm/*", "/include/tree_sitter/api.h", - "/LICENSE", ] [package.metadata.docs.rs] @@ -47,20 +45,20 @@ std = ["regex/std", "regex/perf", "regex-syntax/unicode"] wasm = ["std", "wasmtime-c-api"] [dependencies] -regex = { version = "1.11.3", default-features = false, features = ["unicode"] } -regex-syntax = { version = "0.8.6", default-features = false } -tree-sitter-language.workspace = true +regex = { version = "1.11.1", default-features = false, features = ["unicode"] } +regex-syntax = { version = "0.8.5", default-features = false } +tree-sitter-language = { version = "0.1", path = "language" } streaming-iterator = "0.1.9" [dependencies.wasmtime-c-api] -version = "33.0.2" +version = "29.0.1" optional = true package = "wasmtime-c-api-impl" default-features = false features = ["cranelift", "gc-drc"] [build-dependencies] -bindgen = { version = "0.72.0", optional = true } +bindgen = { version = "0.71.1", optional = true } cc.workspace = true serde_json.workspace = true diff --git a/lib/LICENSE b/lib/LICENSE deleted file mode 100644 index 971b81f9..00000000 --- a/lib/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2018 Max Brunsfeld - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/lib/binding_rust/README.md b/lib/binding_rust/README.md index 39c13b7c..602f647b 100644 --- a/lib/binding_rust/README.md +++ b/lib/binding_rust/README.md @@ -17,6 +17,13 @@ use tree_sitter::{InputEdit, Language, Parser, Point}; let mut parser = Parser::new(); ``` +Add the `cc` crate to your `Cargo.toml` under `[build-dependencies]`: + +```toml +[build-dependencies] +cc="*" +``` + Then, add a language as a dependency: ```toml @@ -25,7 +32,7 @@ tree-sitter = "0.24" tree-sitter-rust = "0.23" ``` -To use a language, you assign them to the parser. +To then use a language, you assign them to the parser. ```rust parser.set_language(&tree_sitter_rust::LANGUAGE.into()).expect("Error loading Rust grammar"); @@ -98,49 +105,6 @@ assert_eq!( ); ``` -## Using Wasm Grammar Files - -> Requires the feature **wasm** to be enabled. - -First, create a parser with a Wasm store: - -```rust -use tree_sitter::{wasmtime::Engine, Parser, WasmStore}; - -let engine = Engine::default(); -let store = WasmStore::new(&engine).unwrap(); - -let mut parser = Parser::new(); -parser.set_wasm_store(store).unwrap(); -``` - -Then, load the language from a Wasm file: - -```rust -const JAVASCRIPT_GRAMMAR: &[u8] = include_bytes!("path/to/tree-sitter-javascript.wasm"); - -let mut store = WasmStore::new(&engine).unwrap(); -let javascript = store - .load_language("javascript", JAVASCRIPT_GRAMMAR) - .unwrap(); - -// The language may be loaded from a different WasmStore than the one set on -// the parser but it must use the same underlying WasmEngine. -parser.set_language(&javascript).unwrap(); -``` - -Now you can parse source code: - -```rust -let source_code = "let x = 1;"; -let tree = parser.parse(source_code, None).unwrap(); - -assert_eq!( - tree.root_node().to_sexp(), - "(program (lexical_declaration (variable_declarator name: (identifier) value: (number))))" -); -``` - [tree-sitter]: https://github.com/tree-sitter/tree-sitter ## Features diff --git a/lib/binding_rust/bindings.rs b/lib/binding_rust/bindings.rs index ec00a7c6..8323bd06 100644 --- a/lib/binding_rust/bindings.rs +++ b/lib/binding_rust/bindings.rs @@ -1,4 +1,4 @@ -/* automatically generated by rust-bindgen 0.72.1 */ +/* automatically generated by rust-bindgen 0.71.1 */ pub const TREE_SITTER_LANGUAGE_VERSION: u32 = 15; pub const TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION: u32 = 13; @@ -35,7 +35,7 @@ pub struct TSQueryCursor { pub struct TSLookaheadIterator { _unused: [u8; 0], } -pub type TSDecodeFunction = ::core::option::Option< +pub type DecodeFunction = ::core::option::Option< unsafe extern "C" fn(string: *const u8, length: u32, code_point: *mut i32) -> u32, >; pub const TSInputEncodingUTF8: TSInputEncoding = 0; @@ -75,7 +75,7 @@ pub struct TSInput { ) -> *const ::core::ffi::c_char, >, pub encoding: TSInputEncoding, - pub decode: TSDecodeFunction, + pub decode: DecodeFunction, } #[repr(C)] #[derive(Debug, Copy, Clone)] @@ -218,7 +218,7 @@ extern "C" { pub fn ts_parser_included_ranges(self_: *const TSParser, count: *mut u32) -> *const TSRange; } extern "C" { - #[doc = " Use the parser to parse some source code and create a syntax tree.\n\n If you are parsing this document for the first time, pass `NULL` for the\n `old_tree` parameter. Otherwise, if you have already parsed an earlier\n version of this document and the document has since been edited, pass the\n previous syntax tree so that the unchanged parts of it can be reused.\n This will save time and memory. For this to work correctly, you must have\n already edited the old syntax tree using the [`ts_tree_edit`] function in a\n way that exactly matches the source code changes.\n\n The [`TSInput`] parameter lets you specify how to read the text. It has the\n following three fields:\n 1. [`read`]: A function to retrieve a chunk of text at a given byte offset\n and (row, column) position. The function should return a pointer to the\n text and write its length to the [`bytes_read`] pointer. The parser does\n not take ownership of this buffer; it just borrows it until it has\n finished reading it. The function should write a zero value to the\n [`bytes_read`] pointer to indicate the end of the document.\n 2. [`payload`]: An arbitrary pointer that will be passed to each invocation\n of the [`read`] function.\n 3. [`encoding`]: An indication of how the text is encoded. Either\n `TSInputEncodingUTF8` or `TSInputEncodingUTF16`.\n\n This function returns a syntax tree on success, and `NULL` on failure. There\n are four possible reasons for failure:\n 1. The parser does not have a language assigned. Check for this using the\n[`ts_parser_language`] function.\n 2. Parsing was cancelled due to the progress callback returning true. This callback\n is passed in [`ts_parser_parse_with_options`] inside the [`TSParseOptions`] struct.\n\n [`read`]: TSInput::read\n [`payload`]: TSInput::payload\n [`encoding`]: TSInput::encoding\n [`bytes_read`]: TSInput::read"] + #[doc = " Use the parser to parse some source code and create a syntax tree.\n\n If you are parsing this document for the first time, pass `NULL` for the\n `old_tree` parameter. Otherwise, if you have already parsed an earlier\n version of this document and the document has since been edited, pass the\n previous syntax tree so that the unchanged parts of it can be reused.\n This will save time and memory. For this to work correctly, you must have\n already edited the old syntax tree using the [`ts_tree_edit`] function in a\n way that exactly matches the source code changes.\n\n The [`TSInput`] parameter lets you specify how to read the text. It has the\n following three fields:\n 1. [`read`]: A function to retrieve a chunk of text at a given byte offset\n and (row, column) position. The function should return a pointer to the\n text and write its length to the [`bytes_read`] pointer. The parser does\n not take ownership of this buffer; it just borrows it until it has\n finished reading it. The function should write a zero value to the\n [`bytes_read`] pointer to indicate the end of the document.\n 2. [`payload`]: An arbitrary pointer that will be passed to each invocation\n of the [`read`] function.\n 3. [`encoding`]: An indication of how the text is encoded. Either\n `TSInputEncodingUTF8` or `TSInputEncodingUTF16`.\n\n This function returns a syntax tree on success, and `NULL` on failure. There\n are four possible reasons for failure:\n 1. The parser does not have a language assigned. Check for this using the\n[`ts_parser_language`] function.\n 2. Parsing was cancelled due to a timeout that was set by an earlier call to\n the [`ts_parser_set_timeout_micros`] function. You can resume parsing from\n where the parser left out by calling [`ts_parser_parse`] again with the\n same arguments. Or you can start parsing from scratch by first calling\n [`ts_parser_reset`].\n 3. Parsing was cancelled using a cancellation flag that was set by an\n earlier call to [`ts_parser_set_cancellation_flag`]. You can resume parsing\n from where the parser left out by calling [`ts_parser_parse`] again with\n the same arguments.\n 4. Parsing was cancelled due to the progress callback returning true. This callback\n is passed in [`ts_parser_parse_with_options`] inside the [`TSParseOptions`] struct.\n\n [`read`]: TSInput::read\n [`payload`]: TSInput::payload\n [`encoding`]: TSInput::encoding\n [`bytes_read`]: TSInput::read"] pub fn ts_parser_parse( self_: *mut TSParser, old_tree: *const TSTree, @@ -254,9 +254,25 @@ extern "C" { ) -> *mut TSTree; } extern "C" { - #[doc = " Instruct the parser to start the next parse from the beginning.\n\n If the parser previously failed because of the progress callback, then\n by default, it will resume where it left off on the next call to\n [`ts_parser_parse`] or other parsing functions. If you don't want to resume,\n and instead intend to use this parser to parse some other document, you must\n call [`ts_parser_reset`] first."] + #[doc = " Instruct the parser to start the next parse from the beginning.\n\n If the parser previously failed because of a timeout or a cancellation, then\n by default, it will resume where it left off on the next call to\n [`ts_parser_parse`] or other parsing functions. If you don't want to resume,\n and instead intend to use this parser to parse some other document, you must\n call [`ts_parser_reset`] first."] pub fn ts_parser_reset(self_: *mut TSParser); } +extern "C" { + #[doc = " @deprecated use [`ts_parser_parse_with_options`] and pass in a callback instead, this will be removed in 0.26.\n\n Set the maximum duration in microseconds that parsing should be allowed to\n take before halting.\n\n If parsing takes longer than this, it will halt early, returning NULL.\n See [`ts_parser_parse`] for more information."] + pub fn ts_parser_set_timeout_micros(self_: *mut TSParser, timeout_micros: u64); +} +extern "C" { + #[doc = " @deprecated use [`ts_parser_parse_with_options`] and pass in a callback instead, this will be removed in 0.26.\n\n Get the duration in microseconds that parsing is allowed to take."] + pub fn ts_parser_timeout_micros(self_: *const TSParser) -> u64; +} +extern "C" { + #[doc = " @deprecated use [`ts_parser_parse_with_options`] and pass in a callback instead, this will be removed in 0.26.\n\n Set the parser's current cancellation flag pointer.\n\n If a non-null pointer is assigned, then the parser will periodically read\n from this pointer during parsing. If it reads a non-zero value, it will\n halt early, returning NULL. See [`ts_parser_parse`] for more information."] + pub fn ts_parser_set_cancellation_flag(self_: *mut TSParser, flag: *const usize); +} +extern "C" { + #[doc = " @deprecated use [`ts_parser_parse_with_options`] and pass in a callback instead, this will be removed in 0.26.\n\n Get the parser's current cancellation flag pointer."] + pub fn ts_parser_cancellation_flag(self_: *const TSParser) -> *const usize; +} extern "C" { #[doc = " Set the logger that a parser should use during parsing.\n\n The parser does not take ownership over the logger payload. If a logger was\n previously assigned, the caller is responsible for releasing any memory\n owned by the previous logger."] pub fn ts_parser_set_logger(self_: *mut TSParser, logger: TSLogger); @@ -495,14 +511,6 @@ extern "C" { #[doc = " Check if two nodes are identical."] pub fn ts_node_eq(self_: TSNode, other: TSNode) -> bool; } -extern "C" { - #[doc = " Edit a point to keep it in-sync with source code that has been edited.\n\n This function updates a single point's byte offset and row/column position\n based on an edit operation. This is useful for editing points without\n requiring a tree or node instance."] - pub fn ts_point_edit(point: *mut TSPoint, point_byte: *mut u32, edit: *const TSInputEdit); -} -extern "C" { - #[doc = " Edit a range to keep it in-sync with source code that has been edited.\n\n This function updates a range's start and end positions based on an edit\n operation. This is useful for editing ranges without requiring a tree\n or node instance."] - pub fn ts_range_edit(range: *mut TSRange, edit: *const TSInputEdit); -} extern "C" { #[doc = " Create a new tree cursor starting from the given node.\n\n A tree cursor allows you to walk a syntax tree more efficiently than is\n possible using the [`TSNode`] functions. It is a mutable object that is always\n on a certain syntax node, and can be moved imperatively to different nodes.\n\n Note that the given node is considered the root of the cursor,\n and the cursor cannot walk outside this node."] pub fn ts_tree_cursor_new(node: TSNode) -> TSTreeCursor; @@ -696,6 +704,14 @@ extern "C" { extern "C" { pub fn ts_query_cursor_set_match_limit(self_: *mut TSQueryCursor, limit: u32); } +extern "C" { + #[doc = " @deprecated use [`ts_query_cursor_exec_with_options`] and pass in a callback instead, this will be removed in 0.26.\n\n Set the maximum duration in microseconds that query execution should be allowed to\n take before halting.\n\n If query execution takes longer than this, it will halt early, returning NULL.\n See [`ts_query_cursor_next_match`] or [`ts_query_cursor_next_capture`] for more information."] + pub fn ts_query_cursor_set_timeout_micros(self_: *mut TSQueryCursor, timeout_micros: u64); +} +extern "C" { + #[doc = " @deprecated use [`ts_query_cursor_exec_with_options`] and pass in a callback instead, this will be removed in 0.26.\n\n Get the duration in microseconds that query execution is allowed to take.\n\n This is set via [`ts_query_cursor_set_timeout_micros`]."] + pub fn ts_query_cursor_timeout_micros(self_: *const TSQueryCursor) -> u64; +} extern "C" { #[doc = " Set the range of bytes in which the query will be executed.\n\n The query cursor will return matches that intersect with the given point range.\n This means that a match may be returned even if some of its captures fall\n outside the specified range, as long as at least part of the match\n overlaps with the range.\n\n For example, if a query pattern matches a node that spans a larger area\n than the specified range, but part of that node intersects with the range,\n the entire match will be returned.\n\n This will return `false` if the start byte is greater than the end byte, otherwise\n it will return `true`."] pub fn ts_query_cursor_set_byte_range( @@ -712,22 +728,6 @@ extern "C" { end_point: TSPoint, ) -> bool; } -extern "C" { - #[doc = " Set the byte range within which all matches must be fully contained.\n\n Set the range of bytes in which matches will be searched for. In contrast to\n `ts_query_cursor_set_byte_range`, this will restrict the query cursor to only return\n matches where _all_ nodes are _fully_ contained within the given range. Both functions\n can be used together, e.g. to search for any matches that intersect line 5000, as\n long as they are fully contained within lines 4500-5500"] - pub fn ts_query_cursor_set_containing_byte_range( - self_: *mut TSQueryCursor, - start_byte: u32, - end_byte: u32, - ) -> bool; -} -extern "C" { - #[doc = " Set the point range within which all matches must be fully contained.\n\n Set the range of bytes in which matches will be searched for. In contrast to\n `ts_query_cursor_set_point_range`, this will restrict the query cursor to only return\n matches where _all_ nodes are _fully_ contained within the given range. Both functions\n can be used together, e.g. to search for any matches that intersect line 5000, as\n long as they are fully contained within lines 4500-5500"] - pub fn ts_query_cursor_set_containing_point_range( - self_: *mut TSQueryCursor, - start_point: TSPoint, - end_point: TSPoint, - ) -> bool; -} extern "C" { #[doc = " Advance to the next match of the currently running query.\n\n If there is a match, write it to `*match` and return `true`.\n Otherwise, return `false`."] pub fn ts_query_cursor_next_match(self_: *mut TSQueryCursor, match_: *mut TSQueryMatch) @@ -815,6 +815,10 @@ extern "C" { #[doc = " Check whether the given node type id belongs to named nodes, anonymous nodes,\n or a hidden nodes.\n\n See also [`ts_node_is_named`]. Hidden nodes are never returned from the API."] pub fn ts_language_symbol_type(self_: *const TSLanguage, symbol: TSSymbol) -> TSSymbolType; } +extern "C" { + #[doc = " @deprecated use [`ts_language_abi_version`] instead, this will be removed in 0.26.\n\n Get the ABI version number for this language. This version number is used\n to ensure that languages were generated by a compatible version of\n Tree-sitter.\n\n See also [`ts_parser_set_language`]."] + pub fn ts_language_version(self_: *const TSLanguage) -> u32; +} extern "C" { #[doc = " Get the ABI version number for this language. This version number is used\n to ensure that languages were generated by a compatible version of\n Tree-sitter.\n\n See also [`ts_parser_set_language`]."] pub fn ts_language_abi_version(self_: *const TSLanguage) -> u32; @@ -924,7 +928,7 @@ extern "C" { ) -> *const TSLanguage; } extern "C" { - #[doc = " Get the number of languages instantiated in the given Wasm store."] + #[doc = " Get the number of languages instantiated in the given wasm store."] pub fn ts_wasm_store_language_count(arg1: *const TSWasmStore) -> usize; } extern "C" { diff --git a/lib/binding_rust/build.rs b/lib/binding_rust/build.rs index 57c5bc94..5e918034 100644 --- a/lib/binding_rust/build.rs +++ b/lib/binding_rust/build.rs @@ -2,7 +2,6 @@ use std::{env, fs, path::PathBuf}; fn main() { let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); - let target = env::var("TARGET").unwrap(); #[cfg(feature = "bindgen")] generate_bindings(&out_dir); @@ -27,11 +26,6 @@ fn main() { let include_path = manifest_path.join("include"); let src_path = manifest_path.join("src"); let wasm_path = src_path.join("wasm"); - - if target.starts_with("wasm32-unknown") { - configure_wasm_build(&mut config); - } - for entry in fs::read_dir(&src_path).unwrap() { let entry = entry.unwrap(); let path = src_path.join(entry.file_name()); @@ -49,8 +43,6 @@ fn main() { .include(&include_path) .define("_POSIX_C_SOURCE", "200112L") .define("_DEFAULT_SOURCE", None) - .define("_BSD_SOURCE", None) - .define("_DARWIN_C_SOURCE", None) .warnings(false) .file(src_path.join("lib.c")) .compile("tree-sitter"); @@ -58,28 +50,35 @@ fn main() { println!("cargo:include={}", include_path.display()); } -fn configure_wasm_build(config: &mut cc::Build) { - let Ok(wasm_headers) = env::var("DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS") else { - panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS must be set by the language crate"); - }; - let Ok(wasm_src) = env::var("DEP_TREE_SITTER_LANGUAGE_WASM_SRC").map(PathBuf::from) else { - panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_SRC must be set by the language crate"); - }; - - config.include(&wasm_headers); - config.files([ - wasm_src.join("stdio.c"), - wasm_src.join("stdlib.c"), - wasm_src.join("string.c"), - ]); -} - #[cfg(feature = "bindgen")] fn generate_bindings(out_dir: &std::path::Path) { - use std::str::FromStr; + use std::{process::Command, str::FromStr}; use bindgen::RustTarget; + let output = Command::new("cargo") + .args(["metadata", "--format-version", "1"]) + .output() + .unwrap(); + + let metadata = serde_json::from_slice::(&output.stdout).unwrap(); + + let Some(rust_version) = metadata + .get("packages") + .and_then(|packages| packages.as_array()) + .and_then(|packages| { + packages.iter().find_map(|package| { + if package["name"] == "tree-sitter" { + package.get("rust_version").and_then(|v| v.as_str()) + } else { + None + } + }) + }) + else { + panic!("Failed to find tree-sitter package in cargo metadata"); + }; + const HEADER_PATH: &str = "include/tree_sitter/api.h"; println!("cargo:rerun-if-changed={HEADER_PATH}"); @@ -98,8 +97,6 @@ fn generate_bindings(out_dir: &std::path::Path) { "TSQueryPredicateStep", ]; - let rust_version = env!("CARGO_PKG_RUST_VERSION"); - let bindings = bindgen::Builder::default() .header(HEADER_PATH) .layout_tests(false) diff --git a/lib/binding_rust/lib.rs b/lib/binding_rust/lib.rs index 51fd7f25..3574391f 100644 --- a/lib/binding_rust/lib.rs +++ b/lib/binding_rust/lib.rs @@ -1,4 +1,4 @@ -#![cfg_attr(not(any(test, doctest)), doc = include_str!("./README.md"))] +#![doc = include_str!("./README.md")] #![cfg_attr(not(feature = "std"), no_std)] #![cfg_attr(docsrs, feature(doc_cfg))] @@ -16,9 +16,10 @@ use core::{ marker::PhantomData, mem::MaybeUninit, num::NonZeroU16, - ops::{self, ControlFlow, Deref}, + ops::{self, Deref}, ptr::{self, NonNull}, slice, str, + sync::atomic::AtomicUsize, }; #[cfg(feature = "std")] use std::error; @@ -120,48 +121,6 @@ pub struct InputEdit { pub new_end_position: Point, } -impl InputEdit { - /// Edit a point to keep it in-sync with source code that has been edited. - /// - /// This function updates a single point's byte offset and row/column position - /// based on this edit operation. This is useful for editing points without - /// requiring a tree or node instance. - #[doc(alias = "ts_point_edit")] - pub fn edit_point(&self, point: &mut Point, byte: &mut usize) { - let edit = self.into(); - let mut ts_point = (*point).into(); - let mut ts_byte = *byte as u32; - - unsafe { - ffi::ts_point_edit( - core::ptr::addr_of_mut!(ts_point), - core::ptr::addr_of_mut!(ts_byte), - &edit, - ); - } - - *point = ts_point.into(); - *byte = ts_byte as usize; - } - - /// Edit a range to keep it in-sync with source code that has been edited. - /// - /// This function updates a range's start and end positions based on this edit - /// operation. This is useful for editing ranges without requiring a tree - /// or node instance. - #[doc(alias = "ts_range_edit")] - pub fn edit_range(&self, range: &mut Range) { - let edit = self.into(); - let mut ts_range = (*range).into(); - - unsafe { - ffi::ts_range_edit(core::ptr::addr_of_mut!(ts_range), &edit); - } - - *range = ts_range.into(); - } -} - /// A single node within a syntax [`Tree`]. #[doc(alias = "TSNode")] #[derive(Clone, Copy)] @@ -218,27 +177,10 @@ impl<'a> ParseOptions<'a> { } #[must_use] - pub fn progress_callback ControlFlow<()>>( - mut self, - callback: &'a mut F, - ) -> Self { + pub fn progress_callback bool>(mut self, callback: &'a mut F) -> Self { self.progress_callback = Some(callback); self } - - /// Create a new `ParseOptions` with a shorter lifetime, borrowing from this one. - /// - /// This is useful when you need to reuse parse options multiple times, e.g., calling - /// [`Parser::parse_with_options`] multiple times with the same options. - #[must_use] - pub fn reborrow(&mut self) -> ParseOptions { - ParseOptions { - progress_callback: match &mut self.progress_callback { - Some(cb) => Some(*cb), - None => None, - }, - } - } } #[derive(Default)] @@ -253,27 +195,13 @@ impl<'a> QueryCursorOptions<'a> { } #[must_use] - pub fn progress_callback ControlFlow<()>>( + pub fn progress_callback bool>( mut self, callback: &'a mut F, ) -> Self { self.progress_callback = Some(callback); self } - - /// Create a new `QueryCursorOptions` with a shorter lifetime, borrowing from this one. - /// - /// This is useful when you need to reuse query cursor options multiple times, e.g., calling - /// [`QueryCursor::matches`] multiple times with the same options. - #[must_use] - pub fn reborrow(&mut self) -> QueryCursorOptions { - QueryCursorOptions { - progress_callback: match &mut self.progress_callback { - Some(cb) => Some(*cb), - None => None, - }, - } - } } struct QueryCursorOptionsDrop(*mut ffi::TSQueryCursorOptions); @@ -304,10 +232,10 @@ type FieldId = NonZeroU16; type Logger<'a> = Box; /// A callback that receives the parse state during parsing. -type ParseProgressCallback<'a> = &'a mut dyn FnMut(&ParseState) -> ControlFlow<()>; +type ParseProgressCallback<'a> = &'a mut dyn FnMut(&ParseState) -> bool; /// A callback that receives the query state during query execution. -type QueryProgressCallback<'a> = &'a mut dyn FnMut(&QueryCursorState) -> ControlFlow<()>; +type QueryProgressCallback<'a> = &'a mut dyn FnMut(&QueryCursorState) -> bool; pub trait Decode { /// A callback that decodes the next code point from the input slice. It should return the code @@ -317,7 +245,7 @@ pub trait Decode { /// A stateful object for walking a syntax [`Tree`] efficiently. #[doc(alias = "TSTreeCursor")] -pub struct TreeCursor<'tree>(ffi::TSTreeCursor, PhantomData<&'tree ()>); +pub struct TreeCursor<'cursor>(ffi::TSTreeCursor, PhantomData<&'cursor ()>); /// A set of patterns that match nodes in a syntax tree. #[doc(alias = "TSQuery")] @@ -351,7 +279,7 @@ impl From for CaptureQuantifier { ffi::TSQuantifierZeroOrMore => Self::ZeroOrMore, ffi::TSQuantifierOne => Self::One, ffi::TSQuantifierOneOrMore => Self::OneOrMore, - _ => unreachable!(), + _ => panic!("Unrecognized quantifier: {value}"), } } } @@ -392,7 +320,7 @@ pub struct QueryMatch<'cursor, 'tree> { } /// A sequence of [`QueryMatch`]es associated with a given [`QueryCursor`]. -pub struct QueryMatches<'query, 'tree, T: TextProvider, I: AsRef<[u8]>> { +pub struct QueryMatches<'query, 'tree: 'query, T: TextProvider, I: AsRef<[u8]>> { ptr: *mut ffi::TSQueryCursor, query: &'query Query, text_provider: T, @@ -407,7 +335,7 @@ pub struct QueryMatches<'query, 'tree, T: TextProvider, I: AsRef<[u8]>> { /// /// During iteration, each element contains a [`QueryMatch`] and index. The index can /// be used to access the new capture inside of the [`QueryMatch::captures`]'s [`captures`]. -pub struct QueryCaptures<'query, 'tree, T: TextProvider, I: AsRef<[u8]>> { +pub struct QueryCaptures<'query, 'tree: 'query, T: TextProvider, I: AsRef<[u8]>> { ptr: *mut ffi::TSQueryCursor, query: &'query Query, text_provider: T, @@ -436,13 +364,10 @@ pub struct QueryCapture<'tree> { } /// An error that occurred when trying to assign an incompatible [`Language`] to -/// a [`Parser`]. If the `wasm` feature is enabled, this can also indicate a failure -/// to load the Wasm store. +/// a [`Parser`]. #[derive(Debug, PartialEq, Eq)] -pub enum LanguageError { - Version(usize), - #[cfg(feature = "wasm")] - Wasm, +pub struct LanguageError { + version: usize, } /// An error that occurred in [`Parser::set_included_ranges`]. @@ -504,6 +429,15 @@ impl Language { (!ptr.is_null()).then(|| unsafe { CStr::from_ptr(ptr) }.to_str().unwrap()) } + /// Get the ABI version number that indicates which version of the + /// Tree-sitter CLI that was used to generate this [`Language`]. + #[doc(alias = "ts_language_version")] + #[deprecated(since = "0.25.0", note = "Use abi_version instead")] + #[must_use] + pub fn version(&self) -> usize { + unsafe { ffi::ts_language_version(self.0) as usize } + } + /// Get the ABI version number that indicates which version of the /// Tree-sitter CLI that was used to generate this [`Language`]. #[doc(alias = "ts_language_abi_version")] @@ -647,7 +581,7 @@ impl Language { /// generate completion suggestions or valid symbols in error nodes. /// /// Example: - /// ```ignore + /// ``` /// let state = language.next_state(node.parse_state(), node.grammar_id()); /// ``` #[doc(alias = "ts_language_next_state")] @@ -732,15 +666,12 @@ impl Parser { pub fn set_language(&mut self, language: &Language) -> Result<(), LanguageError> { let version = language.abi_version(); if (MIN_COMPATIBLE_LANGUAGE_VERSION..=LANGUAGE_VERSION).contains(&version) { - #[allow(unused_variables)] - let success = unsafe { ffi::ts_parser_set_language(self.0.as_ptr(), language.0) }; - #[cfg(feature = "wasm")] - if !success { - return Err(LanguageError::Wasm); + unsafe { + ffi::ts_parser_set_language(self.0.as_ptr(), language.0); } Ok(()) } else { - Err(LanguageError::Version(version)) + Err(LanguageError { version }) } } @@ -768,7 +699,8 @@ impl Parser { drop(unsafe { Box::from_raw(prev_logger.payload.cast::()) }); } - let c_logger = if let Some(logger) = logger { + let c_logger; + if let Some(logger) = logger { let container = Box::new(logger); unsafe extern "C" fn log( @@ -789,16 +721,16 @@ impl Parser { let raw_container = Box::into_raw(container); - ffi::TSLogger { + c_logger = ffi::TSLogger { payload: raw_container.cast::(), log: Some(log), - } + }; } else { - ffi::TSLogger { + c_logger = ffi::TSLogger { payload: ptr::null_mut(), log: None, - } - }; + }; + } unsafe { ffi::ts_parser_set_logger(self.0.as_ptr(), c_logger) }; } @@ -852,6 +784,8 @@ impl Parser { /// /// Returns a [`Tree`] if parsing succeeded, or `None` if: /// * The parser has not yet had a language assigned with [`Parser::set_language`] + /// * The timeout set with [`Parser::set_timeout_micros`] expired (deprecated) + /// * The cancellation flag set with [`Parser::set_cancellation_flag`] was flipped (deprecated) #[doc(alias = "ts_parser_parse")] pub fn parse(&mut self, text: impl AsRef<[u8]>, old_tree: Option<&Tree>) -> Option { let bytes = text.as_ref(); @@ -863,6 +797,47 @@ impl Parser { ) } + /// Parse a slice of UTF16 text. + /// + /// # Arguments: + /// * `text` The UTF16-encoded text to parse. + /// * `old_tree` A previous syntax tree parsed from the same document. If the text of the + /// document has changed since `old_tree` was created, then you must edit `old_tree` to match + /// the new text using [`Tree::edit`]. + #[deprecated(since = "0.25.0", note = "Prefer parse_utf16_le instead")] + pub fn parse_utf16( + &mut self, + input: impl AsRef<[u16]>, + old_tree: Option<&Tree>, + ) -> Option { + let code_points = input.as_ref(); + let len = code_points.len(); + self.parse_utf16_le_with_options( + &mut |i, _| (i < len).then(|| &code_points[i..]).unwrap_or_default(), + old_tree, + None, + ) + } + + /// Parse UTF8 text provided in chunks by a callback. + /// + /// # Arguments: + /// * `callback` A function that takes a byte offset and position and returns a slice of + /// UTF8-encoded text starting at that byte offset and position. The slices can be of any + /// length. If the given position is at the end of the text, the callback should return an + /// empty slice. + /// * `old_tree` A previous syntax tree parsed from the same document. If the text of the + /// document has changed since `old_tree` was created, then you must edit `old_tree` to match + /// the new text using [`Tree::edit`]. + #[deprecated(since = "0.25.0", note = "Prefer `parse_with_options` instead")] + pub fn parse_with, F: FnMut(usize, Point) -> T>( + &mut self, + callback: &mut F, + old_tree: Option<&Tree>, + ) -> Option { + self.parse_with_options(callback, old_tree, None) + } + /// Parse text provided in chunks by a callback. /// /// # Arguments: @@ -889,10 +864,7 @@ impl Parser { .cast::() .as_mut() .unwrap(); - match callback(&ParseState::from_raw(state)) { - ControlFlow::Continue(()) => false, - ControlFlow::Break(()) => true, - } + callback(&ParseState::from_raw(state)) } // This C function is passed to Tree-sitter as the input callback. @@ -956,6 +928,28 @@ impl Parser { } } + /// Parse UTF16 text provided in chunks by a callback. + /// + /// # Arguments: + /// * `callback` A function that takes a code point offset and position and returns a slice of + /// UTF16-encoded text starting at that byte offset and position. The slices can be of any + /// length. If the given position is at the end of the text, the callback should return an + /// empty slice. + /// * `old_tree` A previous syntax tree parsed from the same document. If the text of the + /// document has changed since `old_tree` was created, then you must edit `old_tree` to match + /// the new text using [`Tree::edit`]. + #[deprecated( + since = "0.25.0", + note = "Prefer `parse_utf16_le_with_options` instead" + )] + pub fn parse_utf16_with, F: FnMut(usize, Point) -> T>( + &mut self, + callback: &mut F, + old_tree: Option<&Tree>, + ) -> Option { + self.parse_utf16_le_with_options(callback, old_tree, None) + } + /// Parse a slice of UTF16 little-endian text. /// /// # Arguments: @@ -1002,10 +996,7 @@ impl Parser { .cast::() .as_mut() .unwrap(); - match callback(&ParseState::from_raw(state)) { - ControlFlow::Continue(()) => false, - ControlFlow::Break(()) => true, - } + callback(&ParseState::from_raw(state)) } // This C function is passed to Tree-sitter as the input callback. @@ -1122,10 +1113,7 @@ impl Parser { .cast::() .as_mut() .unwrap(); - match callback(&ParseState::from_raw(state)) { - ControlFlow::Continue(()) => false, - ControlFlow::Break(()) => true, - } + callback(&ParseState::from_raw(state)) } // This C function is passed to Tree-sitter as the input callback. @@ -1225,10 +1213,7 @@ impl Parser { .cast::() .as_mut() .unwrap(); - match callback(&ParseState::from_raw(state)) { - ControlFlow::Continue(()) => false, - ControlFlow::Break(()) => true, - } + callback(&ParseState::from_raw(state)) } // At compile time, create a C-compatible callback that calls the custom `decode` method. @@ -1308,15 +1293,43 @@ impl Parser { /// Instruct the parser to start the next parse from the beginning. /// - /// If the parser previously failed because of a callback, then by default, - /// it will resume where it left off on the next call to [`parse`](Parser::parse) - /// or other parsing functions. If you don't want to resume, and instead intend to use - /// this parser to parse some other document, you must call `reset` first. + /// If the parser previously failed because of a timeout, cancellation, + /// or callback, then by default, it will resume where it left off on the + /// next call to [`parse`](Parser::parse) or other parsing functions. + /// If you don't want to resume, and instead intend to use this parser to + /// parse some other document, you must call `reset` first. #[doc(alias = "ts_parser_reset")] pub fn reset(&mut self) { unsafe { ffi::ts_parser_reset(self.0.as_ptr()) } } + /// Get the duration in microseconds that parsing is allowed to take. + /// + /// This is set via [`set_timeout_micros`](Parser::set_timeout_micros). + #[doc(alias = "ts_parser_timeout_micros")] + #[deprecated( + since = "0.25.0", + note = "Prefer using `parse_with_options` and using a callback" + )] + #[must_use] + pub fn timeout_micros(&self) -> u64 { + unsafe { ffi::ts_parser_timeout_micros(self.0.as_ptr()) } + } + + /// Set the maximum duration in microseconds that parsing should be allowed + /// to take before halting. + /// + /// If parsing takes longer than this, it will halt early, returning `None`. + /// See [`parse`](Parser::parse) for more information. + #[doc(alias = "ts_parser_set_timeout_micros")] + #[deprecated( + since = "0.25.0", + note = "Prefer using `parse_with_options` and using a callback" + )] + pub fn set_timeout_micros(&mut self, timeout_micros: u64) { + unsafe { ffi::ts_parser_set_timeout_micros(self.0.as_ptr(), timeout_micros) } + } + /// Set the ranges of text that the parser should include when parsing. /// /// By default, the parser will always include entire documents. This @@ -1372,6 +1385,49 @@ impl Parser { result } } + + /// Get the parser's current cancellation flag pointer. + /// + /// # Safety + /// + /// It uses FFI + #[doc(alias = "ts_parser_cancellation_flag")] + #[deprecated( + since = "0.25.0", + note = "Prefer using `parse_with_options` and using a callback" + )] + #[must_use] + pub unsafe fn cancellation_flag(&self) -> Option<&AtomicUsize> { + ffi::ts_parser_cancellation_flag(self.0.as_ptr()) + .cast::() + .as_ref() + } + + /// Set the parser's current cancellation flag pointer. + /// + /// If a pointer is assigned, then the parser will periodically read from + /// this pointer during parsing. If it reads a non-zero value, it will halt + /// early, returning `None`. See [`parse`](Parser::parse) for more + /// information. + /// + /// # Safety + /// + /// It uses FFI + #[doc(alias = "ts_parser_set_cancellation_flag")] + #[deprecated( + since = "0.25.0", + note = "Prefer using `parse_with_options` and using a callback" + )] + pub unsafe fn set_cancellation_flag(&mut self, flag: Option<&AtomicUsize>) { + if let Some(flag) = flag { + ffi::ts_parser_set_cancellation_flag( + self.0.as_ptr(), + core::ptr::from_ref::(flag).cast::(), + ); + } else { + ffi::ts_parser_set_cancellation_flag(self.0.as_ptr(), ptr::null()); + } + } } impl Drop for Parser { @@ -1386,11 +1442,6 @@ impl Drop for Parser { } } -#[cfg(windows)] -extern "C" { - fn _open_osfhandle(osfhandle: isize, flags: core::ffi::c_int) -> core::ffi::c_int; -} - impl Tree { /// Get the root node of the syntax tree. #[doc(alias = "ts_tree_root_node")] @@ -1500,8 +1551,7 @@ impl Tree { #[cfg(windows)] { let handle = file.as_raw_handle(); - let fd = unsafe { _open_osfhandle(handle as isize, 0) }; - unsafe { ffi::ts_tree_print_dot_graph(self.0.as_ptr(), fd) } + unsafe { ffi::ts_tree_print_dot_graph(self.0.as_ptr(), handle as i32) } } } } @@ -1581,7 +1631,7 @@ impl<'tree> Node<'tree> { /// Get the [`Language`] that was used to parse this node's syntax tree. #[doc(alias = "ts_node_language")] #[must_use] - pub fn language(&self) -> LanguageRef<'tree> { + pub fn language(&self) -> LanguageRef { LanguageRef(unsafe { ffi::ts_node_language(self.0) }, PhantomData) } @@ -1710,8 +1760,8 @@ impl<'tree> Node<'tree> { /// [`Node::children`] instead. #[doc(alias = "ts_node_child")] #[must_use] - pub fn child(&self, i: u32) -> Option { - Self::new(unsafe { ffi::ts_node_child(self.0, i) }) + pub fn child(&self, i: usize) -> Option { + Self::new(unsafe { ffi::ts_node_child(self.0, i as u32) }) } /// Get this node's number of children. @@ -1729,8 +1779,8 @@ impl<'tree> Node<'tree> { /// [`Node::named_children`] instead. #[doc(alias = "ts_node_named_child")] #[must_use] - pub fn named_child(&self, i: u32) -> Option { - Self::new(unsafe { ffi::ts_node_named_child(self.0, i) }) + pub fn named_child(&self, i: usize) -> Option { + Self::new(unsafe { ffi::ts_node_named_child(self.0, i as u32) }) } /// Get this node's number of *named* children. @@ -2082,11 +2132,11 @@ impl fmt::Display for Node<'_> { } } -impl<'tree> TreeCursor<'tree> { +impl<'cursor> TreeCursor<'cursor> { /// Get the tree cursor's current [`Node`]. #[doc(alias = "ts_tree_cursor_current_node")] #[must_use] - pub fn node(&self) -> Node<'tree> { + pub fn node(&self) -> Node<'cursor> { Node( unsafe { ffi::ts_tree_cursor_current_node(&self.0) }, PhantomData, @@ -2227,7 +2277,7 @@ impl<'tree> TreeCursor<'tree> { /// Re-initialize this tree cursor to start at the original node that the /// cursor was constructed with. #[doc(alias = "ts_tree_cursor_reset")] - pub fn reset(&mut self, node: Node<'tree>) { + pub fn reset(&mut self, node: Node<'cursor>) { unsafe { ffi::ts_tree_cursor_reset(&mut self.0, node.0) }; } @@ -2343,16 +2393,6 @@ impl Query { /// on syntax nodes parsed with that language. References to Queries can be /// shared between multiple threads. pub fn new(language: &Language, source: &str) -> Result { - let ptr = Self::new_raw(language, source)?; - unsafe { Self::from_raw_parts(ptr, source) } - } - - /// Constructs a raw [`TSQuery`](ffi::TSQuery) pointer without performing extra checks specific to the rust - /// bindings, such as predicate validation. A [`Query`] object can be constructed from the - /// returned pointer using [`from_raw_parts`](Query::from_raw_parts). The caller is - /// responsible for ensuring that the returned pointer is eventually freed by calling - /// [`ts_query_delete`](ffi::ts_query_delete). - pub fn new_raw(language: &Language, source: &str) -> Result<*mut ffi::TSQuery, QueryError> { let mut error_offset = 0u32; let mut error_type: ffi::TSQueryError = 0; let bytes = source.as_bytes(); @@ -2368,90 +2408,93 @@ impl Query { ) }; - if !ptr.is_null() { - return Ok(ptr); - } - // On failure, build an error based on the error code and offset. - if error_type == ffi::TSQueryErrorLanguage { + if ptr.is_null() { + if error_type == ffi::TSQueryErrorLanguage { + return Err(QueryError { + row: 0, + column: 0, + offset: 0, + message: LanguageError { + version: language.abi_version(), + } + .to_string(), + kind: QueryErrorKind::Language, + }); + } + + let offset = error_offset as usize; + let mut line_start = 0; + let mut row = 0; + let mut line_containing_error = None; + for line in source.lines() { + let line_end = line_start + line.len() + 1; + if line_end > offset { + line_containing_error = Some(line); + break; + } + line_start = line_end; + row += 1; + } + let column = offset - line_start; + + let kind; + let message; + match error_type { + // Error types that report names + ffi::TSQueryErrorNodeType | ffi::TSQueryErrorField | ffi::TSQueryErrorCapture => { + let suffix = source.split_at(offset).1; + let in_quotes = offset > 0 && source.as_bytes()[offset - 1] == b'"'; + let mut backslashes = 0; + let end_offset = suffix + .find(|c| { + if in_quotes { + if c == '"' && backslashes % 2 == 0 { + true + } else if c == '\\' { + backslashes += 1; + false + } else { + backslashes = 0; + false + } + } else { + !char::is_alphanumeric(c) && c != '_' && c != '-' + } + }) + .unwrap_or(suffix.len()); + message = suffix.split_at(end_offset).0.to_string(); + kind = match error_type { + ffi::TSQueryErrorNodeType => QueryErrorKind::NodeType, + ffi::TSQueryErrorField => QueryErrorKind::Field, + ffi::TSQueryErrorCapture => QueryErrorKind::Capture, + _ => unreachable!(), + }; + } + + // Error types that report positions + _ => { + message = line_containing_error.map_or_else( + || "Unexpected EOF".to_string(), + |line| line.to_string() + "\n" + &" ".repeat(offset - line_start) + "^", + ); + kind = match error_type { + ffi::TSQueryErrorStructure => QueryErrorKind::Structure, + _ => QueryErrorKind::Syntax, + }; + } + } + return Err(QueryError { - row: 0, - column: 0, - offset: 0, - message: LanguageError::Version(language.abi_version()).to_string(), - kind: QueryErrorKind::Language, + row, + column, + offset, + message, + kind, }); } - let offset = error_offset as usize; - let mut line_start = 0; - let mut row = 0; - let mut line_containing_error = None; - for line in source.lines() { - let line_end = line_start + line.len() + 1; - if line_end > offset { - line_containing_error = Some(line); - break; - } - line_start = line_end; - row += 1; - } - let column = offset - line_start; - - let kind; - let message; - match error_type { - // Error types that report names - ffi::TSQueryErrorNodeType | ffi::TSQueryErrorField | ffi::TSQueryErrorCapture => { - let suffix = source.split_at(offset).1; - let in_quotes = offset > 0 && source.as_bytes()[offset - 1] == b'"'; - let mut backslashes = 0; - let end_offset = suffix - .find(|c| { - if in_quotes { - if c == '"' && backslashes % 2 == 0 { - true - } else if c == '\\' { - backslashes += 1; - false - } else { - backslashes = 0; - false - } - } else { - !char::is_alphanumeric(c) && c != '_' && c != '-' - } - }) - .unwrap_or(suffix.len()); - message = format!("\"{}\"", suffix.split_at(end_offset).0); - kind = match error_type { - ffi::TSQueryErrorNodeType => QueryErrorKind::NodeType, - ffi::TSQueryErrorField => QueryErrorKind::Field, - ffi::TSQueryErrorCapture => QueryErrorKind::Capture, - _ => unreachable!(), - }; - } - - // Error types that report positions - _ => { - message = line_containing_error.map_or_else( - || "Unexpected EOF".to_string(), - |line| line.to_string() + "\n" + &" ".repeat(offset - line_start) + "^", - ); - kind = match error_type { - ffi::TSQueryErrorStructure => QueryErrorKind::Structure, - _ => QueryErrorKind::Syntax, - }; - } - } - - Err(QueryError { - row, - column, - offset, - message, - kind, - }) + unsafe { Self::from_raw_parts(ptr, source) } } #[doc(hidden)] @@ -2965,6 +3008,34 @@ impl QueryCursor { } } + /// Set the maximum duration in microseconds that query execution should be allowed to + /// take before halting. + /// + /// If query execution takes longer than this, it will halt early, returning None. + #[doc(alias = "ts_query_cursor_set_timeout_micros")] + #[deprecated( + since = "0.25.0", + note = "Prefer using `matches_with_options` or `captures_with_options` and using a callback" + )] + pub fn set_timeout_micros(&mut self, timeout: u64) { + unsafe { + ffi::ts_query_cursor_set_timeout_micros(self.ptr.as_ptr(), timeout); + } + } + + /// Get the duration in microseconds that query execution is allowed to take. + /// + /// This is set via [`set_timeout_micros`](QueryCursor::set_timeout_micros). + #[doc(alias = "ts_query_cursor_timeout_micros")] + #[deprecated( + since = "0.25.0", + note = "Prefer using `matches_with_options` or `captures_with_options` and using a callback" + )] + #[must_use] + pub fn timeout_micros(&self) -> u64 { + unsafe { ffi::ts_query_cursor_timeout_micros(self.ptr.as_ptr()) } + } + /// Check if, on its last execution, this cursor exceeded its maximum number /// of in-progress matches. #[doc(alias = "ts_query_cursor_did_exceed_match_limit")] @@ -3030,10 +3101,7 @@ impl QueryCursor { .cast::() .as_mut() .unwrap(); - match callback(&QueryCursorState::from_raw(state)) { - ControlFlow::Continue(()) => false, - ControlFlow::Break(()) => true, - } + (callback)(&QueryCursorState::from_raw(state)) } let query_options = options.progress_callback.map(|cb| { @@ -3119,10 +3187,7 @@ impl QueryCursor { .cast::() .as_mut() .unwrap(); - match callback(&QueryCursorState::from_raw(state)) { - ControlFlow::Continue(()) => false, - ControlFlow::Break(()) => true, - } + (callback)(&QueryCursorState::from_raw(state)) } let query_options = options.progress_callback.map(|cb| { @@ -3181,44 +3246,6 @@ impl QueryCursor { self } - /// Set the byte range within which all matches must be fully contained. - /// - /// Set the range of bytes in which matches will be searched for. In contrast to - /// `ts_query_cursor_set_byte_range`, this will restrict the query cursor to only return - /// matches where _all_ nodes are _fully_ contained within the given range. Both functions - /// can be used together, e.g. to search for any matches that intersect line 5000, as - /// long as they are fully contained within lines 4500-5500 - #[doc(alias = "ts_query_cursor_set_containing_byte_range")] - pub fn set_containing_byte_range(&mut self, range: ops::Range) -> &mut Self { - unsafe { - ffi::ts_query_cursor_set_containing_byte_range( - self.ptr.as_ptr(), - range.start as u32, - range.end as u32, - ); - } - self - } - - /// Set the point range within which all matches must be fully contained. - /// - /// Set the range of bytes in which matches will be searched for. In contrast to - /// `ts_query_cursor_set_point_range`, this will restrict the query cursor to only return - /// matches where _all_ nodes are _fully_ contained within the given range. Both functions - /// can be used together, e.g. to search for any matches that intersect line 5000, as - /// long as they are fully contained within lines 4500-5500 - #[doc(alias = "ts_query_cursor_set_containing_point_range")] - pub fn set_containing_point_range(&mut self, range: ops::Range) -> &mut Self { - unsafe { - ffi::ts_query_cursor_set_containing_point_range( - self.ptr.as_ptr(), - range.start.into(), - range.end.into(), - ); - } - self - } - /// Set the maximum start depth for a query cursor. /// /// This prevents cursors from exploring children nodes at a certain depth. @@ -3228,9 +3255,9 @@ impl QueryCursor { /// The zero max start depth value can be used as a special behavior and /// it helps to destructure a subtree by staying on a node and using /// captures for interested parts. Note that the zero max start depth - /// only limits a search depth for a pattern's root node but other nodes - /// that are parts of the pattern may be searched at any depth depending on - /// what is defined by the pattern structure. + /// only limit a search depth for a pattern's root node but other nodes + /// that are parts of the pattern may be searched at any depth what + /// defined by the pattern structure. /// /// Set to `None` to remove the maximum start depth. #[doc(alias = "ts_query_cursor_set_max_start_depth")] @@ -3404,7 +3431,7 @@ impl QueryProperty { /// Provide a `StreamingIterator` instead of the traditional `Iterator`, as the /// underlying object in the C library gets updated on each iteration. Copies would /// have their internal state overwritten, leading to Undefined Behavior -impl<'query, 'tree, T: TextProvider, I: AsRef<[u8]>> StreamingIterator +impl<'query, 'tree: 'query, T: TextProvider, I: AsRef<[u8]>> StreamingIterator for QueryMatches<'query, 'tree, T, I> { type Item = QueryMatch<'query, 'tree>; @@ -3435,13 +3462,15 @@ impl<'query, 'tree, T: TextProvider, I: AsRef<[u8]>> StreamingIterator } } -impl, I: AsRef<[u8]>> StreamingIteratorMut for QueryMatches<'_, '_, T, I> { +impl<'query, 'tree: 'query, T: TextProvider, I: AsRef<[u8]>> StreamingIteratorMut + for QueryMatches<'query, 'tree, T, I> +{ fn get_mut(&mut self) -> Option<&mut Self::Item> { self.current_match.as_mut() } } -impl<'query, 'tree, T: TextProvider, I: AsRef<[u8]>> StreamingIterator +impl<'query, 'tree: 'query, T: TextProvider, I: AsRef<[u8]>> StreamingIterator for QueryCaptures<'query, 'tree, T, I> { type Item = (QueryMatch<'query, 'tree>, usize); @@ -3478,7 +3507,9 @@ impl<'query, 'tree, T: TextProvider, I: AsRef<[u8]>> StreamingIterator } } -impl, I: AsRef<[u8]>> StreamingIteratorMut for QueryCaptures<'_, '_, T, I> { +impl<'query, 'tree: 'query, T: TextProvider, I: AsRef<[u8]>> StreamingIteratorMut + for QueryCaptures<'query, 'tree, T, I> +{ fn get_mut(&mut self) -> Option<&mut Self::Item> { self.current_match.as_mut() } @@ -3618,8 +3649,8 @@ impl From for Range { } } -impl From<&InputEdit> for ffi::TSInputEdit { - fn from(val: &InputEdit) -> Self { +impl From<&'_ InputEdit> for ffi::TSInputEdit { + fn from(val: &'_ InputEdit) -> Self { Self { start_byte: val.start_byte as u32, old_end_byte: val.old_end_byte as u32, @@ -3697,18 +3728,11 @@ impl fmt::Display for IncludedRangesError { impl fmt::Display for LanguageError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - Self::Version(version) => { - write!( - f, - "Incompatible language version {version}. Expected minimum {MIN_COMPATIBLE_LANGUAGE_VERSION}, maximum {LANGUAGE_VERSION}", - ) - } - #[cfg(feature = "wasm")] - Self::Wasm => { - write!(f, "Failed to load the Wasm store.") - } - } + write!( + f, + "Incompatible language version {}. Expected minimum {}, maximum {}", + self.version, MIN_COMPATIBLE_LANGUAGE_VERSION, LANGUAGE_VERSION, + ) } } diff --git a/lib/binding_rust/wasm_language.rs b/lib/binding_rust/wasm_language.rs index 66df377a..76aff50a 100644 --- a/lib/binding_rust/wasm_language.rs +++ b/lib/binding_rust/wasm_language.rs @@ -135,9 +135,9 @@ impl Drop for WasmStore { impl fmt::Display for WasmError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let kind = match self.kind { - WasmErrorKind::Parse => "Failed to parse Wasm", - WasmErrorKind::Compile => "Failed to compile Wasm", - WasmErrorKind::Instantiate => "Failed to instantiate Wasm module", + WasmErrorKind::Parse => "Failed to parse wasm", + WasmErrorKind::Compile => "Failed to compile wasm", + WasmErrorKind::Instantiate => "Failed to instantiate wasm module", WasmErrorKind::Other => "Unknown error", }; write!(f, "{kind}: {}", self.message) diff --git a/lib/binding_web/.gitattributes b/lib/binding_web/.gitattributes index ab19847d..3fa60d26 100644 --- a/lib/binding_web/.gitattributes +++ b/lib/binding_web/.gitattributes @@ -1 +1 @@ -lib/web-tree-sitter.d.ts linguist-generated +/lib/tree-sitter.d.ts linguist-generated diff --git a/lib/binding_web/.gitignore b/lib/binding_web/.gitignore index 8f45e5f9..d7edb1e8 100644 --- a/lib/binding_web/.gitignore +++ b/lib/binding_web/.gitignore @@ -1,10 +1,11 @@ debug/ dist/ -web-tree-sitter* +tree-sitter* +lib/tree-sitter* lib/*.c lib/*.h !lib/tree-sitter.c -!lib/web-tree-sitter.d.ts +!lib/tree-sitter.d.ts node_modules *.tgz LICENSE diff --git a/lib/binding_web/CONTRIBUTING.md b/lib/binding_web/CONTRIBUTING.md index eb4e5fc3..7072abe0 100644 --- a/lib/binding_web/CONTRIBUTING.md +++ b/lib/binding_web/CONTRIBUTING.md @@ -13,7 +13,7 @@ To make changes to Web-tree-sitter, you should have: 1. A [Rust toolchain][rust], for running the xtasks necessary to build the library. 2. Node.js and NPM (or an equivalent package manager). 3. Either [Emscripten][emscripten], [Docker][docker], or [podman][podman] for -compiling the library to Wasm. +compiling the library to WASM. ### Building @@ -41,7 +41,7 @@ by visiting the [Rust website][rust] and following the instructions there. > [!NOTE] > By default, the build process will emit an ES6 module. If you need a CommonJS module, export `CJS` to `true`, or just -> run `CJS=true npm run build` (or the equivalent command for Windows). +> run `CJS=true npm run build`. > [!TIP] > To build the library with debug information, you can run `npm run build:debug`. The `CJS` environment variable is still @@ -51,19 +51,19 @@ by visiting the [Rust website][rust] and following the instructions there. #### The C side -There are several components that come together to build the final JS and Wasm files. First, we use `emscripten` in our -xtask located at `xtask/src/build_wasm.rs` from the root directory to compile the Wasm files. This Wasm module is output into the -local `lib` folder, and is used only in [`src/bindings.ts`][bindings.ts] to handle loading the Wasm module. The C code that -is compiled into the Wasm module is located in at [`lib/tree-sitter.c`][tree-sitter.c], and contains all the necessary +There are several components that come together to build the final JS and WASM files. First, we use `emscripten` in our +xtask located at `xtask/src/build_wasm.rs` from the root directory to compile the WASM files. This WASM module is output into the +local `lib` folder, and is used only in [`src/bindings.ts`][bindings.ts] to handle loading the WASM module. The C code that +is compiled into the WASM module is located in at [`lib/tree-sitter.c`][tree-sitter.c], and contains all the necessary glue code to interact with the JS environment. If you need to update the imported functions from the tree-sitter library, -or anywhere else, you must update [`lib/exports.txt`][exports.txt]. Lastly, the type information for the Wasm module is +or anywhere else, you must update [`lib/exports.txt`][exports.txt]. Lastly, the type information for the WASM module is located at [`lib/tree-sitter.d.ts`][tree-sitter.d.ts], and can be updated by running `cargo xtask build-wasm --emit-tsd` from the root directory. #### The TypeScript side -The TypeScript library is a higher level abstraction over the Wasm module, and is located in `src`. This is where the -public API is defined, and where the Wasm module is loaded and initialized. The TypeScript library is built into a single +The TypeScript library is a higher level abstraction over the WASM module, and is located in `src`. This is where the +public API is defined, and where the WASM module is loaded and initialized. The TypeScript library is built into a single ES6 (or CommonJS) module, and is output into the same directory as `package.json`. If you need to update the public API, you can do so by editing the files in `src`. @@ -80,7 +80,7 @@ to the TypeScript source code. This TypeScript code is then compiled into a single JavaScript file with `esbuild`. The build configuration for this can be found in [`script/build.js`][build.js], but this shouldn't need to be updated. This step is responsible for emitting -the final JS and Wasm files that are shipped with the library, as well as their sourcemaps. +the final JS and WASM files that are shipped with the library, as well as their sourcemaps. ### Testing @@ -97,7 +97,7 @@ Optionally, to update the generated parser.c files: cargo xtask generate-fixtures ``` -Then you can build the Wasm modules: +Then you can build the WASM modules: ```sh cargo xtask generate-fixtures --wasm @@ -120,7 +120,7 @@ npm test ### Debugging You might have noticed that when you ran `npm build`, the build process generated a couple of [sourcemaps][sourcemap]: -`web-tree-sitter.js.map` and `web-tree-sitter.wasm.map`. These sourcemaps can be used to debug the library in the browser, and are +`tree-sitter.js.map` and `tree-sitter.wasm.map`. These sourcemaps can be used to debug the library in the browser, and are shipped with the library on both NPM and the GitHub releases. #### Tweaking the Emscripten build diff --git a/lib/binding_web/README.md b/lib/binding_web/README.md index 08a939dc..3713df5e 100644 --- a/lib/binding_web/README.md +++ b/lib/binding_web/README.md @@ -9,11 +9,11 @@ WebAssembly bindings to the [Tree-sitter](https://github.com/tree-sitter/tree-si ## Setup -You can download the `web-tree-sitter.js` and `web-tree-sitter.wasm` files from [the latest GitHub release][gh release] and load +You can download the `tree-sitter.js` and `tree-sitter.wasm` files from [the latest GitHub release][gh release] and load them using a standalone script: ```html - +