diff --git a/.editorconfig b/.editorconfig index 53780b34..0b70460a 100644 --- a/.editorconfig +++ b/.editorconfig @@ -10,6 +10,9 @@ insert_final_newline = true [*.rs] indent_size = 4 +[*.{zig,zon}] +indent_size = 4 + [Makefile] indent_style = tab indent_size = 8 diff --git a/.envrc b/.envrc new file mode 100644 index 00000000..3550a30f --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake diff --git a/.gitattributes b/.gitattributes index 1d9b8cb4..acd15fd1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,5 +3,4 @@ /lib/src/unicode/*.h linguist-vendored /lib/src/unicode/LICENSE linguist-vendored -/cli/src/generate/prepare_grammar/*.json -diff Cargo.lock -diff diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..6be29e24 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,15 @@ +# These are supported funding model platforms + +github: tree-sitter +patreon: # Replace with a single Patreon username +open_collective: tree-sitter # Replace with a single Open Collective username +ko_fi: amaanq +tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry +polar: # Replace with a single Polar username +buy_me_a_coffee: # Replace with a single Buy Me a Coffee username +thanks_dev: # Replace with a single thanks.dev username +custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 6c8ad05d..88437c52 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -1,6 +1,6 @@ name: Bug Report description: Report a problem -labels: [bug] +type: Bug body: - type: textarea attributes: diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 388f3675..05b37e6f 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -1,6 +1,6 @@ name: Feature request description: Request an enhancement -labels: [enhancement] +type: Feature body: - type: markdown attributes: diff --git a/.github/actions/cache/action.yml b/.github/actions/cache/action.yml index a32cc294..f04dda31 100644 --- a/.github/actions/cache/action.yml +++ b/.github/actions/cache/action.yml @@ -17,7 +17,9 @@ runs: test/fixtures/grammars target/release/tree-sitter-*.wasm key: fixtures-${{ join(matrix.*, '_') }}-${{ hashFiles( - 'cli/generate/src/**', - 'xtask/src/*', + 'crates/generate/src/**', + 'lib/src/parser.h', + 'lib/src/array.h', + 'lib/src/alloc.h', 'test/fixtures/grammars/*/**/src/*.c', '.github/actions/cache/action.yml') }} diff --git a/.github/cliff.toml b/.github/cliff.toml index 95204113..5e0bc8e2 100644 --- a/.github/cliff.toml +++ b/.github/cliff.toml @@ -16,13 +16,13 @@ body = """ {% for commit in commits%}\ {% if not commit.scope %}\ - {{ commit.message | upper_first }}\ - {% if commit.github.pr_number %} (){%- endif %} + {% if commit.remote.pr_number %} (){%- endif %} {% endif %}\ {% endfor %}\ {% for group, commits in commits | group_by(attribute="scope") %}\ {% for commit in commits %}\ - **{{commit.scope}}**: {{ commit.message | upper_first }}\ - {% if commit.github.pr_number %} (){%- endif %} + {% if commit.remote.pr_number %} (){%- endif %} {% endfor %}\ {% endfor %} {% endfor %} diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ccba319b..c75a67e6 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,6 +4,8 @@ updates: directory: "/" schedule: interval: "weekly" + cooldown: + default-days: 3 commit-message: prefix: "build(deps)" labels: @@ -12,10 +14,16 @@ updates: groups: cargo: patterns: ["*"] + ignore: + - dependency-name: "*" + update-types: ["version-update:semver-major", "version-update:semver-minor"] + - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" + cooldown: + default-days: 3 commit-message: prefix: "ci" labels: @@ -24,3 +32,22 @@ updates: groups: actions: patterns: ["*"] + + - package-ecosystem: "npm" + versioning-strategy: increase + directories: + - "/crates/npm" + - "/crates/eslint" + - "/lib/binding_web" + schedule: + interval: "weekly" + cooldown: + default-days: 3 + commit-message: + prefix: "build(deps)" + labels: + - "dependencies" + - "npm" + groups: + npm: + patterns: ["*"] diff --git a/.github/scripts/close_spam.js b/.github/scripts/close_spam.js new file mode 100644 index 00000000..41046964 --- /dev/null +++ b/.github/scripts/close_spam.js @@ -0,0 +1,29 @@ +module.exports = async ({ github, context }) => { + let target = context.payload.issue; + if (target) { + await github.rest.issues.update({ + ...context.repo, + issue_number: target.number, + state: "closed", + state_reason: "not_planned", + title: "[spam]", + body: "", + type: null, + }); + } else { + target = context.payload.pull_request; + await github.rest.pulls.update({ + ...context.repo, + pull_number: target.number, + state: "closed", + title: "[spam]", + body: "", + }); + } + + await github.rest.issues.lock({ + ...context.repo, + issue_number: target.number, + lock_reason: "spam", + }); +}; diff --git a/.github/scripts/cross.sh b/.github/scripts/cross.sh deleted file mode 100755 index de1d4e94..00000000 --- a/.github/scripts/cross.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -eu - -exec docker run --rm -v /home/runner:/home/runner -w "$PWD" "$CROSS_IMAGE" "$@" diff --git a/.github/scripts/make.sh b/.github/scripts/make.sh deleted file mode 100755 index 175074f9..00000000 --- a/.github/scripts/make.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -eu - -tree_sitter="$ROOT"/target/"$TARGET"/release/tree-sitter - -if [[ $BUILD_CMD == cross ]]; then - cross.sh make CC="$CC" AR="$AR" "$@" -else - exec make "$@" -fi diff --git a/.github/scripts/tree-sitter.sh b/.github/scripts/tree-sitter.sh deleted file mode 100755 index 125a2d92..00000000 --- a/.github/scripts/tree-sitter.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -eu - -tree_sitter="$ROOT"/target/"$TARGET"/release/tree-sitter - -if [[ $BUILD_CMD == cross ]]; then - cross.sh "$CROSS_RUNNER" "$tree_sitter" "$@" -else - exec "$tree_sitter" "$@" -fi diff --git a/.github/scripts/wasm_stdlib.js b/.github/scripts/wasm_stdlib.js new file mode 100644 index 00000000..e1350094 --- /dev/null +++ b/.github/scripts/wasm_stdlib.js @@ -0,0 +1,25 @@ +module.exports = async ({ github, context, core }) => { + if (context.eventName !== 'pull_request') return; + + const prNumber = context.payload.pull_request.number; + const owner = context.repo.owner; + const repo = context.repo.repo; + + const { data: files } = await github.rest.pulls.listFiles({ + owner, + repo, + pull_number: prNumber + }); + + const changedFiles = files.map(file => file.filename); + + const wasmStdLibSrc = 'crates/language/wasm/'; + const dirChanged = changedFiles.some(file => file.startsWith(wasmStdLibSrc)); + + if (!dirChanged) return; + + const wasmStdLibHeader = 'lib/src/wasm/wasm-stdlib.h'; + const requiredChanged = changedFiles.includes(wasmStdLibHeader); + + if (!requiredChanged) core.setFailed(`Changes detected in ${wasmStdLibSrc} but ${wasmStdLibHeader} was not modified.`); +}; diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index a0c15e01..7caffa14 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -14,17 +14,17 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Create app token - uses: actions/create-github-app-token@v1 + uses: actions/create-github-app-token@v2 id: app-token with: app-id: ${{ vars.BACKPORT_APP }} private-key: ${{ secrets.BACKPORT_KEY }} - name: Create backport PR - uses: korthout/backport-action@v3 + uses: korthout/backport-action@v4 with: pull_title: "${pull_title}" label_pattern: "^ci:backport ([^ ]+)$" diff --git a/.github/workflows/bindgen.yml b/.github/workflows/bindgen.yml index d2350b31..1b0d20af 100644 --- a/.github/workflows/bindgen.yml +++ b/.github/workflows/bindgen.yml @@ -2,15 +2,21 @@ name: Check Bindgen Output on: pull_request: + paths: + - lib/include/tree_sitter/api.h + - lib/binding_rust/bindings.rs push: branches: [master] + paths: + - lib/include/tree_sitter/api.h + - lib/binding_rust/bindings.rs jobs: check-bindgen: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set up stable Rust toolchain uses: actions-rust-lang/setup-rust-toolchain@v1 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6504f32c..5cde3db9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,10 +1,5 @@ name: Build & Test -env: - CARGO_TERM_COLOR: always - RUSTFLAGS: "-D warnings" - CROSS_DEBUG: 1 - on: workflow_call: inputs: @@ -31,38 +26,41 @@ jobs: - windows-x86 - macos-arm64 - macos-x64 + - wasm32 include: # When adding a new `target`: # 1. Define a new platform alias above - # 2. Add a new record to the matrix map in `cli/npm/install.js` - - { platform: linux-arm64 , target: aarch64-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } - - { platform: linux-arm , target: arm-unknown-linux-gnueabi , os: ubuntu-latest , use-cross: true } - - { platform: linux-x64 , target: x86_64-unknown-linux-gnu , os: ubuntu-20.04 , features: wasm } # See #2272 - - { platform: linux-x86 , target: i686-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } - - { platform: linux-powerpc64 , target: powerpc64-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } - - { platform: windows-arm64 , target: aarch64-pc-windows-msvc , os: windows-latest } - - { platform: windows-x64 , target: x86_64-pc-windows-msvc , os: windows-latest , features: wasm } - - { platform: windows-x86 , target: i686-pc-windows-msvc , os: windows-latest } - - { platform: macos-arm64 , target: aarch64-apple-darwin , os: macos-14 , features: wasm } - - { platform: macos-x64 , target: x86_64-apple-darwin , os: macos-13 , features: wasm } + # 2. Add a new record to the matrix map in `crates/cli/npm/install.js` + - { platform: linux-arm64 , target: aarch64-unknown-linux-gnu , os: ubuntu-24.04-arm } + - { platform: linux-arm , target: armv7-unknown-linux-gnueabihf , os: ubuntu-24.04-arm } + - { platform: linux-x64 , target: x86_64-unknown-linux-gnu , os: ubuntu-24.04 } + - { platform: linux-x86 , target: i686-unknown-linux-gnu , os: ubuntu-24.04 } + - { platform: linux-powerpc64 , target: powerpc64-unknown-linux-gnu , os: ubuntu-24.04 } + - { platform: windows-arm64 , target: aarch64-pc-windows-msvc , os: windows-11-arm } + - { platform: windows-x64 , target: x86_64-pc-windows-msvc , os: windows-2025 } + - { platform: windows-x86 , target: i686-pc-windows-msvc , os: windows-2025 } + - { platform: macos-arm64 , target: aarch64-apple-darwin , os: macos-15 } + - { platform: macos-x64 , target: x86_64-apple-darwin , os: macos-15-intel } + - { platform: wasm32 , target: wasm32-unknown-unknown , os: ubuntu-24.04 } - # Cross compilers for C library - - { platform: linux-arm64 , cc: aarch64-linux-gnu-gcc , ar: aarch64-linux-gnu-ar } - - { platform: linux-arm , cc: arm-linux-gnueabi-gcc , ar: arm-linux-gnueabi-ar } - - { platform: linux-x86 , cc: i686-linux-gnu-gcc , ar: i686-linux-gnu-ar } - - { platform: linux-powerpc64 , cc: powerpc64-linux-gnu-gcc , ar: powerpc64-linux-gnu-ar } + # Extra features + - { platform: linux-arm64 , features: wasm } + - { platform: linux-x64 , features: wasm } + - { platform: macos-arm64 , features: wasm } + - { platform: macos-x64 , features: wasm } - # Prevent race condition (see #2041) - - { platform: windows-x64 , rust-test-threads: 1 } - - { platform: windows-x86 , rust-test-threads: 1 } + # Cross-compilation + - { platform: linux-arm , cross: true } + - { platform: linux-x86 , cross: true } + - { platform: linux-powerpc64 , cross: true } - # Can't natively run CLI on Github runner's host - - { platform: windows-arm64 , no-run: true } + # Compile-only + - { platform: wasm32 , no-run: true } env: - BUILD_CMD: cargo - SUFFIX: ${{ contains(matrix.target, 'windows') && '.exe' || '' }} + CARGO_TERM_COLOR: always + RUSTFLAGS: -D warnings defaults: run: @@ -70,13 +68,28 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 - - name: Read Emscripten version - run: printf 'EMSCRIPTEN_VERSION=%s\n' "$(> $GITHUB_ENV + - name: Set up cross-compilation + if: matrix.cross + run: | + for target in armv7-unknown-linux-gnueabihf i686-unknown-linux-gnu powerpc64-unknown-linux-gnu; do + camel_target=${target//-/_}; target_cc=${target/-unknown/} + printf 'CC_%s=%s\n' "$camel_target" "${target_cc/v7/}-gcc" + printf 'AR_%s=%s\n' "$camel_target" "${target_cc/v7/}-ar" + printf 'CARGO_TARGET_%s_LINKER=%s\n' "${camel_target^^}" "${target_cc/v7/}-gcc" + done >> $GITHUB_ENV + { + printf 'CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_RUNNER=qemu-arm -L /usr/arm-linux-gnueabihf\n' + printf 'CARGO_TARGET_POWERPC64_UNKNOWN_LINUX_GNU_RUNNER=qemu-ppc64 -L /usr/powerpc64-linux-gnu\n' + } >> $GITHUB_ENV + + - name: Get emscripten version + if: contains(matrix.features, 'wasm') + run: printf 'EMSCRIPTEN_VERSION=%s\n' "$(> $GITHUB_ENV - name: Install Emscripten - if: ${{ !matrix.no-run && !matrix.use-cross }} + if: contains(matrix.features, 'wasm') uses: mymindstorm/setup-emsdk@v14 with: version: ${{ env.EMSCRIPTEN_VERSION }} @@ -86,58 +99,84 @@ jobs: with: target: ${{ matrix.target }} - - name: Install cross - if: ${{ matrix.use-cross }} - run: cargo install cross --git https://github.com/cross-rs/cross - - - name: Configure cross - if: ${{ matrix.use-cross }} + - name: Install cross-compilation toolchain + if: matrix.cross run: | - printf '%s\n' > Cross.toml \ - '[target.${{ matrix.target }}]' \ - 'image = "ghcr.io/cross-rs/${{ matrix.target }}:edge"' \ - '[build]' \ - 'pre-build = [' \ - ' "dpkg --add-architecture $CROSS_DEB_ARCH",' \ - ' "curl -fsSL https://deb.nodesource.com/setup_22.x | bash -",' \ - ' "apt-get update && apt-get -y install libssl-dev nodejs"' \ - ']' - cat - Cross.toml <<< 'Cross.toml:' - printf '%s\n' >> $GITHUB_ENV \ - "CROSS_CONFIG=$PWD/Cross.toml" \ - "CROSS_IMAGE=ghcr.io/cross-rs/${{ matrix.target }}:edge" - - - name: Set up environment - env: - RUST_TEST_THREADS: ${{ matrix.rust-test-threads }} - USE_CROSS: ${{ matrix.use-cross }} - TARGET: ${{ matrix.target }} - CC: ${{ matrix.cc }} - AR: ${{ matrix.ar }} - run: | - PATH="$PWD/.github/scripts:$PATH" - printf '%s/.github/scripts\n' "$PWD" >> $GITHUB_PATH - - printf '%s\n' >> $GITHUB_ENV \ - 'TREE_SITTER=tree-sitter.sh' \ - "TARGET=$TARGET" \ - "ROOT=$PWD" - - [[ -n $RUST_TEST_THREADS ]] && \ - printf 'RUST_TEST_THREADS=%s\n' "$RUST_TEST_THREADS" >> $GITHUB_ENV - - [[ -n $CC ]] && printf 'CC=%s\n' "$CC" >> $GITHUB_ENV - [[ -n $AR ]] && printf 'AR=%s\n' "$AR" >> $GITHUB_ENV - - if [[ $USE_CROSS == true ]]; then - printf 'BUILD_CMD=cross\n' >> $GITHUB_ENV - runner=$(cross.sh bash -c "env | sed -n 's/^CARGO_TARGET_.*_RUNNER=//p'") - [[ -n $runner ]] && printf 'CROSS_RUNNER=%s\n' "$runner" >> $GITHUB_ENV + sudo apt-get update -qy + if [[ $PLATFORM == linux-arm ]]; then + sudo apt-get install -qy {binutils,gcc}-arm-linux-gnueabihf qemu-user + elif [[ $PLATFORM == linux-x86 ]]; then + sudo apt-get install -qy {binutils,gcc}-i686-linux-gnu + elif [[ $PLATFORM == linux-powerpc64 ]]; then + sudo apt-get install -qy {binutils,gcc}-powerpc64-linux-gnu qemu-user fi + env: + PLATFORM: ${{ matrix.platform }} - - name: Build wasmtime library - if: ${{ !matrix.use-cross && contains(matrix.features, 'wasm') }} + - name: Install MinGW and Clang (Windows x64 MSYS2) + if: matrix.platform == 'windows-x64' + uses: msys2/setup-msys2@v2 + with: + update: true + install: | + mingw-w64-x86_64-toolchain + mingw-w64-x86_64-clang + mingw-w64-x86_64-make + mingw-w64-x86_64-cmake + + # TODO: Remove RUSTFLAGS="--cap-lints allow" once we use a wasmtime release that addresses + # the `mismatched-lifetime-syntaxes` lint + - name: Build wasmtime library (Windows x64 MSYS2) + if: contains(matrix.features, 'wasm') && matrix.platform == 'windows-x64' run: | + mkdir -p target + WASMTIME_VERSION=$(cargo metadata --format-version=1 --locked --features wasm | \ + jq -r '.packages[] | select(.name == "wasmtime-c-api-impl") | .version') + curl -LSs "$WASMTIME_REPO/archive/refs/tags/v${WASMTIME_VERSION}.tar.gz" | tar xzf - -C target + cd target/wasmtime-${WASMTIME_VERSION} + cmake -S crates/c-api -B target/c-api \ + -DCMAKE_INSTALL_PREFIX="$PWD/artifacts" \ + -DWASMTIME_DISABLE_ALL_FEATURES=ON \ + -DWASMTIME_FEATURE_CRANELIFT=ON \ + -DWASMTIME_TARGET='x86_64-pc-windows-gnu' + cmake --build target/c-api && cmake --install target/c-api + printf 'CMAKE_PREFIX_PATH=%s\n' "$PWD/artifacts" >> $GITHUB_ENV + env: + WASMTIME_REPO: https://github.com/bytecodealliance/wasmtime + RUSTFLAGS: ${{ env.RUSTFLAGS }} --cap-lints allow + + - name: Build C library (Windows x64 MSYS2 CMake) + if: matrix.platform == 'windows-x64' + shell: msys2 {0} + run: | + cmake -G Ninja -S . -B build/static \ + -DBUILD_SHARED_LIBS=OFF \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \ + -DTREE_SITTER_FEATURE_WASM=$WASM \ + -DCMAKE_C_COMPILER=clang + cmake --build build/static + + cmake -G Ninja -S . -B build/shared \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \ + -DTREE_SITTER_FEATURE_WASM=$WASM \ + -DCMAKE_C_COMPILER=clang + cmake --build build/shared + rm -rf \ + build/{static,shared} \ + "${CMAKE_PREFIX_PATH}/artifacts" \ + target/wasmtime-${WASMTIME_VERSION} + env: + WASM: ${{ contains(matrix.features, 'wasm') && 'ON' || 'OFF' }} + + # TODO: Remove RUSTFLAGS="--cap-lints allow" once we use a wasmtime release that addresses + # the `mismatched-lifetime-syntaxes` lint + - name: Build wasmtime library + if: contains(matrix.features, 'wasm') + run: | + mkdir -p target WASMTIME_VERSION=$(cargo metadata --format-version=1 --locked --features wasm | \ jq -r '.packages[] | select(.name == "wasmtime-c-api-impl") | .version') curl -LSs "$WASMTIME_REPO/archive/refs/tags/v${WASMTIME_VERSION}.tar.gz" | tar xzf - -C target @@ -151,86 +190,122 @@ jobs: printf 'CMAKE_PREFIX_PATH=%s\n' "$PWD/artifacts" >> $GITHUB_ENV env: WASMTIME_REPO: https://github.com/bytecodealliance/wasmtime + RUSTFLAGS: ${{ env.RUSTFLAGS }} --cap-lints allow - name: Build C library (make) - if: ${{ runner.os != 'Windows' }} - run: make.sh -j CFLAGS="$CFLAGS" + if: runner.os != 'Windows' + run: | + if [[ $PLATFORM == linux-arm ]]; then + CC=arm-linux-gnueabihf-gcc; AR=arm-linux-gnueabihf-ar + elif [[ $PLATFORM == linux-x86 ]]; then + CC=i686-linux-gnu-gcc; AR=i686-linux-gnu-ar + elif [[ $PLATFORM == linux-powerpc64 ]]; then + CC=powerpc64-linux-gnu-gcc; AR=powerpc64-linux-gnu-ar + else + CC=gcc; AR=ar + fi + make -j CFLAGS="$CFLAGS" CC=$CC AR=$AR env: + PLATFORM: ${{ matrix.platform }} CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types - name: Build C library (CMake) - if: ${{ !matrix.use-cross }} + if: "!matrix.cross" run: | - cmake -S lib -B build/static \ + cmake -S . -B build/static \ -DBUILD_SHARED_LIBS=OFF \ -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \ -DTREE_SITTER_FEATURE_WASM=$WASM cmake --build build/static --verbose - cmake -S lib -B build/shared \ + cmake -S . -B build/shared \ -DBUILD_SHARED_LIBS=ON \ -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \ -DTREE_SITTER_FEATURE_WASM=$WASM cmake --build build/shared --verbose env: - CC: ${{ contains(matrix.target, 'linux') && 'clang' || '' }} + CC: ${{ contains(matrix.platform, 'linux') && 'clang' || '' }} WASM: ${{ contains(matrix.features, 'wasm') && 'ON' || 'OFF' }} - - name: Build wasm library - # No reason to build on the same Github runner hosts many times - if: ${{ !matrix.no-run && !matrix.use-cross }} - run: $BUILD_CMD run -p xtask -- build-wasm + - name: Build Wasm library + if: contains(matrix.features, 'wasm') + shell: bash + run: | + cd lib/binding_web + npm ci + CJS=true npm run build + CJS=true npm run build:debug + npm run build + npm run build:debug + + - name: Check no_std builds + if: inputs.run-test && !matrix.no-run + working-directory: lib + shell: bash + run: cargo check --no-default-features --target='${{ matrix.target }}' - name: Build target - run: $BUILD_CMD build --release --target=${{ matrix.target }} --features=${{ matrix.features }} + run: cargo build --release --target='${{ matrix.target }}' --features='${{ matrix.features }}' $PACKAGE + env: + PACKAGE: ${{ matrix.platform == 'wasm32' && '-p tree-sitter' || '' }} - name: Cache fixtures id: cache - if: ${{ !matrix.no-run && inputs.run-test }} + if: inputs.run-test && !matrix.no-run uses: ./.github/actions/cache - name: Fetch fixtures - if: ${{ !matrix.no-run && inputs.run-test }} - run: $BUILD_CMD run -p xtask -- fetch-fixtures + if: inputs.run-test && !matrix.no-run + run: cargo run -p xtask --target='${{ matrix.target }}' -- fetch-fixtures - name: Generate fixtures - if: ${{ !matrix.no-run && inputs.run-test && steps.cache.outputs.cache-hit != 'true' }} - run: $BUILD_CMD run -p xtask -- generate-fixtures + if: inputs.run-test && !matrix.no-run && steps.cache.outputs.cache-hit != 'true' + run: cargo run -p xtask --target='${{ matrix.target }}' -- generate-fixtures - name: Generate Wasm fixtures - if: ${{ !matrix.no-run && !matrix.use-cross && inputs.run-test && steps.cache.outputs.cache-hit != 'true' }} - run: $BUILD_CMD run -p xtask -- generate-fixtures --wasm + if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && steps.cache.outputs.cache-hit != 'true' + run: cargo run -p xtask --target='${{ matrix.target }}' -- generate-fixtures --wasm - name: Run main tests - if: ${{ !matrix.no-run && inputs.run-test }} - run: $BUILD_CMD test --target=${{ matrix.target }} --features=${{ matrix.features }} + if: inputs.run-test && !matrix.no-run + run: cargo test --target='${{ matrix.target }}' --features='${{ matrix.features }}' - - name: Run wasm tests - if: ${{ !matrix.no-run && !matrix.use-cross && inputs.run-test }} - run: $BUILD_CMD run -p xtask -- test-wasm - - - name: Run benchmarks - # Cross-compiled benchmarks are pointless - if: ${{ !matrix.no-run && !matrix.use-cross && inputs.run-test }} - run: $BUILD_CMD bench benchmark -p tree-sitter-cli --target=${{ matrix.target }} + - name: Run Wasm tests + if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') + run: cargo run -p xtask --target='${{ matrix.target }}' -- test-wasm - name: Upload CLI artifact - uses: actions/upload-artifact@v4 + if: "!matrix.no-run" + uses: actions/upload-artifact@v6 with: name: tree-sitter.${{ matrix.platform }} - path: target/${{ matrix.target }}/release/tree-sitter${{ env.SUFFIX }} + path: target/${{ matrix.target }}/release/tree-sitter${{ contains(matrix.target, 'windows') && '.exe' || '' }} if-no-files-found: error retention-days: 7 - name: Upload Wasm artifacts - if: ${{ matrix.platform == 'linux-x64' }} - uses: actions/upload-artifact@v4 + if: matrix.platform == 'linux-x64' + uses: actions/upload-artifact@v6 with: name: tree-sitter.wasm path: | - lib/binding_web/tree-sitter.js - lib/binding_web/tree-sitter.wasm + lib/binding_web/web-tree-sitter.js + lib/binding_web/web-tree-sitter.js.map + lib/binding_web/web-tree-sitter.cjs + lib/binding_web/web-tree-sitter.cjs.map + lib/binding_web/web-tree-sitter.wasm + lib/binding_web/web-tree-sitter.wasm.map + lib/binding_web/debug/web-tree-sitter.cjs + lib/binding_web/debug/web-tree-sitter.cjs.map + lib/binding_web/debug/web-tree-sitter.js + lib/binding_web/debug/web-tree-sitter.js.map + lib/binding_web/debug/web-tree-sitter.wasm + lib/binding_web/debug/web-tree-sitter.wasm.map + lib/binding_web/lib/*.c + lib/binding_web/lib/*.h + lib/binding_web/lib/*.ts + lib/binding_web/src/*.ts if-no-files-found: error retention-days: 7 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dac4cdbf..a60c93f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,8 +2,20 @@ name: CI on: pull_request: + paths-ignore: + - docs/** + - "**/README.md" + - CONTRIBUTING.md + - LICENSE + - cli/src/templates push: branches: [master] + paths-ignore: + - docs/** + - "**/README.md" + - CONTRIBUTING.md + - LICENSE + - cli/src/templates concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -14,24 +26,24 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set up stable Rust toolchain uses: actions-rust-lang/setup-rust-toolchain@v1 with: toolchain: stable - - - name: Set up nightly Rust toolchain - uses: actions-rust-lang/setup-rust-toolchain@v1 - with: - toolchain: nightly components: clippy, rustfmt - name: Lint files - run: make lint + run: | + make lint + make lint-web sanitize: uses: ./.github/workflows/sanitize.yml build: uses: ./.github/workflows/build.yml + + check-wasm-stdlib: + uses: ./.github/workflows/wasm_stdlib.yml diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..0e4baebf --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,50 @@ +name: Deploy Docs +on: + push: + branches: [master] + paths: [docs/**] + workflow_dispatch: + +jobs: + deploy-docs: + runs-on: ubuntu-latest + + permissions: + contents: write + pages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Rust + uses: actions-rust-lang/setup-rust-toolchain@v1 + + - name: Install mdbook + env: + GH_TOKEN: ${{ github.token }} + run: | + jq_expr='.assets[] | select(.name | contains("x86_64-unknown-linux-gnu")) | .browser_download_url' + url=$(gh api repos/rust-lang/mdbook/releases/tags/v0.4.52 --jq "$jq_expr") + mkdir mdbook + curl -sSL "$url" | tar -xz -C mdbook + printf '%s/mdbook\n' "$PWD" >> "$GITHUB_PATH" + + - name: Install mdbook-admonish + run: cargo install mdbook-admonish + + - name: Build Book + run: mdbook build docs + + - name: Setup Pages + uses: actions/configure-pages@v5 + + - name: Upload artifact + uses: actions/upload-pages-artifact@v4 + with: + path: docs/book + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/nvim_ts.yml b/.github/workflows/nvim_ts.yml index 4bf39366..88e3371f 100644 --- a/.github/workflows/nvim_ts.yml +++ b/.github/workflows/nvim_ts.yml @@ -3,7 +3,10 @@ name: nvim-treesitter parser tests on: pull_request: paths: - - 'cli/**' + - 'crates/cli/**' + - 'crates/config/**' + - 'crates/generate/**' + - 'crates/loader/**' - '.github/workflows/nvim_ts.yml' workflow_dispatch: @@ -13,7 +16,7 @@ concurrency: jobs: check_compilation: - timeout-minutes: 20 + timeout-minutes: 30 strategy: fail-fast: false matrix: @@ -25,9 +28,9 @@ jobs: NVIM: ${{ matrix.os == 'windows-latest' && 'nvim-win64\\bin\\nvim.exe' || 'nvim' }} NVIM_TS_DIR: nvim-treesitter steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: repository: nvim-treesitter/nvim-treesitter path: ${{ env.NVIM_TS_DIR }} @@ -55,7 +58,7 @@ jobs: - if: matrix.type == 'build' name: Compile parsers - run: $NVIM -l ./scripts/install-parsers.lua + run: $NVIM -l ./scripts/install-parsers.lua --max-jobs=10 working-directory: ${{ env.NVIM_TS_DIR }} shell: bash diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7c0152ff..4f6f9d47 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,13 +17,15 @@ jobs: runs-on: ubuntu-latest needs: build permissions: + id-token: write + attestations: write contents: write steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Download build artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v7 with: path: artifacts @@ -33,9 +35,13 @@ jobs: - name: Prepare release artifacts run: | - mkdir -p target - mv artifacts/tree-sitter.wasm/* target/ + mkdir -p target web + mv artifacts/tree-sitter.wasm/* web/ + + tar -czf target/web-tree-sitter.tar.gz -C web . + rm -r artifacts/tree-sitter.wasm + for platform in $(cd artifacts; ls | sed 's/^tree-sitter\.//'); do exe=$(ls artifacts/tree-sitter.$platform/tree-sitter*) gzip --stdout --name $exe > target/tree-sitter-$platform.gz @@ -43,47 +49,65 @@ jobs: rm -rf artifacts ls -l target/ + - name: Generate attestations + uses: actions/attest-build-provenance@v3 + with: + subject-path: | + target/tree-sitter-*.gz + target/web-tree-sitter.tar.gz + - name: Create release run: |- - gh release create \ + gh release create $GITHUB_REF_NAME \ target/tree-sitter-*.gz \ - target/tree-sitter.wasm \ - target/tree-sitter.js + target/web-tree-sitter.tar.gz env: GH_TOKEN: ${{ github.token }} crates_io: name: Publish packages to Crates.io runs-on: ubuntu-latest + environment: crates + permissions: + id-token: write + contents: read needs: release steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set up Rust uses: actions-rust-lang/setup-rust-toolchain@v1 + - name: Set up registry token + id: auth + uses: rust-lang/crates-io-auth-action@v1 + - name: Publish crates to Crates.io uses: katyo/publish-crates@v2 with: - registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }} + registry-token: ${{ steps.auth.outputs.token }} npm: name: Publish packages to npmjs.com runs-on: ubuntu-latest + environment: npm + permissions: + id-token: write + contents: read needs: release strategy: fail-fast: false matrix: - directory: [cli/npm, lib/binding_web] + directory: [crates/cli/npm, lib/binding_web] steps: - - name: CHeckout repository - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v6 - name: Set up Node - uses: actions/setup-node@v4 + uses: actions/setup-node@v6 with: - node-version: 20 + node-version: 24 registry-url: https://registry.npmjs.org - name: Set up Rust @@ -91,10 +115,15 @@ jobs: - name: Build wasm if: matrix.directory == 'lib/binding_web' - run: cargo xtask build-wasm + run: | + cd ${{ matrix.directory }} + npm ci + npm run build + npm run build:debug + CJS=true npm run build + CJS=true npm run build:debug + npm run build:dts - name: Publish to npmjs.com working-directory: ${{ matrix.directory }} run: npm publish - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.github/workflows/response.yml b/.github/workflows/response.yml index 576b9474..54dd2021 100644 --- a/.github/workflows/response.yml +++ b/.github/workflows/response.yml @@ -17,13 +17,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout script - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: sparse-checkout: .github/scripts/close_unresponsive.js sparse-checkout-cone-mode: false - name: Run script - uses: actions/github-script@v7 + uses: actions/github-script@v8 with: script: | const script = require('./.github/scripts/close_unresponsive.js') @@ -35,13 +35,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout script - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: sparse-checkout: .github/scripts/remove_response_label.js sparse-checkout-cone-mode: false - name: Run script - uses: actions/github-script@v7 + uses: actions/github-script@v8 with: script: | const script = require('./.github/scripts/remove_response_label.js') diff --git a/.github/workflows/reviewers_remove.yml b/.github/workflows/reviewers_remove.yml index b99f0caa..3a389ed4 100644 --- a/.github/workflows/reviewers_remove.yml +++ b/.github/workflows/reviewers_remove.yml @@ -12,13 +12,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout script - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: sparse-checkout: .github/scripts/reviewers_remove.js sparse-checkout-cone-mode: false - name: Run script - uses: actions/github-script@v7 + uses: actions/github-script@v8 with: script: | const script = require('./.github/scripts/reviewers_remove.js') diff --git a/.github/workflows/sanitize.yml b/.github/workflows/sanitize.yml index 875b8278..2f8851dc 100644 --- a/.github/workflows/sanitize.yml +++ b/.github/workflows/sanitize.yml @@ -15,7 +15,7 @@ jobs: TREE_SITTER: ${{ github.workspace }}/target/release/tree-sitter steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Install UBSAN library run: sudo apt-get update -y && sudo apt-get install -y libubsan1 diff --git a/.github/workflows/spam.yml b/.github/workflows/spam.yml new file mode 100644 index 00000000..eb1d4a46 --- /dev/null +++ b/.github/workflows/spam.yml @@ -0,0 +1,29 @@ +name: Close as spam + +on: + issues: + types: [labeled] + pull_request_target: + types: [labeled] + +permissions: + issues: write + pull-requests: write + +jobs: + spam: + runs-on: ubuntu-latest + if: github.event.label.name == 'spam' + steps: + - name: Checkout script + uses: actions/checkout@v6 + with: + sparse-checkout: .github/scripts/close_spam.js + sparse-checkout-cone-mode: false + + - name: Run script + uses: actions/github-script@v8 + with: + script: | + const script = require('./.github/scripts/close_spam.js') + await script({github, context}) diff --git a/.github/workflows/wasm_exports.yml b/.github/workflows/wasm_exports.yml new file mode 100644 index 00000000..af48cf8a --- /dev/null +++ b/.github/workflows/wasm_exports.yml @@ -0,0 +1,41 @@ +name: Check Wasm Exports + +on: + pull_request: + paths: + - lib/include/tree_sitter/api.h + - lib/binding_web/** + - xtask/src/** + push: + branches: [master] + paths: + - lib/include/tree_sitter/api.h + - lib/binding_rust/bindings.rs + - CMakeLists.txt + +jobs: + check-wasm-exports: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up stable Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: stable + + - name: Install wasm-objdump + run: sudo apt-get update -y && sudo apt-get install -y wabt + + - name: Build C library (make) + run: make -j CFLAGS="$CFLAGS" + env: + CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types + + - name: Build Wasm Library + working-directory: lib/binding_web + run: npm ci && npm run build:debug + + - name: Check Wasm exports + run: cargo xtask check-wasm-exports diff --git a/.github/workflows/wasm_stdlib.yml b/.github/workflows/wasm_stdlib.yml new file mode 100644 index 00000000..adec8411 --- /dev/null +++ b/.github/workflows/wasm_stdlib.yml @@ -0,0 +1,19 @@ +name: Check Wasm Stdlib build + +on: + workflow_call: + +jobs: + check: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Check directory changes + uses: actions/github-script@v8 + with: + script: | + const scriptPath = `${process.env.GITHUB_WORKSPACE}/.github/scripts/wasm_stdlib.js`; + const script = require(scriptPath); + return script({ github, context, core }); diff --git a/.gitignore b/.gitignore index 25738984..ca47139e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,12 @@ log*.html +.direnv .idea *.xcodeproj .vscode .cache .zig-cache +.direnv profile* fuzz-results @@ -12,7 +14,6 @@ test/fuzz/out test/fixtures/grammars/* !test/fixtures/grammars/.gitkeep -package-lock.json node_modules docs/assets/js/tree-sitter.js @@ -25,6 +26,7 @@ docs/assets/js/tree-sitter.js *.dylib *.so *.so.[0-9]* +*.dll *.o *.obj *.exp @@ -34,3 +36,5 @@ docs/assets/js/tree-sitter.js .build build zig-* + +/result diff --git a/.zed/settings.json b/.zed/settings.json new file mode 100644 index 00000000..15f15d9a --- /dev/null +++ b/.zed/settings.json @@ -0,0 +1,11 @@ +{ + "lsp": { + "rust-analyzer": { + "initialization_options": { + "cargo": { + "features": "all" + } + } + } + } +} diff --git a/lib/CMakeLists.txt b/CMakeLists.txt similarity index 86% rename from lib/CMakeLists.txt rename to CMakeLists.txt index ce0b34e3..f11895c0 100644 --- a/lib/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.13) project(tree-sitter - VERSION "0.25.0" + VERSION "0.27.0" DESCRIPTION "An incremental parsing system for programming tools" HOMEPAGE_URL "https://tree-sitter.github.io/tree-sitter/" LANGUAGES C) @@ -11,15 +11,15 @@ option(TREE_SITTER_FEATURE_WASM "Enable the Wasm feature" OFF) option(AMALGAMATED "Build using an amalgamated source" OFF) if(AMALGAMATED) - set(TS_SOURCE_FILES "${PROJECT_SOURCE_DIR}/src/lib.c") + set(TS_SOURCE_FILES "${PROJECT_SOURCE_DIR}/lib/src/lib.c") else() - file(GLOB TS_SOURCE_FILES src/*.c) - list(REMOVE_ITEM TS_SOURCE_FILES "${PROJECT_SOURCE_DIR}/src/lib.c") + file(GLOB TS_SOURCE_FILES lib/src/*.c) + list(REMOVE_ITEM TS_SOURCE_FILES "${PROJECT_SOURCE_DIR}/lib/src/lib.c") endif() add_library(tree-sitter ${TS_SOURCE_FILES}) -target_include_directories(tree-sitter PRIVATE src src/wasm include) +target_include_directories(tree-sitter PRIVATE lib/src lib/src/wasm PUBLIC lib/include) if(MSVC) target_compile_options(tree-sitter PRIVATE @@ -81,15 +81,15 @@ set_target_properties(tree-sitter SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}" DEFINE_SYMBOL "") -target_compile_definitions(tree-sitter PRIVATE _POSIX_C_SOURCE=200112L _DEFAULT_SOURCE) - -configure_file(tree-sitter.pc.in "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter.pc" @ONLY) +target_compile_definitions(tree-sitter PRIVATE _POSIX_C_SOURCE=200112L _DEFAULT_SOURCE _BSD_SOURCE _DARWIN_C_SOURCE) include(GNUInstallDirs) -install(FILES include/tree_sitter/api.h +configure_file(lib/tree-sitter.pc.in "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter.pc" @ONLY) + +install(FILES lib/include/tree_sitter/api.h DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/tree_sitter") install(FILES "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter.pc" - DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/pkgconfig") + DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") install(TARGETS tree-sitter LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}") diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 42bc7b75..503955b0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1 +1 @@ -See [section-6-contributing.md](./docs/section-6-contributing.md) +See [docs/src/6-contributing.md](./docs/src/6-contributing.md) diff --git a/Cargo.lock b/Cargo.lock index f03977e7..ae7803c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3,31 +3,43 @@ version = 3 [[package]] -name = "ahash" -version = "0.8.11" +name = "addr2line" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ - "cfg-if", - "once_cell", - "version_check", - "zerocopy", + "gimli", ] [[package]] name = "aho-corasick" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] [[package]] -name = "anstream" -version = "0.6.18" +name = "allocator-api2" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "ansi_colours" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14eec43e0298190790f41679fe69ef7a829d2a2ddd78c8c00339e84710e435fe" +dependencies = [ + "rgb", +] + +[[package]] +name = "anstream" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" dependencies = [ "anstyle", "anstyle-parse", @@ -40,49 +52,59 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.10" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" [[package]] name = "anstyle-parse" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.2" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] name = "anstyle-wincon" -version = "3.0.6" +version = "3.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", - "windows-sys 0.59.0", + "once_cell_polyfill", + "windows-sys 0.61.2", ] [[package]] name = "anyhow" -version = "1.0.93" +version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + +[[package]] +name = "ar_archive_writer" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c269894b6fe5e9d7ada0cf69b5bf847ff35bc25fc271f08e1d080fce80339a" +dependencies = [ + "object 0.32.2", +] [[package]] name = "arbitrary" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "ascii" @@ -92,11 +114,11 @@ checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" [[package]] name = "bindgen" -version = "0.70.1" +version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "bitflags", + "bitflags 2.10.0", "cexpr", "clang-sys", "itertools 0.13.0", @@ -105,31 +127,37 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash 1.1.0", + "rustc-hash", "shlex", "syn", ] [[package]] name = "bitflags" -version = "2.6.0" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" [[package]] name = "block2" -version = "0.5.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c132eebf10f5cad5289222520a4a058514204aed6d791f1cf4fe8088b82d15f" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" dependencies = [ "objc2", ] [[package]] name = "bstr" -version = "1.11.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a68f1f47cdf0ec8ee4b941b2eee2a80cb796db73118c0dd09ac63fbe405be22" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" dependencies = [ "memchr", "regex-automata", @@ -138,30 +166,32 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.16.0" +version = "3.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" +dependencies = [ + "allocator-api2", +] [[package]] -name = "byteorder" -version = "1.5.0" +name = "bytemuck" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" [[package]] name = "bytes" -version = "1.8.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" [[package]] name = "cc" -version = "1.2.1" +version = "1.2.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd9de9f2205d5ef3fd67e685b0df337994ddd4495e2a28d185500d0e1edfea47" +checksum = "755d2fce177175ffca841e9a06afdb2c4ab0f593d53b4dee48147dfaade85932" dependencies = [ - "jobserver", - "libc", + "find-msvc-tools", "shlex", ] @@ -182,9 +212,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" @@ -206,14 +236,14 @@ checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", - "libloading", + "libloading 0.8.9", ] [[package]] name = "clap" -version = "4.5.21" +version = "4.5.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb3b4b9e5a7c7514dfa52869339ee98b3156b0bfb4e8a77c4ff4babb64b1604f" +checksum = "c6e6ff9dcd79cff5cd969a17a545d79e84ab086e444102a591e288a8aa3ce394" dependencies = [ "clap_builder", "clap_derive", @@ -221,9 +251,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.21" +version = "4.5.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b17a95aa67cc7b5ebd32aa5370189aa0d79069ef1c64ce893bd30fb24bff20ec" +checksum = "fa42cf4d2b7a41bc8f663a7cab4031ebafa1bf3875705bfaf8466dc60ab52c00" dependencies = [ "anstream", "anstyle", @@ -233,18 +263,28 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.5.38" +version = "4.5.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9647a559c112175f17cf724dc72d3645680a883c58481332779192b0d8e7a01" +checksum = "430b4dc2b5e3861848de79627b2bedc9f3342c7da5173a14eaa5d0f8dc18ae5d" dependencies = [ "clap", ] [[package]] -name = "clap_derive" -version = "4.5.18" +name = "clap_complete_nushell" +version = "4.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +checksum = "685bc86fd34b7467e0532a4f8435ab107960d69a243785ef0275e571b35b641a" +dependencies = [ + "clap", + "clap_complete", +] + +[[package]] +name = "clap_derive" +version = "4.5.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" dependencies = [ "heck", "proc-macro2", @@ -254,21 +294,24 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.2" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" [[package]] name = "cobs" -version = "0.2.3" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.17", +] [[package]] name = "colorchoice" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "combine" @@ -282,22 +325,31 @@ dependencies = [ [[package]] name = "console" -version = "0.15.8" +version = "0.15.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" dependencies = [ "encode_unicode", - "lazy_static", "libc", + "once_cell", "unicode-width", - "windows-sys 0.52.0", + "windows-sys 0.59.0", +] + +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", ] [[package]] name = "core-foundation" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" dependencies = [ "core-foundation-sys", "libc", @@ -310,19 +362,37 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] -name = "cranelift-bforest" -version = "0.113.1" +name = "cranelift-assembler-x64" +version = "0.120.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "540b193ff98b825a1f250a75b3118911af918a734154c69d80bcfcf91e7e9522" +checksum = "a5023e06632d8f351c2891793ccccfe4aef957954904392434038745fb6f1f68" +dependencies = [ + "cranelift-assembler-x64-meta", +] + +[[package]] +name = "cranelift-assembler-x64-meta" +version = "0.120.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c4012b4c8c1f6eb05c0a0a540e3e1ee992631af51aa2bbb3e712903ce4fd65" +dependencies = [ + "cranelift-srcgen", +] + +[[package]] +name = "cranelift-bforest" +version = "0.120.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d6d883b4942ef3a7104096b8bc6f2d1a41393f159ac8de12aed27b25d67f895" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-bitset" -version = "0.113.1" +version = "0.120.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7cb269598b9557ab942d687d3c1086d77c4b50dcf35813f3a65ba306fd42279" +checksum = "db7b2ee9eec6ca8a716d900d5264d678fb2c290c58c46c8da7f94ee268175d17" dependencies = [ "serde", "serde_derive", @@ -330,11 +400,12 @@ dependencies = [ [[package]] name = "cranelift-codegen" -version = "0.113.1" +version = "0.120.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46566d7c83a8bff4150748d66020f4c7224091952aa4b4df1ec4959c39d937a1" +checksum = "aeda0892577afdce1ac2e9a983a55f8c5b87a59334e1f79d8f735a2d7ba4f4b4" dependencies = [ "bumpalo", + "cranelift-assembler-x64", "cranelift-bforest", "cranelift-bitset", "cranelift-codegen-meta", @@ -343,43 +414,48 @@ dependencies = [ "cranelift-entity", "cranelift-isle", "gimli", - "hashbrown 0.14.5", + "hashbrown 0.15.5", "log", + "pulley-interpreter", "regalloc2", - "rustc-hash 2.0.0", + "rustc-hash", + "serde", "smallvec", "target-lexicon", ] [[package]] name = "cranelift-codegen-meta" -version = "0.113.1" +version = "0.120.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2df8a86a34236cc75a8a6a271973da779c2aeb36c43b6e14da474cf931317082" +checksum = "e461480d87f920c2787422463313326f67664e68108c14788ba1676f5edfcd15" dependencies = [ + "cranelift-assembler-x64-meta", "cranelift-codegen-shared", + "cranelift-srcgen", + "pulley-interpreter", ] [[package]] name = "cranelift-codegen-shared" -version = "0.113.1" +version = "0.120.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf75340b6a57b7c7c1b74f10d3d90883ee6d43a554be8131a4046c2ebcf5eb65" +checksum = "976584d09f200c6c84c4b9ff7af64fc9ad0cb64dffa5780991edd3fe143a30a1" [[package]] name = "cranelift-control" -version = "0.113.1" +version = "0.120.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e84495bc5d23d86aad8c86f8ade4af765b94882af60d60e271d3153942f1978" +checksum = "46d43d70f4e17c545aa88dbf4c84d4200755d27c6e3272ebe4de65802fa6a955" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.113.1" +version = "0.120.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "963c17147b80df351965e57c04d20dbedc85bcaf44c3436780a59a3f1ff1b1c2" +checksum = "d75418674520cb400c8772bfd6e11a62736c78fc1b6e418195696841d1bf91f1" dependencies = [ "cranelift-bitset", "serde", @@ -388,9 +464,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.113.1" +version = "0.120.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727f02acbc4b4cb2ba38a6637101d579db50190df1dd05168c68e762851a3dd5" +checksum = "3c8b1a91c86687a344f3c52dd6dfb6e50db0dfa7f2e9c7711b060b3623e1fdeb" dependencies = [ "cranelift-codegen", "log", @@ -400,15 +476,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.113.1" +version = "0.120.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b00cc2e03c748f2531eea01c871f502b909d30295fdcad43aec7bf5c5b4667" +checksum = "711baa4e3432d4129295b39ec2b4040cc1b558874ba0a37d08e832e857db7285" [[package]] name = "cranelift-native" -version = "0.113.1" +version = "0.120.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbeaf978dc7c1a2de8bbb9162510ed218eb156697bc45590b8fbdd69bb08e8de" +checksum = "41c83e8666e3bcc5ffeaf6f01f356f0e1f9dcd69ce5511a1efd7ca5722001a3f" dependencies = [ "cranelift-codegen", "libc", @@ -416,19 +492,25 @@ dependencies = [ ] [[package]] -name = "crc32fast" -version = "1.4.2" +name = "cranelift-srcgen" +version = "0.120.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +checksum = "02e3f4d783a55c64266d17dc67d2708852235732a100fc40dd9f1051adc64d7b" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ "cfg-if", ] [[package]] name = "ctor" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f" +checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501" dependencies = [ "quote", "syn", @@ -436,12 +518,13 @@ dependencies = [ [[package]] name = "ctrlc" -version = "3.4.5" +version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90eeab0aa92f3f9b4e87f258c72b139c207d251f9cbc1080a0086b86a8870dd3" +checksum = "73736a89c4aff73035ba2ed2e565061954da00d4970fc9ac25dcc85a2a20d790" dependencies = [ + "dispatch2", "nix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -454,7 +537,7 @@ dependencies = [ "fuzzy-matcher", "shell-words", "tempfile", - "thiserror", + "thiserror 1.0.69", "zeroize", ] @@ -465,24 +548,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" [[package]] -name = "dirs" -version = "5.0.1" +name = "dispatch2" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" -dependencies = [ - "dirs-sys", -] - -[[package]] -name = "dirs-sys" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +checksum = "89a09f22a6c6069a18470eb92d2298acf25463f14256d24778e1230d789a2aec" dependencies = [ + "bitflags 2.10.0", + "block2", "libc", - "option-ext", - "redox_users", - "windows-sys 0.48.0", + "objc2", ] [[package]] @@ -497,10 +571,22 @@ dependencies = [ ] [[package]] -name = "either" -version = "1.13.0" +name = "dunce" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] name = "embedded-io" @@ -516,9 +602,9 @@ checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" [[package]] name = "encode_unicode" -version = "0.3.6" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" [[package]] name = "encoding_rs" @@ -531,18 +617,28 @@ dependencies = [ [[package]] name = "equivalent" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.9" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", +] + +[[package]] +name = "etcetera" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de48cc4d1c1d97a20fd819def54b890cadde72ed3ad0c614822a0a433361be96" +dependencies = [ + "cfg-if", + "windows-sys 0.61.2", ] [[package]] @@ -553,47 +649,71 @@ checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" [[package]] name = "fastrand" -version = "2.1.1" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] -name = "filetime" -version = "0.2.25" +name = "file-id" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" +checksum = "e1fc6a637b6dc58414714eddd9170ff187ecb0933d4c7024d1abbd23a3cc26e9" dependencies = [ - "cfg-if", - "libc", - "libredox", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] -name = "foldhash" -version = "0.1.3" +name = "find-msvc-tools" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" +checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] name = "form_urlencoded" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" dependencies = [ "percent-encoding", ] [[package]] name = "fs4" -version = "0.9.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8c6b3bd49c37d2aa3f3f2220233b29a7cd23f79d1fe70e5337d25fb390793de" +checksum = "c29c30684418547d476f0b48e84f4821639119c483b1eccd566c8cd0cd05f521" dependencies = [ - "rustix", + "rustix 0.38.44", "windows-sys 0.52.0", ] +[[package]] +name = "fsevent-sys" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" +dependencies = [ + "libc", +] + [[package]] name = "fuzzy-matcher" version = "0.3.7" @@ -605,15 +725,27 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", "libc", "wasi", ] +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + [[package]] name = "gimli" version = "0.31.1" @@ -625,44 +757,31 @@ dependencies = [ "stable_deref_trait", ] -[[package]] -name = "git2" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b903b73e45dc0c6c596f2d37eccece7c1c8bb6e4407b001096387c63d0d93724" -dependencies = [ - "bitflags", - "libc", - "libgit2-sys", - "log", - "openssl-probe", - "openssl-sys", - "url", -] - [[package]] name = "glob" -version = "0.3.1" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "hashbrown" -version = "0.14.5" +version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "ahash", + "foldhash 0.1.5", "serde", ] [[package]] name = "hashbrown" -version = "0.15.1" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ - "foldhash", + "allocator-api2", + "equivalent", + "foldhash 0.2.0", ] [[package]] @@ -671,15 +790,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "home" -version = "0.5.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" -dependencies = [ - "windows-sys 0.52.0", -] - [[package]] name = "html-escape" version = "0.2.13" @@ -697,21 +807,22 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "icu_collections" -version = "1.5.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" dependencies = [ "displaydoc", + "potential_utf", "yoke", "zerofrom", "zerovec", ] [[package]] -name = "icu_locid" -version = "1.5.0" +name = "icu_locale_core" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" dependencies = [ "displaydoc", "litemap", @@ -720,110 +831,72 @@ dependencies = [ "zerovec", ] -[[package]] -name = "icu_locid_transform" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_locid_transform_data", - "icu_provider", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_locid_transform_data" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" - [[package]] name = "icu_normalizer" -version = "1.5.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" dependencies = [ - "displaydoc", "icu_collections", "icu_normalizer_data", "icu_properties", "icu_provider", "smallvec", - "utf16_iter", - "utf8_iter", - "write16", "zerovec", ] [[package]] name = "icu_normalizer_data" -version = "1.5.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" [[package]] name = "icu_properties" -version = "1.5.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" dependencies = [ - "displaydoc", "icu_collections", - "icu_locid_transform", + "icu_locale_core", "icu_properties_data", "icu_provider", - "tinystr", + "zerotrie", "zerovec", ] [[package]] name = "icu_properties_data" -version = "1.5.0" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" [[package]] name = "icu_provider" -version = "1.5.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" dependencies = [ "displaydoc", - "icu_locid", - "icu_provider_macros", - "stable_deref_trait", - "tinystr", + "icu_locale_core", "writeable", "yoke", "zerofrom", + "zerotrie", "zerovec", ] [[package]] -name = "icu_provider_macros" -version = "1.5.0" +name = "ident_case" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "id-arena" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "1.0.3" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" dependencies = [ "idna_adapter", "smallvec", @@ -832,9 +905,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" dependencies = [ "icu_normalizer", "icu_properties", @@ -842,35 +915,50 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.6.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" dependencies = [ "equivalent", - "hashbrown 0.15.1", + "hashbrown 0.16.1", "serde", + "serde_core", ] [[package]] name = "indoc" -version = "2.0.5" +version = "2.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + +[[package]] +name = "inotify" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f37dccff2791ab604f9babef0ba14fbe0be30bd368dc541e2b08d07c8aa908f3" +dependencies = [ + "bitflags 2.10.0", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb" +dependencies = [ + "libc", +] [[package]] name = "is_terminal_polyfill" -version = "1.70.1" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" - -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" [[package]] name = "itertools" @@ -882,10 +970,19 @@ dependencies = [ ] [[package]] -name = "itoa" -version = "1.0.11" +name = "itertools" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" [[package]] name = "jni" @@ -898,7 +995,7 @@ dependencies = [ "combine", "jni-sys", "log", - "thiserror", + "thiserror 1.0.69", "walkdir", "windows-sys 0.45.0", ] @@ -909,149 +1006,120 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" -[[package]] -name = "jobserver" -version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" -dependencies = [ - "libc", -] - [[package]] name = "js-sys" -version = "0.3.72" +version = "0.3.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" +checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" dependencies = [ + "once_cell", "wasm-bindgen", ] [[package]] -name = "lazy_static" -version = "1.5.0" +name = "kqueue" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +checksum = "eac30106d7dce88daf4a3fcb4879ea939476d5074a9b7ddd0fb97fa4bed5596a" +dependencies = [ + "kqueue-sys", + "libc", +] [[package]] -name = "leb128" -version = "0.2.5" +name = "kqueue-sys" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" +checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b" +dependencies = [ + "bitflags 1.3.2", + "libc", +] + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libc" -version = "0.2.161" +version = "0.2.178" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" +checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" [[package]] -name = "libgit2-sys" -version = "0.17.0+1.8.1" +name = "libloading" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10472326a8a6477c3c20a64547b0059e4b0d086869eee31e6d7da728a8eb7224" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" dependencies = [ - "cc", - "libc", - "libssh2-sys", - "libz-sys", - "openssl-sys", - "pkg-config", + "cfg-if", + "windows-link", ] [[package]] name = "libloading" -version = "0.8.5" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-link", ] [[package]] name = "libm" -version = "0.2.11" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" - -[[package]] -name = "libredox" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" -dependencies = [ - "bitflags", - "libc", - "redox_syscall", -] - -[[package]] -name = "libssh2-sys" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dc8a030b787e2119a731f1951d6a773e2280c660f8ec4b0f5e1505a386e71ee" -dependencies = [ - "cc", - "libc", - "libz-sys", - "openssl-sys", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "libz-sys" -version = "1.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" [[package]] name = "linux-raw-sys" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "litemap" -version = "0.7.3" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" [[package]] name = "log" -version = "0.4.22" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "mach2" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" +checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44" dependencies = [ "libc", ] [[package]] name = "memchr" -version = "2.7.4" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" [[package]] name = "memfd" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2cffa4ad52c6f791f4f8b15f0c05f9824b2ced1160e88cc393d64fff9a8ac64" +checksum = "ad38eb12aea514a0466ea40a80fd8cc83637065948eb4a426e4aa46261175227" dependencies = [ - "rustix", + "rustix 1.1.3", ] [[package]] @@ -1060,6 +1128,18 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "mio" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.61.2", +] + [[package]] name = "ndk-context" version = "0.1.1" @@ -1068,11 +1148,11 @@ checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" [[package]] name = "nix" -version = "0.29.0" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags", + "bitflags 2.10.0", "cfg-if", "cfg_aliases", "libc", @@ -1089,116 +1169,152 @@ dependencies = [ ] [[package]] -name = "objc-sys" -version = "0.3.5" +name = "notify" +version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb91bdd390c7ce1a8607f35f3ca7151b65afc0ff5ff3b34fa350f7d7c7e4310" +checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3" +dependencies = [ + "bitflags 2.10.0", + "fsevent-sys", + "inotify", + "kqueue", + "libc", + "log", + "mio", + "notify-types", + "walkdir", + "windows-sys 0.60.2", +] + +[[package]] +name = "notify-debouncer-full" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "375bd3a138be7bfeff3480e4a623df4cbfb55b79df617c055cd810ba466fa078" +dependencies = [ + "file-id", + "log", + "notify", + "notify-types", + "walkdir", +] + +[[package]] +name = "notify-types" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e0826a989adedc2a244799e823aece04662b66609d96af8dff7ac6df9a8925d" [[package]] name = "objc2" -version = "0.5.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46a785d4eeff09c14c487497c162e92766fbb3e4059a71840cecc03d9a50b804" +checksum = "b7c2599ce0ec54857b29ce62166b0ed9b4f6f1a70ccc9a71165b6154caca8c05" dependencies = [ - "objc-sys", "objc2-encode", ] [[package]] name = "objc2-encode" -version = "4.0.3" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7891e71393cd1f227313c9379a26a584ff3d7e6e7159e988851f0934c993f0f8" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" [[package]] name = "objc2-foundation" -version = "0.2.2" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ee638a5da3799329310ad4cfa62fbf045d5f56e3ef5ba4149e7452dcf89d5a8" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ - "bitflags", - "block2", - "libc", + "bitflags 2.10.0", "objc2", ] [[package]] name = "object" -version = "0.36.5" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +dependencies = [ + "memchr", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" dependencies = [ "crc32fast", - "hashbrown 0.15.1", + "hashbrown 0.15.5", "indexmap", "memchr", ] [[package]] name = "once_cell" -version = "1.20.2" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] -name = "openssl-probe" -version = "0.1.5" +name = "once_cell_polyfill" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] -name = "openssl-sys" -version = "0.9.104" +name = "pathdiff" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "option-ext" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" - -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - -[[package]] -name = "path-slash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e91099d4268b0e11973f036e885d652fb0b21fedcf69738c627f94db6a44f42" +checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" [[package]] name = "percent-encoding" -version = "2.3.1" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_shared", + "serde", +] + +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand", + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] [[package]] name = "pin-project-lite" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" - -[[package]] -name = "pkg-config" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "postcard" -version = "1.0.10" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f7f0a8d620d71c457dd1d47df76bb18960378da56af4527aaa10f515eee732e" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" dependencies = [ "cobs", "embedded-io 0.4.0", @@ -1207,10 +1323,19 @@ dependencies = [ ] [[package]] -name = "ppv-lite86" -version = "0.2.20" +name = "potential_utf" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ "zerocopy", ] @@ -1227,52 +1352,68 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.25" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", "syn", ] [[package]] -name = "proc-macro2" -version = "1.0.89" +name = "proc-macro-crate" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" +checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9695f8df41bb4f3d222c95a67532365f569318332d03d5f3f67f37b20e6ebdf0" dependencies = [ "unicode-ident", ] [[package]] name = "psm" -version = "0.1.23" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa37f80ca58604976033fae9515a8a2989fc13797d953f7c04fb8fa36a11f205" +checksum = "d11f2fedc3b7dafdc2851bc52f277377c5473d378859be234bc7ebb593144d01" dependencies = [ + "ar_archive_writer", "cc", ] [[package]] name = "pulley-interpreter" -version = "26.0.1" +version = "33.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df33e7f8a43ccc7f93b330fef4baf271764674926f3f4d40f4a196d54de8af26" +checksum = "986beaef947a51d17b42b0ea18ceaa88450d35b6994737065ed505c39172db71" dependencies = [ "cranelift-bitset", "log", - "sptr", + "wasmtime-math", ] [[package]] name = "quote" -version = "1.0.37" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "rand" version = "0.8.5" @@ -1300,47 +1441,48 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.16", ] [[package]] -name = "redox_syscall" -version = "0.5.7" +name = "ref-cast" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" dependencies = [ - "bitflags", + "ref-cast-impl", ] [[package]] -name = "redox_users" -version = "0.4.6" +name = "ref-cast-impl" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ - "getrandom", - "libredox", - "thiserror", + "proc-macro2", + "quote", + "syn", ] [[package]] name = "regalloc2" -version = "0.10.2" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12908dbeb234370af84d0579b9f68258a0f67e201412dd9a2814e6f45b2fc0f0" +checksum = "5216b1837de2149f8bc8e6d5f88a9326b63b8c836ed58ce4a0a29ec736a59734" dependencies = [ - "hashbrown 0.14.5", + "allocator-api2", + "bumpalo", + "hashbrown 0.15.5", "log", - "rustc-hash 2.0.0", - "slice-group-by", + "rustc-hash", "smallvec", ] [[package]] name = "regex" -version = "1.11.1" +version = "1.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" dependencies = [ "aho-corasick", "memchr", @@ -1350,9 +1492,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.8" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" dependencies = [ "aho-corasick", "memchr", @@ -1361,40 +1503,116 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.5" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] -name = "rustc-hash" -version = "1.1.0" +name = "relative-path" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - -[[package]] -name = "rustc-hash" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" - -[[package]] -name = "rustix" -version = "0.38.39" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "375116bee2be9ed569afe2154ea6a99dfdffd257f533f187498c2a8f5feaf4ee" +checksum = "bca40a312222d8ba74837cb474edef44b37f561da5f773981007a10bbaa992b0" dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys", - "windows-sys 0.52.0", + "serde", ] [[package]] -name = "ryu" -version = "1.0.18" +name = "rgb" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +checksum = "0c6a884d2998352bb4daf0183589aec883f16a6da1f4dde84d8e2e9a5409a1ce" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "rquickjs" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c50dc6d6c587c339edb4769cf705867497a2baf0eca8b4645fa6ecd22f02c77a" +dependencies = [ + "rquickjs-core", + "rquickjs-macro", +] + +[[package]] +name = "rquickjs-core" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8bf7840285c321c3ab20e752a9afb95548c75cd7f4632a0627cea3507e310c1" +dependencies = [ + "hashbrown 0.16.1", + "phf", + "relative-path", + "rquickjs-sys", +] + +[[package]] +name = "rquickjs-macro" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7106215ff41a5677b104906a13e1a440b880f4b6362b5dc4f3978c267fad2b80" +dependencies = [ + "convert_case", + "fnv", + "ident_case", + "indexmap", + "phf_generator", + "phf_shared", + "proc-macro-crate", + "proc-macro2", + "quote", + "rquickjs-core", + "syn", +] + +[[package]] +name = "rquickjs-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27344601ef27460e82d6a4e1ecb9e7e99f518122095f3c51296da8e9be2b9d83" +dependencies = [ + "bindgen", + "cc", +] + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.10.0", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +dependencies = [ + "bitflags 2.10.0", + "errno", + "libc", + "linux-raw-sys 0.11.0", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "same-file" @@ -1406,28 +1624,75 @@ dependencies = [ ] [[package]] -name = "semver" -version = "1.0.23" +name = "schemars" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +checksum = "54e910108742c57a770f492731f99be216a52fadd361b06c8fb59d74ccc267d2" +dependencies = [ + "dyn-clone", + "ref-cast", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4908ad288c5035a8eb12cfdf0d49270def0a268ee162b75eeee0f85d155a7c45" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn", +] + +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" dependencies = [ "serde", + "serde_core", ] [[package]] name = "serde" -version = "1.0.215" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.215" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_derive_internals" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", @@ -1436,31 +1701,23 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.133" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ "indexmap", "itoa", "memchr", - "ryu", - "serde", -] - -[[package]] -name = "serde_spanned" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" -dependencies = [ "serde", + "serde_core", + "zmij", ] [[package]] name = "shell-words" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" +checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" [[package]] name = "shlex" @@ -1470,27 +1727,27 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "similar" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1de1d4f81173b03af4c0cbed3c898f6bff5b870e4a7f5d6f4057d62a7a4b686e" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" [[package]] -name = "slice-group-by" -version = "0.3.1" +name = "siphasher" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" [[package]] name = "smallbitvec" -version = "2.5.3" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcc3fc564a4b53fd1e8589628efafe57602d91bde78be18186b5f61e8faea470" +checksum = "d31d263dd118560e1a492922182ab6ca6dc1d03a3bf54e7699993f31a4150e3f" [[package]] name = "smallvec" -version = "1.13.2" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" dependencies = [ "serde", ] @@ -1503,9 +1760,9 @@ checksum = "3b9b39299b249ad65f3b7e96443bad61c02ca5cd3589f46cb6d610a0fd6c0d6a" [[package]] name = "stable_deref_trait" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "streaming-iterator" @@ -1521,9 +1778,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" -version = "2.0.87" +version = "2.0.112" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" +checksum = "21f182278bf2d2bcb3c88b1b08a37df029d71ce3d3ae26168e3c653b213b99d4" dependencies = [ "proc-macro2", "quote", @@ -1532,9 +1789,9 @@ dependencies = [ [[package]] name = "synstructure" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", @@ -1543,21 +1800,21 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.16" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" +checksum = "b1dd07eb858a2067e2f3c7155d54e929265c264e6f37efe3ee7a8d1b5a1dd0ba" [[package]] name = "tempfile" -version = "3.14.0" +version = "3.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" +checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" dependencies = [ - "cfg-if", "fastrand", + "getrandom 0.3.4", "once_cell", - "rustix", - "windows-sys 0.59.0", + "rustix 1.1.3", + "windows-sys 0.61.2", ] [[package]] @@ -1575,7 +1832,16 @@ version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +dependencies = [ + "thiserror-impl 2.0.17", ] [[package]] @@ -1590,13 +1856,23 @@ dependencies = [ ] [[package]] -name = "thread_local" -version = "1.1.8" +name = "thiserror-impl" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ "cfg-if", - "once_cell", ] [[package]] @@ -1613,53 +1889,55 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.7.6" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" dependencies = [ "displaydoc", "zerovec", ] -[[package]] -name = "toml" -version = "0.8.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit", -] - [[package]] name = "toml_datetime" -version = "0.6.8" +version = "0.7.5+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" dependencies = [ - "serde", + "serde_core", ] [[package]] name = "toml_edit" -version = "0.22.22" +version = "0.23.10+spec-1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +checksum = "84c8b9f757e028cee9fa244aea147aab2a9ec09d5325a9b01e0a49730c2b5269" dependencies = [ "indexmap", - "serde", - "serde_spanned", "toml_datetime", + "toml_parser", "winnow", ] [[package]] -name = "tracing" -version = "0.1.40" +name = "toml_parser" +version = "1.0.6+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "a3198b4b0a8e11f09dd03e133c0280504d0801269e9afa46362ffde1cbeebf44" +dependencies = [ + "winnow", +] + +[[package]] +name = "topological-sort" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea68304e134ecd095ac6c3574494fc62b909f416c4fca77e440530221e549d3d" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ "pin-project-lite", "tracing-attributes", @@ -1668,9 +1946,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.27" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", @@ -1679,21 +1957,22 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.32" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", ] [[package]] name = "tree-sitter" -version = "0.25.0" +version = "0.27.0" dependencies = [ "bindgen", "cc", "regex", "regex-syntax", + "serde_json", "streaming-iterator", "tree-sitter-language", "wasmtime-c-api-impl", @@ -1701,40 +1980,37 @@ dependencies = [ [[package]] name = "tree-sitter-cli" -version = "0.25.0" +version = "0.27.0" dependencies = [ + "ansi_colours", "anstyle", "anyhow", "bstr", "clap", "clap_complete", + "clap_complete_nushell", + "crc32fast", "ctor", "ctrlc", "dialoguer", - "dirs", "encoding_rs", - "filetime", "glob", "heck", "html-escape", - "indexmap", "indoc", - "lazy_static", "log", "memchr", "pretty_assertions", "rand", "regex", - "regex-syntax", - "rustc-hash 2.0.0", + "schemars", "semver", "serde", - "serde_derive", "serde_json", "similar", - "smallbitvec", "streaming-iterator", "tempfile", + "thiserror 2.0.17", "tiny_http", "tree-sitter", "tree-sitter-config", @@ -1744,91 +2020,90 @@ dependencies = [ "tree-sitter-tags", "tree-sitter-tests-proc-macro", "unindent", - "url", "walkdir", - "wasmparser", + "wasmparser 0.243.0", "webbrowser", "widestring", ] [[package]] name = "tree-sitter-config" -version = "0.25.0" +version = "0.27.0" dependencies = [ - "anyhow", - "dirs", + "etcetera", + "log", "serde", "serde_json", + "thiserror 2.0.17", ] [[package]] name = "tree-sitter-generate" -version = "0.25.0" +version = "0.27.0" dependencies = [ - "anyhow", - "heck", + "bitflags 2.10.0", + "dunce", "indexmap", "indoc", - "lazy_static", "log", + "pathdiff", "regex", "regex-syntax", - "rustc-hash 2.0.0", + "rquickjs", + "rustc-hash", "semver", "serde", "serde_json", "smallbitvec", - "tree-sitter", - "url", + "tempfile", + "thiserror 2.0.17", + "topological-sort", ] [[package]] name = "tree-sitter-highlight" -version = "0.25.0" +version = "0.27.0" dependencies = [ - "lazy_static", "regex", "streaming-iterator", - "thiserror", + "thiserror 2.0.17", "tree-sitter", ] [[package]] name = "tree-sitter-language" -version = "0.1.2" +version = "0.1.7" [[package]] name = "tree-sitter-loader" -version = "0.25.0" +version = "0.27.0" dependencies = [ - "anyhow", "cc", - "dirs", + "etcetera", "fs4", "indoc", - "lazy_static", - "libloading", + "libloading 0.9.0", + "log", "once_cell", - "path-slash", "regex", "semver", "serde", "serde_json", "tempfile", + "thiserror 2.0.17", "tree-sitter", "tree-sitter-highlight", "tree-sitter-tags", - "url", ] [[package]] name = "tree-sitter-tags" -version = "0.25.0" +version = "0.27.0" dependencies = [ "memchr", "regex", "streaming-iterator", - "thiserror", + "thiserror 2.0.17", "tree-sitter", ] @@ -1838,39 +2113,38 @@ version = "0.0.0" dependencies = [ "proc-macro2", "quote", - "rand", "syn", ] [[package]] name = "unicode-ident" -version = "1.0.13" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" + +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "unicode-width" -version = "0.1.14" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" - -[[package]] -name = "unicode-xid" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" [[package]] name = "unindent" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" +checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" [[package]] name = "url" -version = "2.5.3" +version = "2.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" +checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" dependencies = [ "form_urlencoded", "idna", @@ -1878,17 +2152,11 @@ dependencies = [ "serde", ] -[[package]] -name = "utf16_iter" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" - [[package]] name = "utf8-width" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3" +checksum = "1292c0d970b54115d14f2492fe0170adf21d68a1de108eebc51c1df4f346a091" [[package]] name = "utf8_iter" @@ -1902,18 +2170,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - [[package]] name = "walkdir" version = "2.5.0" @@ -1926,41 +2182,37 @@ dependencies = [ [[package]] name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" +version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] -name = "wasm-bindgen" -version = "0.2.95" +name = "wasip2" +version = "1.0.1+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" dependencies = [ - "cfg-if", - "once_cell", - "wasm-bindgen-macro", + "wit-bindgen", ] [[package]] -name = "wasm-bindgen-backend" -version = "0.2.95" +name = "wasm-bindgen" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" +checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" dependencies = [ - "bumpalo", - "log", + "cfg-if", "once_cell", - "proc-macro2", - "quote", - "syn", + "rustversion", + "wasm-bindgen-macro", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.95" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" +checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1968,41 +2220,57 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.95" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" +checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" dependencies = [ + "bumpalo", "proc-macro2", "quote", "syn", - "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.95" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" +checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +dependencies = [ + "unicode-ident", +] [[package]] name = "wasm-encoder" -version = "0.218.0" +version = "0.229.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22b896fa8ceb71091ace9bcb81e853f54043183a1c9667cf93422c40252ffa0a" +checksum = "38ba1d491ecacb085a2552025c10a675a6fddcbd03b1fc9b36c536010ce265d2" dependencies = [ - "leb128", + "leb128fmt", + "wasmparser 0.229.0", ] [[package]] name = "wasmparser" -version = "0.218.0" +version = "0.229.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b09e46c7fceceaa72b2dd1a8a137ea7fd8f93dfaa69806010a709918e496c5dc" +checksum = "0cc3b1f053f5d41aa55640a1fa9b6d1b8a9e4418d118ce308d20e24ff3575a8c" dependencies = [ - "ahash", - "bitflags", - "hashbrown 0.14.5", + "bitflags 2.10.0", + "hashbrown 0.15.5", + "indexmap", + "semver", + "serde", +] + +[[package]] +name = "wasmparser" +version = "0.243.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6d8db401b0528ec316dfbe579e6ab4152d61739cfe076706d2009127970159d" +dependencies = [ + "bitflags 2.10.0", + "hashbrown 0.15.5", "indexmap", "semver", "serde", @@ -2010,74 +2278,74 @@ dependencies = [ [[package]] name = "wasmprinter" -version = "0.218.0" +version = "0.229.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ace089155491837b75f474bf47c99073246d1b737393fe722d6dee311595ddc" +checksum = "d25dac01892684a99b8fbfaf670eb6b56edea8a096438c75392daeb83156ae2e" dependencies = [ "anyhow", "termcolor", - "wasmparser", + "wasmparser 0.229.0", ] [[package]] name = "wasmtime" -version = "26.0.1" +version = "33.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e762e163fd305770c6c341df3290f0cabb3c264e7952943018e9a1ced8d917" +checksum = "57373e1d8699662fb791270ac5dfac9da5c14f618ecf940cdb29dc3ad9472a3c" dependencies = [ + "addr2line", "anyhow", - "bitflags", + "bitflags 2.10.0", "bumpalo", "cc", "cfg-if", - "hashbrown 0.14.5", + "hashbrown 0.15.5", "indexmap", "libc", - "libm", "log", "mach2", "memfd", - "object", + "object 0.36.7", "once_cell", - "paste", "postcard", "psm", "pulley-interpreter", - "rustix", + "rustix 1.1.3", "serde", "serde_derive", "smallvec", "sptr", "target-lexicon", - "wasmparser", + "wasmparser 0.229.0", "wasmtime-asm-macros", - "wasmtime-component-macro", "wasmtime-cranelift", "wasmtime-environ", + "wasmtime-fiber", "wasmtime-jit-icache-coherence", + "wasmtime-math", "wasmtime-slab", "wasmtime-versioned-export-macros", + "wasmtime-winch", "windows-sys 0.59.0", ] [[package]] name = "wasmtime-asm-macros" -version = "26.0.1" +version = "33.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63caa7aebb546374e26257a1900fb93579171e7c02514cde26805b9ece3ef812" +checksum = "bd0fc91372865167a695dc98d0d6771799a388a7541d3f34e939d0539d6583de" dependencies = [ "cfg-if", ] [[package]] name = "wasmtime-c-api-impl" -version = "26.0.1" +version = "33.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "956075190611786d56f4a92573d1993c77e6a18926a84be6006ac70c2c6da93d" +checksum = "46db556f1dccdd88e0672bd407162ab0036b72e5eccb0f4398d8251cba32dba1" dependencies = [ "anyhow", "log", - "once_cell", "tracing", "wasmtime", "wasmtime-c-api-macros", @@ -2085,40 +2353,19 @@ dependencies = [ [[package]] name = "wasmtime-c-api-macros" -version = "26.0.1" +version = "33.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3301b09a17a65280543a48e81e9ae5952f53c94e0f1c74aa92bbd80c044d318" +checksum = "315cc6bc8cdc66f296accb26d7625ae64c1c7b6da6f189e8a72ce6594bf7bd36" dependencies = [ "proc-macro2", "quote", ] -[[package]] -name = "wasmtime-component-macro" -version = "26.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61a4b5ce2ad9c15655e830f0eac0c38b8def30c74ecac71f452d3901e491b68" -dependencies = [ - "anyhow", - "proc-macro2", - "quote", - "syn", - "wasmtime-component-util", - "wasmtime-wit-bindgen", - "wit-parser", -] - -[[package]] -name = "wasmtime-component-util" -version = "26.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35e87a1212270dbb84a49af13d82594e00a92769d6952b0ea7fc4366c949f6ad" - [[package]] name = "wasmtime-cranelift" -version = "26.0.1" +version = "33.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cb40dddf38c6a5eefd5ce7c1baf43b00fe44eada11a319fab22e993a960262f" +checksum = "b2bd72f0a6a0ffcc6a184ec86ac35c174e48ea0e97bbae277c8f15f8bf77a566" dependencies = [ "anyhow", "cfg-if", @@ -2128,22 +2375,23 @@ dependencies = [ "cranelift-frontend", "cranelift-native", "gimli", - "itertools 0.12.1", + "itertools 0.14.0", "log", - "object", + "object 0.36.7", + "pulley-interpreter", "smallvec", "target-lexicon", - "thiserror", - "wasmparser", + "thiserror 2.0.17", + "wasmparser 0.229.0", "wasmtime-environ", "wasmtime-versioned-export-macros", ] [[package]] name = "wasmtime-environ" -version = "26.0.1" +version = "33.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8613075e89e94a48c05862243c2b718eef1b9c337f51493ebf951e149a10fa19" +checksum = "e6187bb108a23eb25d2a92aa65d6c89fb5ed53433a319038a2558567f3011ff2" dependencies = [ "anyhow", "cranelift-bitset", @@ -2151,22 +2399,37 @@ dependencies = [ "gimli", "indexmap", "log", - "object", + "object 0.36.7", "postcard", "serde", "serde_derive", "smallvec", "target-lexicon", "wasm-encoder", - "wasmparser", + "wasmparser 0.229.0", "wasmprinter", ] [[package]] -name = "wasmtime-jit-icache-coherence" -version = "26.0.1" +name = "wasmtime-fiber" +version = "33.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da47fba49af72581bc0dc67c8faaf5ee550e6f106e285122a184a675193701a5" +checksum = "dc8965d2128c012329f390e24b8b2758dd93d01bf67e1a1a0dd3d8fd72f56873" +dependencies = [ + "anyhow", + "cc", + "cfg-if", + "rustix 1.1.3", + "wasmtime-asm-macros", + "wasmtime-versioned-export-macros", + "windows-sys 0.59.0", +] + +[[package]] +name = "wasmtime-jit-icache-coherence" +version = "33.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7af0e940cb062a45c0b3f01a926f77da5947149e99beb4e3dd9846d5b8f11619" dependencies = [ "anyhow", "cfg-if", @@ -2175,16 +2438,25 @@ dependencies = [ ] [[package]] -name = "wasmtime-slab" -version = "26.0.1" +name = "wasmtime-math" +version = "33.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "770e10cdefb15f2b6304152978e115bd062753c1ebe7221c0b6b104fa0419ff6" +checksum = "acfca360e719dda9a27e26944f2754ff2fd5bad88e21919c42c5a5f38ddd93cb" +dependencies = [ + "libm", +] + +[[package]] +name = "wasmtime-slab" +version = "33.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e240559cada55c4b24af979d5f6c95e0029f5772f32027ec3c62b258aaff65" [[package]] name = "wasmtime-versioned-export-macros" -version = "26.0.1" +version = "33.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db8efb877c9e5e67239d4553bb44dd2a34ae5cfb728f3cf2c5e64439c6ca6ee7" +checksum = "d0963c1438357a3d8c0efe152b4ef5259846c1cf8b864340270744fe5b3bae5e" dependencies = [ "proc-macro2", "quote", @@ -2192,22 +2464,27 @@ dependencies = [ ] [[package]] -name = "wasmtime-wit-bindgen" -version = "26.0.1" +name = "wasmtime-winch" +version = "33.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bef2a726fd8d1ee9b0144655e16c492dc32eb4c7c9f7e3309fcffe637870933" +checksum = "cbc3b117d03d6eeabfa005a880c5c22c06503bb8820f3aa2e30f0e8d87b6752f" dependencies = [ "anyhow", - "heck", - "indexmap", - "wit-parser", + "cranelift-codegen", + "gimli", + "object 0.36.7", + "target-lexicon", + "wasmparser 0.229.0", + "wasmtime-cranelift", + "wasmtime-environ", + "winch-codegen", ] [[package]] name = "web-sys" -version = "0.3.72" +version = "0.3.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" +checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" dependencies = [ "js-sys", "wasm-bindgen", @@ -2215,13 +2492,11 @@ dependencies = [ [[package]] name = "webbrowser" -version = "1.0.2" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e5f07fb9bc8de2ddfe6b24a71a75430673fd679e568c48b52716cef1cfae923" +checksum = "00f1243ef785213e3a32fa0396093424a3a6ea566f9948497e5a2309261a4c97" dependencies = [ - "block2", "core-foundation", - "home", "jni", "log", "ndk-context", @@ -2233,19 +2508,44 @@ dependencies = [ [[package]] name = "widestring" -version = "1.1.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7219d36b6eac893fa81e84ebe06485e7dcbb616177469b142df14f1f4deb1311" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" [[package]] name = "winapi-util" -version = "0.1.9" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] +[[package]] +name = "winch-codegen" +version = "33.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7914c296fbcef59d1b89a15e82384d34dc9669bc09763f2ef068a28dd3a64ebf" +dependencies = [ + "anyhow", + "cranelift-assembler-x64", + "cranelift-codegen", + "gimli", + "regalloc2", + "smallvec", + "target-lexicon", + "thiserror 2.0.17", + "wasmparser 0.229.0", + "wasmtime-cranelift", + "wasmtime-environ", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + [[package]] name = "windows-sys" version = "0.45.0" @@ -2255,15 +2555,6 @@ dependencies = [ "windows-targets 0.42.2", ] -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", -] - [[package]] name = "windows-sys" version = "0.52.0" @@ -2282,6 +2573,24 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-targets" version = "0.42.2" @@ -2297,21 +2606,6 @@ dependencies = [ "windows_x86_64_msvc 0.42.2", ] -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", -] - [[package]] name = "windows-targets" version = "0.52.6" @@ -2321,139 +2615,156 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -2461,43 +2772,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] -name = "winnow" -version = "0.6.20" +name = "windows_x86_64_msvc" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "winnow" +version = "0.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" dependencies = [ "memchr", ] [[package]] -name = "wit-parser" -version = "0.218.0" +name = "wit-bindgen" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d3d1066ab761b115f97fef2b191090faabcb0f37b555b758d3caf42d4ed9e55" -dependencies = [ - "anyhow", - "id-arena", - "indexmap", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", -] - -[[package]] -name = "write16" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" [[package]] name = "writeable" -version = "0.5.5" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" [[package]] name = "xtask" @@ -2506,15 +2805,17 @@ dependencies = [ "anstyle", "anyhow", "bindgen", - "cc", "clap", - "git2", + "etcetera", "indoc", + "notify", + "notify-debouncer-full", "regex", + "schemars", "semver", - "serde", "serde_json", - "toml", + "tree-sitter-cli", + "tree-sitter-loader", ] [[package]] @@ -2525,11 +2826,10 @@ checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "yoke" -version = "0.7.4" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" dependencies = [ - "serde", "stable_deref_trait", "yoke-derive", "zerofrom", @@ -2537,9 +2837,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.7.4" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", @@ -2549,19 +2849,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.35" +version = "0.8.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" dependencies = [ - "byteorder", "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.35" +version = "0.8.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" dependencies = [ "proc-macro2", "quote", @@ -2570,18 +2869,18 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.4" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.4" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", @@ -2591,15 +2890,26 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.8.1" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] [[package]] name = "zerovec" -version = "0.10.4" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" dependencies = [ "yoke", "zerofrom", @@ -2608,11 +2918,17 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.10.3" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", "syn", ] + +[[package]] +name = "zmij" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9747e91771f56fd7893e1164abd78febd14a670ceec257caad15e051de35f06" diff --git a/Cargo.toml b/Cargo.toml index e20911cc..ca0d644a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,22 +1,26 @@ [workspace] -default-members = ["cli"] +default-members = ["crates/cli"] members = [ - "cli", - "cli/config", - "cli/loader", + "crates/cli", + "crates/config", + "crates/generate", + "crates/highlight", + "crates/loader", + "crates/tags", + "crates/xtask", + "crates/language", "lib", - "lib/language", - "tags", - "highlight", - "xtask", ] resolver = "2" [workspace.package] -version = "0.25.0" -authors = ["Max Brunsfeld "] +version = "0.27.0" +authors = [ + "Max Brunsfeld ", + "Amaan Qureshi ", +] edition = "2021" -rust-version = "1.74.1" +rust-version = "1.85" homepage = "https://tree-sitter.github.io/tree-sitter" repository = "https://github.com/tree-sitter/tree-sitter" license = "MIT" @@ -56,6 +60,8 @@ missing_errors_doc = "allow" missing_panics_doc = "allow" module_name_repetitions = "allow" multiple_crate_versions = "allow" +needless_for_each = "allow" +obfuscated_if_else = "allow" option_if_let_else = "allow" or_fun_call = "allow" range_plus_one = "allow" @@ -72,6 +78,9 @@ unnecessary_wraps = "allow" unused_self = "allow" used_underscore_items = "allow" +[workspace.lints.rust] +mismatched_lifetime_syntaxes = "allow" + [profile.optimize] inherits = "release" strip = true # Automatically strip symbols from the binary. @@ -93,61 +102,62 @@ incremental = true codegen-units = 256 [workspace.dependencies] -anstyle = "1.0.8" -anyhow = "1.0.89" -bstr = "1.11.0" -cc = "1.2.1" -clap = { version = "4.5.21", features = [ +ansi_colours = "1.2.3" +anstyle = "1.0.13" +anyhow = "1.0.100" +bstr = "1.12.0" +cc = "1.2.53" +clap = { version = "4.5.54", features = [ "cargo", "derive", "env", "help", + "string", "unstable-styles", ] } -clap_complete = "4.5.38" -ctor = "0.2.8" -ctrlc = { version = "3.4.5", features = ["termination"] } +clap_complete = "4.5.65" +clap_complete_nushell = "4.5.10" +crc32fast = "1.5.0" +ctor = "0.2.9" +ctrlc = { version = "3.5.0", features = ["termination"] } dialoguer = { version = "0.11.0", features = ["fuzzy-select"] } -dirs = "5.0.1" -filetime = "0.2.25" -fs4 = "0.9.1" -git2 = "0.19.0" -glob = "0.3.1" +etcetera = "0.11.0" +fs4 = "0.12.0" +glob = "0.3.3" heck = "0.5.0" html-escape = "0.2.13" -indexmap = "2.5.0" -indoc = "2.0.5" -lazy_static = "1.5.0" -libloading = "0.8.5" -log = { version = "0.4.22", features = ["std"] } -memchr = "2.7.4" -once_cell = "1.19.0" -path-slash = "0.2.1" +indexmap = "2.12.1" +indoc = "2.0.6" +libloading = "0.9.0" +log = { version = "0.4.28", features = ["std"] } +memchr = "2.7.6" +once_cell = "1.21.3" pretty_assertions = "1.4.1" rand = "0.8.5" -regex = "1.10.6" -regex-syntax = "0.8.4" -rustc-hash = "2.0.0" -semver = { version = "1.0.23", features = ["serde"] } -serde = { version = "1.0.215", features = ["derive"] } -serde_derive = "1.0.210" -serde_json = { version = "1.0.133", features = ["preserve_order"] } -similar = "2.6.0" -smallbitvec = "2.5.3" +regex = "1.11.3" +regex-syntax = "0.8.6" +rustc-hash = "2.1.1" +schemars = "1.0.5" +semver = { version = "1.0.27", features = ["serde"] } +serde = { version = "1.0.219", features = ["derive"] } +serde_json = { version = "1.0.149", features = ["preserve_order"] } +similar = "2.7.0" +smallbitvec = "2.6.0" streaming-iterator = "0.1.9" -tempfile = "3.14.0" -thiserror = "1.0.69" +tempfile = "3.23.0" +thiserror = "2.0.17" tiny_http = "0.12.0" -toml = "0.8.19" -unindent = "0.2.3" -url = { version = "2.5.2", features = ["serde"] } +topological-sort = "0.2.2" +unindent = "0.2.4" walkdir = "2.5.0" -wasmparser = "0.218.0" -webbrowser = "1.0.2" +wasmparser = "0.243.0" +webbrowser = "1.0.5" -tree-sitter = { version = "0.25.0", path = "./lib" } -tree-sitter-generate = { version = "0.25.0", path = "./cli/generate" } -tree-sitter-loader = { version = "0.25.0", path = "./cli/loader" } -tree-sitter-config = { version = "0.25.0", path = "./cli/config" } -tree-sitter-highlight = { version = "0.25.0", path = "./highlight" } -tree-sitter-tags = { version = "0.25.0", path = "./tags" } +tree-sitter = { version = "0.27.0", path = "./lib" } +tree-sitter-generate = { version = "0.27.0", path = "./crates/generate" } +tree-sitter-loader = { version = "0.27.0", path = "./crates/loader" } +tree-sitter-config = { version = "0.27.0", path = "./crates/config" } +tree-sitter-highlight = { version = "0.27.0", path = "./crates/highlight" } +tree-sitter-tags = { version = "0.27.0", path = "./crates/tags" } + +tree-sitter-language = { version = "0.1", path = "./crates/language" } diff --git a/LICENSE b/LICENSE index 451fe1d2..971b81f9 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2018-2024 Max Brunsfeld +Copyright (c) 2018 Max Brunsfeld Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Makefile b/Makefile index 7b3ae4a5..2098d275 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,4 @@ -ifeq ($(OS),Windows_NT) -$(error Windows is not supported) -endif - -VERSION := 0.25.0 +VERSION := 0.27.0 DESCRIPTION := An incremental parsing system for programming tools HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/ @@ -10,6 +6,7 @@ HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/ PREFIX ?= /usr/local INCLUDEDIR ?= $(PREFIX)/include LIBDIR ?= $(PREFIX)/lib +BINDIR ?= $(PREFIX)/bin PCLIBDIR ?= $(LIBDIR)/pkgconfig # collect sources @@ -27,7 +24,7 @@ OBJ := $(SRC:.c=.o) ARFLAGS := rcs CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types override CFLAGS += -std=c11 -fPIC -fvisibility=hidden -override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE +override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE -D_BSD_SOURCE -D_DARWIN_C_SOURCE override CFLAGS += -Ilib/src -Ilib/src/wasm -Ilib/include # ABI versioning @@ -35,20 +32,25 @@ SONAME_MAJOR := $(word 1,$(subst ., ,$(VERSION))) SONAME_MINOR := $(word 2,$(subst ., ,$(VERSION))) # OS-specific bits -ifneq ($(findstring darwin,$(shell $(CC) -dumpmachine)),) +MACHINE := $(shell $(CC) -dumpmachine) + +ifneq ($(findstring darwin,$(MACHINE)),) SOEXT = dylib SOEXTVER_MAJOR = $(SONAME_MAJOR).$(SOEXT) SOEXTVER = $(SONAME_MAJOR).$(SONAME_MINOR).$(SOEXT) LINKSHARED += -dynamiclib -Wl,-install_name,$(LIBDIR)/libtree-sitter.$(SOEXTVER) +else ifneq ($(findstring mingw32,$(MACHINE)),) + SOEXT = dll + LINKSHARED += -s -shared -Wl,--out-implib,libtree-sitter.dll.a else SOEXT = so SOEXTVER_MAJOR = $(SOEXT).$(SONAME_MAJOR) SOEXTVER = $(SOEXT).$(SONAME_MAJOR).$(SONAME_MINOR) LINKSHARED += -shared -Wl,-soname,libtree-sitter.$(SOEXTVER) -endif ifneq ($(filter $(shell uname),FreeBSD NetBSD DragonFly),) PCLIBDIR := $(PREFIX)/libdata/pkgconfig endif +endif all: libtree-sitter.a libtree-sitter.$(SOEXT) tree-sitter.pc @@ -61,6 +63,10 @@ ifneq ($(STRIP),) $(STRIP) $@ endif +ifneq ($(findstring mingw32,$(MACHINE)),) +libtree-sitter.dll.a: libtree-sitter.$(SOEXT) +endif + tree-sitter.pc: lib/tree-sitter.pc.in sed -e 's|@PROJECT_VERSION@|$(VERSION)|' \ -e 's|@CMAKE_INSTALL_LIBDIR@|$(LIBDIR:$(PREFIX)/%=%)|' \ @@ -69,17 +75,27 @@ tree-sitter.pc: lib/tree-sitter.pc.in -e 's|@PROJECT_HOMEPAGE_URL@|$(HOMEPAGE_URL)|' \ -e 's|@CMAKE_INSTALL_PREFIX@|$(PREFIX)|' $< > $@ +shared: libtree-sitter.$(SOEXT) + +static: libtree-sitter.a + clean: - $(RM) $(OBJ) tree-sitter.pc libtree-sitter.a libtree-sitter.$(SOEXT) + $(RM) $(OBJ) tree-sitter.pc libtree-sitter.a libtree-sitter.$(SOEXT) libtree-stitter.dll.a install: all install -d '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter '$(DESTDIR)$(PCLIBDIR)' '$(DESTDIR)$(LIBDIR)' install -m644 lib/include/tree_sitter/api.h '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/api.h install -m644 tree-sitter.pc '$(DESTDIR)$(PCLIBDIR)'/tree-sitter.pc install -m644 libtree-sitter.a '$(DESTDIR)$(LIBDIR)'/libtree-sitter.a +ifneq ($(findstring mingw32,$(MACHINE)),) + install -d '$(DESTDIR)$(BINDIR)' + install -m755 libtree-sitter.dll '$(DESTDIR)$(BINDIR)'/libtree-sitter.dll + install -m755 libtree-sitter.dll.a '$(DESTDIR)$(LIBDIR)'/libtree-sitter.dll.a +else install -m755 libtree-sitter.$(SOEXT) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXTVER) - ln -sf libtree-sitter.$(SOEXTVER) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXTVER_MAJOR) - ln -sf libtree-sitter.$(SOEXTVER_MAJOR) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXT) + cd '$(DESTDIR)$(LIBDIR)' && ln -sf libtree-sitter.$(SOEXTVER) libtree-sitter.$(SOEXTVER_MAJOR) + cd '$(DESTDIR)$(LIBDIR)' && ln -sf libtree-sitter.$(SOEXTVER_MAJOR) libtree-sitter.$(SOEXT) +endif uninstall: $(RM) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.a \ @@ -88,8 +104,9 @@ uninstall: '$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXT) \ '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/api.h \ '$(DESTDIR)$(PCLIBDIR)'/tree-sitter.pc + rmdir '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter -.PHONY: all install uninstall clean +.PHONY: all shared static install uninstall clean ##### Dev targets ##### @@ -99,20 +116,24 @@ test: cargo xtask generate-fixtures cargo xtask test -test_wasm: - cargo xtask generate-fixtures-wasm +test-wasm: + cargo xtask generate-fixtures --wasm cargo xtask test-wasm lint: cargo update --workspace --locked --quiet cargo check --workspace --all-targets - cargo +nightly fmt --all --check - cargo +nightly clippy --workspace --all-targets -- -D warnings + cargo fmt --all --check + cargo clippy --workspace --all-targets -- -D warnings + +lint-web: + npm --prefix lib/binding_web ci + npm --prefix lib/binding_web run lint format: - cargo +nightly fmt --all + cargo fmt --all changelog: @git-cliff --config .github/cliff.toml --prepend CHANGELOG.md --latest --github-token $(shell gh auth token) -.PHONY: test test_wasm lint format changelog +.PHONY: test test-wasm lint format changelog diff --git a/Package.swift b/Package.swift index 86e1ef01..fb6c6e95 100644 --- a/Package.swift +++ b/Package.swift @@ -14,11 +14,21 @@ let package = Package( targets: [ .target(name: "TreeSitter", path: "lib", - sources: ["src/lib.c"], + exclude: [ + "src/unicode/ICU_SHA", + "src/unicode/README.md", + "src/unicode/LICENSE", + "src/wasm/stdlib-symbols.txt", + "src/lib.c", + ], + sources: ["src"], + publicHeadersPath: "include", cSettings: [ .headerSearchPath("src"), .define("_POSIX_C_SOURCE", to: "200112L"), .define("_DEFAULT_SOURCE"), + .define("_BSD_SOURCE"), + .define("_DARWIN_C_SOURCE"), ]), ], cLanguageStandard: .c11 diff --git a/README.md b/README.md index d378215e..b347c880 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,8 @@ Tree-sitter is a parser generator tool and an incremental parsing library. It ca ## Links - [Documentation](https://tree-sitter.github.io) - [Rust binding](lib/binding_rust/README.md) -- [WASM binding](lib/binding_web/README.md) -- [Command-line interface](cli/README.md) +- [Wasm binding](lib/binding_web/README.md) +- [Command-line interface](crates/cli/README.md) [discord]: https://img.shields.io/discord/1063097320771698699?logo=discord&label=discord [matrix]: https://img.shields.io/matrix/tree-sitter-chat%3Amatrix.org?logo=matrix&label=matrix diff --git a/build.zig b/build.zig index 7496b4ac..9bb1e818 100644 --- a/build.zig +++ b/build.zig @@ -1,116 +1,142 @@ const std = @import("std"); pub fn build(b: *std.Build) !void { - const target = b.standardTargetOptions(.{}); - const optimize = b.standardOptimizeOption(.{}); + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); - const wasm = b.option(bool, "enable-wasm", "Enable Wasm support") orelse false; - const shared = b.option(bool, "build-shared", "Build a shared library") orelse false; - const amalgamated = b.option(bool, "amalgamated", "Build using an amalgamated source") orelse false; + const wasm = b.option(bool, "enable-wasm", "Enable Wasm support") orelse false; + const shared = b.option(bool, "build-shared", "Build a shared library") orelse false; + const amalgamated = b.option(bool, "amalgamated", "Build using an amalgamated source") orelse false; - const lib: *std.Build.Step.Compile = if (!shared) b.addStaticLibrary(.{ - .name = "tree-sitter", - .target = target, - .optimize = optimize, - .link_libc = true, - }) else b.addSharedLibrary(.{ - .name = "tree-sitter", - .pic = true, - .target = target, - .optimize = optimize, - .link_libc = true, - }); - - if (amalgamated) { - lib.addCSourceFile(.{ - .file = b.path("lib/src/lib.c"), - .flags = &.{"-std=c11"}, + const lib: *std.Build.Step.Compile = b.addLibrary(.{ + .name = "tree-sitter", + .linkage = if (shared) .dynamic else .static, + .root_module = b.createModule(.{ + .target = target, + .optimize = optimize, + .link_libc = true, + .pic = if (shared) true else null, + }), }); - } else { - lib.addCSourceFiles(.{ - .root = b.path("lib/src"), - .files = try findSourceFiles(b), - .flags = &.{"-std=c11"}, - }); - } - lib.addIncludePath(b.path("lib/include")); - lib.addIncludePath(b.path("lib/src")); - lib.addIncludePath(b.path("lib/src/wasm")); - - lib.root_module.addCMacro("_POSIX_C_SOURCE", "200112L"); - lib.root_module.addCMacro("_DEFAULT_SOURCE", ""); - - if (wasm) { - if (b.lazyDependency(wasmtimeDep(target.result), .{})) |wasmtime| { - lib.root_module.addCMacro("TREE_SITTER_FEATURE_WASM", ""); - lib.addSystemIncludePath(wasmtime.path("include")); - lib.addLibraryPath(wasmtime.path("lib")); - lib.linkSystemLibrary("wasmtime"); + if (amalgamated) { + lib.addCSourceFile(.{ + .file = b.path("lib/src/lib.c"), + .flags = &.{"-std=c11"}, + }); + } else { + const files = try findSourceFiles(b); + defer b.allocator.free(files); + lib.addCSourceFiles(.{ + .root = b.path("lib/src"), + .files = files, + .flags = &.{"-std=c11"}, + }); } - } - lib.installHeadersDirectory(b.path("lib/include"), ".", .{}); + lib.addIncludePath(b.path("lib/include")); + lib.addIncludePath(b.path("lib/src")); + lib.addIncludePath(b.path("lib/src/wasm")); - b.installArtifact(lib); + lib.root_module.addCMacro("_POSIX_C_SOURCE", "200112L"); + lib.root_module.addCMacro("_DEFAULT_SOURCE", ""); + lib.root_module.addCMacro("_BSD_SOURCE", ""); + lib.root_module.addCMacro("_DARWIN_C_SOURCE", ""); + + if (wasm) { + if (b.lazyDependency(wasmtimeDep(target.result), .{})) |wasmtime| { + lib.root_module.addCMacro("TREE_SITTER_FEATURE_WASM", ""); + lib.addSystemIncludePath(wasmtime.path("include")); + lib.addLibraryPath(wasmtime.path("lib")); + if (shared) lib.linkSystemLibrary("wasmtime"); + } + } + + lib.installHeadersDirectory(b.path("lib/include"), ".", .{}); + + b.installArtifact(lib); } -fn wasmtimeDep(target: std.Target) []const u8 { - const arch = target.cpu.arch; - const os = target.os.tag; - const abi = target.abi; - return switch (os) { - .linux => switch (arch) { - .x86_64 => switch (abi) { - .gnu => "wasmtime_c_api_x86_64_linux", - .musl => "wasmtime_c_api_x86_64_musl", - .android => "wasmtime_c_api_x86_64_android", - else => null - }, - .aarch64 => switch (abi) { - .gnu => "wasmtime_c_api_aarch64_linux", - .android => "wasmtime_c_api_aarch64_android", - else => null - }, - .s390x => "wasmtime_c_api_s390x_linux", - .riscv64 => "wasmtime_c_api_riscv64gc_linux", - else => null - }, - .windows => switch (arch) { - .x86_64 => switch (abi) { - .gnu => "wasmtime_c_api_x86_64_mingw", - .msvc => "wasmtime_c_api_x86_64_windows", - else => null - }, - else => null - }, - .macos => switch (arch) { - .x86_64 => "wasmtime_c_api_x86_64_macos", - .aarch64 => "wasmtime_c_api_aarch64_macos", - else => null - }, - else => null - } orelse std.debug.panic( - "Unsupported target for wasmtime: {s}-{s}-{s}", - .{ @tagName(arch), @tagName(os), @tagName(abi) } - ); +/// Get the name of the wasmtime dependency for this target. +pub fn wasmtimeDep(target: std.Target) []const u8 { + const arch = target.cpu.arch; + const os = target.os.tag; + const abi = target.abi; + return @as(?[]const u8, switch (os) { + .linux => switch (arch) { + .x86_64 => switch (abi) { + .gnu => "wasmtime_c_api_x86_64_linux", + .musl => "wasmtime_c_api_x86_64_musl", + .android => "wasmtime_c_api_x86_64_android", + else => null, + }, + .aarch64 => switch (abi) { + .gnu => "wasmtime_c_api_aarch64_linux", + .musl => "wasmtime_c_api_aarch64_musl", + .android => "wasmtime_c_api_aarch64_android", + else => null, + }, + .x86 => switch (abi) { + .gnu => "wasmtime_c_api_i686_linux", + else => null, + }, + .arm => switch (abi) { + .gnueabi => "wasmtime_c_api_armv7_linux", + else => null, + }, + .s390x => switch (abi) { + .gnu => "wasmtime_c_api_s390x_linux", + else => null, + }, + .riscv64 => switch (abi) { + .gnu => "wasmtime_c_api_riscv64gc_linux", + else => null, + }, + else => null, + }, + .windows => switch (arch) { + .x86_64 => switch (abi) { + .gnu => "wasmtime_c_api_x86_64_mingw", + .msvc => "wasmtime_c_api_x86_64_windows", + else => null, + }, + .aarch64 => switch (abi) { + .msvc => "wasmtime_c_api_aarch64_windows", + else => null, + }, + .x86 => switch (abi) { + .msvc => "wasmtime_c_api_i686_windows", + else => null, + }, + else => null, + }, + .macos => switch (arch) { + .x86_64 => "wasmtime_c_api_x86_64_macos", + .aarch64 => "wasmtime_c_api_aarch64_macos", + else => null, + }, + else => null, + }) orelse std.debug.panic( + "Unsupported target for wasmtime: {s}-{s}-{s}", + .{ @tagName(arch), @tagName(os), @tagName(abi) }, + ); } fn findSourceFiles(b: *std.Build) ![]const []const u8 { - var sources = std.ArrayList([]const u8).init(b.allocator); + var sources: std.ArrayListUnmanaged([]const u8) = .empty; - var dir = try b.build_root.handle.openDir("lib/src", .{ .iterate = true }); - var iter = dir.iterate(); - defer dir.close(); + var dir = try b.build_root.handle.openDir("lib/src", .{ .iterate = true }); + var iter = dir.iterate(); + defer dir.close(); - while (try iter.next()) |entry| { - if (entry.kind != .file) continue; - const file = entry.name; - const ext = std.fs.path.extension(file); - if (std.mem.eql(u8, ext, ".c") and !std.mem.eql(u8, file, "lib.c")) { - try sources.append(b.dupe(file)); + while (try iter.next()) |entry| { + if (entry.kind != .file) continue; + const file = entry.name; + const ext = std.fs.path.extension(file); + if (std.mem.eql(u8, ext, ".c") and !std.mem.eql(u8, file, "lib.c")) { + try sources.append(b.allocator, b.dupe(file)); + } } - } - return sources.items; + return sources.toOwnedSlice(b.allocator); } diff --git a/build.zig.zon b/build.zig.zon index 0537744a..4ef5de16 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,69 +1,96 @@ .{ - .name = "tree-sitter", - .version = "0.25.0", - .paths = .{ - "build.zig", - "build.zig.zon", - "lib/src", - "lib/include", - "README.md", - "LICENSE", - }, - .dependencies = .{ - .wasmtime_c_api_aarch64_android = .{ - .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v26.0.1/wasmtime-v26.0.1-aarch64-android-c-api.tar.xz", - .hash = "12208b1c6fc26df81b3bf6b82ba38a2099bcbfb3eea21b93c9cca797d8f0067d891f", - .lazy = true, + .name = .tree_sitter, + .fingerprint = 0x841224b447ac0d4f, + .version = "0.27.0", + .minimum_zig_version = "0.14.1", + .paths = .{ + "build.zig", + "build.zig.zon", + "lib/src", + "lib/include", + "README.md", + "LICENSE", }, - .wasmtime_c_api_aarch64_linux = .{ - .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v26.0.1/wasmtime-v26.0.1-aarch64-linux-c-api.tar.xz", - .hash = "12209aaa1bd480ad8674b8d9cc89300e8b045f0fc626938b64158a09e87597705a45", - .lazy = true, + .dependencies = .{ + .wasmtime_c_api_aarch64_android = .{ + .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-android-c-api.tar.xz", + .hash = "N-V-__8AAIfPIgdw2YnV3QyiFQ2NHdrxrXzzCdjYJyxJDOta", + .lazy = true, + }, + .wasmtime_c_api_aarch64_linux = .{ + .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-linux-c-api.tar.xz", + .hash = "N-V-__8AAIt97QZi7Pf7nNJ2mVY6uxA80Klyuvvtop3pLMRK", + .lazy = true, + }, + .wasmtime_c_api_aarch64_macos = .{ + .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-macos-c-api.tar.xz", + .hash = "N-V-__8AAAO48QQf91w9RmmUDHTja8DrXZA1n6Bmc8waW3qe", + .lazy = true, + }, + .wasmtime_c_api_aarch64_musl = .{ + .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-musl-c-api.tar.xz", + .hash = "N-V-__8AAI196wa9pwADoA2RbCDp5F7bKQg1iOPq6gIh8-FH", + .lazy = true, + }, + .wasmtime_c_api_aarch64_windows = .{ + .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-windows-c-api.zip", + .hash = "N-V-__8AAC9u4wXfqd1Q6XyQaC8_DbQZClXux60Vu5743N05", + .lazy = true, + }, + .wasmtime_c_api_armv7_linux = .{ + .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-armv7-linux-c-api.tar.xz", + .hash = "N-V-__8AAHXe8gWs3s83Cc5G6SIq0_jWxj8fGTT5xG4vb6-x", + .lazy = true, + }, + .wasmtime_c_api_i686_linux = .{ + .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-i686-linux-c-api.tar.xz", + .hash = "N-V-__8AAN2pzgUUfulRCYnipSfis9IIYHoTHVlieLRmKuct", + .lazy = true, + }, + .wasmtime_c_api_i686_windows = .{ + .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-i686-windows-c-api.zip", + .hash = "N-V-__8AAJu0YAUUTFBLxFIOi-MSQVezA6MMkpoFtuaf2Quf", + .lazy = true, + }, + .wasmtime_c_api_riscv64gc_linux = .{ + .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-riscv64gc-linux-c-api.tar.xz", + .hash = "N-V-__8AAG8m-gc3E3AIImtTZ3l1c7HC6HUWazQ9OH5KACX4", + .lazy = true, + }, + .wasmtime_c_api_s390x_linux = .{ + .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-s390x-linux-c-api.tar.xz", + .hash = "N-V-__8AAH314gd-gE4IBp2uvAL3gHeuW1uUZjMiLLeUdXL_", + .lazy = true, + }, + .wasmtime_c_api_x86_64_android = .{ + .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-android-c-api.tar.xz", + .hash = "N-V-__8AAIPNRwfNkznebrcGb0IKUe7f35bkuZEYOjcx6q3f", + .lazy = true, + }, + .wasmtime_c_api_x86_64_linux = .{ + .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-linux-c-api.tar.xz", + .hash = "N-V-__8AAI8EDwcyTtk_Afhk47SEaqfpoRqGkJeZpGs69ChF", + .lazy = true, + }, + .wasmtime_c_api_x86_64_macos = .{ + .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-macos-c-api.tar.xz", + .hash = "N-V-__8AAGtGNgVaOpHSxC22IjrampbRIy6lLwscdcAE8nG1", + .lazy = true, + }, + .wasmtime_c_api_x86_64_mingw = .{ + .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-mingw-c-api.zip", + .hash = "N-V-__8AAPS2PAbVix50L6lnddlgazCPTz3whLUFk1qnRtnZ", + .lazy = true, + }, + .wasmtime_c_api_x86_64_musl = .{ + .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-musl-c-api.tar.xz", + .hash = "N-V-__8AAF-WEQe0nzvi09PgusM5i46FIuCKJmIDWUleWgQ3", + .lazy = true, + }, + .wasmtime_c_api_x86_64_windows = .{ + .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-windows-c-api.zip", + .hash = "N-V-__8AAKGNXwbpJQsn0_6kwSIVDDWifSg8cBzf7T2RzsC9", + .lazy = true, + }, }, - .wasmtime_c_api_aarch64_macos = .{ - .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v26.0.1/wasmtime-v26.0.1-aarch64-macos-c-api.tar.xz", - .hash = "12206de8f3ce815b0cd9fd735fc61ac73f338e7601e973916b06ae050b4fa7118baf", - .lazy = true, - }, - .wasmtime_c_api_riscv64gc_linux = .{ - .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v26.0.1/wasmtime-v26.0.1-riscv64gc-linux-c-api.tar.xz", - .hash = "122005e52855c8be82f574b6f35c1e2f5bc6d74ec1e12f16852654e4edd6ac7e2fc1", - .lazy = true, - }, - .wasmtime_c_api_s390x_linux = .{ - .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v26.0.1/wasmtime-v26.0.1-s390x-linux-c-api.tar.xz", - .hash = "1220a4643445f5e67daffe6473c8e68267682aa92e4d612355b7ac6d46be41d8511e", - .lazy = true, - }, - .wasmtime_c_api_x86_64_android = .{ - .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v26.0.1/wasmtime-v26.0.1-x86_64-android-c-api.tar.xz", - .hash = "122082a6f5db4787a639d8fa587087d3452aa53a92137fef701dfd2be4d62a70102f", - .lazy = true, - }, - .wasmtime_c_api_x86_64_linux = .{ - .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v26.0.1/wasmtime-v26.0.1-x86_64-linux-c-api.tar.xz", - .hash = "12201e8daa6057abd4ce5d25d29a053f4be66a81b695f32f65a14f999bf075ddc0f2", - .lazy = true, - }, - .wasmtime_c_api_x86_64_macos = .{ - .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v26.0.1/wasmtime-v26.0.1-x86_64-macos-c-api.tar.xz", - .hash = "122063a6a6811cf6a3ae6838a61abb66ff4c348447c657a5ed2348c0d310efc2edbb", - .lazy = true, - }, - .wasmtime_c_api_x86_64_mingw = .{ - .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v26.0.1/wasmtime-v26.0.1-x86_64-mingw-c-api.zip", - .hash = "1220bdd5c3711af386ca07795c7ee8917f58365b0bb6b95255424aa86e08a7fcb4fa", - .lazy = true, - }, - .wasmtime_c_api_x86_64_musl = .{ - .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v26.0.1/wasmtime-v26.0.1-x86_64-musl-c-api.tar.xz", - .hash = "12200037419e1a5f8a529d42e0ec289919dc5baf06981bc98295e61df4976563566d", - .lazy = true, - }, - .wasmtime_c_api_x86_64_windows = .{ - .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v26.0.1/wasmtime-v26.0.1-x86_64-windows-c-api.zip", - .hash = "122069341103b7d16b1f47c3bb96101614af0845ba63a0664e5cc857e9feb369a772", - .lazy = true, - }, - } } diff --git a/cli/generate/build.rs b/cli/generate/build.rs deleted file mode 100644 index 6fdbc45b..00000000 --- a/cli/generate/build.rs +++ /dev/null @@ -1,32 +0,0 @@ -use std::{env, path::PathBuf, process::Command}; - -fn main() { - if let Some(git_sha) = read_git_sha() { - println!("cargo:rustc-env=BUILD_SHA={git_sha}"); - } -} - -// This is copied from the build.rs in parent directory. This should be updated if the -// parent build.rs gets fixes. -fn read_git_sha() -> Option { - let crate_path = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); - - if !crate_path - .parent()? - .parent() - .is_some_and(|p| p.join(".git").exists()) - { - return None; - } - - Command::new("git") - .args(["rev-parse", "HEAD"]) - .current_dir(crate_path) - .output() - .map_or(None, |output| { - if !output.status.success() { - return None; - } - Some(String::from_utf8_lossy(&output.stdout).to_string()) - }) -} diff --git a/cli/generate/src/grammar_files.rs b/cli/generate/src/grammar_files.rs deleted file mode 100644 index 8b137891..00000000 --- a/cli/generate/src/grammar_files.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/cli/generate/src/lib.rs b/cli/generate/src/lib.rs deleted file mode 100644 index 14f20672..00000000 --- a/cli/generate/src/lib.rs +++ /dev/null @@ -1,251 +0,0 @@ -use std::{ - env, fs, - io::Write, - path::{Path, PathBuf}, - process::{Command, Stdio}, -}; - -use anyhow::{anyhow, Context, Result}; -use build_tables::build_tables; -use grammars::InputGrammar; -use lazy_static::lazy_static; -use parse_grammar::parse_grammar; -use prepare_grammar::prepare_grammar; -use regex::{Regex, RegexBuilder}; -use render::render_c_code; -use semver::Version; - -mod build_tables; -mod dedup; -mod grammar_files; -mod grammars; -mod nfa; -mod node_types; -pub mod parse_grammar; -mod prepare_grammar; -mod render; -mod rules; -mod tables; - -lazy_static! { - static ref JSON_COMMENT_REGEX: Regex = RegexBuilder::new("^\\s*//.*") - .multi_line(true) - .build() - .unwrap(); -} - -struct GeneratedParser { - c_code: String, - node_types_json: String, -} - -pub const ALLOC_HEADER: &str = include_str!("templates/alloc.h"); -pub const ARRAY_HEADER: &str = include_str!("templates/array.h"); - -pub fn generate_parser_in_directory( - repo_path: &Path, - out_path: Option<&str>, - grammar_path: Option<&str>, - abi_version: usize, - report_symbol_name: Option<&str>, - js_runtime: Option<&str>, -) -> Result<()> { - let mut repo_path = repo_path.to_owned(); - let mut grammar_path = grammar_path; - - // Populate a new empty grammar directory. - if let Some(path) = grammar_path { - let path = PathBuf::from(path); - if !path - .try_exists() - .with_context(|| "Some error with specified path")? - { - fs::create_dir_all(&path)?; - grammar_path = None; - repo_path = path; - } - } - - let grammar_path = grammar_path.map_or_else(|| repo_path.join("grammar.js"), PathBuf::from); - - // Read the grammar file. - let grammar_json = load_grammar_file(&grammar_path, js_runtime)?; - - let src_path = out_path.map_or_else(|| repo_path.join("src"), PathBuf::from); - let header_path = src_path.join("tree_sitter"); - - // Ensure that the output directories exist. - fs::create_dir_all(&src_path)?; - fs::create_dir_all(&header_path)?; - - if grammar_path.file_name().unwrap() != "grammar.json" { - fs::write(src_path.join("grammar.json"), &grammar_json) - .with_context(|| format!("Failed to write grammar.json to {src_path:?}"))?; - } - - // Parse and preprocess the grammar. - let input_grammar = parse_grammar(&grammar_json)?; - - // Generate the parser and related files. - let GeneratedParser { - c_code, - node_types_json, - } = generate_parser_for_grammar_with_opts(&input_grammar, abi_version, report_symbol_name)?; - - write_file(&src_path.join("parser.c"), c_code)?; - write_file(&src_path.join("node-types.json"), node_types_json)?; - write_file(&header_path.join("alloc.h"), ALLOC_HEADER)?; - write_file(&header_path.join("array.h"), ARRAY_HEADER)?; - write_file(&header_path.join("parser.h"), tree_sitter::PARSER_HEADER)?; - - Ok(()) -} - -pub fn generate_parser_for_grammar(grammar_json: &str) -> Result<(String, String)> { - let grammar_json = JSON_COMMENT_REGEX.replace_all(grammar_json, "\n"); - let input_grammar = parse_grammar(&grammar_json)?; - let parser = - generate_parser_for_grammar_with_opts(&input_grammar, tree_sitter::LANGUAGE_VERSION, None)?; - Ok((input_grammar.name, parser.c_code)) -} - -fn generate_parser_for_grammar_with_opts( - input_grammar: &InputGrammar, - abi_version: usize, - report_symbol_name: Option<&str>, -) -> Result { - let (syntax_grammar, lexical_grammar, inlines, simple_aliases) = - prepare_grammar(input_grammar)?; - let variable_info = - node_types::get_variable_info(&syntax_grammar, &lexical_grammar, &simple_aliases)?; - let node_types_json = node_types::generate_node_types_json( - &syntax_grammar, - &lexical_grammar, - &simple_aliases, - &variable_info, - ); - let tables = build_tables( - &syntax_grammar, - &lexical_grammar, - &simple_aliases, - &variable_info, - &inlines, - report_symbol_name, - )?; - let c_code = render_c_code( - &input_grammar.name, - tables, - syntax_grammar, - lexical_grammar, - simple_aliases, - abi_version, - ); - Ok(GeneratedParser { - c_code, - node_types_json: serde_json::to_string_pretty(&node_types_json).unwrap(), - }) -} - -pub fn load_grammar_file(grammar_path: &Path, js_runtime: Option<&str>) -> Result { - if grammar_path.is_dir() { - return Err(anyhow!( - "Path to a grammar file with `.js` or `.json` extension is required" - )); - } - match grammar_path.extension().and_then(|e| e.to_str()) { - Some("js") => Ok(load_js_grammar_file(grammar_path, js_runtime) - .with_context(|| "Failed to load grammar.js")?), - Some("json") => { - Ok(fs::read_to_string(grammar_path).with_context(|| "Failed to load grammar.json")?) - } - _ => Err(anyhow!("Unknown grammar file extension: {grammar_path:?}",)), - } -} - -fn load_js_grammar_file(grammar_path: &Path, js_runtime: Option<&str>) -> Result { - let grammar_path = fs::canonicalize(grammar_path)?; - - #[cfg(windows)] - let grammar_path = url::Url::from_file_path(grammar_path) - .expect("Failed to convert path to URL") - .to_string(); - - let js_runtime = js_runtime.unwrap_or("node"); - - let mut js_command = Command::new(js_runtime); - match js_runtime { - "node" => { - js_command.args(["--input-type=module", "-"]); - } - "bun" => { - js_command.arg("-"); - } - "deno" => { - js_command.args(["run", "--allow-all", "-"]); - } - _ => {} - } - - let mut js_process = js_command - .env("TREE_SITTER_GRAMMAR_PATH", grammar_path) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .spawn() - .with_context(|| format!("Failed to run `{js_runtime}`"))?; - - let mut js_stdin = js_process - .stdin - .take() - .with_context(|| format!("Failed to open stdin for {js_runtime}"))?; - let cli_version = Version::parse(env!("CARGO_PKG_VERSION")) - .with_context(|| "Could not parse this package's version as semver.")?; - write!( - js_stdin, - "globalThis.TREE_SITTER_CLI_VERSION_MAJOR = {}; - globalThis.TREE_SITTER_CLI_VERSION_MINOR = {}; - globalThis.TREE_SITTER_CLI_VERSION_PATCH = {};", - cli_version.major, cli_version.minor, cli_version.patch, - ) - .with_context(|| format!("Failed to write tree-sitter version to {js_runtime}'s stdin"))?; - js_stdin - .write(include_bytes!("./dsl.js")) - .with_context(|| format!("Failed to write grammar dsl to {js_runtime}'s stdin"))?; - drop(js_stdin); - - let output = js_process - .wait_with_output() - .with_context(|| format!("Failed to read output from {js_runtime}"))?; - match output.status.code() { - None => panic!("{js_runtime} process was killed"), - Some(0) => { - let stdout = String::from_utf8(output.stdout) - .with_context(|| format!("Got invalid UTF8 from {js_runtime}"))?; - - let mut grammar_json = &stdout[..]; - - if let Some(pos) = stdout.rfind('\n') { - // If there's a newline, split the last line from the rest of the output - let node_output = &stdout[..pos]; - grammar_json = &stdout[pos + 1..]; - - let mut stdout = std::io::stdout().lock(); - stdout.write_all(node_output.as_bytes())?; - stdout.write_all(b"\n")?; - stdout.flush()?; - } - - Ok(serde_json::to_string_pretty( - &serde_json::from_str::(grammar_json) - .with_context(|| "Failed to parse grammar JSON")?, - ) - .with_context(|| "Failed to serialize grammar JSON")? - + "\n") - } - Some(code) => Err(anyhow!("{js_runtime} process exited with status {code}")), - } -} - -pub fn write_file(path: &Path, body: impl AsRef<[u8]>) -> Result<()> { - fs::write(path, body) - .with_context(|| format!("Failed to write {:?}", path.file_name().unwrap())) -} diff --git a/cli/generate/src/parse_grammar.rs b/cli/generate/src/parse_grammar.rs deleted file mode 100644 index fedb6382..00000000 --- a/cli/generate/src/parse_grammar.rs +++ /dev/null @@ -1,343 +0,0 @@ -use std::collections::HashSet; - -use anyhow::{anyhow, Result}; -use serde::Deserialize; -use serde_json::{Map, Value}; - -use super::{ - grammars::{InputGrammar, PrecedenceEntry, Variable, VariableType}, - rules::{Precedence, Rule}, -}; - -#[derive(Deserialize)] -#[serde(tag = "type")] -#[allow(non_camel_case_types)] -#[allow(clippy::upper_case_acronyms)] -enum RuleJSON { - ALIAS { - content: Box, - named: bool, - value: String, - }, - BLANK, - STRING { - value: String, - }, - PATTERN { - value: String, - flags: Option, - }, - SYMBOL { - name: String, - }, - CHOICE { - members: Vec, - }, - FIELD { - name: String, - content: Box, - }, - SEQ { - members: Vec, - }, - REPEAT { - content: Box, - }, - REPEAT1 { - content: Box, - }, - PREC_DYNAMIC { - value: i32, - content: Box, - }, - PREC_LEFT { - value: PrecedenceValueJSON, - content: Box, - }, - PREC_RIGHT { - value: PrecedenceValueJSON, - content: Box, - }, - PREC { - value: PrecedenceValueJSON, - content: Box, - }, - TOKEN { - content: Box, - }, - IMMEDIATE_TOKEN { - content: Box, - }, -} - -#[derive(Deserialize)] -#[serde(untagged)] -enum PrecedenceValueJSON { - Integer(i32), - Name(String), -} - -#[derive(Deserialize)] -pub struct GrammarJSON { - pub name: String, - rules: Map, - #[serde(default)] - precedences: Vec>, - #[serde(default)] - conflicts: Vec>, - #[serde(default)] - externals: Vec, - #[serde(default)] - extras: Vec, - #[serde(default)] - inline: Vec, - #[serde(default)] - supertypes: Vec, - word: Option, -} - -fn rule_is_referenced(rule: &Rule, target: &str) -> bool { - match rule { - Rule::NamedSymbol(name) => name == target, - Rule::Choice(rules) | Rule::Seq(rules) => { - rules.iter().any(|r| rule_is_referenced(r, target)) - } - Rule::Metadata { rule, .. } => rule_is_referenced(rule, target), - Rule::Repeat(inner) => rule_is_referenced(inner, target), - Rule::Blank | Rule::String(_) | Rule::Pattern(_, _) | Rule::Symbol(_) => false, - } -} - -fn variable_is_used( - grammar_rules: &[(String, Rule)], - other_rules: (&[Rule], &[Rule]), - target_name: &str, - in_progress: &mut HashSet, -) -> bool { - let root = &grammar_rules.first().unwrap().0; - if target_name == root { - return true; - } - - if other_rules - .0 - .iter() - .chain(other_rules.1.iter()) - .any(|rule| rule_is_referenced(rule, target_name)) - { - return true; - } - - in_progress.insert(target_name.to_string()); - let result = grammar_rules - .iter() - .filter(|(key, _)| *key != target_name) - .any(|(name, rule)| { - if !rule_is_referenced(rule, target_name) || in_progress.contains(name) { - return false; - } - variable_is_used(grammar_rules, other_rules, name, in_progress) - }); - in_progress.remove(target_name); - - result -} - -pub(crate) fn parse_grammar(input: &str) -> Result { - let mut grammar_json = serde_json::from_str::(input)?; - - let mut extra_symbols = - grammar_json - .extras - .into_iter() - .try_fold(Vec::new(), |mut acc, item| { - let rule = parse_rule(item); - if let Rule::String(ref value) = rule { - if value.is_empty() { - return Err(anyhow!( - "Rules in the `extras` array must not contain empty strings" - )); - } - } - acc.push(rule); - Ok(acc) - })?; - - let mut external_tokens = grammar_json - .externals - .into_iter() - .map(parse_rule) - .collect::>(); - - let mut precedence_orderings = Vec::with_capacity(grammar_json.precedences.len()); - for list in grammar_json.precedences { - let mut ordering = Vec::with_capacity(list.len()); - for entry in list { - ordering.push(match entry { - RuleJSON::STRING { value } => PrecedenceEntry::Name(value), - RuleJSON::SYMBOL { name } => PrecedenceEntry::Symbol(name), - _ => { - return Err(anyhow!( - "Invalid rule in precedences array. Only strings and symbols are allowed" - )) - } - }); - } - precedence_orderings.push(ordering); - } - - let mut variables = Vec::with_capacity(grammar_json.rules.len()); - - let rules = grammar_json - .rules - .into_iter() - .map(|(n, r)| Ok((n, parse_rule(serde_json::from_value(r)?)))) - .collect::>>()?; - - let mut in_progress = HashSet::new(); - - for (name, rule) in &rules { - if !variable_is_used( - &rules, - (&extra_symbols, &external_tokens), - name, - &mut in_progress, - ) && grammar_json.word.as_ref().is_some_and(|w| w != name) - { - grammar_json.conflicts.retain(|r| !r.contains(name)); - grammar_json.supertypes.retain(|r| r != name); - grammar_json.inline.retain(|r| r != name); - extra_symbols.retain(|r| !rule_is_referenced(r, name)); - external_tokens.retain(|r| !rule_is_referenced(r, name)); - precedence_orderings.retain(|r| { - !r.iter().any(|e| { - let PrecedenceEntry::Symbol(s) = e else { - return false; - }; - s == name - }) - }); - continue; - } - variables.push(Variable { - name: name.clone(), - kind: VariableType::Named, - rule: rule.clone(), - }); - } - - Ok(InputGrammar { - name: grammar_json.name, - word_token: grammar_json.word, - expected_conflicts: grammar_json.conflicts, - supertype_symbols: grammar_json.supertypes, - variables_to_inline: grammar_json.inline, - precedence_orderings, - variables, - extra_symbols, - external_tokens, - }) -} - -fn parse_rule(json: RuleJSON) -> Rule { - match json { - RuleJSON::ALIAS { - content, - value, - named, - } => Rule::alias(parse_rule(*content), value, named), - RuleJSON::BLANK => Rule::Blank, - RuleJSON::STRING { value } => Rule::String(value), - RuleJSON::PATTERN { value, flags } => Rule::Pattern( - value, - flags.map_or(String::new(), |f| { - f.matches(|c| { - if c == 'i' { - true - } else { - // silently ignore unicode flags - if c != 'u' && c != 'v' { - eprintln!("Warning: unsupported flag {c}"); - } - false - } - }) - .collect() - }), - ), - RuleJSON::SYMBOL { name } => Rule::NamedSymbol(name), - RuleJSON::CHOICE { members } => Rule::choice(members.into_iter().map(parse_rule).collect()), - RuleJSON::FIELD { content, name } => Rule::field(name, parse_rule(*content)), - RuleJSON::SEQ { members } => Rule::seq(members.into_iter().map(parse_rule).collect()), - RuleJSON::REPEAT1 { content } => Rule::repeat(parse_rule(*content)), - RuleJSON::REPEAT { content } => { - Rule::choice(vec![Rule::repeat(parse_rule(*content)), Rule::Blank]) - } - RuleJSON::PREC { value, content } => Rule::prec(value.into(), parse_rule(*content)), - RuleJSON::PREC_LEFT { value, content } => { - Rule::prec_left(value.into(), parse_rule(*content)) - } - RuleJSON::PREC_RIGHT { value, content } => { - Rule::prec_right(value.into(), parse_rule(*content)) - } - RuleJSON::PREC_DYNAMIC { value, content } => { - Rule::prec_dynamic(value, parse_rule(*content)) - } - RuleJSON::TOKEN { content } => Rule::token(parse_rule(*content)), - RuleJSON::IMMEDIATE_TOKEN { content } => Rule::immediate_token(parse_rule(*content)), - } -} - -impl From for Precedence { - fn from(val: PrecedenceValueJSON) -> Self { - match val { - PrecedenceValueJSON::Integer(i) => Self::Integer(i), - PrecedenceValueJSON::Name(i) => Self::Name(i), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_parse_grammar() { - let grammar = parse_grammar( - r#"{ - "name": "my_lang", - "rules": { - "file": { - "type": "REPEAT1", - "content": { - "type": "SYMBOL", - "name": "statement" - } - }, - "statement": { - "type": "STRING", - "value": "foo" - } - } - }"#, - ) - .unwrap(); - - assert_eq!(grammar.name, "my_lang"); - assert_eq!( - grammar.variables, - vec![ - Variable { - name: "file".to_string(), - kind: VariableType::Named, - rule: Rule::repeat(Rule::NamedSymbol("statement".to_string())) - }, - Variable { - name: "statement".to_string(), - kind: VariableType::Named, - rule: Rule::String("foo".to_string()) - }, - ] - ); - } -} diff --git a/cli/loader/emscripten-version b/cli/loader/emscripten-version deleted file mode 100644 index 96cef7dd..00000000 --- a/cli/loader/emscripten-version +++ /dev/null @@ -1 +0,0 @@ -3.1.64 \ No newline at end of file diff --git a/cli/src/init.rs b/cli/src/init.rs deleted file mode 100644 index 819842c8..00000000 --- a/cli/src/init.rs +++ /dev/null @@ -1,992 +0,0 @@ -use std::{ - fs, - path::{Path, PathBuf}, - str::{self, FromStr}, -}; - -use anyhow::{anyhow, Context, Result}; -use heck::{ToKebabCase, ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase}; -use regex::Regex; -use semver::Version; -use serde::{Deserialize, Serialize}; -use serde_json::{Map, Value}; -use tree_sitter_generate::write_file; -use tree_sitter_loader::{ - Author, Bindings, Grammar, Links, Metadata, PackageJSON, PackageJSONAuthor, - PackageJSONRepository, PathsJSON, TreeSitterJSON, -}; -use url::Url; - -const CLI_VERSION: &str = env!("CARGO_PKG_VERSION"); -const CLI_VERSION_PLACEHOLDER: &str = "CLI_VERSION"; - -const ABI_VERSION_MAX: usize = tree_sitter::LANGUAGE_VERSION; -const ABI_VERSION_MAX_PLACEHOLDER: &str = "ABI_VERSION_MAX"; - -const PARSER_NAME_PLACEHOLDER: &str = "PARSER_NAME"; -const CAMEL_PARSER_NAME_PLACEHOLDER: &str = "CAMEL_PARSER_NAME"; -const UPPER_PARSER_NAME_PLACEHOLDER: &str = "UPPER_PARSER_NAME"; -const LOWER_PARSER_NAME_PLACEHOLDER: &str = "LOWER_PARSER_NAME"; - -const PARSER_DESCRIPTION_PLACEHOLDER: &str = "PARSER_DESCRIPTION"; -const PARSER_LICENSE_PLACEHOLDER: &str = "PARSER_LICENSE"; -const PARSER_URL_PLACEHOLDER: &str = "PARSER_URL"; -const PARSER_URL_STRIPPED_PLACEHOLDER: &str = "PARSER_URL_STRIPPED"; -const PARSER_VERSION_PLACEHOLDER: &str = "PARSER_VERSION"; - -const AUTHOR_NAME_PLACEHOLDER: &str = "PARSER_AUTHOR_NAME"; -const AUTHOR_EMAIL_PLACEHOLDER: &str = "PARSER_AUTHOR_EMAIL"; -const AUTHOR_URL_PLACEHOLDER: &str = "PARSER_AUTHOR_URL"; - -const AUTHOR_BLOCK_JS: &str = "\n \"author\": {"; -const AUTHOR_NAME_PLACEHOLDER_JS: &str = "\n \"name\": \"PARSER_AUTHOR_NAME\","; -const AUTHOR_EMAIL_PLACEHOLDER_JS: &str = ",\n \"email\": \"PARSER_AUTHOR_EMAIL\""; -const AUTHOR_URL_PLACEHOLDER_JS: &str = ",\n \"url\": \"PARSER_AUTHOR_URL\""; - -const AUTHOR_BLOCK_PY: &str = "\nauthors = [{"; -const AUTHOR_NAME_PLACEHOLDER_PY: &str = "name = \"PARSER_AUTHOR_NAME\""; -const AUTHOR_EMAIL_PLACEHOLDER_PY: &str = ", email = \"PARSER_AUTHOR_EMAIL\""; - -const AUTHOR_BLOCK_RS: &str = "\nauthors = ["; -const AUTHOR_NAME_PLACEHOLDER_RS: &str = "PARSER_AUTHOR_NAME"; -const AUTHOR_EMAIL_PLACEHOLDER_RS: &str = " PARSER_AUTHOR_EMAIL"; - -const AUTHOR_BLOCK_GRAMMAR: &str = "\n * @author "; -const AUTHOR_NAME_PLACEHOLDER_GRAMMAR: &str = "PARSER_AUTHOR_NAME"; -const AUTHOR_EMAIL_PLACEHOLDER_GRAMMAR: &str = " PARSER_AUTHOR_EMAIL"; - -const GRAMMAR_JS_TEMPLATE: &str = include_str!("./templates/grammar.js"); -const PACKAGE_JSON_TEMPLATE: &str = include_str!("./templates/package.json"); -const GITIGNORE_TEMPLATE: &str = include_str!("./templates/gitignore"); -const GITATTRIBUTES_TEMPLATE: &str = include_str!("./templates/gitattributes"); -const EDITORCONFIG_TEMPLATE: &str = include_str!("./templates/.editorconfig"); - -const RUST_BINDING_VERSION: &str = env!("CARGO_PKG_VERSION"); -const RUST_BINDING_VERSION_PLACEHOLDER: &str = "RUST_BINDING_VERSION"; - -const LIB_RS_TEMPLATE: &str = include_str!("./templates/lib.rs"); -const BUILD_RS_TEMPLATE: &str = include_str!("./templates/build.rs"); -const CARGO_TOML_TEMPLATE: &str = include_str!("./templates/_cargo.toml"); - -const INDEX_JS_TEMPLATE: &str = include_str!("./templates/index.js"); -const INDEX_D_TS_TEMPLATE: &str = include_str!("./templates/index.d.ts"); -const JS_BINDING_CC_TEMPLATE: &str = include_str!("./templates/js-binding.cc"); -const BINDING_GYP_TEMPLATE: &str = include_str!("./templates/binding.gyp"); -const BINDING_TEST_JS_TEMPLATE: &str = include_str!("./templates/binding_test.js"); - -const MAKEFILE_TEMPLATE: &str = include_str!("./templates/makefile"); -const CMAKELISTS_TXT_TEMPLATE: &str = include_str!("./templates/cmakelists.cmake"); -const PARSER_NAME_H_TEMPLATE: &str = include_str!("./templates/PARSER_NAME.h"); -const PARSER_NAME_PC_IN_TEMPLATE: &str = include_str!("./templates/PARSER_NAME.pc.in"); - -const GO_MOD_TEMPLATE: &str = include_str!("./templates/go.mod"); -const BINDING_GO_TEMPLATE: &str = include_str!("./templates/binding.go"); -const BINDING_TEST_GO_TEMPLATE: &str = include_str!("./templates/binding_test.go"); - -const SETUP_PY_TEMPLATE: &str = include_str!("./templates/setup.py"); -const INIT_PY_TEMPLATE: &str = include_str!("./templates/__init__.py"); -const INIT_PYI_TEMPLATE: &str = include_str!("./templates/__init__.pyi"); -const PYPROJECT_TOML_TEMPLATE: &str = include_str!("./templates/pyproject.toml"); -const PY_BINDING_C_TEMPLATE: &str = include_str!("./templates/py-binding.c"); -const TEST_BINDING_PY_TEMPLATE: &str = include_str!("./templates/test_binding.py"); - -const PACKAGE_SWIFT_TEMPLATE: &str = include_str!("./templates/package.swift"); -const TESTS_SWIFT_TEMPLATE: &str = include_str!("./templates/tests.swift"); - -const TREE_SITTER_JSON_SCHEMA: &str = - "https://tree-sitter.github.io/tree-sitter/assets/schemas/config.schema.json"; - -#[must_use] -pub fn path_in_ignore(repo_path: &Path) -> bool { - [ - "bindings", - "build", - "examples", - "node_modules", - "queries", - "script", - "src", - "target", - "test", - "types", - ] - .iter() - .any(|dir| repo_path.ends_with(dir)) -} - -#[derive(Serialize, Deserialize, Clone)] -pub struct JsonConfigOpts { - pub name: String, - pub camelcase: String, - pub description: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub repository: Option, - pub scope: String, - pub file_types: Vec, - pub version: Version, - pub license: String, - pub author: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub email: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub url: Option, -} - -impl JsonConfigOpts { - #[must_use] - pub fn to_tree_sitter_json(self) -> TreeSitterJSON { - TreeSitterJSON { - schema: Some(TREE_SITTER_JSON_SCHEMA.to_string()), - grammars: vec![Grammar { - name: self.name.clone(), - camelcase: Some(self.camelcase), - scope: self.scope, - path: None, - external_files: PathsJSON::Empty, - file_types: Some(self.file_types), - highlights: PathsJSON::Empty, - injections: PathsJSON::Empty, - locals: PathsJSON::Empty, - tags: PathsJSON::Empty, - injection_regex: Some(format!("^{}$", self.name)), - first_line_regex: None, - content_regex: None, - }], - metadata: Metadata { - version: self.version, - license: Some(self.license), - description: Some(self.description), - authors: Some(vec![Author { - name: self.author, - email: self.email, - url: self.url.map(|url| url.to_string()), - }]), - links: Some(Links { - repository: self.repository.unwrap_or_else(|| { - Url::parse(&format!( - "https://github.com/tree-sitter/tree-sitter-{}", - self.name - )) - .expect("Failed to parse default repository URL") - }), - homepage: None, - }), - namespace: None, - }, - bindings: Bindings::default(), - } - } -} - -impl Default for JsonConfigOpts { - fn default() -> Self { - Self { - name: String::new(), - camelcase: String::new(), - description: String::new(), - repository: None, - scope: String::new(), - file_types: vec![], - version: Version::from_str("0.1.0").unwrap(), - license: String::new(), - author: String::new(), - email: None, - url: None, - } - } -} - -struct GenerateOpts<'a> { - author_name: Option<&'a str>, - author_email: Option<&'a str>, - author_url: Option<&'a str>, - license: Option<&'a str>, - description: Option<&'a str>, - repository: Option<&'a str>, - version: &'a Version, - camel_parser_name: &'a str, -} - -// TODO: remove in 0.25 -// A return value of true means migration was successful, and false if not. -pub fn migrate_package_json(repo_path: &Path) -> Result { - let root_path = - get_root_path(&repo_path.join("package.json")).unwrap_or_else(|_| repo_path.to_path_buf()); - let (package_json_path, tree_sitter_json_path) = ( - root_path.join("package.json"), - root_path.join("tree-sitter.json"), - ); - - let old_config = serde_json::from_str::( - &fs::read_to_string(&package_json_path) - .with_context(|| format!("Failed to read package.json in {}", root_path.display()))?, - )?; - - if old_config.tree_sitter.is_none() { - eprintln!("Failed to find `tree-sitter` section in package.json, unable to migrate"); - return Ok(false); - } - - let name = old_config.name.replace("tree-sitter-", ""); - - let new_config = TreeSitterJSON { - schema: Some(TREE_SITTER_JSON_SCHEMA.to_string()), - grammars: old_config - .tree_sitter - .unwrap() - .into_iter() - .map(|l| Grammar { - name: name.clone(), - camelcase: Some(name.to_upper_camel_case()), - scope: l.scope.unwrap_or_else(|| format!("source.{name}")), - path: Some(l.path), - external_files: l.external_files, - file_types: l.file_types, - highlights: l.highlights, - injections: l.injections, - locals: l.locals, - tags: l.tags, - injection_regex: l.injection_regex, - first_line_regex: l.first_line_regex, - content_regex: l.content_regex, - }) - .collect(), - metadata: Metadata { - version: old_config.version, - license: old_config - .license - .map_or_else(|| Some("MIT".to_string()), Some), - description: old_config - .description - .map_or_else(|| Some(format!("{name} grammar for tree-sitter")), Some), - authors: { - let authors = old_config - .author - .map_or_else(|| vec![].into_iter(), |a| vec![a].into_iter()) - .chain(old_config.maintainers.unwrap_or_default()) - .filter_map(|a| match a { - PackageJSONAuthor::String(s) => { - let mut name = s.trim().to_string(); - if name.is_empty() { - return None; - } - - let mut email = None; - let mut url = None; - - if let Some(url_start) = name.rfind('(') { - if let Some(url_end) = name.rfind(')') { - url = Some(name[url_start + 1..url_end].trim().to_string()); - name = name[..url_start].trim().to_string(); - } - } - - if let Some(email_start) = name.rfind('<') { - if let Some(email_end) = name.rfind('>') { - email = - Some(name[email_start + 1..email_end].trim().to_string()); - name = name[..email_start].trim().to_string(); - } - } - - Some(Author { name, email, url }) - } - PackageJSONAuthor::Object { name, email, url } => { - if name.is_empty() { - None - } else { - Some(Author { name, email, url }) - } - } - }) - .collect::>(); - if authors.is_empty() { - None - } else { - Some(authors) - } - }, - links: Some(Links { - repository: old_config - .repository - .map(|r| match r { - PackageJSONRepository::String(s) => { - if let Some(stripped) = s.strip_prefix("github:") { - Url::parse(&format!("https://github.com/{stripped}")) - } else if Regex::new(r"^[\w.-]+/[\w.-]+$").unwrap().is_match(&s) { - Url::parse(&format!("https://github.com/{s}")) - } else if let Some(stripped) = s.strip_prefix("gitlab:") { - Url::parse(&format!("https://gitlab.com/{stripped}")) - } else if let Some(stripped) = s.strip_prefix("bitbucket:") { - Url::parse(&format!("https://bitbucket.org/{stripped}")) - } else { - Url::parse(&s) - } - } - PackageJSONRepository::Object { url, .. } => Url::parse(&url), - }) - .transpose()? - .unwrap_or_else(|| { - Url::parse(&format!( - "https://github.com/tree-sitter/tree-sitter-{name}" - )) - .expect("Failed to parse default repository URL") - }), - homepage: None, - }), - namespace: None, - }, - bindings: Bindings::default(), - }; - - write_file( - &tree_sitter_json_path, - serde_json::to_string_pretty(&new_config)? + "\n", - )?; - - // Remove the `tree-sitter` field in-place - let mut package_json = serde_json::from_str::>( - &fs::read_to_string(&package_json_path) - .with_context(|| format!("Failed to read package.json in {}", root_path.display()))?, - ) - .unwrap(); - package_json.remove("tree-sitter"); - write_file( - &root_path.join("package.json"), - serde_json::to_string_pretty(&package_json)? + "\n", - )?; - - println!("Warning: your package.json's `tree-sitter` field has been automatically migrated to the new `tree-sitter.json` config file"); - println!( - "For more information, visit https://tree-sitter.github.io/tree-sitter/creating-parsers" - ); - - Ok(true) -} - -pub fn generate_grammar_files( - repo_path: &Path, - language_name: &str, - allow_update: bool, - opts: Option<&JsonConfigOpts>, -) -> Result<()> { - let dashed_language_name = language_name.to_kebab_case(); - - let tree_sitter_config = missing_path_else( - repo_path.join("tree-sitter.json"), - true, - |path| { - // invariant: opts is always Some when `tree-sitter.json` doesn't exist - let Some(opts) = opts else { unreachable!() }; - - let tree_sitter_json = opts.clone().to_tree_sitter_json(); - write_file(path, serde_json::to_string_pretty(&tree_sitter_json)?) - }, - |path| { - // updating the config, if needed - if let Some(opts) = opts { - let tree_sitter_json = opts.clone().to_tree_sitter_json(); - write_file(path, serde_json::to_string_pretty(&tree_sitter_json)?)?; - } - Ok(()) - }, - )?; - - let tree_sitter_config = serde_json::from_str::( - &fs::read_to_string(tree_sitter_config.as_path()) - .with_context(|| "Failed to read tree-sitter.json")?, - )?; - - let authors = tree_sitter_config.metadata.authors.as_ref(); - let camel_name = tree_sitter_config.grammars[0] - .camelcase - .clone() - .unwrap_or_else(|| language_name.to_upper_camel_case()); - - let generate_opts = GenerateOpts { - author_name: authors - .map(|a| a.first().map(|a| a.name.as_str())) - .unwrap_or_default(), - author_email: authors - .map(|a| a.first().and_then(|a| a.email.as_deref())) - .unwrap_or_default(), - author_url: authors - .map(|a| a.first().and_then(|a| a.url.as_deref())) - .unwrap_or_default(), - license: tree_sitter_config.metadata.license.as_deref(), - description: tree_sitter_config.metadata.description.as_deref(), - repository: tree_sitter_config - .metadata - .links - .as_ref() - .map(|l| l.repository.as_str()), - version: &tree_sitter_config.metadata.version, - camel_parser_name: &camel_name, - }; - - // Create package.json - missing_path(repo_path.join("package.json"), |path| { - generate_file( - path, - PACKAGE_JSON_TEMPLATE, - dashed_language_name.as_str(), - &generate_opts, - ) - })?; - - // Do not create a grammar.js file in a repo with multiple language configs - if !tree_sitter_config.has_multiple_language_configs() { - missing_path(repo_path.join("grammar.js"), |path| { - generate_file(path, GRAMMAR_JS_TEMPLATE, language_name, &generate_opts) - })?; - } - - // Write .gitignore file - missing_path(repo_path.join(".gitignore"), |path| { - generate_file(path, GITIGNORE_TEMPLATE, language_name, &generate_opts) - })?; - - // Write .gitattributes file - missing_path(repo_path.join(".gitattributes"), |path| { - generate_file(path, GITATTRIBUTES_TEMPLATE, language_name, &generate_opts) - })?; - - // Write .editorconfig file - missing_path(repo_path.join(".editorconfig"), |path| { - generate_file(path, EDITORCONFIG_TEMPLATE, language_name, &generate_opts) - })?; - - let bindings_dir = repo_path.join("bindings"); - - // Generate Rust bindings - if tree_sitter_config.bindings.rust { - missing_path(bindings_dir.join("rust"), create_dir)?.apply(|path| { - missing_path(path.join("lib.rs"), |path| { - generate_file(path, LIB_RS_TEMPLATE, language_name, &generate_opts) - })?; - - missing_path(path.join("build.rs"), |path| { - generate_file(path, BUILD_RS_TEMPLATE, language_name, &generate_opts) - })?; - - missing_path(repo_path.join("Cargo.toml"), |path| { - generate_file( - path, - CARGO_TOML_TEMPLATE, - dashed_language_name.as_str(), - &generate_opts, - ) - })?; - - Ok(()) - })?; - } - - // Generate Node bindings - if tree_sitter_config.bindings.node { - missing_path(bindings_dir.join("node"), create_dir)?.apply(|path| { - missing_path_else( - path.join("index.js"), - allow_update, - |path| generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts), - |path| { - let contents = fs::read_to_string(path)?; - if !contents.contains("bun") { - generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts)?; - } - Ok(()) - }, - )?; - - missing_path(path.join("index.d.ts"), |path| { - generate_file(path, INDEX_D_TS_TEMPLATE, language_name, &generate_opts) - })?; - - missing_path(path.join("binding_test.js"), |path| { - generate_file( - path, - BINDING_TEST_JS_TEMPLATE, - language_name, - &generate_opts, - ) - })?; - - missing_path(path.join("binding.cc"), |path| { - generate_file(path, JS_BINDING_CC_TEMPLATE, language_name, &generate_opts) - })?; - - missing_path(repo_path.join("binding.gyp"), |path| { - generate_file(path, BINDING_GYP_TEMPLATE, language_name, &generate_opts) - })?; - - Ok(()) - })?; - } - - // Generate C bindings - if tree_sitter_config.bindings.c { - missing_path(bindings_dir.join("c"), create_dir)?.apply(|path| { - missing_path( - path.join(format!("tree-sitter-{language_name}.h")), - |path| generate_file(path, PARSER_NAME_H_TEMPLATE, language_name, &generate_opts), - )?; - - missing_path( - path.join(format!("tree-sitter-{language_name}.pc.in")), - |path| { - generate_file( - path, - PARSER_NAME_PC_IN_TEMPLATE, - language_name, - &generate_opts, - ) - }, - )?; - - missing_path(repo_path.join("Makefile"), |path| { - generate_file(path, MAKEFILE_TEMPLATE, language_name, &generate_opts) - })?; - - missing_path_else( - repo_path.join("CMakeLists.txt"), - allow_update, - |path| generate_file(path, CMAKELISTS_TXT_TEMPLATE, language_name, &generate_opts), - |path| { - let contents = fs::read_to_string(path)?; - let old = "add_custom_target(test"; - if contents.contains(old) { - write_file(path, contents.replace(old, "add_custom_target(ts-test")) - } else { - Ok(()) - } - }, - )?; - - Ok(()) - })?; - } - - // Generate Go bindings - if tree_sitter_config.bindings.go { - missing_path(bindings_dir.join("go"), create_dir)?.apply(|path| { - missing_path(path.join("binding.go"), |path| { - generate_file(path, BINDING_GO_TEMPLATE, language_name, &generate_opts) - })?; - - missing_path(path.join("binding_test.go"), |path| { - generate_file( - path, - BINDING_TEST_GO_TEMPLATE, - language_name, - &generate_opts, - ) - })?; - - missing_path(repo_path.join("go.mod"), |path| { - generate_file(path, GO_MOD_TEMPLATE, language_name, &generate_opts) - })?; - - Ok(()) - })?; - } - - // Generate Python bindings - if tree_sitter_config.bindings.python { - missing_path(bindings_dir.join("python"), create_dir)?.apply(|path| { - let lang_path = path.join(format!("tree_sitter_{}", language_name.to_snake_case())); - missing_path(&lang_path, create_dir)?; - - missing_path(lang_path.join("binding.c"), |path| { - generate_file(path, PY_BINDING_C_TEMPLATE, language_name, &generate_opts) - })?; - - missing_path(lang_path.join("__init__.py"), |path| { - generate_file(path, INIT_PY_TEMPLATE, language_name, &generate_opts) - })?; - - missing_path(lang_path.join("__init__.pyi"), |path| { - generate_file(path, INIT_PYI_TEMPLATE, language_name, &generate_opts) - })?; - - missing_path(lang_path.join("py.typed"), |path| { - generate_file(path, "", language_name, &generate_opts) // py.typed is empty - })?; - - missing_path(path.join("tests"), create_dir)?.apply(|path| { - missing_path(path.join("test_binding.py"), |path| { - generate_file( - path, - TEST_BINDING_PY_TEMPLATE, - language_name, - &generate_opts, - ) - })?; - Ok(()) - })?; - - missing_path(repo_path.join("setup.py"), |path| { - generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts) - })?; - - missing_path(repo_path.join("pyproject.toml"), |path| { - generate_file( - path, - PYPROJECT_TOML_TEMPLATE, - dashed_language_name.as_str(), - &generate_opts, - ) - })?; - - Ok(()) - })?; - } - - // Generate Swift bindings - if tree_sitter_config.bindings.swift { - missing_path(bindings_dir.join("swift"), create_dir)?.apply(|path| { - let lang_path = path.join(format!("TreeSitter{camel_name}",)); - missing_path(&lang_path, create_dir)?; - - missing_path(lang_path.join(format!("{language_name}.h")), |path| { - generate_file(path, PARSER_NAME_H_TEMPLATE, language_name, &generate_opts) - })?; - - missing_path( - path.join(format!("TreeSitter{camel_name}Tests",)), - create_dir, - )? - .apply(|path| { - missing_path( - path.join(format!("TreeSitter{camel_name}Tests.swift")), - |path| generate_file(path, TESTS_SWIFT_TEMPLATE, language_name, &generate_opts), - )?; - - Ok(()) - })?; - - missing_path(repo_path.join("Package.swift"), |path| { - generate_file(path, PACKAGE_SWIFT_TEMPLATE, language_name, &generate_opts) - })?; - - Ok(()) - })?; - } - - Ok(()) -} - -pub fn get_root_path(path: &Path) -> Result { - let mut pathbuf = path.to_owned(); - let filename = path.file_name().unwrap().to_str().unwrap(); - let is_package_json = filename == "package.json"; - loop { - let json = pathbuf - .exists() - .then(|| { - let contents = fs::read_to_string(pathbuf.as_path()) - .with_context(|| format!("Failed to read {filename}"))?; - if is_package_json { - serde_json::from_str::>(&contents) - .context(format!("Failed to parse {filename}")) - .map(|v| v.contains_key("tree-sitter")) - } else { - serde_json::from_str::(&contents) - .context(format!("Failed to parse {filename}")) - .map(|_| true) - } - }) - .transpose()?; - if json == Some(true) { - return Ok(pathbuf.parent().unwrap().to_path_buf()); - } - pathbuf.pop(); // filename - if !pathbuf.pop() { - return Err(anyhow!(format!( - concat!( - "Failed to locate a {} file,", - " please ensure you have one, and if you don't then consult the docs", - ), - filename - ))); - } - pathbuf.push(filename); - } -} - -fn generate_file( - path: &Path, - template: &str, - language_name: &str, - generate_opts: &GenerateOpts, -) -> Result<()> { - let filename = path.file_name().unwrap().to_str().unwrap(); - - let mut replacement = template - .replace( - CAMEL_PARSER_NAME_PLACEHOLDER, - generate_opts.camel_parser_name, - ) - .replace( - UPPER_PARSER_NAME_PLACEHOLDER, - &language_name.to_shouty_snake_case(), - ) - .replace( - LOWER_PARSER_NAME_PLACEHOLDER, - &language_name.to_snake_case(), - ) - .replace(PARSER_NAME_PLACEHOLDER, language_name) - .replace(CLI_VERSION_PLACEHOLDER, CLI_VERSION) - .replace(RUST_BINDING_VERSION_PLACEHOLDER, RUST_BINDING_VERSION) - .replace(ABI_VERSION_MAX_PLACEHOLDER, &ABI_VERSION_MAX.to_string()) - .replace( - PARSER_VERSION_PLACEHOLDER, - &generate_opts.version.to_string(), - ); - - if let Some(name) = generate_opts.author_name { - replacement = replacement.replace(AUTHOR_NAME_PLACEHOLDER, name); - } else { - match filename { - "package.json" => { - replacement = replacement.replace(AUTHOR_NAME_PLACEHOLDER_JS, ""); - } - "pyproject.toml" => { - replacement = replacement.replace(AUTHOR_NAME_PLACEHOLDER_PY, ""); - } - "grammar.js" => { - replacement = replacement.replace(AUTHOR_NAME_PLACEHOLDER_GRAMMAR, ""); - } - "Cargo.toml" => { - replacement = replacement.replace(AUTHOR_NAME_PLACEHOLDER_RS, ""); - } - _ => {} - } - } - - if let Some(email) = generate_opts.author_email { - replacement = match filename { - "Cargo.toml" | "grammar.js" => { - replacement.replace(AUTHOR_EMAIL_PLACEHOLDER, &format!("<{email}>")) - } - _ => replacement.replace(AUTHOR_EMAIL_PLACEHOLDER, email), - } - } else { - match filename { - "package.json" => { - replacement = replacement.replace(AUTHOR_EMAIL_PLACEHOLDER_JS, ""); - } - "pyproject.toml" => { - replacement = replacement.replace(AUTHOR_EMAIL_PLACEHOLDER_PY, ""); - } - "grammar.js" => { - replacement = replacement.replace(AUTHOR_EMAIL_PLACEHOLDER_GRAMMAR, ""); - } - "Cargo.toml" => { - replacement = replacement.replace(AUTHOR_EMAIL_PLACEHOLDER_RS, ""); - } - _ => {} - } - } - - if filename == "package.json" { - if let Some(url) = generate_opts.author_url { - replacement = replacement.replace(AUTHOR_URL_PLACEHOLDER, url); - } else { - replacement = replacement.replace(AUTHOR_URL_PLACEHOLDER_JS, ""); - } - } - - if generate_opts.author_name.is_none() - && generate_opts.author_email.is_none() - && generate_opts.author_url.is_none() - && filename == "package.json" - { - if let Some(start_idx) = replacement.find(AUTHOR_BLOCK_JS) { - if let Some(end_idx) = replacement[start_idx..] - .find("},") - .map(|i| i + start_idx + 2) - { - replacement.replace_range(start_idx..end_idx, ""); - } - } - } else if generate_opts.author_name.is_none() && generate_opts.author_email.is_none() { - match filename { - "pyproject.toml" => { - if let Some(start_idx) = replacement.find(AUTHOR_BLOCK_PY) { - if let Some(end_idx) = replacement[start_idx..] - .find("}]") - .map(|i| i + start_idx + 2) - { - replacement.replace_range(start_idx..end_idx, ""); - } else { - println!("none 2"); - } - } else { - println!("none 1"); - } - } - "grammar.js" => { - if let Some(start_idx) = replacement.find(AUTHOR_BLOCK_GRAMMAR) { - if let Some(end_idx) = replacement[start_idx..] - .find(" \n") - .map(|i| i + start_idx + 1) - { - replacement.replace_range(start_idx..end_idx, ""); - } else { - println!("none 2"); - } - } else { - println!("none 1"); - } - } - "Cargo.toml" => { - if let Some(start_idx) = replacement.find(AUTHOR_BLOCK_RS) { - if let Some(end_idx) = replacement[start_idx..] - .find("\"]") - .map(|i| i + start_idx + 2) - { - replacement.replace_range(start_idx..end_idx, ""); - } - } - } - _ => {} - } - } - - match generate_opts.license { - Some(license) => replacement = replacement.replace(PARSER_LICENSE_PLACEHOLDER, license), - _ => replacement = replacement.replace(PARSER_LICENSE_PLACEHOLDER, "MIT"), - } - - match generate_opts.description { - Some(description) => { - replacement = replacement.replace(PARSER_DESCRIPTION_PLACEHOLDER, description); - } - _ => { - replacement = replacement.replace( - PARSER_DESCRIPTION_PLACEHOLDER, - &format!( - "{} grammar for tree-sitter", - generate_opts.camel_parser_name, - ), - ); - } - } - - match generate_opts.repository { - Some(repository) => { - replacement = replacement - .replace( - PARSER_URL_STRIPPED_PLACEHOLDER, - &repository.replace("https://", "").to_lowercase(), - ) - .replace(PARSER_URL_PLACEHOLDER, &repository.to_lowercase()); - } - _ => { - replacement = replacement - .replace( - PARSER_URL_STRIPPED_PLACEHOLDER, - &format!( - "github.com/tree-sitter/tree-sitter-{}", - language_name.to_lowercase() - ), - ) - .replace( - PARSER_URL_PLACEHOLDER, - &format!( - "https://github.com/tree-sitter/tree-sitter-{}", - language_name.to_lowercase() - ), - ); - } - } - - write_file(path, replacement) -} - -fn create_dir(path: &Path) -> Result<()> { - fs::create_dir_all(path) - .with_context(|| format!("Failed to create {:?}", path.to_string_lossy())) -} - -#[derive(PartialEq, Eq, Debug)] -enum PathState

-where - P: AsRef, -{ - Exists(P), - Missing(P), -} - -#[allow(dead_code)] -impl

PathState

-where - P: AsRef, -{ - fn exists(&self, mut action: impl FnMut(&Path) -> Result<()>) -> Result<&Self> { - if let Self::Exists(path) = self { - action(path.as_ref())?; - } - Ok(self) - } - - fn missing(&self, mut action: impl FnMut(&Path) -> Result<()>) -> Result<&Self> { - if let Self::Missing(path) = self { - action(path.as_ref())?; - } - Ok(self) - } - - fn apply(&self, mut action: impl FnMut(&Path) -> Result<()>) -> Result<&Self> { - action(self.as_path())?; - Ok(self) - } - - fn apply_state(&self, mut action: impl FnMut(&Self) -> Result<()>) -> Result<&Self> { - action(self)?; - Ok(self) - } - - fn as_path(&self) -> &Path { - match self { - Self::Exists(path) | Self::Missing(path) => path.as_ref(), - } - } -} - -fn missing_path(path: P, mut action: F) -> Result> -where - P: AsRef, - F: FnMut(&Path) -> Result<()>, -{ - let path_ref = path.as_ref(); - if !path_ref.exists() { - action(path_ref)?; - Ok(PathState::Missing(path)) - } else { - Ok(PathState::Exists(path)) - } -} - -fn missing_path_else( - path: P, - allow_update: bool, - mut action: T, - mut else_action: F, -) -> Result> -where - P: AsRef, - T: FnMut(&Path) -> Result<()>, - F: FnMut(&Path) -> Result<()>, -{ - let path_ref = path.as_ref(); - if !path_ref.exists() { - action(path_ref)?; - Ok(PathState::Missing(path)) - } else { - if allow_update { - else_action(path_ref)?; - } - Ok(PathState::Exists(path)) - } -} diff --git a/cli/src/logger.rs b/cli/src/logger.rs deleted file mode 100644 index ce4f74a3..00000000 --- a/cli/src/logger.rs +++ /dev/null @@ -1,30 +0,0 @@ -use log::{LevelFilter, Log, Metadata, Record}; - -#[allow(dead_code)] -struct Logger { - pub filter: Option, -} - -impl Log for Logger { - fn enabled(&self, _: &Metadata) -> bool { - true - } - - fn log(&self, record: &Record) { - eprintln!( - "[{}] {}", - record - .module_path() - .unwrap_or_default() - .trim_start_matches("rust_tree_sitter_cli::"), - record.args() - ); - } - - fn flush(&self) {} -} - -pub fn init() { - log::set_boxed_logger(Box::new(Logger { filter: None })).unwrap(); - log::set_max_level(LevelFilter::Info); -} diff --git a/cli/src/main.rs b/cli/src/main.rs deleted file mode 100644 index ae24f3fa..00000000 --- a/cli/src/main.rs +++ /dev/null @@ -1,1461 +0,0 @@ -use std::{ - collections::HashSet, - env, fs, - path::{Path, PathBuf}, -}; - -use anstyle::{AnsiColor, Color, Style}; -use anyhow::{anyhow, Context, Result}; -use clap::{crate_authors, Args, Command, FromArgMatches as _, Subcommand, ValueEnum}; -use clap_complete::{generate, Shell}; -use dialoguer::{theme::ColorfulTheme, Confirm, FuzzySelect, Input}; -use glob::glob; -use heck::ToUpperCamelCase; -use regex::Regex; -use semver::Version as SemverVersion; -use tree_sitter::{ffi, Parser, Point}; -use tree_sitter_cli::{ - fuzz::{ - fuzz_language_corpus, FuzzOptions, EDIT_COUNT, ITERATION_COUNT, LOG_ENABLED, - LOG_GRAPH_ENABLED, START_SEED, - }, - highlight, - init::{generate_grammar_files, get_root_path, migrate_package_json, JsonConfigOpts}, - logger, - parse::{self, ParseFileOptions, ParseOutput, ParseTheme}, - playground, query, tags, - test::{self, TestOptions}, - test_highlight, test_tags, util, version, wasm, -}; -use tree_sitter_config::Config; -use tree_sitter_highlight::Highlighter; -use tree_sitter_loader::{self as loader, TreeSitterJSON}; -use tree_sitter_tags::TagsContext; -use url::Url; - -const BUILD_VERSION: &str = env!("CARGO_PKG_VERSION"); -const BUILD_SHA: Option<&'static str> = option_env!("BUILD_SHA"); -const DEFAULT_GENERATE_ABI_VERSION: usize = 15; - -#[derive(Subcommand)] -#[command(about="Generates and tests parsers", author=crate_authors!("\n"), styles=get_styles())] -enum Commands { - InitConfig(InitConfig), - Init(Init), - Generate(Generate), - Build(Build), - Parse(Parse), - Test(Test), - Version(Version), - Fuzz(Fuzz), - Query(Query), - Highlight(Highlight), - Tags(Tags), - Playground(Playground), - DumpLanguages(DumpLanguages), - Complete(Complete), -} - -#[derive(Args)] -#[command(about = "Generate a default config file")] -struct InitConfig; - -#[derive(Args)] -#[command(about = "Initialize a grammar repository", alias = "i")] -struct Init { - #[arg(long, short, help = "Update outdated files")] - pub update: bool, -} - -#[derive(Args)] -#[command(about = "Generate a parser", alias = "gen", alias = "g")] -struct Generate { - #[arg(index = 1, help = "The path to the grammar file")] - pub grammar_path: Option, - #[arg(long, short, help = "Show debug log during generation")] - pub log: bool, - #[arg(long, help = "Deprecated (no-op)")] - pub no_bindings: bool, - #[arg( - long = "abi", - value_name = "VERSION", - help = format!(concat!( - "Select the language ABI version to generate (default {}).\n", - "Use --abi=latest to generate the newest supported version ({}).", - ), - DEFAULT_GENERATE_ABI_VERSION, - tree_sitter::LANGUAGE_VERSION, - ) - )] - pub abi_version: Option, - #[arg( - long, - short = 'b', - help = "Compile all defined languages in the current dir" - )] - pub build: bool, - #[arg(long, short = '0', help = "Compile a parser in debug mode")] - pub debug_build: bool, - #[arg( - long, - value_name = "PATH", - help = "The path to the directory containing the parser library" - )] - pub libdir: Option, - #[arg( - long, - short, - value_name = "DIRECTORY", - help = "The path to output the generated source files" - )] - pub output: Option, - #[arg( - long, - help = "Produce a report of the states for the given rule, use `-` to report every rule" - )] - pub report_states_for_rule: Option, - - #[arg( - long, - value_name = "EXECUTABLE", - env = "TREE_SITTER_JS_RUNTIME", - default_value = "node", - help = "The name or path of the JavaScript runtime to use for generating parsers" - )] - pub js_runtime: Option, -} - -#[derive(Args)] -#[command(about = "Compile a parser", alias = "b")] -struct Build { - #[arg(short, long, help = "Build a WASM module instead of a dynamic library")] - pub wasm: bool, - #[arg( - short, - long, - help = "Run emscripten via docker even if it is installed locally (only if building a WASM module with --wasm)" - )] - pub docker: bool, - #[arg(short, long, help = "The path to output the compiled file")] - pub output: Option, - #[arg(index = 1, num_args = 1, help = "The path to the grammar directory")] - pub path: Option, - #[arg(long, help = "Make the parser reuse the same allocator as the library")] - pub reuse_allocator: bool, - #[arg(long, short = '0', help = "Compile a parser in debug mode")] - pub debug: bool, -} - -#[derive(Args)] -#[command(about = "Parse files", alias = "p")] -struct Parse { - #[arg( - long = "paths", - help = "The path to a file with paths to source file(s)" - )] - pub paths_file: Option, - #[arg(num_args=1.., help = "The source file(s) to use")] - pub paths: Option>, - #[arg( - long, - help = "Select a language by the scope instead of a file extension" - )] - pub scope: Option, - #[arg(long, short = 'd', help = "Show parsing debug log")] - pub debug: bool, - #[arg(long, short = '0', help = "Compile a parser in debug mode")] - pub debug_build: bool, - #[arg( - long, - short = 'D', - help = "Produce the log.html file with debug graphs" - )] - pub debug_graph: bool, - #[arg( - long, - help = "Compile parsers to wasm instead of native dynamic libraries" - )] - pub wasm: bool, - #[arg(long = "dot", help = "Output the parse data with graphviz dot")] - pub output_dot: bool, - #[arg( - long = "xml", - short = 'x', - help = "Output the parse data in XML format" - )] - pub output_xml: bool, - #[arg( - long = "cst", - short = 'c', - help = "Output the parse data in a pretty-printed CST format" - )] - pub output_cst: bool, - #[arg(long, short, help = "Show parsing statistic")] - pub stat: bool, - #[arg(long, help = "Interrupt the parsing process by timeout (µs)")] - pub timeout: Option, - #[arg(long, short, help = "Measure execution time")] - pub time: bool, - #[arg(long, short, help = "Suppress main output")] - pub quiet: bool, - #[arg( - long, - num_args = 1.., - help = "Apply edits in the format: \"row, col delcount insert_text\"" - )] - pub edits: Option>, - #[arg(long, help = "The encoding of the input files")] - pub encoding: Option, - #[arg( - long, - help = "Open `log.html` in the default browser, if `--debug-graph` is supplied" - )] - pub open_log: bool, - #[arg(long, help = "The path to an alternative config.json file")] - pub config_path: Option, - #[arg(long, short = 'n', help = "Parse the contents of a specific test")] - #[clap(conflicts_with = "paths", conflicts_with = "paths_file")] - pub test_number: Option, - #[arg(short, long, help = "Force rebuild the parser")] - pub rebuild: bool, - #[arg(long, help = "Omit ranges in the output")] - pub no_ranges: bool, -} - -#[derive(ValueEnum, Clone)] -pub enum Encoding { - Utf8, - Utf16LE, - Utf16BE, -} - -#[derive(Args)] -#[command(about = "Run a parser's tests", alias = "t")] -struct Test { - #[arg( - long, - short, - help = "Only run corpus test cases whose name matches the given regex" - )] - pub include: Option, - #[arg( - long, - short, - help = "Only run corpus test cases whose name does not match the given regex" - )] - pub exclude: Option, - #[arg( - long, - short, - help = "Update all syntax trees in corpus files with current parser output" - )] - pub update: bool, - #[arg(long, short = 'd', help = "Show parsing debug log")] - pub debug: bool, - #[arg(long, short = '0', help = "Compile a parser in debug mode")] - pub debug_build: bool, - #[arg( - long, - short = 'D', - help = "Produce the log.html file with debug graphs" - )] - pub debug_graph: bool, - #[arg( - long, - help = "Compile parsers to wasm instead of native dynamic libraries" - )] - pub wasm: bool, - #[arg( - long, - help = "Open `log.html` in the default browser, if `--debug-graph` is supplied" - )] - pub open_log: bool, - #[arg(long, help = "The path to an alternative config.json file")] - pub config_path: Option, - #[arg(long, help = "Force showing fields in test diffs")] - pub show_fields: bool, - #[arg(short, long, help = "Force rebuild the parser")] - pub rebuild: bool, - #[arg(long, help = "Show only the pass-fail overview tree")] - pub overview_only: bool, -} - -#[derive(Args)] -#[command(alias = "publish")] -/// Increment the version of a grammar -struct Version { - #[arg(num_args = 1)] - /// The version to bump to - pub version: SemverVersion, -} - -#[derive(Args)] -#[command(about = "Fuzz a parser", alias = "f")] -struct Fuzz { - #[arg(long, short, help = "List of test names to skip")] - pub skip: Option>, - #[arg(long, help = "Subdirectory to the language")] - pub subdir: Option, - #[arg(long, help = "Maximum number of edits to perform per fuzz test")] - pub edits: Option, - #[arg(long, help = "Number of fuzzing iterations to run per test")] - pub iterations: Option, - #[arg( - long, - short, - help = "Only fuzz corpus test cases whose name matches the given regex" - )] - pub include: Option, - #[arg( - long, - short, - help = "Only fuzz corpus test cases whose name does not match the given regex" - )] - pub exclude: Option, - #[arg(long, help = "Enable logging of graphs and input")] - pub log_graphs: bool, - #[arg(long, short, help = "Enable parser logging")] - pub log: bool, - #[arg(short, long, help = "Force rebuild the parser")] - pub rebuild: bool, -} - -#[derive(Args)] -#[command(about = "Search files using a syntax tree query", alias = "q")] -struct Query { - #[arg(help = "Path to a file with queries", index = 1, required = true)] - query_path: String, - #[arg(long, short, help = "Measure execution time")] - pub time: bool, - #[arg(long, short, help = "Suppress main output")] - pub quiet: bool, - #[arg( - long = "paths", - help = "The path to a file with paths to source file(s)" - )] - pub paths_file: Option, - #[arg(index = 2, num_args=1.., help = "The source file(s) to use")] - pub paths: Option>, - #[arg( - long, - help = "The range of byte offsets in which the query will be executed" - )] - pub byte_range: Option, - #[arg(long, help = "The range of rows in which the query will be executed")] - pub row_range: Option, - #[arg( - long, - help = "Select a language by the scope instead of a file extension" - )] - pub scope: Option, - #[arg(long, short, help = "Order by captures instead of matches")] - pub captures: bool, - #[arg(long, help = "Whether to run query tests or not")] - pub test: bool, - #[arg(long, help = "The path to an alternative config.json file")] - pub config_path: Option, -} - -#[derive(Args)] -#[command(about = "Highlight a file", alias = "hi")] -struct Highlight { - #[arg(long, short = 'H', help = "Generate highlighting as an HTML document")] - pub html: bool, - #[arg( - long, - help = "Check that highlighting captures conform strictly to standards" - )] - pub check: bool, - #[arg(long, help = "The path to a file with captures")] - pub captures_path: Option, - #[arg(long, num_args = 1.., help = "The paths to files with queries")] - pub query_paths: Option>, - #[arg( - long, - help = "Select a language by the scope instead of a file extension" - )] - pub scope: Option, - #[arg(long, short, help = "Measure execution time")] - pub time: bool, - #[arg(long, short, help = "Suppress main output")] - pub quiet: bool, - #[arg( - long = "paths", - help = "The path to a file with paths to source file(s)" - )] - pub paths_file: Option, - #[arg(num_args = 1.., help = "The source file(s) to use")] - pub paths: Option>, - #[arg(long, help = "The path to an alternative config.json file")] - pub config_path: Option, -} - -#[derive(Args)] -#[command(about = "Generate a list of tags")] -struct Tags { - #[arg( - long, - help = "Select a language by the scope instead of a file extension" - )] - pub scope: Option, - #[arg(long, short, help = "Measure execution time")] - pub time: bool, - #[arg(long, short, help = "Suppress main output")] - pub quiet: bool, - #[arg( - long = "paths", - help = "The path to a file with paths to source file(s)" - )] - pub paths_file: Option, - #[arg(num_args = 1.., help = "The source file(s) to use")] - pub paths: Option>, - #[arg(long, help = "The path to an alternative config.json file")] - pub config_path: Option, -} - -#[derive(Args)] -#[command( - about = "Start local playground for a parser in the browser", - alias = "play", - alias = "pg", - alias = "web-ui" -)] -struct Playground { - #[arg(long, short, help = "Don't open in default browser")] - pub quiet: bool, - #[arg( - long, - help = "Path to the directory containing the grammar and wasm files" - )] - pub grammar_path: Option, -} - -#[derive(Args)] -#[command(about = "Print info about all known language parsers", alias = "langs")] -struct DumpLanguages { - #[arg(long, help = "The path to an alternative config.json file")] - pub config_path: Option, -} - -#[derive(Args)] -#[command(about = "Generate shell completions", alias = "comp")] -struct Complete { - #[arg( - long, - short, - value_enum, - help = "The shell to generate completions for" - )] - pub shell: Shell, -} - -impl InitConfig { - fn run() -> Result<()> { - if let Ok(Some(config_path)) = Config::find_config_file() { - return Err(anyhow!( - "Remove your existing config file first: {}", - config_path.to_string_lossy() - )); - } - let mut config = Config::initial()?; - config.add(tree_sitter_loader::Config::initial())?; - config.add(tree_sitter_cli::highlight::ThemeConfig::default())?; - config.save()?; - println!( - "Saved initial configuration to {}", - config.location.display() - ); - Ok(()) - } -} - -impl Init { - fn run(self, current_dir: &Path, migrated: bool) -> Result<()> { - let configure_json = !current_dir.join("tree-sitter.json").exists() - && (!current_dir.join("package.json").exists() || !migrated); - - let (language_name, json_config_opts) = if configure_json { - let mut opts = JsonConfigOpts::default(); - - let name = || { - Input::::with_theme(&ColorfulTheme::default()) - .with_prompt("Parser name") - .validate_with(|input: &String| { - if input.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_') { - Ok(()) - } else { - Err("The name must be lowercase and contain only letters, digits, and underscores") - } - }) - .interact_text() - }; - - let camelcase_name = |name: &str| { - Input::::with_theme(&ColorfulTheme::default()) - .with_prompt("CamelCase name") - .default(name.to_upper_camel_case()) - .validate_with(|input: &String| { - if input - .chars() - .all(|c| c.is_ascii_alphabetic() || c.is_ascii_digit() || c == '_') - { - Ok(()) - } else { - Err("The name must contain only letters, digits, and underscores") - } - }) - .interact_text() - }; - - let description = |name: &str| { - Input::::with_theme(&ColorfulTheme::default()) - .with_prompt("Description") - .default(format!( - "{} grammar for tree-sitter", - name.to_upper_camel_case() - )) - .show_default(false) - .allow_empty(true) - .interact_text() - }; - - let repository = |name: &str| { - Input::::with_theme(&ColorfulTheme::default()) - .with_prompt("Repository URL") - .allow_empty(true) - .default( - Url::parse(&format!( - "https://github.com/tree-sitter/tree-sitter-{name}" - )) - .expect("Failed to parse default repository URL"), - ) - .show_default(false) - .interact_text() - }; - - let scope = |name: &str| { - Input::::with_theme(&ColorfulTheme::default()) - .with_prompt("TextMate scope") - .default(format!("source.{name}")) - .validate_with(|input: &String| { - if input.starts_with("source.") || input.starts_with("text.") { - Ok(()) - } else { - Err("The scope must start with 'source.' or 'text.'") - } - }) - .interact_text() - }; - - let file_types = |name: &str| { - Input::::with_theme(&ColorfulTheme::default()) - .with_prompt("File types (space-separated)") - .default(format!(".{name}")) - .interact_text() - .map(|ft| { - let mut set = HashSet::new(); - for ext in ft.split(' ') { - let ext = ext.trim(); - if !ext.is_empty() { - set.insert(ext.to_string()); - } - } - set.into_iter().collect::>() - }) - }; - - let initial_version = || { - Input::::with_theme(&ColorfulTheme::default()) - .with_prompt("Version") - .default(SemverVersion::new(0, 1, 0)) - .interact_text() - }; - - let license = || { - Input::::with_theme(&ColorfulTheme::default()) - .with_prompt("License") - .default("MIT".to_string()) - .allow_empty(true) - .interact() - }; - - let author = || { - Input::::with_theme(&ColorfulTheme::default()) - .with_prompt("Author name") - .interact_text() - }; - - let email = || { - Input::with_theme(&ColorfulTheme::default()) - .with_prompt("Author email") - .validate_with({ - move |input: &String| -> Result<(), &str> { - if (input.contains('@') && input.contains('.')) - || input.trim().is_empty() - { - Ok(()) - } else { - Err("This is not a valid email address") - } - } - }) - .allow_empty(true) - .interact_text() - .map(|e| (!e.trim().is_empty()).then_some(e)) - }; - - let url = || { - Input::::with_theme(&ColorfulTheme::default()) - .with_prompt("Author URL") - .allow_empty(true) - .validate_with(|input: &String| -> Result<(), &str> { - if input.trim().is_empty() || Url::parse(input).is_ok() { - Ok(()) - } else { - Err("This is not a valid URL") - } - }) - .interact_text() - .map(|e| (!e.trim().is_empty()).then(|| Url::parse(&e).unwrap())) - }; - - let choices = [ - "name", - "camelcase", - "description", - "repository", - "scope", - "file_types", - "version", - "license", - "author", - "email", - "url", - "exit", - ]; - - macro_rules! set_choice { - ($choice:expr) => { - match $choice { - "name" => opts.name = name()?, - "camelcase" => opts.camelcase = camelcase_name(&opts.name)?, - "description" => opts.description = description(&opts.name)?, - "repository" => opts.repository = Some(repository(&opts.name)?), - "scope" => opts.scope = scope(&opts.name)?, - "file_types" => opts.file_types = file_types(&opts.name)?, - "version" => opts.version = initial_version()?, - "license" => opts.license = license()?, - "author" => opts.author = author()?, - "email" => opts.email = email()?, - "url" => opts.url = url()?, - "exit" => break, - _ => unreachable!(), - } - }; - } - - // Initial configuration - for choice in choices.iter().take(choices.len() - 1) { - set_choice!(*choice); - } - - // Loop for editing the configuration - loop { - println!( - "Your current configuration:\n{}", - serde_json::to_string_pretty(&opts)? - ); - - if Confirm::with_theme(&ColorfulTheme::default()) - .with_prompt("Does the config above look correct?") - .interact()? - { - break; - } - - let idx = FuzzySelect::with_theme(&ColorfulTheme::default()) - .with_prompt("Which field would you like to change?") - .items(&choices) - .interact()?; - - set_choice!(choices[idx]); - } - - (opts.name.clone(), Some(opts)) - } else { - let mut json = serde_json::from_str::( - &fs::read_to_string(current_dir.join("tree-sitter.json")) - .with_context(|| "Failed to read tree-sitter.json")?, - )?; - (json.grammars.swap_remove(0).name, None) - }; - - generate_grammar_files( - current_dir, - &language_name, - self.update, - json_config_opts.as_ref(), - )?; - - Ok(()) - } -} - -impl Generate { - fn run(self, mut loader: loader::Loader, current_dir: &Path) -> Result<()> { - if self.no_bindings { - eprint!("The --no-bindings flag is no longer used and will be removed in v0.25.0"); - } - if self.log { - logger::init(); - } - let abi_version = - self.abi_version - .as_ref() - .map_or(DEFAULT_GENERATE_ABI_VERSION, |version| { - if version == "latest" { - tree_sitter::LANGUAGE_VERSION - } else { - version.parse().expect("invalid abi version flag") - } - }); - tree_sitter_generate::generate_parser_in_directory( - current_dir, - self.output.as_deref(), - self.grammar_path.as_deref(), - abi_version, - self.report_states_for_rule.as_deref(), - self.js_runtime.as_deref(), - )?; - if self.build { - if let Some(path) = self.libdir { - loader = loader::Loader::with_parser_lib_path(PathBuf::from(path)); - } - loader.debug_build(self.debug_build); - loader.languages_at_path(current_dir)?; - } - Ok(()) - } -} - -impl Build { - fn run(self, mut loader: loader::Loader, current_dir: &Path) -> Result<()> { - let grammar_path = current_dir.join(self.path.as_deref().unwrap_or_default()); - - if self.wasm { - let output_path = self.output.map(|path| current_dir.join(path)); - let root_path = get_root_path(&grammar_path.join("tree-sitter.json"))?; - wasm::compile_language_to_wasm( - &loader, - Some(&root_path), - &grammar_path, - current_dir, - output_path, - self.docker, - )?; - } else { - let output_path = if let Some(ref path) = self.output { - let path = Path::new(path); - if path.is_absolute() { - path.to_path_buf() - } else { - current_dir.join(path) - } - } else { - let file_name = grammar_path - .file_stem() - .unwrap() - .to_str() - .unwrap() - .strip_prefix("tree-sitter-") - .unwrap_or("parser"); - current_dir - .join(file_name) - .with_extension(env::consts::DLL_EXTENSION) - }; - - let flags: &[&str] = match (self.reuse_allocator, self.debug) { - (true, true) => &["TREE_SITTER_REUSE_ALLOCATOR", "TREE_SITTER_DEBUG"], - (true, false) => &["TREE_SITTER_REUSE_ALLOCATOR"], - (false, true) => &["TREE_SITTER_DEBUG"], - (false, false) => &[], - }; - - loader.debug_build(self.debug); - loader.force_rebuild(true); - - let config = Config::load(None)?; - let loader_config = config.get()?; - loader.find_all_languages(&loader_config).unwrap(); - loader - .compile_parser_at_path(&grammar_path, output_path, flags) - .unwrap(); - } - Ok(()) - } -} - -impl Parse { - fn run(self, mut loader: loader::Loader, current_dir: &Path) -> Result<()> { - let config = Config::load(self.config_path)?; - let color = env::var("NO_COLOR").map_or(true, |v| v != "1"); - let output = if self.output_dot { - ParseOutput::Dot - } else if self.output_xml { - ParseOutput::Xml - } else if self.output_cst { - ParseOutput::Cst - } else if self.quiet { - ParseOutput::Quiet - } else { - ParseOutput::Normal - }; - - let parse_theme = if color { - config - .get::() - .with_context(|| "Failed to parse CST theme")? - .parse_theme - .unwrap_or_default() - .into() - } else { - ParseTheme::empty() - }; - - let encoding = self.encoding.map(|e| match e { - Encoding::Utf8 => ffi::TSInputEncodingUTF8, - Encoding::Utf16LE => ffi::TSInputEncodingUTF16LE, - Encoding::Utf16BE => ffi::TSInputEncodingUTF16BE, - }); - - let time = self.time; - let edits = self.edits.unwrap_or_default(); - let cancellation_flag = util::cancel_on_signal(); - let mut parser = Parser::new(); - - loader.debug_build(self.debug_build); - loader.force_rebuild(self.rebuild); - - #[cfg(feature = "wasm")] - if self.wasm { - let engine = tree_sitter::wasmtime::Engine::default(); - parser - .set_wasm_store(tree_sitter::WasmStore::new(&engine).unwrap()) - .unwrap(); - loader.use_wasm(&engine); - } - - let timeout = self.timeout.unwrap_or_default(); - - let (paths, language) = if let Some(target_test) = self.test_number { - let (test_path, language_names) = test::get_tmp_test_file(target_test, color)?; - let languages = loader.languages_at_path(current_dir)?; - let language = languages - .iter() - .find(|(_, n)| language_names.contains(&Box::from(n.as_str()))) - .map(|(l, _)| l.clone()); - let paths = collect_paths(None, Some(vec![test_path.to_str().unwrap().to_owned()]))?; - (paths, language) - } else { - (collect_paths(self.paths_file.as_deref(), self.paths)?, None) - }; - - let max_path_length = paths.iter().map(|p| p.chars().count()).max().unwrap_or(0); - let mut has_error = false; - let loader_config = config.get()?; - loader.find_all_languages(&loader_config)?; - - let should_track_stats = self.stat; - let mut stats = parse::Stats::default(); - - for path in &paths { - let path = Path::new(&path); - - let language = if let Some(ref language) = language { - language.clone() - } else { - loader.select_language(path, current_dir, self.scope.as_deref())? - }; - parser - .set_language(&language) - .context("incompatible language")?; - - let opts = ParseFileOptions { - language: language.clone(), - path, - edits: &edits - .iter() - .map(std::string::String::as_str) - .collect::>(), - max_path_length, - output, - print_time: time, - timeout, - debug: self.debug, - debug_graph: self.debug_graph, - cancellation_flag: Some(&cancellation_flag), - encoding, - open_log: self.open_log, - no_ranges: self.no_ranges, - parse_theme: &parse_theme, - }; - - let parse_result = parse::parse_file_at_path(&mut parser, &opts)?; - - if should_track_stats { - stats.total_parses += 1; - if parse_result.successful { - stats.successful_parses += 1; - } - if let Some(duration) = parse_result.duration { - stats.total_bytes += parse_result.bytes; - stats.total_duration += duration; - } - } - - has_error |= !parse_result.successful; - } - - if should_track_stats { - println!("\n{stats}"); - } - - if has_error { - return Err(anyhow!("")); - } - - Ok(()) - } -} - -impl Test { - fn run(self, mut loader: loader::Loader, current_dir: &Path) -> Result<()> { - let config = Config::load(self.config_path)?; - let color = env::var("NO_COLOR").map_or(true, |v| v != "1"); - - loader.debug_build(self.debug_build); - loader.force_rebuild(self.rebuild); - - let mut parser = Parser::new(); - - #[cfg(feature = "wasm")] - if self.wasm { - let engine = tree_sitter::wasmtime::Engine::default(); - parser - .set_wasm_store(tree_sitter::WasmStore::new(&engine).unwrap()) - .unwrap(); - loader.use_wasm(&engine); - } - - let languages = loader.languages_at_path(current_dir)?; - let language = &languages - .first() - .ok_or_else(|| anyhow!("No language found"))? - .0; - parser.set_language(language)?; - - let test_dir = current_dir.join("test"); - - // Run the corpus tests. Look for them in `test/corpus`. - let test_corpus_dir = test_dir.join("corpus"); - if test_corpus_dir.is_dir() { - let mut opts = TestOptions { - path: test_corpus_dir, - debug: self.debug, - debug_graph: self.debug_graph, - include: self.include, - exclude: self.exclude, - update: self.update, - open_log: self.open_log, - languages: languages.iter().map(|(l, n)| (n.as_str(), l)).collect(), - color, - test_num: 1, - show_fields: self.show_fields, - overview_only: self.overview_only, - }; - - test::run_tests_at_path(&mut parser, &mut opts)?; - } - - // Check that all of the queries are valid. - test::check_queries_at_path(language, ¤t_dir.join("queries"))?; - - // Run the syntax highlighting tests. - let test_highlight_dir = test_dir.join("highlight"); - if test_highlight_dir.is_dir() { - let mut highlighter = Highlighter::new(); - highlighter.parser = parser; - test_highlight::test_highlights( - &loader, - &config.get()?, - &mut highlighter, - &test_highlight_dir, - color, - )?; - parser = highlighter.parser; - } - - let test_tag_dir = test_dir.join("tags"); - if test_tag_dir.is_dir() { - let mut tags_context = TagsContext::new(); - tags_context.parser = parser; - test_tags::test_tags( - &loader, - &config.get()?, - &mut tags_context, - &test_tag_dir, - color, - )?; - } - - // For the rest of the queries, find their tests and run them - for entry in walkdir::WalkDir::new(current_dir.join("queries")) - .into_iter() - .filter_map(|e| e.ok()) - .filter(|e| e.file_type().is_file()) - { - let stem = entry - .path() - .file_stem() - .map(|s| s.to_str().unwrap_or_default()) - .unwrap_or_default(); - if stem != "highlights" && stem != "tags" { - let paths = walkdir::WalkDir::new(test_dir.join(stem)) - .into_iter() - .filter_map(|e| { - let entry = e.ok()?; - if entry.file_type().is_file() { - Some(String::from(entry.path().to_string_lossy())) - } else { - None - } - }) - .collect::>(); - if !paths.is_empty() { - println!("{stem}:"); - } - query::query_files_at_paths( - language, - paths, - entry.path(), - false, - None, - None, - true, - false, - false, - )?; - } - } - Ok(()) - } -} - -impl Version { - fn run(self, current_dir: PathBuf) -> Result<()> { - version::Version::new(self.version.to_string(), current_dir).run() - } -} - -impl Fuzz { - fn run(self, mut loader: loader::Loader, current_dir: &Path) -> Result<()> { - loader.sanitize_build(true); - loader.force_rebuild(self.rebuild); - - let languages = loader.languages_at_path(current_dir)?; - let (language, language_name) = &languages - .first() - .ok_or_else(|| anyhow!("No language found"))?; - - let mut fuzz_options = FuzzOptions { - skipped: self.skip, - subdir: self.subdir, - edits: self.edits.unwrap_or(*EDIT_COUNT), - iterations: self.iterations.unwrap_or(*ITERATION_COUNT), - include: self.include, - exclude: self.exclude, - log_graphs: self.log_graphs || *LOG_GRAPH_ENABLED, - log: self.log || *LOG_ENABLED, - }; - - fuzz_language_corpus( - language, - language_name, - *START_SEED, - current_dir, - &mut fuzz_options, - ); - Ok(()) - } -} - -impl Query { - fn run(self, mut loader: loader::Loader, current_dir: &Path) -> Result<()> { - let config = Config::load(self.config_path)?; - let paths = collect_paths(self.paths_file.as_deref(), self.paths)?; - let loader_config = config.get()?; - loader.find_all_languages(&loader_config)?; - let language = - loader.select_language(Path::new(&paths[0]), current_dir, self.scope.as_deref())?; - let query_path = Path::new(&self.query_path); - - let byte_range = self.byte_range.as_ref().and_then(|range| { - let mut parts = range.split(':'); - let start = parts.next()?.parse().ok()?; - let end = parts.next().unwrap().parse().ok()?; - Some(start..end) - }); - let point_range = self.row_range.as_ref().and_then(|range| { - let mut parts = range.split(':'); - let start = parts.next()?.parse().ok()?; - let end = parts.next().unwrap().parse().ok()?; - Some(Point::new(start, 0)..Point::new(end, 0)) - }); - - query::query_files_at_paths( - &language, - paths, - query_path, - self.captures, - byte_range, - point_range, - self.test, - self.quiet, - self.time, - )?; - Ok(()) - } -} - -impl Highlight { - fn run(self, mut loader: loader::Loader) -> Result<()> { - let config = Config::load(self.config_path)?; - let theme_config: tree_sitter_cli::highlight::ThemeConfig = config.get()?; - loader.configure_highlights(&theme_config.theme.highlight_names); - let loader_config = config.get()?; - loader.find_all_languages(&loader_config)?; - - let quiet = self.quiet; - let html_mode = quiet || self.html; - let paths = collect_paths(self.paths_file.as_deref(), self.paths)?; - - if html_mode && !quiet { - println!("{}", highlight::HTML_HEADER); - } - - let cancellation_flag = util::cancel_on_signal(); - - let mut language = None; - if let Some(scope) = self.scope.as_deref() { - language = loader.language_configuration_for_scope(scope)?; - if language.is_none() { - return Err(anyhow!("Unknown scope '{scope}'")); - } - } - - for path in paths { - let path = Path::new(&path); - let (language, language_config) = match language.clone() { - Some(v) => v, - None => { - if let Some(v) = loader.language_configuration_for_file_name(path)? { - v - } else { - eprintln!("{}", util::lang_not_found_for_path(path, &loader_config)); - continue; - } - } - }; - - if let Some(highlight_config) = - language_config.highlight_config(language, self.query_paths.as_deref())? - { - if self.check { - let names = if let Some(path) = self.captures_path.as_deref() { - let path = Path::new(path); - let file = fs::read_to_string(path)?; - let capture_names = file - .lines() - .filter_map(|line| { - if line.trim().is_empty() || line.trim().starts_with(';') { - return None; - } - line.split(';').next().map(|s| s.trim().trim_matches('"')) - }) - .collect::>(); - highlight_config.nonconformant_capture_names(&capture_names) - } else { - highlight_config.nonconformant_capture_names(&HashSet::new()) - }; - if names.is_empty() { - eprintln!("All highlight captures conform to standards."); - } else { - eprintln!( - "Non-standard highlight {} detected:", - if names.len() > 1 { - "captures" - } else { - "capture" - } - ); - for name in names { - eprintln!("* {name}"); - } - } - } - - let source = fs::read(path)?; - if html_mode { - highlight::html( - &loader, - &theme_config.theme, - &source, - highlight_config, - quiet, - self.time, - Some(&cancellation_flag), - )?; - } else { - highlight::ansi( - &loader, - &theme_config.theme, - &source, - highlight_config, - self.time, - Some(&cancellation_flag), - )?; - } - } else { - eprintln!("No syntax highlighting config found for path {path:?}"); - } - } - - if html_mode && !quiet { - println!("{}", highlight::HTML_FOOTER); - } - Ok(()) - } -} - -impl Tags { - fn run(self, mut loader: loader::Loader) -> Result<()> { - let config = Config::load(self.config_path)?; - let loader_config = config.get()?; - loader.find_all_languages(&loader_config)?; - let paths = collect_paths(self.paths_file.as_deref(), self.paths)?; - tags::generate_tags( - &loader, - &config.get()?, - self.scope.as_deref(), - &paths, - self.quiet, - self.time, - )?; - Ok(()) - } -} - -impl Playground { - fn run(self, current_dir: &Path) -> Result<()> { - let open_in_browser = !self.quiet; - let grammar_path = self.grammar_path.as_deref().map_or(current_dir, Path::new); - playground::serve(grammar_path, open_in_browser)?; - Ok(()) - } -} - -impl DumpLanguages { - fn run(self, mut loader: loader::Loader) -> Result<()> { - let config = Config::load(self.config_path)?; - let loader_config = config.get()?; - loader.find_all_languages(&loader_config)?; - for (configuration, language_path) in loader.get_all_language_configurations() { - println!( - concat!( - "scope: {}\n", - "parser: {:?}\n", - "highlights: {:?}\n", - "file_types: {:?}\n", - "content_regex: {:?}\n", - "injection_regex: {:?}\n", - ), - configuration.scope.as_ref().unwrap_or(&String::new()), - language_path, - configuration.highlights_filenames, - configuration.file_types, - configuration.content_regex, - configuration.injection_regex, - ); - } - Ok(()) - } -} - -impl Complete { - fn run(self, cli: &mut Command) { - generate( - self.shell, - cli, - cli.get_name().to_string(), - &mut std::io::stdout(), - ); - } -} - -fn main() { - let result = run(); - if let Err(err) = &result { - // Ignore BrokenPipe errors - if let Some(error) = err.downcast_ref::() { - if error.kind() == std::io::ErrorKind::BrokenPipe { - return; - } - } - if !err.to_string().is_empty() { - eprintln!("{err:?}"); - } - std::process::exit(1); - } -} - -fn run() -> Result<()> { - let version = BUILD_SHA.map_or_else( - || BUILD_VERSION.to_string(), - |build_sha| format!("{BUILD_VERSION} ({build_sha})"), - ); - let version: &'static str = Box::leak(version.into_boxed_str()); - - let cli = Command::new("tree-sitter") - .help_template( - "\ -{before-help}{name} {version} -{author-with-newline}{about-with-newline} -{usage-heading} {usage} - -{all-args}{after-help} -", - ) - .version(version) - .subcommand_required(true) - .arg_required_else_help(true) - .disable_help_subcommand(true) - .disable_colored_help(false); - let mut cli = Commands::augment_subcommands(cli); - - let command = Commands::from_arg_matches(&cli.clone().get_matches())?; - - let current_dir = env::current_dir().unwrap(); - let loader = loader::Loader::new()?; - - let migrated = if !current_dir.join("tree-sitter.json").exists() - && current_dir.join("package.json").exists() - { - migrate_package_json(¤t_dir).unwrap_or(false) - } else { - false - }; - - match command { - Commands::InitConfig(_) => InitConfig::run()?, - Commands::Init(init_options) => init_options.run(¤t_dir, migrated)?, - Commands::Generate(generate_options) => generate_options.run(loader, ¤t_dir)?, - Commands::Build(build_options) => build_options.run(loader, ¤t_dir)?, - Commands::Parse(parse_options) => parse_options.run(loader, ¤t_dir)?, - Commands::Test(test_options) => test_options.run(loader, ¤t_dir)?, - Commands::Version(version_options) => version_options.run(current_dir)?, - Commands::Fuzz(fuzz_options) => fuzz_options.run(loader, ¤t_dir)?, - Commands::Query(query_options) => query_options.run(loader, ¤t_dir)?, - Commands::Highlight(highlight_options) => highlight_options.run(loader)?, - Commands::Tags(tags_options) => tags_options.run(loader)?, - Commands::Playground(playground_options) => playground_options.run(¤t_dir)?, - Commands::DumpLanguages(dump_options) => dump_options.run(loader)?, - Commands::Complete(complete_options) => complete_options.run(&mut cli), - } - - Ok(()) -} - -#[must_use] -const fn get_styles() -> clap::builder::Styles { - clap::builder::Styles::styled() - .usage( - Style::new() - .bold() - .fg_color(Some(Color::Ansi(AnsiColor::Yellow))), - ) - .header( - Style::new() - .bold() - .fg_color(Some(Color::Ansi(AnsiColor::Yellow))), - ) - .literal(Style::new().fg_color(Some(Color::Ansi(AnsiColor::Green)))) - .invalid( - Style::new() - .bold() - .fg_color(Some(Color::Ansi(AnsiColor::Red))), - ) - .error( - Style::new() - .bold() - .fg_color(Some(Color::Ansi(AnsiColor::Red))), - ) - .valid( - Style::new() - .bold() - .fg_color(Some(Color::Ansi(AnsiColor::Green))), - ) - .placeholder(Style::new().fg_color(Some(Color::Ansi(AnsiColor::White)))) -} - -fn collect_paths(paths_file: Option<&str>, paths: Option>) -> Result> { - if let Some(paths_file) = paths_file { - return Ok(fs::read_to_string(paths_file) - .with_context(|| format!("Failed to read paths file {paths_file}"))? - .trim() - .lines() - .map(String::from) - .collect::>()); - } - - if let Some(paths) = paths { - let mut result = Vec::new(); - - let mut incorporate_path = |path: &str, positive| { - if positive { - result.push(path.to_string()); - } else if let Some(index) = result.iter().position(|p| p == path) { - result.remove(index); - } - }; - - for mut path in paths { - let mut positive = true; - if path.starts_with('!') { - positive = false; - path = path.trim_start_matches('!').to_string(); - } - - if Path::new(&path).exists() { - incorporate_path(&path, positive); - } else { - let paths = - glob(&path).with_context(|| format!("Invalid glob pattern {path:?}"))?; - for path in paths { - if let Some(path) = path?.to_str() { - incorporate_path(path, positive); - } - } - } - } - - if result.is_empty() { - return Err(anyhow!( - "No files were found at or matched by the provided pathname/glob" - )); - } - - return Ok(result); - } - - Err(anyhow!("Must provide one or more paths")) -} diff --git a/cli/src/playground.html b/cli/src/playground.html deleted file mode 100644 index 420cd28d..00000000 --- a/cli/src/playground.html +++ /dev/null @@ -1,176 +0,0 @@ - - - tree-sitter THE_LANGUAGE_NAME - - - - - - - -

- - - - - - - - - - - - diff --git a/cli/src/query.rs b/cli/src/query.rs deleted file mode 100644 index 2d8a1013..00000000 --- a/cli/src/query.rs +++ /dev/null @@ -1,146 +0,0 @@ -use std::{ - fs, - io::{self, Write}, - ops::Range, - path::Path, - time::Instant, -}; - -use anstyle::AnsiColor; -use anyhow::{Context, Result}; -use streaming_iterator::StreamingIterator; -use tree_sitter::{Language, Parser, Point, Query, QueryCursor}; - -use crate::{ - query_testing::{self, to_utf8_point}, - test::paint, -}; - -#[allow(clippy::too_many_arguments)] -pub fn query_files_at_paths( - language: &Language, - paths: Vec, - query_path: &Path, - ordered_captures: bool, - byte_range: Option>, - point_range: Option>, - should_test: bool, - quiet: bool, - print_time: bool, -) -> Result<()> { - let stdout = io::stdout(); - let mut stdout = stdout.lock(); - - let query_source = fs::read_to_string(query_path) - .with_context(|| format!("Error reading query file {query_path:?}"))?; - let query = Query::new(language, &query_source).with_context(|| "Query compilation failed")?; - - let mut query_cursor = QueryCursor::new(); - if let Some(range) = byte_range { - query_cursor.set_byte_range(range); - } - if let Some(range) = point_range { - query_cursor.set_point_range(range); - } - - let mut parser = Parser::new(); - parser.set_language(language)?; - - for path in paths { - let mut results = Vec::new(); - - if !should_test { - writeln!(&mut stdout, "{path}")?; - } - - let source_code = - fs::read(&path).with_context(|| format!("Error reading source file {path:?}"))?; - let tree = parser.parse(&source_code, None).unwrap(); - - let start = Instant::now(); - if ordered_captures { - let mut captures = - query_cursor.captures(&query, tree.root_node(), source_code.as_slice()); - while let Some((mat, capture_index)) = captures.next() { - let capture = mat.captures[*capture_index]; - let capture_name = &query.capture_names()[capture.index as usize]; - if !quiet && !should_test { - writeln!( - &mut stdout, - " pattern: {:>2}, capture: {} - {capture_name}, start: {}, end: {}, text: `{}`", - mat.pattern_index, - capture.index, - capture.node.start_position(), - capture.node.end_position(), - capture.node.utf8_text(&source_code).unwrap_or("") - )?; - } - results.push(query_testing::CaptureInfo { - name: (*capture_name).to_string(), - start: to_utf8_point(capture.node.start_position(), source_code.as_slice()), - end: to_utf8_point(capture.node.end_position(), source_code.as_slice()), - }); - } - } else { - let mut matches = - query_cursor.matches(&query, tree.root_node(), source_code.as_slice()); - while let Some(m) = matches.next() { - if !quiet && !should_test { - writeln!(&mut stdout, " pattern: {}", m.pattern_index)?; - } - for capture in m.captures { - let start = capture.node.start_position(); - let end = capture.node.end_position(); - let capture_name = &query.capture_names()[capture.index as usize]; - if !quiet && !should_test { - if end.row == start.row { - writeln!( - &mut stdout, - " capture: {} - {capture_name}, start: {start}, end: {end}, text: `{}`", - capture.index, - capture.node.utf8_text(&source_code).unwrap_or("") - )?; - } else { - writeln!( - &mut stdout, - " capture: {capture_name}, start: {start}, end: {end}", - )?; - } - } - results.push(query_testing::CaptureInfo { - name: (*capture_name).to_string(), - start: to_utf8_point(capture.node.start_position(), source_code.as_slice()), - end: to_utf8_point(capture.node.end_position(), source_code.as_slice()), - }); - } - } - } - if query_cursor.did_exceed_match_limit() { - writeln!( - &mut stdout, - " WARNING: Query exceeded maximum number of in-progress captures!" - )?; - } - if should_test { - let path_name = Path::new(&path).file_name().unwrap().to_str().unwrap(); - match query_testing::assert_expected_captures(&results, &path, &mut parser, language) { - Ok(assertion_count) => { - println!( - " ✓ {} ({} assertions)", - paint(Some(AnsiColor::Green), path_name), - assertion_count - ); - } - Err(e) => { - println!(" ✗ {}", paint(Some(AnsiColor::Red), path_name)); - return Err(e); - } - } - } - if print_time { - writeln!(&mut stdout, "{:?}", start.elapsed())?; - } - } - - Ok(()) -} diff --git a/cli/src/tags.rs b/cli/src/tags.rs deleted file mode 100644 index 4e2058c7..00000000 --- a/cli/src/tags.rs +++ /dev/null @@ -1,98 +0,0 @@ -use std::{ - fs, - io::{self, Write}, - path::Path, - str, - time::Instant, -}; - -use anyhow::{anyhow, Result}; -use tree_sitter_loader::{Config, Loader}; -use tree_sitter_tags::TagsContext; - -use super::util; - -pub fn generate_tags( - loader: &Loader, - loader_config: &Config, - scope: Option<&str>, - paths: &[String], - quiet: bool, - time: bool, -) -> Result<()> { - let mut lang = None; - if let Some(scope) = scope { - lang = loader.language_configuration_for_scope(scope)?; - if lang.is_none() { - return Err(anyhow!("Unknown scope '{scope}'")); - } - } - - let mut context = TagsContext::new(); - let cancellation_flag = util::cancel_on_signal(); - let stdout = io::stdout(); - let mut stdout = stdout.lock(); - - for path in paths { - let path = Path::new(&path); - let (language, language_config) = match lang.clone() { - Some(v) => v, - None => { - if let Some(v) = loader.language_configuration_for_file_name(path)? { - v - } else { - eprintln!("{}", util::lang_not_found_for_path(path, loader_config)); - continue; - } - } - }; - - if let Some(tags_config) = language_config.tags_config(language)? { - let indent = if paths.len() > 1 { - if !quiet { - writeln!(&mut stdout, "{}", path.to_string_lossy())?; - } - "\t" - } else { - "" - }; - - let source = fs::read(path)?; - let t0 = Instant::now(); - for tag in context - .generate_tags(tags_config, &source, Some(&cancellation_flag))? - .0 - { - let tag = tag?; - if !quiet { - write!( - &mut stdout, - "{indent}{:<10}\t | {:<8}\t{} {} - {} `{}`", - str::from_utf8(&source[tag.name_range]).unwrap_or(""), - &tags_config.syntax_type_name(tag.syntax_type_id), - if tag.is_definition { "def" } else { "ref" }, - tag.span.start, - tag.span.end, - str::from_utf8(&source[tag.line_range]).unwrap_or(""), - )?; - if let Some(docs) = tag.docs { - if docs.len() > 120 { - write!(&mut stdout, "\t{:?}...", docs.get(0..120).unwrap_or(""))?; - } else { - write!(&mut stdout, "\t{:?}", &docs)?; - } - } - writeln!(&mut stdout)?; - } - } - - if time { - writeln!(&mut stdout, "{indent}time: {}ms", t0.elapsed().as_millis(),)?; - } - } else { - eprintln!("No tags config found for path {path:?}"); - } - } - - Ok(()) -} diff --git a/cli/src/templates/__init__.py b/cli/src/templates/__init__.py deleted file mode 100644 index fd137b0f..00000000 --- a/cli/src/templates/__init__.py +++ /dev/null @@ -1,42 +0,0 @@ -"""PARSER_DESCRIPTION""" - -from importlib.resources import files as _files - -from ._binding import language - - -def _get_query(name, file): - query = _files(f"{__package__}.queries") / file - globals()[name] = query.read_text() - return globals()[name] - - -def __getattr__(name): - # NOTE: uncomment these to include any queries that this grammar contains: - - # if name == "HIGHLIGHTS_QUERY": - # return _get_query("HIGHLIGHTS_QUERY", "highlights.scm") - # if name == "INJECTIONS_QUERY": - # return _get_query("INJECTIONS_QUERY", "injections.scm") - # if name == "LOCALS_QUERY": - # return _get_query("LOCALS_QUERY", "locals.scm") - # if name == "TAGS_QUERY": - # return _get_query("TAGS_QUERY", "tags.scm") - - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - - -__all__ = [ - "language", - # "HIGHLIGHTS_QUERY", - # "INJECTIONS_QUERY", - # "LOCALS_QUERY", - # "TAGS_QUERY", -] - - -def __dir__(): - return sorted(__all__ + [ - "__all__", "__builtins__", "__cached__", "__doc__", "__file__", - "__loader__", "__name__", "__package__", "__path__", "__spec__", - ]) diff --git a/cli/src/templates/__init__.pyi b/cli/src/templates/__init__.pyi deleted file mode 100644 index abf6633f..00000000 --- a/cli/src/templates/__init__.pyi +++ /dev/null @@ -1,10 +0,0 @@ -from typing import Final - -# NOTE: uncomment these to include any queries that this grammar contains: - -# HIGHLIGHTS_QUERY: Final[str] -# INJECTIONS_QUERY: Final[str] -# LOCALS_QUERY: Final[str] -# TAGS_QUERY: Final[str] - -def language() -> object: ... diff --git a/cli/src/templates/binding_test.js b/cli/src/templates/binding_test.js deleted file mode 100644 index 55becacf..00000000 --- a/cli/src/templates/binding_test.js +++ /dev/null @@ -1,9 +0,0 @@ -const assert = require("node:assert"); -const { test } = require("node:test"); - -const Parser = require("tree-sitter"); - -test("can load grammar", () => { - const parser = new Parser(); - assert.doesNotThrow(() => parser.setLanguage(require("."))); -}); diff --git a/cli/src/templates/build.rs b/cli/src/templates/build.rs deleted file mode 100644 index 858e80c2..00000000 --- a/cli/src/templates/build.rs +++ /dev/null @@ -1,21 +0,0 @@ -fn main() { - let src_dir = std::path::Path::new("src"); - - let mut c_config = cc::Build::new(); - c_config.std("c11").include(src_dir); - - #[cfg(target_env = "msvc")] - c_config.flag("-utf-8"); - - let parser_path = src_dir.join("parser.c"); - c_config.file(&parser_path); - println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap()); - - let scanner_path = src_dir.join("scanner.c"); - if scanner_path.exists() { - c_config.file(&scanner_path); - println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap()); - } - - c_config.compile("tree-sitter-PARSER_NAME"); -} diff --git a/cli/src/templates/cmakelists.cmake b/cli/src/templates/cmakelists.cmake deleted file mode 100644 index 3ce70239..00000000 --- a/cli/src/templates/cmakelists.cmake +++ /dev/null @@ -1,58 +0,0 @@ -cmake_minimum_required(VERSION 3.13) - -project(tree-sitter-PARSER_NAME - VERSION "PARSER_VERSION" - DESCRIPTION "PARSER_DESCRIPTION" - HOMEPAGE_URL "PARSER_URL" - LANGUAGES C) - -option(BUILD_SHARED_LIBS "Build using shared libraries" ON) -option(TREE_SITTER_REUSE_ALLOCATOR "Reuse the library allocator" OFF) - -set(TREE_SITTER_ABI_VERSION ABI_VERSION_MAX CACHE STRING "Tree-sitter ABI version") -if(NOT ${TREE_SITTER_ABI_VERSION} MATCHES "^[0-9]+$") - unset(TREE_SITTER_ABI_VERSION CACHE) - message(FATAL_ERROR "TREE_SITTER_ABI_VERSION must be an integer") -endif() - -find_program(TREE_SITTER_CLI tree-sitter DOC "Tree-sitter CLI") - -add_custom_command(OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/src/parser.c" - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/grammar.json" - COMMAND "${TREE_SITTER_CLI}" generate src/grammar.json - --abi=${TREE_SITTER_ABI_VERSION} - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" - COMMENT "Generating parser.c") - -add_library(tree-sitter-PARSER_NAME src/parser.c) -if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/src/scanner.c) - target_sources(tree-sitter-PARSER_NAME PRIVATE src/scanner.c) -endif() -target_include_directories(tree-sitter-PARSER_NAME PRIVATE src) - -target_compile_definitions(tree-sitter-PARSER_NAME PRIVATE - $<$:TREE_SITTER_REUSE_ALLOCATOR> - $<$:TREE_SITTER_DEBUG>) - -set_target_properties(tree-sitter-PARSER_NAME - PROPERTIES - C_STANDARD 11 - POSITION_INDEPENDENT_CODE ON - SOVERSION "${TREE_SITTER_ABI_VERSION}.${PROJECT_VERSION_MAJOR}" - DEFINE_SYMBOL "") - -configure_file(bindings/c/tree-sitter-PARSER_NAME.pc.in - "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter-PARSER_NAME.pc" @ONLY) - -include(GNUInstallDirs) - -install(FILES bindings/c/tree-sitter-PARSER_NAME.h - DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/tree_sitter") -install(FILES "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter-PARSER_NAME.pc" - DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/pkgconfig") -install(TARGETS tree-sitter-PARSER_NAME - LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}") - -add_custom_target(ts-test "${TREE_SITTER_CLI}" test - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" - COMMENT "tree-sitter test") diff --git a/cli/src/templates/index.d.ts b/cli/src/templates/index.d.ts deleted file mode 100644 index 528e060f..00000000 --- a/cli/src/templates/index.d.ts +++ /dev/null @@ -1,27 +0,0 @@ -type BaseNode = { - type: string; - named: boolean; -}; - -type ChildNode = { - multiple: boolean; - required: boolean; - types: BaseNode[]; -}; - -type NodeInfo = - | (BaseNode & { - subtypes: BaseNode[]; - }) - | (BaseNode & { - fields: { [name: string]: ChildNode }; - children: ChildNode[]; - }); - -type Language = { - language: unknown; - nodeTypeInfo: NodeInfo[]; -}; - -declare const language: Language; -export = language; diff --git a/cli/src/templates/index.js b/cli/src/templates/index.js deleted file mode 100644 index 88437495..00000000 --- a/cli/src/templates/index.js +++ /dev/null @@ -1,11 +0,0 @@ -const root = require("path").join(__dirname, "..", ".."); - -module.exports = - typeof process.versions.bun === "string" - // Support `bun build --compile` by being statically analyzable enough to find the .node file at build-time - ? require(`../../prebuilds/${process.platform}-${process.arch}/tree-sitter-PARSER_NAME.node`) - : require("node-gyp-build")(root); - -try { - module.exports.nodeTypeInfo = require("../../src/node-types.json"); -} catch (_) {} diff --git a/cli/src/templates/lib.rs b/cli/src/templates/lib.rs deleted file mode 100644 index 4e3522ae..00000000 --- a/cli/src/templates/lib.rs +++ /dev/null @@ -1,53 +0,0 @@ -//! This crate provides CAMEL_PARSER_NAME language support for the [tree-sitter][] parsing library. -//! -//! Typically, you will use the [LANGUAGE][] constant to add this language to a -//! tree-sitter [Parser][], and then use the parser to parse some code: -//! -//! ``` -//! let code = r#" -//! "#; -//! let mut parser = tree_sitter::Parser::new(); -//! let language = tree_sitter_PARSER_NAME::LANGUAGE; -//! parser -//! .set_language(&language.into()) -//! .expect("Error loading CAMEL_PARSER_NAME parser"); -//! let tree = parser.parse(code, None).unwrap(); -//! assert!(!tree.root_node().has_error()); -//! ``` -//! -//! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html -//! [tree-sitter]: https://tree-sitter.github.io/ - -use tree_sitter_language::LanguageFn; - -extern "C" { - fn tree_sitter_PARSER_NAME() -> *const (); -} - -/// The tree-sitter [`LanguageFn`][LanguageFn] for this grammar. -/// -/// [LanguageFn]: https://docs.rs/tree-sitter-language/*/tree_sitter_language/struct.LanguageFn.html -pub const LANGUAGE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_PARSER_NAME) }; - -/// The content of the [`node-types.json`][] file for this grammar. -/// -/// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types -pub const NODE_TYPES: &str = include_str!("../../src/node-types.json"); - -// NOTE: uncomment these to include any queries that this grammar contains: - -// pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm"); -// pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm"); -// pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm"); -// pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm"); - -#[cfg(test)] -mod tests { - #[test] - fn test_can_load_grammar() { - let mut parser = tree_sitter::Parser::new(); - parser - .set_language(&super::LANGUAGE.into()) - .expect("Error loading CAMEL_PARSER_NAME parser"); - } -} diff --git a/cli/src/templates/setup.py b/cli/src/templates/setup.py deleted file mode 100644 index ab6a7a47..00000000 --- a/cli/src/templates/setup.py +++ /dev/null @@ -1,64 +0,0 @@ -from os import path -from platform import system - -from setuptools import Extension, find_packages, setup -from setuptools.command.build import build -from wheel.bdist_wheel import bdist_wheel - -sources = [ - "bindings/python/tree_sitter_LOWER_PARSER_NAME/binding.c", - "src/parser.c", -] -if path.exists("src/scanner.c"): - sources.extend("src/scanner.c") - -if system() != "Windows": - cflags = ["-std=c11", "-fvisibility=hidden"] -else: - cflags = ["/std:c11", "/utf-8"] - - -class Build(build): - def run(self): - if path.isdir("queries"): - dest = path.join(self.build_lib, "tree_sitter_PARSER_NAME", "queries") - self.copy_tree("queries", dest) - super().run() - - -class BdistWheel(bdist_wheel): - def get_tag(self): - python, abi, platform = super().get_tag() - if python.startswith("cp"): - python, abi = "cp39", "abi3" - return python, abi, platform - - -setup( - packages=find_packages("bindings/python"), - package_dir={"": "bindings/python"}, - package_data={ - "tree_sitter_LOWER_PARSER_NAME": ["*.pyi", "py.typed"], - "tree_sitter_LOWER_PARSER_NAME.queries": ["*.scm"], - }, - ext_package="tree_sitter_LOWER_PARSER_NAME", - ext_modules=[ - Extension( - name="_binding", - sources=sources, - extra_compile_args=cflags, - define_macros=[ - ("Py_LIMITED_API", "0x03090000"), - ("PY_SSIZE_T_CLEAN", None), - ("TREE_SITTER_HIDE_SYMBOLS", None), - ], - include_dirs=["src"], - py_limited_api=True, - ) - ], - cmdclass={ - "build": Build, - "bdist_wheel": BdistWheel - }, - zip_safe=False -) diff --git a/cli/src/templates/test_binding.py b/cli/src/templates/test_binding.py deleted file mode 100644 index 6be6cf4b..00000000 --- a/cli/src/templates/test_binding.py +++ /dev/null @@ -1,11 +0,0 @@ -from unittest import TestCase - -import tree_sitter, tree_sitter_LOWER_PARSER_NAME - - -class TestLanguage(TestCase): - def test_can_load_grammar(self): - try: - tree_sitter.Language(tree_sitter_LOWER_PARSER_NAME.language()) - except Exception: - self.fail("Error loading CAMEL_PARSER_NAME grammar") diff --git a/cli/src/test.rs b/cli/src/test.rs deleted file mode 100644 index aca94574..00000000 --- a/cli/src/test.rs +++ /dev/null @@ -1,1512 +0,0 @@ -use std::{ - collections::BTreeMap, - ffi::OsStr, - fs, - io::{self, Write}, - path::{Path, PathBuf}, - str, -}; - -use anstyle::{AnsiColor, Color, Style}; -use anyhow::{anyhow, Context, Result}; -use indoc::indoc; -use lazy_static::lazy_static; -use regex::{ - bytes::{Regex as ByteRegex, RegexBuilder as ByteRegexBuilder}, - Regex, -}; -use similar::{ChangeTag, TextDiff}; -use tree_sitter::{format_sexp, Language, LogType, Parser, Query}; -use walkdir::WalkDir; - -use super::util; - -lazy_static! { - static ref HEADER_REGEX: ByteRegex = ByteRegexBuilder::new( - r"^(?x) - (?P(?:=+){3,}) - (?P[^=\r\n][^\r\n]*)? - \r?\n - (?P(?:([^=\r\n]|\s+:)[^\r\n]*\r?\n)+) - ===+ - (?P[^=\r\n][^\r\n]*)?\r?\n" - ) - .multi_line(true) - .build() - .unwrap(); - static ref DIVIDER_REGEX: ByteRegex = - ByteRegexBuilder::new(r"^(?P(?:-+){3,})(?P[^-\r\n][^\r\n]*)?\r?\n") - .multi_line(true) - .build() - .unwrap(); - static ref COMMENT_REGEX: Regex = Regex::new(r"(?m)^\s*;.*$").unwrap(); - static ref WHITESPACE_REGEX: Regex = Regex::new(r"\s+").unwrap(); - static ref SEXP_FIELD_REGEX: Regex = Regex::new(r" \w+: \(").unwrap(); - static ref POINT_REGEX: Regex = Regex::new(r"\s*\[\s*\d+\s*,\s*\d+\s*\]\s*").unwrap(); -} - -#[derive(Debug, PartialEq, Eq)] -pub enum TestEntry { - Group { - name: String, - children: Vec, - file_path: Option, - }, - Example { - name: String, - input: Vec, - output: String, - header_delim_len: usize, - divider_delim_len: usize, - has_fields: bool, - attributes_str: String, - attributes: TestAttributes, - }, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct TestAttributes { - pub skip: bool, - pub platform: bool, - pub fail_fast: bool, - pub error: bool, - pub languages: Vec>, -} - -impl Default for TestEntry { - fn default() -> Self { - Self::Group { - name: String::new(), - children: Vec::new(), - file_path: None, - } - } -} - -impl Default for TestAttributes { - fn default() -> Self { - Self { - skip: false, - platform: true, - fail_fast: false, - error: false, - languages: vec!["".into()], - } - } -} - -pub struct TestOptions<'a> { - pub path: PathBuf, - pub debug: bool, - pub debug_graph: bool, - pub include: Option, - pub exclude: Option, - pub update: bool, - pub open_log: bool, - pub languages: BTreeMap<&'a str, &'a Language>, - pub color: bool, - pub test_num: usize, - pub show_fields: bool, - pub overview_only: bool, -} - -pub fn run_tests_at_path(parser: &mut Parser, opts: &mut TestOptions) -> Result<()> { - let test_entry = parse_tests(&opts.path)?; - let mut _log_session = None; - - if opts.debug_graph { - _log_session = Some(util::log_graphs(parser, "log.html", opts.open_log)?); - } else if opts.debug { - parser.set_logger(Some(Box::new(|log_type, message| { - if log_type == LogType::Lex { - io::stderr().write_all(b" ").unwrap(); - } - writeln!(&mut io::stderr(), "{message}").unwrap(); - }))); - } - - let mut failures = Vec::new(); - let mut corrected_entries = Vec::new(); - let mut has_parse_errors = false; - run_tests( - parser, - test_entry, - opts, - 0, - &mut failures, - &mut corrected_entries, - &mut has_parse_errors, - )?; - - parser.stop_printing_dot_graphs(); - - if failures.is_empty() { - Ok(()) - } else { - println!(); - - if opts.update && !has_parse_errors { - if failures.len() == 1 { - println!("1 update:\n"); - } else { - println!("{} updates:\n", failures.len()); - } - - for (i, (name, ..)) in failures.iter().enumerate() { - println!(" {}. {name}", i + 1); - } - - Ok(()) - } else { - has_parse_errors = opts.update && has_parse_errors; - - if !opts.overview_only { - if !has_parse_errors { - if failures.len() == 1 { - println!("1 failure:"); - } else { - println!("{} failures:", failures.len()); - } - } - - if opts.color { - print_diff_key(); - } - for (i, (name, actual, expected)) in failures.iter().enumerate() { - if expected == "NO ERROR" { - println!("\n {}. {name}:\n", i + 1); - println!(" Expected an ERROR node, but got:"); - println!( - " {}", - paint( - opts.color.then_some(AnsiColor::Red), - &format_sexp(actual, 2) - ) - ); - } else { - println!("\n {}. {name}:", i + 1); - let actual = format_sexp(actual, 2); - let expected = format_sexp(expected, 2); - print_diff(&actual, &expected, opts.color); - } - } - } - - if has_parse_errors { - Err(anyhow!(indoc! {" - Some tests failed to parse with unexpected `ERROR` or `MISSING` nodes, as shown above, and cannot be updated automatically. - Either fix the grammar or manually update the tests if this is expected."})) - } else { - Err(anyhow!("")) - } - } - } -} - -#[allow(clippy::type_complexity)] -pub fn get_test_info<'test>( - test_entry: &'test TestEntry, - target_test: u32, - test_num: &mut u32, -) -> Option<(&'test str, &'test [u8], Vec>)> { - match test_entry { - TestEntry::Example { - name, - input, - attributes, - .. - } => { - if *test_num == target_test { - return Some((name, input, attributes.languages.clone())); - } - *test_num += 1; - } - TestEntry::Group { children, .. } => { - for child in children { - if let Some((name, input, languages)) = get_test_info(child, target_test, test_num) - { - return Some((name, input, languages)); - } - } - } - } - - None -} - -/// Writes the input of `target_test` to a temporary file and returns the path -pub fn get_tmp_test_file(target_test: u32, color: bool) -> Result<(PathBuf, Vec>)> { - let current_dir = std::env::current_dir().unwrap(); - let test_dir = current_dir.join("test").join("corpus"); - - // Get the input of the target test - let test_entry = parse_tests(&test_dir)?; - let mut test_num = 0; - let Some((test_name, test_contents, languages)) = - get_test_info(&test_entry, target_test - 1, &mut test_num) - else { - return Err(anyhow!("Failed to fetch contents of test #{target_test}")); - }; - - // Write the test contents to a temporary file - let test_path = std::env::temp_dir().join(".tree-sitter-test"); - let mut test_file = std::fs::File::create(&test_path)?; - test_file.write_all(test_contents)?; - - println!( - "{target_test}. {}\n", - paint(color.then_some(AnsiColor::Green), test_name) - ); - - Ok((test_path, languages)) -} - -pub fn check_queries_at_path(language: &Language, path: &Path) -> Result<()> { - if path.exists() { - for entry in WalkDir::new(path) - .into_iter() - .filter_map(std::result::Result::ok) - .filter(|e| { - e.file_type().is_file() - && e.path().extension().and_then(OsStr::to_str) == Some("scm") - && !e.path().starts_with(".") - }) - { - let filepath = entry.file_name().to_str().unwrap_or(""); - let content = fs::read_to_string(entry.path()) - .with_context(|| format!("Error reading query file {filepath:?}"))?; - Query::new(language, &content) - .with_context(|| format!("Error in query file {filepath:?}"))?; - } - } - Ok(()) -} - -pub fn print_diff_key() { - println!( - "\ncorrect / {} / {}", - paint(Some(AnsiColor::Green), "expected"), - paint(Some(AnsiColor::Red), "unexpected") - ); -} - -pub fn print_diff(actual: &str, expected: &str, use_color: bool) { - let diff = TextDiff::from_lines(actual, expected); - for diff in diff.iter_all_changes() { - match diff.tag() { - ChangeTag::Equal => { - if use_color { - print!("{diff}"); - } else { - print!(" {diff}"); - } - } - ChangeTag::Insert => { - if use_color { - print!("{}", paint(Some(AnsiColor::Green), diff.as_str().unwrap())); - } else { - print!("+{diff}"); - } - if diff.missing_newline() { - println!(); - } - } - ChangeTag::Delete => { - if use_color { - print!("{}", paint(Some(AnsiColor::Red), diff.as_str().unwrap())); - } else { - print!("-{diff}"); - } - if diff.missing_newline() { - println!(); - } - } - } - } - - println!(); -} - -pub fn paint(color: Option>, text: &str) -> String { - let style = Style::new().fg_color(color.map(Into::into)); - format!("{style}{text}{style:#}") -} - -/// This will return false if we want to "fail fast". It will bail and not parse any more tests. -#[allow(clippy::too_many_arguments)] -fn run_tests( - parser: &mut Parser, - test_entry: TestEntry, - opts: &mut TestOptions, - mut indent_level: u32, - failures: &mut Vec<(String, String, String)>, - corrected_entries: &mut Vec<(String, String, String, String, usize, usize)>, - has_parse_errors: &mut bool, -) -> Result { - match test_entry { - TestEntry::Example { - name, - input, - output, - header_delim_len, - divider_delim_len, - has_fields, - attributes_str, - attributes, - } => { - print!("{}", " ".repeat(indent_level as usize)); - - if attributes.skip { - println!( - "{:>3}.  {}", - opts.test_num, - paint(opts.color.then_some(AnsiColor::Yellow), &name), - ); - return Ok(true); - } - - if !attributes.platform { - println!( - "{:>3}.  {}", - opts.test_num, - paint(opts.color.then_some(AnsiColor::Magenta), &name), - ); - return Ok(true); - } - - for (i, language_name) in attributes.languages.iter().enumerate() { - if !language_name.is_empty() { - let language = opts - .languages - .get(language_name.as_ref()) - .ok_or_else(|| anyhow!("Language not found: {language_name}"))?; - parser.set_language(language)?; - } - let tree = parser.parse(&input, None).unwrap(); - - if attributes.error { - if tree.root_node().has_error() { - println!( - "{:>3}.  {}", - opts.test_num, - paint(opts.color.then_some(AnsiColor::Green), &name) - ); - if opts.update { - let input = String::from_utf8(input.clone()).unwrap(); - let output = format_sexp(&output, 0); - corrected_entries.push(( - name.clone(), - input, - output, - attributes_str.clone(), - header_delim_len, - divider_delim_len, - )); - } - } else { - if opts.update { - let input = String::from_utf8(input.clone()).unwrap(); - // Keep the original `expected` output if the actual output has no error - let output = format_sexp(&output, 0); - corrected_entries.push(( - name.clone(), - input, - output, - attributes_str.clone(), - header_delim_len, - divider_delim_len, - )); - } - println!( - "{:>3}.  {}", - opts.test_num, - paint(opts.color.then_some(AnsiColor::Red), &name) - ); - failures.push(( - name.clone(), - tree.root_node().to_sexp(), - "NO ERROR".to_string(), - )); - } - - if attributes.fail_fast { - return Ok(false); - } - } else { - let mut actual = tree.root_node().to_sexp(); - if !(opts.show_fields || has_fields) { - actual = strip_sexp_fields(&actual); - } - - if actual == output { - println!( - "{:>3}. ✓ {}", - opts.test_num, - paint(opts.color.then_some(AnsiColor::Green), &name) - ); - if opts.update { - let input = String::from_utf8(input.clone()).unwrap(); - let output = format_sexp(&output, 0); - corrected_entries.push(( - name.clone(), - input, - output, - attributes_str.clone(), - header_delim_len, - divider_delim_len, - )); - } - } else { - if opts.update { - let input = String::from_utf8(input.clone()).unwrap(); - let expected_output = format_sexp(&output, 0); - let actual_output = format_sexp(&actual, 0); - - // Only bail early before updating if the actual is not the output, - // sometimes users want to test cases that - // are intended to have errors, hence why this - // check isn't shown above - if actual.contains("ERROR") || actual.contains("MISSING") { - *has_parse_errors = true; - - // keep the original `expected` output if the actual output has an - // error - corrected_entries.push(( - name.clone(), - input, - expected_output, - attributes_str.clone(), - header_delim_len, - divider_delim_len, - )); - } else { - corrected_entries.push(( - name.clone(), - input, - actual_output, - attributes_str.clone(), - header_delim_len, - divider_delim_len, - )); - println!( - "{:>3}. ✓ {}", - opts.test_num, - paint(opts.color.then_some(AnsiColor::Blue), &name), - ); - } - } else { - println!( - "{:>3}. ✗ {}", - opts.test_num, - paint(opts.color.then_some(AnsiColor::Red), &name), - ); - } - failures.push((name.clone(), actual, output.clone())); - - if attributes.fail_fast { - return Ok(false); - } - } - } - - if i == attributes.languages.len() - 1 { - // reset to the first language - parser.set_language(opts.languages.values().next().unwrap())?; - } - } - opts.test_num += 1; - } - TestEntry::Group { - name, - children, - file_path, - } => { - if children.is_empty() { - return Ok(true); - } - - indent_level += 1; - let mut advance_counter = opts.test_num; - let failure_count = failures.len(); - let mut has_printed = false; - let mut skipped_tests = 0; - - let matches_filter = |name: &str, opts: &TestOptions| { - if let Some(include) = &opts.include { - include.is_match(name) - } else if let Some(exclude) = &opts.exclude { - !exclude.is_match(name) - } else { - true - } - }; - - let mut should_skip = |entry: &TestEntry, opts: &TestOptions| match entry { - TestEntry::Example { name, .. } => { - advance_counter += 1; - !matches_filter(name, opts) - } - TestEntry::Group { .. } => { - advance_counter += count_subtests(entry); - false - } - }; - - for child in children { - if let TestEntry::Example { - ref name, - ref input, - ref output, - ref attributes_str, - header_delim_len, - divider_delim_len, - .. - } = child - { - if should_skip(&child, opts) { - let input = String::from_utf8(input.clone()).unwrap(); - let output = format_sexp(output, 0); - corrected_entries.push(( - name.clone(), - input, - output, - attributes_str.clone(), - header_delim_len, - divider_delim_len, - )); - - opts.test_num += 1; - skipped_tests += 1; - - continue; - } - } - if !has_printed && indent_level > 1 { - has_printed = true; - print!("{}", " ".repeat((indent_level - 1) as usize)); - println!("{name}:"); - } - if !run_tests( - parser, - child, - opts, - indent_level, - failures, - corrected_entries, - has_parse_errors, - )? { - // fail fast - return Ok(false); - } - } - - opts.test_num += skipped_tests; - - if let Some(file_path) = file_path { - if opts.update && failures.len() - failure_count > 0 { - write_tests(&file_path, corrected_entries)?; - } - corrected_entries.clear(); - } - } - } - Ok(true) -} - -fn count_subtests(test_entry: &TestEntry) -> usize { - match test_entry { - TestEntry::Example { .. } => 1, - TestEntry::Group { children, .. } => children - .iter() - .fold(0, |count, child| count + count_subtests(child)), - } -} - -fn write_tests( - file_path: &Path, - corrected_entries: &[(String, String, String, String, usize, usize)], -) -> Result<()> { - let mut buffer = fs::File::create(file_path)?; - write_tests_to_buffer(&mut buffer, corrected_entries) -} - -fn write_tests_to_buffer( - buffer: &mut impl Write, - corrected_entries: &[(String, String, String, String, usize, usize)], -) -> Result<()> { - for (i, (name, input, output, attributes_str, header_delim_len, divider_delim_len)) in - corrected_entries.iter().enumerate() - { - if i > 0 { - writeln!(buffer)?; - } - writeln!( - buffer, - "{}\n{name}\n{}{}\n{input}\n{}\n\n{}", - "=".repeat(*header_delim_len), - if attributes_str.is_empty() { - attributes_str.clone() - } else { - format!("{attributes_str}\n") - }, - "=".repeat(*header_delim_len), - "-".repeat(*divider_delim_len), - output.trim() - )?; - } - Ok(()) -} - -pub fn parse_tests(path: &Path) -> io::Result { - let name = path - .file_stem() - .and_then(|s| s.to_str()) - .unwrap_or("") - .to_string(); - if path.is_dir() { - let mut children = Vec::new(); - for entry in fs::read_dir(path)? { - let entry = entry?; - let hidden = entry.file_name().to_str().unwrap_or("").starts_with('.'); - if !hidden { - children.push(entry.path()); - } - } - children.sort_by(|a, b| { - a.file_name() - .unwrap_or_default() - .cmp(b.file_name().unwrap_or_default()) - }); - let children = children - .iter() - .map(|path| parse_tests(path)) - .collect::>>()?; - Ok(TestEntry::Group { - name, - children, - file_path: None, - }) - } else { - let content = fs::read_to_string(path)?; - Ok(parse_test_content(name, &content, Some(path.to_path_buf()))) - } -} - -#[must_use] -pub fn strip_sexp_fields(sexp: &str) -> String { - SEXP_FIELD_REGEX.replace_all(sexp, " (").to_string() -} - -#[must_use] -pub fn strip_points(sexp: &str) -> String { - POINT_REGEX.replace_all(sexp, "").to_string() -} - -fn parse_test_content(name: String, content: &str, file_path: Option) -> TestEntry { - let mut children = Vec::new(); - let bytes = content.as_bytes(); - let mut prev_name = String::new(); - let mut prev_attributes_str = String::new(); - let mut prev_header_end = 0; - - // Find the first test header in the file, and determine if it has a - // custom suffix. If so, then this suffix will be used to identify - // all subsequent headers and divider lines in the file. - let first_suffix = HEADER_REGEX - .captures(bytes) - .and_then(|c| c.name("suffix1")) - .map(|m| String::from_utf8_lossy(m.as_bytes())); - - // Find all of the `===` test headers, which contain the test names. - // Ignore any matches whose suffix does not match the first header - // suffix in the file. - let header_matches = HEADER_REGEX.captures_iter(bytes).filter_map(|c| { - let header_delim_len = c.name("equals").map_or(80, |m| m.as_bytes().len()); - let suffix1 = c - .name("suffix1") - .map(|m| String::from_utf8_lossy(m.as_bytes())); - let suffix2 = c - .name("suffix2") - .map(|m| String::from_utf8_lossy(m.as_bytes())); - - let (mut skip, mut platform, mut fail_fast, mut error, mut languages) = - (false, None, false, false, vec![]); - - let test_name_and_markers = c - .name("test_name_and_markers") - .map_or("".as_bytes(), |m| m.as_bytes()); - - let mut test_name = String::new(); - let mut attributes_str = String::new(); - - let mut seen_marker = false; - - let test_name_and_markers = str::from_utf8(test_name_and_markers).unwrap(); - for line in test_name_and_markers - .split_inclusive('\n') - .filter(|s| !s.is_empty()) - { - let trimmed_line = line.trim(); - match trimmed_line.split('(').next().unwrap() { - ":skip" => (seen_marker, skip) = (true, true), - ":platform" => { - if let Some(platforms) = trimmed_line.strip_prefix(':').and_then(|s| { - s.strip_prefix("platform(") - .and_then(|s| s.strip_suffix(')')) - }) { - seen_marker = true; - platform = Some( - platform.unwrap_or(false) || platforms.trim() == std::env::consts::OS, - ); - } - } - ":fail-fast" => (seen_marker, fail_fast) = (true, true), - ":error" => (seen_marker, error) = (true, true), - ":language" => { - if let Some(lang) = trimmed_line.strip_prefix(':').and_then(|s| { - s.strip_prefix("language(") - .and_then(|s| s.strip_suffix(')')) - }) { - seen_marker = true; - languages.push(lang.into()); - } - } - _ if !seen_marker => { - test_name.push_str(line); - } - _ => {} - } - } - attributes_str.push_str(test_name_and_markers.strip_prefix(&test_name).unwrap()); - - // prefer skip over error, both shouldn't be set - if skip { - error = false; - } - - // add a default language if none are specified, will defer to the first language - if languages.is_empty() { - languages.push("".into()); - } - - if suffix1 == first_suffix && suffix2 == first_suffix { - let header_range = c.get(0).unwrap().range(); - let test_name = if test_name.is_empty() { - None - } else { - Some(test_name.trim_end().to_string()) - }; - let attributes_str = if attributes_str.is_empty() { - None - } else { - Some(attributes_str.trim_end().to_string()) - }; - Some(( - header_delim_len, - header_range, - test_name, - attributes_str, - TestAttributes { - skip, - platform: platform.unwrap_or(true), - fail_fast, - error, - languages, - }, - )) - } else { - None - } - }); - - let (mut prev_header_len, mut prev_attributes) = (80, TestAttributes::default()); - for (header_delim_len, header_range, test_name, attributes_str, attributes) in header_matches - .chain(Some(( - 80, - bytes.len()..bytes.len(), - None, - None, - TestAttributes::default(), - ))) - { - // Find the longest line of dashes following each test description. That line - // separates the input from the expected output. Ignore any matches whose suffix - // does not match the first suffix in the file. - if prev_header_end > 0 { - let divider_range = DIVIDER_REGEX - .captures_iter(&bytes[prev_header_end..header_range.start]) - .filter_map(|m| { - let divider_delim_len = m.name("hyphens").map_or(80, |m| m.as_bytes().len()); - let suffix = m - .name("suffix") - .map(|m| String::from_utf8_lossy(m.as_bytes())); - if suffix == first_suffix { - let range = m.get(0).unwrap().range(); - Some(( - divider_delim_len, - (prev_header_end + range.start)..(prev_header_end + range.end), - )) - } else { - None - } - }) - .max_by_key(|(_, range)| range.len()); - - if let Some((divider_delim_len, divider_range)) = divider_range { - if let Ok(output) = str::from_utf8(&bytes[divider_range.end..header_range.start]) { - let mut input = bytes[prev_header_end..divider_range.start].to_vec(); - - // Remove trailing newline from the input. - input.pop(); - if input.last() == Some(&b'\r') { - input.pop(); - } - - // Remove all comments - let output = COMMENT_REGEX.replace_all(output, "").to_string(); - - // Normalize the whitespace in the expected output. - let output = WHITESPACE_REGEX.replace_all(output.trim(), " "); - let output = output.replace(" )", ")"); - - // Identify if the expected output has fields indicated. If not, then - // fields will not be checked. - let has_fields = SEXP_FIELD_REGEX.is_match(&output); - - let t = TestEntry::Example { - name: prev_name, - input, - output, - header_delim_len: prev_header_len, - divider_delim_len, - has_fields, - attributes_str: prev_attributes_str, - attributes: prev_attributes, - }; - - children.push(t); - } - } - } - prev_attributes = attributes; - prev_name = test_name.unwrap_or_default(); - prev_attributes_str = attributes_str.unwrap_or_default(); - prev_header_len = header_delim_len; - prev_header_end = header_range.end; - } - TestEntry::Group { - name, - children, - file_path, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_parse_test_content_simple() { - let entry = parse_test_content( - "the-filename".to_string(), - r" -=============== -The first test -=============== - -a b c - ---- - -(a - (b c)) - -================ -The second test -================ -d ---- -(d) - " - .trim(), - None, - ); - - assert_eq!( - entry, - TestEntry::Group { - name: "the-filename".to_string(), - children: vec![ - TestEntry::Example { - name: "The first test".to_string(), - input: b"\na b c\n".to_vec(), - output: "(a (b c))".to_string(), - header_delim_len: 15, - divider_delim_len: 3, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - }, - TestEntry::Example { - name: "The second test".to_string(), - input: b"d".to_vec(), - output: "(d)".to_string(), - header_delim_len: 16, - divider_delim_len: 3, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - }, - ], - file_path: None, - } - ); - } - - #[test] - fn test_parse_test_content_with_dashes_in_source_code() { - let entry = parse_test_content( - "the-filename".to_string(), - r" -================== -Code with dashes -================== -abc ---- -defg ----- -hijkl -------- - -(a (b)) - -========================= -Code ending with dashes -========================= -abc ------------ -------------------- - -(c (d)) - " - .trim(), - None, - ); - - assert_eq!( - entry, - TestEntry::Group { - name: "the-filename".to_string(), - children: vec![ - TestEntry::Example { - name: "Code with dashes".to_string(), - input: b"abc\n---\ndefg\n----\nhijkl".to_vec(), - output: "(a (b))".to_string(), - header_delim_len: 18, - divider_delim_len: 7, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - }, - TestEntry::Example { - name: "Code ending with dashes".to_string(), - input: b"abc\n-----------".to_vec(), - output: "(c (d))".to_string(), - header_delim_len: 25, - divider_delim_len: 19, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - }, - ], - file_path: None, - } - ); - } - - #[test] - fn test_format_sexp() { - assert_eq!(format_sexp("", 0), ""); - assert_eq!( - format_sexp("(a b: (c) (d) e: (f (g (h (MISSING i)))))", 0), - r" -(a - b: (c) - (d) - e: (f - (g - (h - (MISSING i))))) -" - .trim() - ); - assert_eq!( - format_sexp("(program (ERROR (UNEXPECTED ' ')) (identifier))", 0), - r" -(program - (ERROR - (UNEXPECTED ' ')) - (identifier)) -" - .trim() - ); - assert_eq!( - format_sexp(r#"(source_file (MISSING ")"))"#, 0), - r#" -(source_file - (MISSING ")")) - "# - .trim() - ); - assert_eq!( - format_sexp( - r"(source_file (ERROR (UNEXPECTED 'f') (UNEXPECTED '+')))", - 0 - ), - r" -(source_file - (ERROR - (UNEXPECTED 'f') - (UNEXPECTED '+'))) -" - .trim() - ); - } - - #[test] - fn test_write_tests_to_buffer() { - let mut buffer = Vec::new(); - let corrected_entries = vec![ - ( - "title 1".to_string(), - "input 1".to_string(), - "output 1".to_string(), - String::new(), - 80, - 80, - ), - ( - "title 2".to_string(), - "input 2".to_string(), - "output 2".to_string(), - String::new(), - 80, - 80, - ), - ]; - write_tests_to_buffer(&mut buffer, &corrected_entries).unwrap(); - assert_eq!( - String::from_utf8(buffer).unwrap(), - r" -================================================================================ -title 1 -================================================================================ -input 1 --------------------------------------------------------------------------------- - -output 1 - -================================================================================ -title 2 -================================================================================ -input 2 --------------------------------------------------------------------------------- - -output 2 -" - .trim_start() - .to_string() - ); - } - - #[test] - fn test_parse_test_content_with_comments_in_sexp() { - let entry = parse_test_content( - "the-filename".to_string(), - r#" -================== -sexp with comment -================== -code ---- - -; Line start comment -(a (b)) - -================== -sexp with comment between -================== -code ---- - -; Line start comment -(a -; ignore this - (b) - ; also ignore this -) - -========================= -sexp with ';' -========================= -code ---- - -(MISSING ";") - "# - .trim(), - None, - ); - - assert_eq!( - entry, - TestEntry::Group { - name: "the-filename".to_string(), - children: vec![ - TestEntry::Example { - name: "sexp with comment".to_string(), - input: b"code".to_vec(), - output: "(a (b))".to_string(), - header_delim_len: 18, - divider_delim_len: 3, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - }, - TestEntry::Example { - name: "sexp with comment between".to_string(), - input: b"code".to_vec(), - output: "(a (b))".to_string(), - header_delim_len: 18, - divider_delim_len: 3, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - }, - TestEntry::Example { - name: "sexp with ';'".to_string(), - input: b"code".to_vec(), - output: "(MISSING \";\")".to_string(), - header_delim_len: 25, - divider_delim_len: 3, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - } - ], - file_path: None, - } - ); - } - - #[test] - fn test_parse_test_content_with_suffixes() { - let entry = parse_test_content( - "the-filename".to_string(), - r" -==================asdf\()[]|{}*+?^$.- -First test -==================asdf\()[]|{}*+?^$.- - -========================= -NOT A TEST HEADER -========================= -------------------------- - ----asdf\()[]|{}*+?^$.- - -(a) - -==================asdf\()[]|{}*+?^$.- -Second test -==================asdf\()[]|{}*+?^$.- - -========================= -NOT A TEST HEADER -========================= -------------------------- - ----asdf\()[]|{}*+?^$.- - -(a) - -=========================asdf\()[]|{}*+?^$.- -Test name with = symbol -=========================asdf\()[]|{}*+?^$.- - -========================= -NOT A TEST HEADER -========================= -------------------------- - ----asdf\()[]|{}*+?^$.- - -(a) - -==============================asdf\()[]|{}*+?^$.- -Test containing equals -==============================asdf\()[]|{}*+?^$.- - -=== - -------------------------------asdf\()[]|{}*+?^$.- - -(a) - -==============================asdf\()[]|{}*+?^$.- -Subsequent test containing equals -==============================asdf\()[]|{}*+?^$.- - -=== - -------------------------------asdf\()[]|{}*+?^$.- - -(a) -" - .trim(), - None, - ); - - let expected_input = b"\n=========================\n\ - NOT A TEST HEADER\n\ - =========================\n\ - -------------------------\n" - .to_vec(); - pretty_assertions::assert_eq!( - entry, - TestEntry::Group { - name: "the-filename".to_string(), - children: vec![ - TestEntry::Example { - name: "First test".to_string(), - input: expected_input.clone(), - output: "(a)".to_string(), - header_delim_len: 18, - divider_delim_len: 3, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - }, - TestEntry::Example { - name: "Second test".to_string(), - input: expected_input.clone(), - output: "(a)".to_string(), - header_delim_len: 18, - divider_delim_len: 3, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - }, - TestEntry::Example { - name: "Test name with = symbol".to_string(), - input: expected_input, - output: "(a)".to_string(), - header_delim_len: 25, - divider_delim_len: 3, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - }, - TestEntry::Example { - name: "Test containing equals".to_string(), - input: "\n===\n".into(), - output: "(a)".into(), - header_delim_len: 30, - divider_delim_len: 30, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - }, - TestEntry::Example { - name: "Subsequent test containing equals".to_string(), - input: "\n===\n".into(), - output: "(a)".into(), - header_delim_len: 30, - divider_delim_len: 30, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - } - ], - file_path: None, - } - ); - } - - #[test] - fn test_parse_test_content_with_newlines_in_test_names() { - let entry = parse_test_content( - "the-filename".to_string(), - r" -=============== -name -with -newlines -=============== -a ---- -(b) - -==================== -name with === signs -==================== -code with ---- ---- -(d) -", - None, - ); - - assert_eq!( - entry, - TestEntry::Group { - name: "the-filename".to_string(), - file_path: None, - children: vec![ - TestEntry::Example { - name: "name\nwith\nnewlines".to_string(), - input: b"a".to_vec(), - output: "(b)".to_string(), - header_delim_len: 15, - divider_delim_len: 3, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - }, - TestEntry::Example { - name: "name with === signs".to_string(), - input: b"code with ----".to_vec(), - output: "(d)".to_string(), - header_delim_len: 20, - divider_delim_len: 3, - has_fields: false, - attributes_str: String::new(), - attributes: TestAttributes::default(), - } - ] - } - ); - } - - #[test] - fn test_parse_test_with_markers() { - // do one with :skip, we should not see it in the entry output - - let entry = parse_test_content( - "the-filename".to_string(), - r" -===================== -Test with skip marker -:skip -===================== -a ---- -(b) -", - None, - ); - - assert_eq!( - entry, - TestEntry::Group { - name: "the-filename".to_string(), - file_path: None, - children: vec![TestEntry::Example { - name: "Test with skip marker".to_string(), - input: b"a".to_vec(), - output: "(b)".to_string(), - header_delim_len: 21, - divider_delim_len: 3, - has_fields: false, - attributes_str: ":skip".to_string(), - attributes: TestAttributes { - skip: true, - platform: true, - fail_fast: false, - error: false, - languages: vec!["".into()] - }, - }] - } - ); - - let entry = parse_test_content( - "the-filename".to_string(), - &format!( - r" -========================= -Test with platform marker -:platform({}) -:fail-fast -========================= -a ---- -(b) - -============================= -Test with bad platform marker -:platform({}) - -:language(foo) -============================= -a ---- -(b) -", - std::env::consts::OS, - if std::env::consts::OS == "linux" { - "macos" - } else { - "linux" - } - ), - None, - ); - - assert_eq!( - entry, - TestEntry::Group { - name: "the-filename".to_string(), - file_path: None, - children: vec![ - TestEntry::Example { - name: "Test with platform marker".to_string(), - input: b"a".to_vec(), - output: "(b)".to_string(), - header_delim_len: 25, - divider_delim_len: 3, - has_fields: false, - attributes_str: format!(":platform({})\n:fail-fast", std::env::consts::OS), - attributes: TestAttributes { - skip: false, - platform: true, - fail_fast: true, - error: false, - languages: vec!["".into()] - }, - }, - TestEntry::Example { - name: "Test with bad platform marker".to_string(), - input: b"a".to_vec(), - output: "(b)".to_string(), - header_delim_len: 29, - divider_delim_len: 3, - has_fields: false, - attributes_str: if std::env::consts::OS == "linux" { - ":platform(macos)\n\n:language(foo)".to_string() - } else { - ":platform(linux)\n\n:language(foo)".to_string() - }, - attributes: TestAttributes { - skip: false, - platform: false, - fail_fast: false, - error: false, - languages: vec!["foo".into()] - }, - } - ] - } - ); - } -} diff --git a/cli/src/tests/async_context_test.rs b/cli/src/tests/async_context_test.rs deleted file mode 100644 index edcd5e4c..00000000 --- a/cli/src/tests/async_context_test.rs +++ /dev/null @@ -1,278 +0,0 @@ -use std::{ - future::Future, - pin::{pin, Pin}, - ptr, - task::{self, Context, Poll, RawWaker, RawWakerVTable, Waker}, -}; - -use tree_sitter::Parser; - -use super::helpers::fixtures::get_language; - -#[test] -fn test_node_in_fut() { - let (ret, pended) = tokio_like_spawn(async { - let mut parser = Parser::new(); - let language = get_language("bash"); - parser.set_language(&language).unwrap(); - - let tree = parser.parse("#", None).unwrap(); - - let root = tree.root_node(); - let root_ref = &root; - - let fut_val_fn = || async { - yield_now().await; - root.child(0).unwrap().kind() - }; - - yield_now().await; - - let fut_ref_fn = || async { - yield_now().await; - root_ref.child(0).unwrap().kind() - }; - - let f1 = fut_val_fn().await; - let f2 = fut_ref_fn().await; - assert_eq!(f1, f2); - - let fut_val = async { - yield_now().await; - root.child(0).unwrap().kind() - }; - - let fut_ref = async { - yield_now().await; - root_ref.child(0).unwrap().kind() - }; - - let f1 = fut_val.await; - let f2 = fut_ref.await; - assert_eq!(f1, f2); - - f1 - }) - .join(); - assert_eq!(ret, "comment"); - assert_eq!(pended, 5); -} - -#[test] -fn test_node_and_cursor_ref_in_fut() { - let ((), pended) = tokio_like_spawn(async { - let mut parser = Parser::new(); - let language = get_language("c"); - parser.set_language(&language).unwrap(); - - let tree = parser.parse("#", None).unwrap(); - - let root = tree.root_node(); - let root_ref = &root; - - let mut cursor = tree.walk(); - let cursor_ref = &mut cursor; - - cursor_ref.goto_first_child(); - - let fut_val = async { - yield_now().await; - let _ = root.to_sexp(); - }; - - yield_now().await; - - let fut_ref = async { - yield_now().await; - let _ = root_ref.to_sexp(); - cursor_ref.goto_first_child(); - }; - - fut_val.await; - fut_ref.await; - - cursor_ref.goto_first_child(); - }) - .join(); - assert_eq!(pended, 3); -} - -#[test] -fn test_node_and_cursor_ref_in_fut_with_fut_fabrics() { - let ((), pended) = tokio_like_spawn(async { - let mut parser = Parser::new(); - let language = get_language("javascript"); - parser.set_language(&language).unwrap(); - - let tree = parser.parse("#", None).unwrap(); - - let root = tree.root_node(); - let root_ref = &root; - - let mut cursor = tree.walk(); - let cursor_ref = &mut cursor; - - cursor_ref.goto_first_child(); - - let fut_val = || async { - yield_now().await; - let _ = root.to_sexp(); - }; - - yield_now().await; - - let fut_ref = || async move { - yield_now().await; - let _ = root_ref.to_sexp(); - cursor_ref.goto_first_child(); - }; - - fut_val().await; - fut_val().await; - fut_ref().await; - }) - .join(); - assert_eq!(pended, 4); -} - -#[test] -fn test_node_and_cursor_ref_in_fut_with_inner_spawns() { - let (ret, pended) = tokio_like_spawn(async { - let mut parser = Parser::new(); - let language = get_language("rust"); - parser.set_language(&language).unwrap(); - - let tree = parser.parse("#", None).unwrap(); - - let mut cursor = tree.walk(); - let cursor_ref = &mut cursor; - - cursor_ref.goto_first_child(); - - let fut_val = || { - let tree = tree.clone(); - async move { - let root = tree.root_node(); - let mut cursor = tree.walk(); - let cursor_ref = &mut cursor; - yield_now().await; - let _ = root.to_sexp(); - cursor_ref.goto_first_child(); - } - }; - - yield_now().await; - - let fut_ref = || { - let tree = tree.clone(); - async move { - let root = tree.root_node(); - let root_ref = &root; - let mut cursor = tree.walk(); - let cursor_ref = &mut cursor; - yield_now().await; - let _ = root_ref.to_sexp(); - cursor_ref.goto_first_child(); - } - }; - - let ((), p1) = tokio_like_spawn(fut_val()).await.unwrap(); - let ((), p2) = tokio_like_spawn(fut_ref()).await.unwrap(); - - cursor_ref.goto_first_child(); - - fut_val().await; - fut_val().await; - fut_ref().await; - - cursor_ref.goto_first_child(); - - p1 + p2 - }) - .join(); - assert_eq!(pended, 4); - assert_eq!(ret, 2); -} - -fn tokio_like_spawn(future: T) -> JoinHandle<(T::Output, usize)> -where - T: Future + Send + 'static, - T::Output: Send + 'static, -{ - // No runtime, just noop waker - - let waker = noop_waker(); - let mut cx = task::Context::from_waker(&waker); - - let mut pending = 0; - let mut future = pin!(future); - let ret = loop { - match future.as_mut().poll(&mut cx) { - Poll::Pending => pending += 1, - Poll::Ready(r) => { - break r; - } - } - }; - JoinHandle::new((ret, pending)) -} - -async fn yield_now() { - struct SimpleYieldNow { - yielded: bool, - } - - impl Future for SimpleYieldNow { - type Output = (); - - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { - cx.waker().wake_by_ref(); - if self.yielded { - return Poll::Ready(()); - } - self.yielded = true; - Poll::Pending - } - } - - SimpleYieldNow { yielded: false }.await; -} - -pub fn noop_waker() -> Waker { - const VTABLE: RawWakerVTable = RawWakerVTable::new( - // Cloning just returns a new no-op raw waker - |_| RAW, - // `wake` does nothing - |_| {}, - // `wake_by_ref` does nothing - |_| {}, - // Dropping does nothing as we don't allocate anything - |_| {}, - ); - const RAW: RawWaker = RawWaker::new(ptr::null(), &VTABLE); - unsafe { Waker::from_raw(RAW) } -} - -struct JoinHandle { - data: Option, -} - -impl JoinHandle { - #[must_use] - const fn new(data: T) -> Self { - Self { data: Some(data) } - } - - fn join(&mut self) -> T { - self.data.take().unwrap() - } -} - -impl Future for JoinHandle { - type Output = std::result::Result; - - fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll { - let data = self.get_mut().data.take().unwrap(); - Poll::Ready(Ok(data)) - } -} diff --git a/cli/src/tests/helpers/allocations.rs b/cli/src/tests/helpers/allocations.rs deleted file mode 100644 index 103cb092..00000000 --- a/cli/src/tests/helpers/allocations.rs +++ /dev/null @@ -1,121 +0,0 @@ -use std::{ - collections::HashMap, - os::raw::c_void, - sync::{ - atomic::{AtomicBool, AtomicUsize, Ordering::SeqCst}, - Mutex, - }, -}; - -#[ctor::ctor] -unsafe fn initialize_allocation_recording() { - tree_sitter::set_allocator( - Some(ts_record_malloc), - Some(ts_record_calloc), - Some(ts_record_realloc), - Some(ts_record_free), - ); -} - -#[derive(Debug, PartialEq, Eq, Hash)] -struct Allocation(*const c_void); -unsafe impl Send for Allocation {} -unsafe impl Sync for Allocation {} - -#[derive(Default)] -struct AllocationRecorder { - enabled: AtomicBool, - allocation_count: AtomicUsize, - outstanding_allocations: Mutex>, -} - -thread_local! { - static RECORDER: AllocationRecorder = AllocationRecorder::default(); -} - -extern "C" { - fn malloc(size: usize) -> *mut c_void; - fn calloc(count: usize, size: usize) -> *mut c_void; - fn realloc(ptr: *mut c_void, size: usize) -> *mut c_void; - fn free(ptr: *mut c_void); -} - -pub fn record(f: impl FnOnce() -> T) -> T { - RECORDER.with(|recorder| { - recorder.enabled.store(true, SeqCst); - recorder.allocation_count.store(0, SeqCst); - recorder.outstanding_allocations.lock().unwrap().clear(); - }); - - let value = f(); - - let outstanding_allocation_indices = RECORDER.with(|recorder| { - recorder.enabled.store(false, SeqCst); - recorder.allocation_count.store(0, SeqCst); - recorder - .outstanding_allocations - .lock() - .unwrap() - .drain() - .map(|e| e.1) - .collect::>() - }); - assert!( - outstanding_allocation_indices.is_empty(), - "Leaked allocation indices: {outstanding_allocation_indices:?}" - ); - value -} - -fn record_alloc(ptr: *mut c_void) { - RECORDER.with(|recorder| { - if recorder.enabled.load(SeqCst) { - let count = recorder.allocation_count.fetch_add(1, SeqCst); - recorder - .outstanding_allocations - .lock() - .unwrap() - .insert(Allocation(ptr), count); - } - }); -} - -fn record_dealloc(ptr: *mut c_void) { - RECORDER.with(|recorder| { - if recorder.enabled.load(SeqCst) { - recorder - .outstanding_allocations - .lock() - .unwrap() - .remove(&Allocation(ptr)); - } - }); -} - -unsafe extern "C" fn ts_record_malloc(size: usize) -> *mut c_void { - let result = malloc(size); - record_alloc(result); - result -} - -unsafe extern "C" fn ts_record_calloc(count: usize, size: usize) -> *mut c_void { - let result = calloc(count, size); - record_alloc(result); - result -} - -unsafe extern "C" fn ts_record_realloc(ptr: *mut c_void, size: usize) -> *mut c_void { - let result = realloc(ptr, size); - if ptr.is_null() { - record_alloc(result); - } else if ptr != result { - record_dealloc(ptr); - record_alloc(result); - } - result -} - -unsafe extern "C" fn ts_record_free(ptr: *mut c_void) { - record_dealloc(ptr); - free(ptr); -} diff --git a/cli/src/tests/helpers/dirs.rs b/cli/src/tests/helpers/dirs.rs deleted file mode 100644 index 4d1c4982..00000000 --- a/cli/src/tests/helpers/dirs.rs +++ /dev/null @@ -1,47 +0,0 @@ -lazy_static! { - pub static ref ROOT_DIR: PathBuf = PathBuf::from(env!("CARGO_MANIFEST_DIR")).parent().unwrap().to_owned(); - pub static ref FIXTURES_DIR: PathBuf = ROOT_DIR.join("test").join("fixtures"); - pub static ref HEADER_DIR: PathBuf = ROOT_DIR.join("lib").join("include"); - pub static ref GRAMMARS_DIR: PathBuf = ROOT_DIR.join("test").join("fixtures").join("grammars"); - pub static ref SCRATCH_BASE_DIR: PathBuf = { - let result = ROOT_DIR.join("target").join("scratch"); - fs::create_dir_all(&result).unwrap(); - result - }; - pub static ref WASM_DIR: PathBuf = ROOT_DIR.join("target").join("release"); - pub static ref SCRATCH_DIR: PathBuf = { - // https://doc.rust-lang.org/reference/conditional-compilation.html - let vendor = if cfg!(target_vendor = "apple") { - "apple" - } else if cfg!(target_vendor = "fortanix") { - "fortanix" - } else if cfg!(target_vendor = "pc") { - "pc" - } else { - "unknown" - }; - let env = if cfg!(target_env = "gnu") { - "gnu" - } else if cfg!(target_env = "msvc") { - "msvc" - } else if cfg!(target_env = "musl") { - "musl" - } else if cfg!(target_env = "sgx") { - "sgx" - } else { - "unknown" - }; - let endian = if cfg!(target_endian = "little") { - "little" - } else if cfg!(target_endian = "big") { - "big" - } else { - "unknown" - }; - - let machine = format!("{}-{}-{vendor}-{env}-{endian}", std::env::consts::ARCH, std::env::consts::OS); - let result = SCRATCH_BASE_DIR.join(machine); - fs::create_dir_all(&result).unwrap(); - result - }; -} diff --git a/cli/src/tests/language_test.rs b/cli/src/tests/language_test.rs deleted file mode 100644 index 3def3d60..00000000 --- a/cli/src/tests/language_test.rs +++ /dev/null @@ -1,97 +0,0 @@ -use tree_sitter::{self, Parser}; - -use super::helpers::fixtures::get_language; - -#[test] -fn test_lookahead_iterator() { - let mut parser = Parser::new(); - let language = get_language("rust"); - parser.set_language(&language).unwrap(); - - let tree = parser.parse("struct Stuff {}", None).unwrap(); - - let mut cursor = tree.walk(); - - assert!(cursor.goto_first_child()); // struct - assert!(cursor.goto_first_child()); // struct keyword - - let next_state = cursor.node().next_parse_state(); - assert_ne!(next_state, 0); - assert_eq!( - next_state, - language.next_state(cursor.node().parse_state(), cursor.node().grammar_id()) - ); - assert!((next_state as usize) < language.parse_state_count()); - assert!(cursor.goto_next_sibling()); // type_identifier - assert_eq!(next_state, cursor.node().parse_state()); - assert_eq!(cursor.node().grammar_name(), "identifier"); - assert_ne!(cursor.node().grammar_id(), cursor.node().kind_id()); - - let expected_symbols = ["//", "/*", "identifier", "line_comment", "block_comment"]; - let mut lookahead = language.lookahead_iterator(next_state).unwrap(); - assert_eq!(*lookahead.language(), language); - assert!(lookahead.iter_names().eq(expected_symbols)); - - lookahead.reset_state(next_state); - assert!(lookahead.iter_names().eq(expected_symbols)); - - lookahead.reset(&language, next_state); - assert!(lookahead - .map(|s| language.node_kind_for_id(s).unwrap()) - .eq(expected_symbols)); -} - -#[test] -fn test_lookahead_iterator_modifiable_only_by_mut() { - let mut parser = Parser::new(); - let language = get_language("rust"); - parser.set_language(&language).unwrap(); - - let tree = parser.parse("struct Stuff {}", None).unwrap(); - - let mut cursor = tree.walk(); - - assert!(cursor.goto_first_child()); // struct - assert!(cursor.goto_first_child()); // struct keyword - - let next_state = cursor.node().next_parse_state(); - assert_ne!(next_state, 0); - - let mut lookahead = language.lookahead_iterator(next_state).unwrap(); - let _ = lookahead.next(); - - let mut names = lookahead.iter_names(); - let _ = names.next(); -} - -#[test] -fn test_symbol_metadata_checks() { - let language = get_language("rust"); - for i in 0..language.node_kind_count() { - let sym = i as u16; - let name = language.node_kind_for_id(sym).unwrap(); - match name { - "_type" - | "_expression" - | "_pattern" - | "_literal" - | "_literal_pattern" - | "_declaration_statement" => assert!(language.node_kind_is_supertype(sym)), - - "_raw_string_literal_start" - | "_raw_string_literal_end" - | "_line_doc_comment" - | "_error_sentinel" => assert!(!language.node_kind_is_supertype(sym)), - - "enum_item" | "struct_item" | "type_item" => { - assert!(language.node_kind_is_named(sym)); - } - - "=>" | "[" | "]" | "(" | ")" | "{" | "}" => { - assert!(language.node_kind_is_visible(sym)); - } - - _ => {} - } - } -} diff --git a/cli/src/tests/mod.rs b/cli/src/tests/mod.rs deleted file mode 100644 index 234172e4..00000000 --- a/cli/src/tests/mod.rs +++ /dev/null @@ -1,26 +0,0 @@ -mod async_context_test; -mod corpus_test; -mod detect_language; -mod helpers; -mod highlight_test; -mod language_test; -mod node_test; -mod parser_hang_test; -mod parser_test; -mod pathological_test; -mod query_test; -mod tags_test; -mod test_highlight_test; -mod test_tags_test; -mod text_provider_test; -mod tree_test; - -#[cfg(feature = "wasm")] -mod wasm_language_test; - -pub use crate::fuzz::{ - allocations, - edits::{get_random_edit, invert_edit}, - random::Rand, - ITERATION_COUNT, -}; diff --git a/cli/src/tests/parser_hang_test.rs b/cli/src/tests/parser_hang_test.rs deleted file mode 100644 index 3f7b6394..00000000 --- a/cli/src/tests/parser_hang_test.rs +++ /dev/null @@ -1,103 +0,0 @@ -// For some reasons `Command::spawn` doesn't work in CI env for many exotic arches. -#![cfg(all(any(target_arch = "x86_64", target_arch = "x86"), not(sanitizing)))] - -use std::{ - env::VarError, - process::{Command, Stdio}, -}; - -use tree_sitter::Parser; -use tree_sitter_generate::{generate_parser_for_grammar, load_grammar_file}; - -use crate::tests::helpers::fixtures::{fixtures_dir, get_test_language}; - -// The `sanitizing` cfg is required to don't run tests under specific sunitizer -// because they don't work well with subprocesses _(it's an assumption)_. -// -// Below are two alternative examples of how to disable tests for some arches -// if a way with excluding the whole mod from compilation wouldn't work well. -// -// XXX: Also may be it makes sense to keep such tests as ignored by default -// to omit surprises and enable them on CI by passing an extra option explicitly: -// -// > cargo test -- --include-ignored -// -// #[cfg(all(any(target_arch = "x86_64", target_arch = "x86"), not(sanitizing)))] -// #[cfg_attr(not(all(any(target_arch = "x86_64", target_arch = "x86"), not(sanitizing))), ignore)] -// -#[test] -fn test_grammar_that_should_hang_and_not_segfault() { - let parent_sleep_millis = 1000; - let test_name = "test_grammar_that_should_hang_and_not_segfault"; - let test_var = "CARGO_HANG_TEST"; - - eprintln!(" {test_name}"); - - let tests_exec_path = std::env::args() - .next() - .expect("Failed to get tests executable path"); - - match std::env::var(test_var) { - Ok(v) if v == test_name => { - eprintln!(" child process id {}", std::process::id()); - hang_test(); - } - - Err(VarError::NotPresent) => { - eprintln!(" parent process id {}", std::process::id()); - let mut command = Command::new(tests_exec_path); - command.arg(test_name).env(test_var, test_name); - - if std::env::args().any(|x| x == "--nocapture") { - command.arg("--nocapture"); - } else { - command.stdout(Stdio::null()).stderr(Stdio::null()); - } - - match command.spawn() { - Ok(mut child) => { - std::thread::sleep(std::time::Duration::from_millis(parent_sleep_millis)); - match child.try_wait() { - Ok(Some(status)) if status.success() => { - panic!("Child didn't hang and exited successfully") - } - Ok(Some(status)) => panic!( - "Child didn't hang and exited with status code: {:?}", - status.code() - ), - _ => (), - } - if let Err(e) = child.kill() { - eprintln!( - "Failed to kill hang test's process id: {}, error: {e}", - child.id() - ); - } - } - Err(e) => panic!("{e}"), - } - } - - Err(e) => panic!("Env var error: {e}"), - - _ => unreachable!(), - } -} - -fn hang_test() { - let test_grammar_dir = fixtures_dir() - .join("test_grammars") - .join("get_col_should_hang_not_crash"); - - let grammar_json = load_grammar_file(&test_grammar_dir.join("grammar.js"), None).unwrap(); - let (parser_name, parser_code) = generate_parser_for_grammar(grammar_json.as_str()).unwrap(); - - let language = get_test_language(&parser_name, &parser_code, Some(test_grammar_dir.as_path())); - - let mut parser = Parser::new(); - parser.set_language(&language).unwrap(); - - let code_that_should_hang = "\nHello"; - - parser.parse(code_that_should_hang, None).unwrap(); -} diff --git a/cli/src/version.rs b/cli/src/version.rs deleted file mode 100644 index 11871265..00000000 --- a/cli/src/version.rs +++ /dev/null @@ -1,264 +0,0 @@ -use std::{fs, path::PathBuf, process::Command}; - -use anyhow::{anyhow, Context, Result}; -use regex::Regex; -use tree_sitter_loader::TreeSitterJSON; - -pub struct Version { - pub version: String, - pub current_dir: PathBuf, -} - -impl Version { - #[must_use] - pub const fn new(version: String, current_dir: PathBuf) -> Self { - Self { - version, - current_dir, - } - } - - pub fn run(self) -> Result<()> { - let tree_sitter_json = self.current_dir.join("tree-sitter.json"); - - let tree_sitter_json = - serde_json::from_str::(&fs::read_to_string(tree_sitter_json)?)?; - - let is_multigrammar = tree_sitter_json.grammars.len() > 1; - - self.update_treesitter_json().with_context(|| { - format!( - "Failed to update tree-sitter.json at {}", - self.current_dir.display() - ) - })?; - self.update_cargo_toml().with_context(|| { - format!( - "Failed to update Cargo.toml at {}", - self.current_dir.display() - ) - })?; - self.update_package_json().with_context(|| { - format!( - "Failed to update package.json at {}", - self.current_dir.display() - ) - })?; - self.update_makefile(is_multigrammar).with_context(|| { - format!( - "Failed to update Makefile at {}", - self.current_dir.display() - ) - })?; - self.update_cmakelists_txt().with_context(|| { - format!( - "Failed to update CMakeLists.txt at {}", - self.current_dir.display() - ) - })?; - self.update_pyproject_toml().with_context(|| { - format!( - "Failed to update pyproject.toml at {}", - self.current_dir.display() - ) - })?; - - Ok(()) - } - - fn update_treesitter_json(&self) -> Result<()> { - let tree_sitter_json = &fs::read_to_string(self.current_dir.join("tree-sitter.json"))?; - - let tree_sitter_json = tree_sitter_json - .lines() - .map(|line| { - if line.contains("\"version\":") { - let prefix_index = line.find("\"version\":").unwrap() + "\"version\":".len(); - let start_quote = line[prefix_index..].find('"').unwrap() + prefix_index + 1; - let end_quote = line[start_quote + 1..].find('"').unwrap() + start_quote + 1; - - format!( - "{}{}{}", - &line[..start_quote], - self.version, - &line[end_quote..] - ) - } else { - line.to_string() - } - }) - .collect::>() - .join("\n") - + "\n"; - - fs::write(self.current_dir.join("tree-sitter.json"), tree_sitter_json)?; - - Ok(()) - } - - fn update_cargo_toml(&self) -> Result<()> { - if !self.current_dir.join("Cargo.toml").exists() { - return Ok(()); - } - - let cargo_toml = fs::read_to_string(self.current_dir.join("Cargo.toml"))?; - - let cargo_toml = cargo_toml - .lines() - .map(|line| { - if line.starts_with("version =") { - format!("version = \"{}\"", self.version) - } else { - line.to_string() - } - }) - .collect::>() - .join("\n") - + "\n"; - - fs::write(self.current_dir.join("Cargo.toml"), cargo_toml)?; - - if self.current_dir.join("Cargo.lock").exists() { - let Ok(cmd) = Command::new("cargo") - .arg("generate-lockfile") - .arg("--offline") - .current_dir(&self.current_dir) - .output() - else { - return Ok(()); // cargo is not `executable`, ignore - }; - - if !cmd.status.success() { - let stderr = String::from_utf8_lossy(&cmd.stderr); - return Err(anyhow!( - "Failed to run `cargo generate-lockfile`:\n{stderr}" - )); - } - } - - Ok(()) - } - - fn update_package_json(&self) -> Result<()> { - if !self.current_dir.join("package.json").exists() { - return Ok(()); - } - - let package_json = &fs::read_to_string(self.current_dir.join("package.json"))?; - - let package_json = package_json - .lines() - .map(|line| { - if line.contains("\"version\":") { - let prefix_index = line.find("\"version\":").unwrap() + "\"version\":".len(); - let start_quote = line[prefix_index..].find('"').unwrap() + prefix_index + 1; - let end_quote = line[start_quote + 1..].find('"').unwrap() + start_quote + 1; - - format!( - "{}{}{}", - &line[..start_quote], - self.version, - &line[end_quote..] - ) - } else { - line.to_string() - } - }) - .collect::>() - .join("\n") - + "\n"; - - fs::write(self.current_dir.join("package.json"), package_json)?; - - if self.current_dir.join("package-lock.json").exists() { - let Ok(cmd) = Command::new("npm") - .arg("install") - .arg("--package-lock-only") - .current_dir(&self.current_dir) - .output() - else { - return Ok(()); // npm is not `executable`, ignore - }; - - if !cmd.status.success() { - let stderr = String::from_utf8_lossy(&cmd.stderr); - return Err(anyhow!("Failed to run `npm install`:\n{stderr}")); - } - } - - Ok(()) - } - - fn update_makefile(&self, is_multigrammar: bool) -> Result<()> { - let makefile = if is_multigrammar { - if !self.current_dir.join("common").join("common.mak").exists() { - return Ok(()); - } - - fs::read_to_string(self.current_dir.join("Makefile"))? - } else { - if !self.current_dir.join("Makefile").exists() { - return Ok(()); - } - - fs::read_to_string(self.current_dir.join("Makefile"))? - }; - - let makefile = makefile - .lines() - .map(|line| { - if line.starts_with("VERSION") { - format!("VERSION := {}", self.version) - } else { - line.to_string() - } - }) - .collect::>() - .join("\n") - + "\n"; - - fs::write(self.current_dir.join("Makefile"), makefile)?; - - Ok(()) - } - - fn update_cmakelists_txt(&self) -> Result<()> { - if !self.current_dir.join("CMakeLists.txt").exists() { - return Ok(()); - } - - let cmake = fs::read_to_string(self.current_dir.join("CMakeLists.txt"))?; - - let re = Regex::new(r#"(\s*VERSION\s+)"[0-9]+\.[0-9]+\.[0-9]+""#)?; - let cmake = re.replace(&cmake, format!(r#"$1"{}""#, self.version)); - - fs::write(self.current_dir.join("CMakeLists.txt"), cmake.as_bytes())?; - - Ok(()) - } - - fn update_pyproject_toml(&self) -> Result<()> { - if !self.current_dir.join("pyproject.toml").exists() { - return Ok(()); - } - - let pyproject_toml = fs::read_to_string(self.current_dir.join("pyproject.toml"))?; - - let pyproject_toml = pyproject_toml - .lines() - .map(|line| { - if line.starts_with("version =") { - format!("version = \"{}\"", self.version) - } else { - line.to_string() - } - }) - .collect::>() - .join("\n") - + "\n"; - - fs::write(self.current_dir.join("pyproject.toml"), pyproject_toml)?; - - Ok(()) - } -} diff --git a/cli/vendor/xterm-colors.json b/cli/vendor/xterm-colors.json deleted file mode 100644 index 47994496..00000000 --- a/cli/vendor/xterm-colors.json +++ /dev/null @@ -1,258 +0,0 @@ -[ - "#000000", - "#800000", - "#008000", - "#808000", - "#000080", - "#800080", - "#008080", - "#c0c0c0", - "#808080", - "#ff0000", - "#00ff00", - "#ffff00", - "#0000ff", - "#ff00ff", - "#00ffff", - "#ffffff", - "#000000", - "#00005f", - "#000087", - "#0000af", - "#0000d7", - "#0000ff", - "#005f00", - "#005f5f", - "#005f87", - "#005faf", - "#005fd7", - "#005fff", - "#008700", - "#00875f", - "#008787", - "#0087af", - "#0087d7", - "#0087ff", - "#00af00", - "#00af5f", - "#00af87", - "#00afaf", - "#00afd7", - "#00afff", - "#00d700", - "#00d75f", - "#00d787", - "#00d7af", - "#00d7d7", - "#00d7ff", - "#00ff00", - "#00ff5f", - "#00ff87", - "#00ffaf", - "#00ffd7", - "#00ffff", - "#5f0000", - "#5f005f", - "#5f0087", - "#5f00af", - "#5f00d7", - "#5f00ff", - "#5f5f00", - "#5f5f5f", - "#5f5f87", - "#5f5faf", - "#5f5fd7", - "#5f5fff", - "#5f8700", - "#5f875f", - "#5f8787", - "#5f87af", - "#5f87d7", - "#5f87ff", - "#5faf00", - "#5faf5f", - "#5faf87", - "#5fafaf", - "#5fafd7", - "#5fafff", - "#5fd700", - "#5fd75f", - "#5fd787", - "#5fd7af", - "#5fd7d7", - "#5fd7ff", - "#5fff00", - "#5fff5f", - "#5fff87", - "#5fffaf", - "#5fffd7", - "#5fffff", - "#870000", - "#87005f", - "#870087", - "#8700af", - "#8700d7", - "#8700ff", - "#875f00", - "#875f5f", - "#875f87", - "#875faf", - "#875fd7", - "#875fff", - "#878700", - "#87875f", - "#878787", - "#8787af", - "#8787d7", - "#8787ff", - "#87af00", - "#87af5f", - "#87af87", - "#87afaf", - "#87afd7", - "#87afff", - "#87d700", - "#87d75f", - "#87d787", - "#87d7af", - "#87d7d7", - "#87d7ff", - "#87ff00", - "#87ff5f", - "#87ff87", - "#87ffaf", - "#87ffd7", - "#87ffff", - "#af0000", - "#af005f", - "#af0087", - "#af00af", - "#af00d7", - "#af00ff", - "#af5f00", - "#af5f5f", - "#af5f87", - "#af5faf", - "#af5fd7", - "#af5fff", - "#af8700", - "#af875f", - "#af8787", - "#af87af", - "#af87d7", - "#af87ff", - "#afaf00", - "#afaf5f", - "#afaf87", - "#afafaf", - "#afafd7", - "#afafff", - "#afd700", - "#afd75f", - "#afd787", - "#afd7af", - "#afd7d7", - "#afd7ff", - "#afff00", - "#afff5f", - "#afff87", - "#afffaf", - "#afffd7", - "#afffff", - "#d70000", - "#d7005f", - "#d70087", - "#d700af", - "#d700d7", - "#d700ff", - "#d75f00", - "#d75f5f", - "#d75f87", - "#d75faf", - "#d75fd7", - "#d75fff", - "#d78700", - "#d7875f", - "#d78787", - "#d787af", - "#d787d7", - "#d787ff", - "#d7af00", - "#d7af5f", - "#d7af87", - "#d7afaf", - "#d7afd7", - "#d7afff", - "#d7d700", - "#d7d75f", - "#d7d787", - "#d7d7af", - "#d7d7d7", - "#d7d7ff", - "#d7ff00", - "#d7ff5f", - "#d7ff87", - "#d7ffaf", - "#d7ffd7", - "#d7ffff", - "#ff0000", - "#ff005f", - "#ff0087", - "#ff00af", - "#ff00d7", - "#ff00ff", - "#ff5f00", - "#ff5f5f", - "#ff5f87", - "#ff5faf", - "#ff5fd7", - "#ff5fff", - "#ff8700", - "#ff875f", - "#ff8787", - "#ff87af", - "#ff87d7", - "#ff87ff", - "#ffaf00", - "#ffaf5f", - "#ffaf87", - "#ffafaf", - "#ffafd7", - "#ffafff", - "#ffd700", - "#ffd75f", - "#ffd787", - "#ffd7af", - "#ffd7d7", - "#ffd7ff", - "#ffff00", - "#ffff5f", - "#ffff87", - "#ffffaf", - "#ffffd7", - "#ffffff", - "#080808", - "#121212", - "#1c1c1c", - "#262626", - "#303030", - "#3a3a3a", - "#444444", - "#4e4e4e", - "#585858", - "#626262", - "#6c6c6c", - "#767676", - "#808080", - "#8a8a8a", - "#949494", - "#9e9e9e", - "#a8a8a8", - "#b2b2b2", - "#bcbcbc", - "#c6c6c6", - "#d0d0d0", - "#dadada", - "#e4e4e4", - "#eeeeee" -] diff --git a/cli/Cargo.toml b/crates/cli/Cargo.toml similarity index 80% rename from cli/Cargo.toml rename to crates/cli/Cargo.toml index c6dfa9e8..c10b4652 100644 --- a/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -8,13 +8,18 @@ rust-version.workspace = true readme = "README.md" homepage.workspace = true repository.workspace = true +documentation = "https://docs.rs/tree-sitter-cli" license.workspace = true keywords.workspace = true categories.workspace = true +include = ["build.rs", "README.md", "LICENSE", "benches/*", "src/**"] [lints] workspace = true +[lib] +path = "src/tree_sitter_cli.rs" + [[bin]] name = "tree-sitter" path = "src/main.rs" @@ -25,40 +30,38 @@ name = "benchmark" harness = false [features] +default = ["qjs-rt"] wasm = ["tree-sitter/wasm", "tree-sitter-loader/wasm"] +qjs-rt = ["tree-sitter-generate/qjs-rt"] [dependencies] +ansi_colours.workspace = true anstyle.workspace = true anyhow.workspace = true bstr.workspace = true clap.workspace = true clap_complete.workspace = true +clap_complete_nushell.workspace = true +crc32fast.workspace = true ctor.workspace = true ctrlc.workspace = true dialoguer.workspace = true -dirs.workspace = true -filetime.workspace = true glob.workspace = true heck.workspace = true html-escape.workspace = true -indexmap.workspace = true indoc.workspace = true -lazy_static.workspace = true log.workspace = true memchr.workspace = true rand.workspace = true regex.workspace = true -regex-syntax.workspace = true -rustc-hash.workspace = true +schemars.workspace = true semver.workspace = true serde.workspace = true -serde_derive.workspace = true serde_json.workspace = true similar.workspace = true -smallbitvec.workspace = true streaming-iterator.workspace = true +thiserror.workspace = true tiny_http.workspace = true -url.workspace = true walkdir.workspace = true wasmparser.workspace = true webbrowser.workspace = true @@ -72,7 +75,7 @@ tree-sitter-tags.workspace = true [dev-dependencies] encoding_rs = "0.8.35" -widestring = "1.1.0" +widestring = "1.2.1" tree_sitter_proc_macro = { path = "src/tests/proc_macro", package = "tree-sitter-tests-proc-macro" } tempfile.workspace = true diff --git a/crates/cli/LICENSE b/crates/cli/LICENSE new file mode 100644 index 00000000..971b81f9 --- /dev/null +++ b/crates/cli/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2018 Max Brunsfeld + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cli/README.md b/crates/cli/README.md similarity index 81% rename from cli/README.md rename to crates/cli/README.md index 5a399f08..e3ef899e 100644 --- a/cli/README.md +++ b/crates/cli/README.md @@ -7,7 +7,8 @@ [npmjs.com]: https://www.npmjs.org/package/tree-sitter-cli [npmjs.com badge]: https://img.shields.io/npm/v/tree-sitter-cli.svg?color=%23BF4A4A -The Tree-sitter CLI allows you to develop, test, and use Tree-sitter grammars from the command line. It works on `MacOS`, `Linux`, and `Windows`. +The Tree-sitter CLI allows you to develop, test, and use Tree-sitter grammars from the command line. It works on `MacOS`, +`Linux`, and `Windows`. ### Installation @@ -34,9 +35,11 @@ The `tree-sitter` binary itself has no dependencies, but specific commands have ### Commands -* `generate` - The `tree-sitter generate` command will generate a Tree-sitter parser based on the grammar in the current working directory. See [the documentation] for more information. +* `generate` - The `tree-sitter generate` command will generate a Tree-sitter parser based on the grammar in the current + working directory. See [the documentation] for more information. -* `test` - The `tree-sitter test` command will run the unit tests for the Tree-sitter parser in the current working directory. See [the documentation] for more information. +* `test` - The `tree-sitter test` command will run the unit tests for the Tree-sitter parser in the current working directory. + See [the documentation] for more information. * `parse` - The `tree-sitter parse` command will parse a file (or list of files) using Tree-sitter parsers. diff --git a/cli/benches/benchmark.rs b/crates/cli/benches/benchmark.rs similarity index 58% rename from cli/benches/benchmark.rs rename to crates/cli/benches/benchmark.rs index 7d6cc2fb..3edda96e 100644 --- a/cli/benches/benchmark.rs +++ b/crates/cli/benches/benchmark.rs @@ -3,70 +3,77 @@ use std::{ env, fs, path::{Path, PathBuf}, str, + sync::LazyLock, time::Instant, }; use anyhow::Context; -use lazy_static::lazy_static; +use log::info; use tree_sitter::{Language, Parser, Query}; use tree_sitter_loader::{CompileConfig, Loader}; include!("../src/tests/helpers/dirs.rs"); -lazy_static! { - static ref LANGUAGE_FILTER: Option = - env::var("TREE_SITTER_BENCHMARK_LANGUAGE_FILTER").ok(); - static ref EXAMPLE_FILTER: Option = - env::var("TREE_SITTER_BENCHMARK_EXAMPLE_FILTER").ok(); - static ref REPETITION_COUNT: usize = env::var("TREE_SITTER_BENCHMARK_REPETITION_COUNT") +static LANGUAGE_FILTER: LazyLock> = + LazyLock::new(|| env::var("TREE_SITTER_BENCHMARK_LANGUAGE_FILTER").ok()); +static EXAMPLE_FILTER: LazyLock> = + LazyLock::new(|| env::var("TREE_SITTER_BENCHMARK_EXAMPLE_FILTER").ok()); +static REPETITION_COUNT: LazyLock = LazyLock::new(|| { + env::var("TREE_SITTER_BENCHMARK_REPETITION_COUNT") .map(|s| s.parse::().unwrap()) - .unwrap_or(5); - static ref TEST_LOADER: Loader = Loader::with_parser_lib_path(SCRATCH_DIR.clone()); - static ref EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR: BTreeMap, Vec)> = { - fn process_dir(result: &mut BTreeMap, Vec)>, dir: &Path) { - if dir.join("grammar.js").exists() { - let relative_path = dir.strip_prefix(GRAMMARS_DIR.as_path()).unwrap(); - let (example_paths, query_paths) = - result.entry(relative_path.to_owned()).or_default(); + .unwrap_or(5) +}); +static TEST_LOADER: LazyLock = + LazyLock::new(|| Loader::with_parser_lib_path(SCRATCH_DIR.clone())); - if let Ok(example_files) = fs::read_dir(dir.join("examples")) { - example_paths.extend(example_files.filter_map(|p| { - let p = p.unwrap().path(); - if p.is_file() { - Some(p) - } else { - None - } - })); - } +#[allow(clippy::type_complexity)] +static EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR: LazyLock< + BTreeMap, Vec)>, +> = LazyLock::new(|| { + fn process_dir(result: &mut BTreeMap, Vec)>, dir: &Path) { + if dir.join("grammar.js").exists() { + let relative_path = dir.strip_prefix(GRAMMARS_DIR.as_path()).unwrap(); + let (example_paths, query_paths) = result.entry(relative_path.to_owned()).or_default(); - if let Ok(query_files) = fs::read_dir(dir.join("queries")) { - query_paths.extend(query_files.filter_map(|p| { - let p = p.unwrap().path(); - if p.is_file() { - Some(p) - } else { - None - } - })); - } - } else { - for entry in fs::read_dir(dir).unwrap() { - let entry = entry.unwrap().path(); - if entry.is_dir() { - process_dir(result, &entry); + if let Ok(example_files) = fs::read_dir(dir.join("examples")) { + example_paths.extend(example_files.filter_map(|p| { + let p = p.unwrap().path(); + if p.is_file() { + Some(p) + } else { + None } + })); + } + + if let Ok(query_files) = fs::read_dir(dir.join("queries")) { + query_paths.extend(query_files.filter_map(|p| { + let p = p.unwrap().path(); + if p.is_file() { + Some(p) + } else { + None + } + })); + } + } else { + for entry in fs::read_dir(dir).unwrap() { + let entry = entry.unwrap().path(); + if entry.is_dir() { + process_dir(result, &entry); } } } + } - let mut result = BTreeMap::new(); - process_dir(&mut result, &GRAMMARS_DIR); - result - }; -} + let mut result = BTreeMap::new(); + process_dir(&mut result, &GRAMMARS_DIR); + result +}); fn main() { + tree_sitter_cli::logger::init(); + let max_path_length = EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR .values() .flat_map(|(e, q)| { @@ -77,7 +84,7 @@ fn main() { .max() .unwrap_or(0); - eprintln!("Benchmarking with {} repetitions", *REPETITION_COUNT); + info!("Benchmarking with {} repetitions", *REPETITION_COUNT); let mut parser = Parser::new(); let mut all_normal_speeds = Vec::new(); @@ -94,11 +101,11 @@ fn main() { } } - eprintln!("\nLanguage: {language_name}"); + info!("\nLanguage: {language_name}"); let language = get_language(language_path); parser.set_language(&language).unwrap(); - eprintln!(" Constructing Queries"); + info!(" Constructing Queries"); for path in query_paths { if let Some(filter) = EXAMPLE_FILTER.as_ref() { if !path.to_str().unwrap().contains(filter.as_str()) { @@ -108,12 +115,12 @@ fn main() { parse(path, max_path_length, |source| { Query::new(&language, str::from_utf8(source).unwrap()) - .with_context(|| format!("Query file path: {path:?}")) + .with_context(|| format!("Query file path: {}", path.display())) .expect("Failed to parse query"); }); } - eprintln!(" Parsing Valid Code:"); + info!(" Parsing Valid Code:"); let mut normal_speeds = Vec::new(); for example_path in example_paths { if let Some(filter) = EXAMPLE_FILTER.as_ref() { @@ -127,7 +134,7 @@ fn main() { })); } - eprintln!(" Parsing Invalid Code (mismatched languages):"); + info!(" Parsing Invalid Code (mismatched languages):"); let mut error_speeds = Vec::new(); for (other_language_path, (example_paths, _)) in EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR.iter() @@ -148,30 +155,30 @@ fn main() { } if let Some((average_normal, worst_normal)) = aggregate(&normal_speeds) { - eprintln!(" Average Speed (normal): {average_normal} bytes/ms"); - eprintln!(" Worst Speed (normal): {worst_normal} bytes/ms"); + info!(" Average Speed (normal): {average_normal} bytes/ms"); + info!(" Worst Speed (normal): {worst_normal} bytes/ms"); } if let Some((average_error, worst_error)) = aggregate(&error_speeds) { - eprintln!(" Average Speed (errors): {average_error} bytes/ms"); - eprintln!(" Worst Speed (errors): {worst_error} bytes/ms"); + info!(" Average Speed (errors): {average_error} bytes/ms"); + info!(" Worst Speed (errors): {worst_error} bytes/ms"); } all_normal_speeds.extend(normal_speeds); all_error_speeds.extend(error_speeds); } - eprintln!("\n Overall"); + info!("\n Overall"); if let Some((average_normal, worst_normal)) = aggregate(&all_normal_speeds) { - eprintln!(" Average Speed (normal): {average_normal} bytes/ms"); - eprintln!(" Worst Speed (normal): {worst_normal} bytes/ms"); + info!(" Average Speed (normal): {average_normal} bytes/ms"); + info!(" Worst Speed (normal): {worst_normal} bytes/ms"); } if let Some((average_error, worst_error)) = aggregate(&all_error_speeds) { - eprintln!(" Average Speed (errors): {average_error} bytes/ms"); - eprintln!(" Worst Speed (errors): {worst_error} bytes/ms"); + info!(" Average Speed (errors): {average_error} bytes/ms"); + info!(" Worst Speed (errors): {worst_error} bytes/ms"); } - eprintln!(); + info!(""); } fn aggregate(speeds: &[usize]) -> Option<(usize, usize)> { @@ -190,14 +197,8 @@ fn aggregate(speeds: &[usize]) -> Option<(usize, usize)> { } fn parse(path: &Path, max_path_length: usize, mut action: impl FnMut(&[u8])) -> usize { - eprint!( - " {:width$}\t", - path.file_name().unwrap().to_str().unwrap(), - width = max_path_length - ); - let source_code = fs::read(path) - .with_context(|| format!("Failed to read {path:?}")) + .with_context(|| format!("Failed to read {}", path.display())) .unwrap(); let time = Instant::now(); for _ in 0..*REPETITION_COUNT { @@ -206,8 +207,9 @@ fn parse(path: &Path, max_path_length: usize, mut action: impl FnMut(&[u8])) -> let duration = time.elapsed() / (*REPETITION_COUNT as u32); let duration_ns = duration.as_nanos(); let speed = ((source_code.len() as u128) * 1_000_000) / duration_ns; - eprintln!( - "time {:>7.2} ms\t\tspeed {speed:>6} bytes/ms", + info!( + " {:max_path_length$}\ttime {:>7.2} ms\t\tspeed {speed:>6} bytes/ms", + path.file_name().unwrap().to_str().unwrap(), (duration_ns as f64) / 1e6, ); speed as usize @@ -217,6 +219,6 @@ fn get_language(path: &Path) -> Language { let src_path = GRAMMARS_DIR.join(path).join("src"); TEST_LOADER .load_language_at_path(CompileConfig::new(&src_path, None, None)) - .with_context(|| format!("Failed to load language at path {src_path:?}")) + .with_context(|| format!("Failed to load language at path {}", src_path.display())) .unwrap() } diff --git a/cli/build.rs b/crates/cli/build.rs similarity index 88% rename from cli/build.rs rename to crates/cli/build.rs index d59980a5..45f54634 100644 --- a/cli/build.rs +++ b/crates/cli/build.rs @@ -52,16 +52,14 @@ fn main() { fn web_playground_files_present() -> bool { let paths = [ - "../docs/assets/js/playground.js", - "../lib/binding_web/tree-sitter.js", - "../lib/binding_web/tree-sitter.wasm", + "../../docs/src/assets/js/playground.js", + "../../lib/binding_web/web-tree-sitter.js", + "../../lib/binding_web/web-tree-sitter.wasm", ]; paths.iter().all(|p| Path::new(p).exists()) } -// When updating this function, don't forget to also update generate/build.rs which has a -// near-identical function. fn read_git_sha() -> Option { let crate_path = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); diff --git a/crates/cli/eslint/.gitignore b/crates/cli/eslint/.gitignore new file mode 100644 index 00000000..6b1d0bfa --- /dev/null +++ b/crates/cli/eslint/.gitignore @@ -0,0 +1 @@ +LICENSE diff --git a/cli/eslint/index.js b/crates/cli/eslint/index.js similarity index 100% rename from cli/eslint/index.js rename to crates/cli/eslint/index.js diff --git a/cli/eslint/package-lock.json b/crates/cli/eslint/package-lock.json similarity index 99% rename from cli/eslint/package-lock.json rename to crates/cli/eslint/package-lock.json index 44266c4c..60559b10 100644 --- a/cli/eslint/package-lock.json +++ b/crates/cli/eslint/package-lock.json @@ -305,9 +305,9 @@ "peer": true }, "node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", "license": "MIT", "peer": true, "dependencies": { @@ -805,9 +805,9 @@ "peer": true }, "node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", "license": "MIT", "peer": true, "dependencies": { diff --git a/cli/eslint/package.json b/crates/cli/eslint/package.json similarity index 71% rename from cli/eslint/package.json rename to crates/cli/eslint/package.json index d53bf3a4..a3ef0c10 100644 --- a/cli/eslint/package.json +++ b/crates/cli/eslint/package.json @@ -4,7 +4,8 @@ "description": "Eslint configuration for Tree-sitter grammar files", "repository": { "type": "git", - "url": "git+https://github.com/tree-sitter/tree-sitter.git" + "url": "git+https://github.com/tree-sitter/tree-sitter.git", + "directory": "crates/cli/eslint" }, "license": "MIT", "author": "Amaan Qureshi ", @@ -20,5 +21,9 @@ }, "peerDependencies": { "eslint": ">= 9" + }, + "scripts": { + "prepack": "cp ../../../LICENSE .", + "postpack": "rm LICENSE" } } diff --git a/cli/npm/.gitignore b/crates/cli/npm/.gitignore similarity index 100% rename from cli/npm/.gitignore rename to crates/cli/npm/.gitignore diff --git a/cli/npm/cli.js b/crates/cli/npm/cli.js similarity index 100% rename from cli/npm/cli.js rename to crates/cli/npm/cli.js diff --git a/cli/npm/dsl.d.ts b/crates/cli/npm/dsl.d.ts similarity index 90% rename from cli/npm/dsl.d.ts rename to crates/cli/npm/dsl.d.ts index 174f902f..accdb95f 100644 --- a/cli/npm/dsl.d.ts +++ b/crates/cli/npm/dsl.d.ts @@ -10,6 +10,7 @@ type PrecRightRule = { type: 'PREC_RIGHT'; content: Rule; value: number }; type PrecRule = { type: 'PREC'; content: Rule; value: number }; type Repeat1Rule = { type: 'REPEAT1'; content: Rule }; type RepeatRule = { type: 'REPEAT'; content: Rule }; +type ReservedRule = { type: 'RESERVED'; content: Rule; context_name: string }; type SeqRule = { type: 'SEQ'; members: Rule[] }; type StringRule = { type: 'STRING'; value: string }; type SymbolRule = { type: 'SYMBOL'; name: Name }; @@ -28,12 +29,19 @@ type Rule = | PrecRule | Repeat1Rule | RepeatRule + | ReservedRule | SeqRule | StringRule | SymbolRule | TokenRule; -type RuleOrLiteral = Rule | RegExp | string; +declare class RustRegex { + value: string; + + constructor(pattern: string); +} + +type RuleOrLiteral = Rule | RegExp | RustRegex | string; type GrammarSymbols = { [name in RuleName]: SymbolRule; @@ -105,7 +113,7 @@ interface Grammar< * @param $ grammar rules * @param previous array of externals from the base schema, if any * - * @see https://tree-sitter.github.io/tree-sitter/creating-parsers#external-scanners + * @see https://tree-sitter.github.io/tree-sitter/creating-parsers/4-external-scanners */ externals?: ( $: Record>, @@ -143,7 +151,7 @@ interface Grammar< * * @param $ grammar rules * - * @see https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types + * @see https://tree-sitter.github.io/tree-sitter/using-parsers/6-static-node-types */ supertypes?: ( $: GrammarSymbols, @@ -156,9 +164,20 @@ interface Grammar< * * @param $ grammar rules * - * @see https://tree-sitter.github.io/tree-sitter/creating-parsers#keyword-extraction + * @see https://tree-sitter.github.io/tree-sitter/creating-parsers/3-writing-the-grammar#keyword-extraction */ word?: ($: GrammarSymbols) => RuleOrLiteral; + + + /** + * Mapping of names to reserved word sets. The first reserved word set is the + * global word set, meaning it applies to every rule in every parse state. + * The other word sets can be used with the `reserved` function. + */ + reserved?: Record< + string, + ($: GrammarSymbols) => RuleOrLiteral[] + >; } type GrammarSchema = { @@ -243,7 +262,7 @@ declare function optional(rule: RuleOrLiteral): ChoiceRule; * @see https://docs.oracle.com/cd/E19504-01/802-5880/6i9k05dh3/index.html */ declare const prec: { - (value: String | number, rule: RuleOrLiteral): PrecRule; + (value: string | number, rule: RuleOrLiteral): PrecRule; /** * Marks the given rule as left-associative (and optionally applies a @@ -259,7 +278,7 @@ declare const prec: { * @see https://docs.oracle.com/cd/E19504-01/802-5880/6i9k05dh3/index.html */ left(rule: RuleOrLiteral): PrecLeftRule; - left(value: String | number, rule: RuleOrLiteral): PrecLeftRule; + left(value: string | number, rule: RuleOrLiteral): PrecLeftRule; /** * Marks the given rule as right-associative (and optionally applies a @@ -275,7 +294,7 @@ declare const prec: { * @see https://docs.oracle.com/cd/E19504-01/802-5880/6i9k05dh3/index.html */ right(rule: RuleOrLiteral): PrecRightRule; - right(value: String | number, rule: RuleOrLiteral): PrecRightRule; + right(value: string | number, rule: RuleOrLiteral): PrecRightRule; /** * Marks the given rule with a numerical precedence which will be used to @@ -292,7 +311,7 @@ declare const prec: { * * @see https://www.gnu.org/software/bison/manual/html_node/Generalized-LR-Parsing.html */ - dynamic(value: String | number, rule: RuleOrLiteral): PrecDynamicRule; + dynamic(value: string | number, rule: RuleOrLiteral): PrecDynamicRule; }; /** @@ -312,6 +331,15 @@ declare function repeat(rule: RuleOrLiteral): RepeatRule; */ declare function repeat1(rule: RuleOrLiteral): Repeat1Rule; +/** + * Overrides the global reserved word set for a given rule. The word set name + * should be defined in the `reserved` field in the grammar. + * + * @param wordset name of the reserved word set + * @param rule rule that will use the reserved word set + */ +declare function reserved(wordset: string, rule: RuleOrLiteral): ReservedRule; + /** * Creates a rule that matches any number of other rules, one after another. * It is analogous to simply writing multiple symbols next to each other @@ -330,7 +358,7 @@ declare function sym(name: Name): SymbolRule; /** * Marks the given rule as producing only a single token. Tree-sitter's - * default is to treat each String or RegExp literal in the grammar as a + * default is to treat each string or RegExp literal in the grammar as a * separate token. Each token is matched separately by the lexer and * returned as its own leaf node in the tree. The token function allows * you to express a complex rule using the DSL functions (rather diff --git a/cli/npm/install.js b/crates/cli/npm/install.js old mode 100755 new mode 100644 similarity index 97% rename from cli/npm/install.js rename to crates/cli/npm/install.js index f2a4944d..6d0fbc57 --- a/cli/npm/install.js +++ b/crates/cli/npm/install.js @@ -6,7 +6,8 @@ const http = require('http'); const https = require('https'); const packageJSON = require('./package.json'); -// Look to a results table in https://github.com/tree-sitter/tree-sitter/issues/2196 +https.globalAgent.keepAlive = false; + const matrix = { platform: { 'darwin': { diff --git a/crates/cli/npm/package-lock.json b/crates/cli/npm/package-lock.json new file mode 100644 index 00000000..739e69f1 --- /dev/null +++ b/crates/cli/npm/package-lock.json @@ -0,0 +1,20 @@ +{ + "name": "tree-sitter-cli", + "version": "0.27.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "tree-sitter-cli", + "version": "0.27.0", + "hasInstallScript": true, + "license": "MIT", + "bin": { + "tree-sitter": "cli.js" + }, + "engines": { + "node": ">=12.0.0" + } + } + } +} diff --git a/cli/npm/package.json b/crates/cli/npm/package.json similarity index 51% rename from cli/npm/package.json rename to crates/cli/npm/package.json index 4ba4e0b7..d49abd46 100644 --- a/cli/npm/package.json +++ b/crates/cli/npm/package.json @@ -1,24 +1,33 @@ { "name": "tree-sitter-cli", - "version": "0.25.0", - "author": "Max Brunsfeld", + "version": "0.27.0", + "author": { + "name": "Max Brunsfeld", + "email": "maxbrunsfeld@gmail.com" + }, + "maintainers": [ + { + "name": "Amaan Qureshi", + "email": "amaanq12@gmail.com" + } + ], "license": "MIT", "repository": { "type": "git", - "url": "https://github.com/tree-sitter/tree-sitter.git" + "url": "git+https://github.com/tree-sitter/tree-sitter.git", + "directory": "crates/cli/npm" }, "description": "CLI for generating fast incremental parsers", "keywords": [ "parser", "lexer" ], - "main": "lib/api/index.js", "engines": { "node": ">=12.0.0" }, "scripts": { "install": "node install.js", - "prepack": "cp ../../LICENSE ../README.md .", + "prepack": "cp ../../../LICENSE ../README.md .", "postpack": "rm LICENSE README.md" }, "bin": { diff --git a/crates/cli/package.nix b/crates/cli/package.nix new file mode 100644 index 00000000..eea05e12 --- /dev/null +++ b/crates/cli/package.nix @@ -0,0 +1,69 @@ +{ + lib, + src, + rustPlatform, + version, + clang, + libclang, + cmake, + pkg-config, + nodejs_22, + test-grammars, + stdenv, + installShellFiles, +}: +let + isCross = stdenv.targetPlatform == stdenv.buildPlatform; +in +rustPlatform.buildRustPackage { + pname = "tree-sitter-cli"; + + inherit src version; + + cargoBuildFlags = [ "--all-features" ]; + + nativeBuildInputs = [ + clang + cmake + pkg-config + nodejs_22 + ] + ++ lib.optionals (!isCross) [ installShellFiles ]; + + cargoLock.lockFile = ../../Cargo.lock; + + env.LIBCLANG_PATH = "${libclang.lib}/lib"; + + preBuild = '' + rm -rf test/fixtures + mkdir -p test/fixtures + cp -r ${test-grammars}/fixtures/* test/fixtures/ + chmod -R u+w test/fixtures + ''; + + preCheck = "export HOME=$TMPDIR"; + doCheck = !isCross; + + postInstall = lib.optionalString (!isCross) '' + installShellCompletion --cmd tree-sitter \ + --bash <($out/bin/tree-sitter complete --shell bash) \ + --zsh <($out/bin/tree-sitter complete --shell zsh) \ + --fish <($out/bin/tree-sitter complete --shell fish) + ''; + + meta = { + description = "Tree-sitter CLI - A tool for developing, testing, and using Tree-sitter parsers"; + longDescription = '' + Tree-sitter is a parser generator tool and an incremental parsing library. + It can build a concrete syntax tree for a source file and efficiently update + the syntax tree as the source file is edited. This package provides the CLI + tool for developing, testing, and using Tree-sitter parsers. + ''; + homepage = "https://tree-sitter.github.io/tree-sitter"; + changelog = "https://github.com/tree-sitter/tree-sitter/releases/tag/v${version}"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ amaanq ]; + platforms = lib.platforms.all; + mainProgram = "tree-sitter"; + }; +} diff --git a/cli/src/fuzz/mod.rs b/crates/cli/src/fuzz.rs similarity index 77% rename from cli/src/fuzz/mod.rs rename to crates/cli/src/fuzz.rs index 38993e1f..39ad7691 100644 --- a/cli/src/fuzz/mod.rs +++ b/crates/cli/src/fuzz.rs @@ -1,6 +1,11 @@ -use std::{collections::HashMap, env, fs, path::Path}; +use std::{ + collections::HashMap, + env, fs, + path::{Path, PathBuf}, + sync::LazyLock, +}; -use lazy_static::lazy_static; +use log::{error, info}; use rand::Rng; use regex::Regex; use tree_sitter::{Language, Parser}; @@ -20,19 +25,30 @@ use crate::{ random::Rand, }, parse::perform_edit, - test::{parse_tests, print_diff, print_diff_key, strip_sexp_fields, TestEntry}, + test::{parse_tests, strip_sexp_fields, DiffKey, TestDiff, TestEntry}, }; -lazy_static! { - pub static ref LOG_ENABLED: bool = env::var("TREE_SITTER_LOG").is_ok(); - pub static ref LOG_GRAPH_ENABLED: bool = env::var("TREE_SITTER_LOG_GRAPHS").is_ok(); - pub static ref LANGUAGE_FILTER: Option = env::var("TREE_SITTER_LANGUAGE").ok(); - pub static ref EXAMPLE_INCLUDE: Option = regex_env_var("TREE_SITTER_EXAMPLE_INCLUDE"); - pub static ref EXAMPLE_EXCLUDE: Option = regex_env_var("TREE_SITTER_EXAMPLE_EXCLUDE"); - pub static ref START_SEED: usize = new_seed(); - pub static ref EDIT_COUNT: usize = int_env_var("TREE_SITTER_EDITS").unwrap_or(3); - pub static ref ITERATION_COUNT: usize = int_env_var("TREE_SITTER_ITERATIONS").unwrap_or(10); -} +pub static LOG_ENABLED: LazyLock = LazyLock::new(|| env::var("TREE_SITTER_LOG").is_ok()); + +pub static LOG_GRAPH_ENABLED: LazyLock = + LazyLock::new(|| env::var("TREE_SITTER_LOG_GRAPHS").is_ok()); + +pub static LANGUAGE_FILTER: LazyLock> = + LazyLock::new(|| env::var("TREE_SITTER_LANGUAGE").ok()); + +pub static EXAMPLE_INCLUDE: LazyLock> = + LazyLock::new(|| regex_env_var("TREE_SITTER_EXAMPLE_INCLUDE")); + +pub static EXAMPLE_EXCLUDE: LazyLock> = + LazyLock::new(|| regex_env_var("TREE_SITTER_EXAMPLE_EXCLUDE")); + +pub static START_SEED: LazyLock = LazyLock::new(new_seed); + +pub static EDIT_COUNT: LazyLock = + LazyLock::new(|| int_env_var("TREE_SITTER_EDITS").unwrap_or(3)); + +pub static ITERATION_COUNT: LazyLock = + LazyLock::new(|| int_env_var("TREE_SITTER_ITERATIONS").unwrap_or(10)); fn int_env_var(name: &'static str) -> Option { env::var(name).ok().and_then(|e| e.parse().ok()) @@ -46,13 +62,15 @@ fn regex_env_var(name: &'static str) -> Option { pub fn new_seed() -> usize { int_env_var("TREE_SITTER_SEED").unwrap_or_else(|| { let mut rng = rand::thread_rng(); - rng.gen::() + let seed = rng.gen::(); + info!("Seed: {seed}"); + seed }) } pub struct FuzzOptions { pub skipped: Option>, - pub subdir: Option, + pub subdir: Option, pub edits: usize, pub iterations: usize, pub include: Option, @@ -91,12 +109,12 @@ pub fn fuzz_language_corpus( let corpus_dir = grammar_dir.join(subdir).join("test").join("corpus"); if !corpus_dir.exists() || !corpus_dir.is_dir() { - eprintln!("No corpus directory found, ensure that you have a `test/corpus` directory in your grammar directory with at least one test file."); + error!("No corpus directory found, ensure that you have a `test/corpus` directory in your grammar directory with at least one test file."); return; } if std::fs::read_dir(&corpus_dir).unwrap().count() == 0 { - eprintln!("No corpus files found in `test/corpus`, ensure that you have at least one test file in your corpus directory."); + error!("No corpus files found in `test/corpus`, ensure that you have at least one test file in your corpus directory."); return; } @@ -132,7 +150,7 @@ pub fn fuzz_language_corpus( let dump_edits = env::var("TREE_SITTER_DUMP_EDITS").is_ok(); if log_seed { - println!(" start seed: {start_seed}"); + info!(" start seed: {start_seed}"); } println!(); @@ -146,7 +164,7 @@ pub fn fuzz_language_corpus( println!(" {test_index}. {test_name}"); - let passed = allocations::record(|| { + let passed = allocations::record_checked(|| { let mut log_session = None; let mut parser = get_parser(&mut log_session, "log.html"); parser.set_language(language).unwrap(); @@ -165,8 +183,8 @@ pub fn fuzz_language_corpus( if actual_output != test.output { println!("Incorrect initial parse for {test_name}"); - print_diff_key(); - print_diff(&actual_output, &test.output, true); + DiffKey::print(); + println!("{}", TestDiff::new(&actual_output, &test.output)); println!(); return false; } @@ -174,7 +192,7 @@ pub fn fuzz_language_corpus( true }) .unwrap_or_else(|e| { - eprintln!("Error: {e}"); + error!("{e}"); false }); @@ -190,7 +208,7 @@ pub fn fuzz_language_corpus( for trial in 0..options.iterations { let seed = start_seed + trial; - let passed = allocations::record(|| { + let passed = allocations::record_checked(|| { let mut rand = Rand::new(seed); let mut log_session = None; let mut parser = get_parser(&mut log_session, "log.html"); @@ -199,19 +217,20 @@ pub fn fuzz_language_corpus( let mut input = test.input.clone(); if options.log_graphs { - eprintln!("{}\n", String::from_utf8_lossy(&input)); + info!("{}\n", String::from_utf8_lossy(&input)); } // Perform a random series of edits and reparse. - let mut undo_stack = Vec::new(); - for _ in 0..=rand.unsigned(*EDIT_COUNT) { + let edit_count = rand.unsigned(*EDIT_COUNT); + let mut undo_stack = Vec::with_capacity(edit_count); + for _ in 0..=edit_count { let edit = get_random_edit(&mut rand, &input); undo_stack.push(invert_edit(&input, &edit)); perform_edit(&mut tree, &mut input, &edit).unwrap(); } if log_seed { - println!(" {test_index}.{trial:<2} seed: {seed}"); + info!(" {test_index}.{trial:<2} seed: {seed}"); } if dump_edits { @@ -225,7 +244,7 @@ pub fn fuzz_language_corpus( } if options.log_graphs { - eprintln!("{}\n", String::from_utf8_lossy(&input)); + info!("{}\n", String::from_utf8_lossy(&input)); } set_included_ranges(&mut parser, &input, test.template_delimiters); @@ -234,7 +253,7 @@ pub fn fuzz_language_corpus( // Check that the new tree is consistent. check_consistent_sizes(&tree2, &input); if let Err(message) = check_changed_ranges(&tree, &tree2, &input) { - println!("\nUnexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n",); + error!("\nUnexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n",); return false; } @@ -243,7 +262,7 @@ pub fn fuzz_language_corpus( perform_edit(&mut tree2, &mut input, &edit).unwrap(); } if options.log_graphs { - eprintln!("{}\n", String::from_utf8_lossy(&input)); + info!("{}\n", String::from_utf8_lossy(&input)); } set_included_ranges(&mut parser, &test.input, test.template_delimiters); @@ -257,8 +276,8 @@ pub fn fuzz_language_corpus( if actual_output != test.output && !test.error { println!("Incorrect parse for {test_name} - seed {seed}"); - print_diff_key(); - print_diff(&actual_output, &test.output, true); + DiffKey::print(); + println!("{}", TestDiff::new(&actual_output, &test.output)); println!(); return false; } @@ -266,13 +285,13 @@ pub fn fuzz_language_corpus( // Check that the edited tree is consistent. check_consistent_sizes(&tree3, &input); if let Err(message) = check_changed_ranges(&tree2, &tree3, &input) { - println!("Unexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n"); + error!("Unexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n"); return false; } true }).unwrap_or_else(|e| { - eprintln!("Error: {e}"); + error!("{e}"); false }); @@ -284,17 +303,17 @@ pub fn fuzz_language_corpus( } if failure_count != 0 { - eprintln!("{failure_count} {language_name} corpus tests failed fuzzing"); + info!("{failure_count} {language_name} corpus tests failed fuzzing"); } skipped.retain(|_, v| *v == 0); if !skipped.is_empty() { - println!("Non matchable skip definitions:"); + info!("Non matchable skip definitions:"); for k in skipped.keys() { - println!(" {k}"); + info!(" {k}"); } - panic!("Non matchable skip definitions needs to be removed"); + panic!("Non matchable skip definitions need to be removed"); } } diff --git a/cli/src/fuzz/allocations.rs b/crates/cli/src/fuzz/allocations.rs similarity index 72% rename from cli/src/fuzz/allocations.rs rename to crates/cli/src/fuzz/allocations.rs index fed446c6..ca0c0860 100644 --- a/cli/src/fuzz/allocations.rs +++ b/crates/cli/src/fuzz/allocations.rs @@ -40,7 +40,11 @@ extern "C" { fn free(ptr: *mut c_void); } -pub fn record(f: impl FnOnce() -> T) -> Result { +pub fn record(f: impl FnOnce() -> T) -> T { + record_checked(f).unwrap() +} + +pub fn record_checked(f: impl FnOnce() -> T) -> Result { RECORDER.with(|recorder| { recorder.enabled.store(true, SeqCst); recorder.allocation_count.store(0, SeqCst); @@ -93,30 +97,49 @@ fn record_dealloc(ptr: *mut c_void) { }); } -unsafe extern "C" fn ts_record_malloc(size: usize) -> *mut c_void { +/// # Safety +/// +/// The caller must ensure that the returned pointer is eventually +/// freed by calling `ts_record_free`. +#[must_use] +pub unsafe extern "C" fn ts_record_malloc(size: usize) -> *mut c_void { let result = malloc(size); record_alloc(result); result } -unsafe extern "C" fn ts_record_calloc(count: usize, size: usize) -> *mut c_void { +/// # Safety +/// +/// The caller must ensure that the returned pointer is eventually +/// freed by calling `ts_record_free`. +#[must_use] +pub unsafe extern "C" fn ts_record_calloc(count: usize, size: usize) -> *mut c_void { let result = calloc(count, size); record_alloc(result); result } -unsafe extern "C" fn ts_record_realloc(ptr: *mut c_void, size: usize) -> *mut c_void { +/// # Safety +/// +/// The caller must ensure that the returned pointer is eventually +/// freed by calling `ts_record_free`. +#[must_use] +pub unsafe extern "C" fn ts_record_realloc(ptr: *mut c_void, size: usize) -> *mut c_void { let result = realloc(ptr, size); if ptr.is_null() { record_alloc(result); - } else if ptr != result { + } else if !core::ptr::eq(ptr, result) { record_dealloc(ptr); record_alloc(result); } result } -unsafe extern "C" fn ts_record_free(ptr: *mut c_void) { +/// # Safety +/// +/// The caller must ensure that `ptr` was allocated by a previous call +/// to `ts_record_malloc`, `ts_record_calloc`, or `ts_record_realloc`. +pub unsafe extern "C" fn ts_record_free(ptr: *mut c_void) { record_dealloc(ptr); free(ptr); } diff --git a/cli/src/fuzz/corpus_test.rs b/crates/cli/src/fuzz/corpus_test.rs similarity index 98% rename from cli/src/fuzz/corpus_test.rs rename to crates/cli/src/fuzz/corpus_test.rs index 8807d9a9..e95ab283 100644 --- a/cli/src/fuzz/corpus_test.rs +++ b/crates/cli/src/fuzz/corpus_test.rs @@ -23,7 +23,7 @@ pub fn check_consistent_sizes(tree: &Tree, input: &[u8]) { let mut some_child_has_changes = false; let mut actual_named_child_count = 0; for i in 0..node.child_count() { - let child = node.child(i).unwrap(); + let child = node.child(i as u32).unwrap(); assert!(child.start_byte() >= last_child_end_byte); assert!(child.start_position() >= last_child_end_point); check(child, line_offsets); diff --git a/cli/src/fuzz/edits.rs b/crates/cli/src/fuzz/edits.rs similarity index 100% rename from cli/src/fuzz/edits.rs rename to crates/cli/src/fuzz/edits.rs diff --git a/cli/src/fuzz/random.rs b/crates/cli/src/fuzz/random.rs similarity index 94% rename from cli/src/fuzz/random.rs rename to crates/cli/src/fuzz/random.rs index 8a8410f4..7b2ede62 100644 --- a/cli/src/fuzz/random.rs +++ b/crates/cli/src/fuzz/random.rs @@ -20,8 +20,8 @@ impl Rand { } pub fn words(&mut self, max_count: usize) -> Vec { - let mut result = Vec::new(); let word_count = self.unsigned(max_count); + let mut result = Vec::with_capacity(2 * word_count); for i in 0..word_count { if i > 0 { if self.unsigned(5) == 0 { diff --git a/cli/src/fuzz/scope_sequence.rs b/crates/cli/src/fuzz/scope_sequence.rs similarity index 100% rename from cli/src/fuzz/scope_sequence.rs rename to crates/cli/src/fuzz/scope_sequence.rs diff --git a/cli/src/highlight.rs b/crates/cli/src/highlight.rs similarity index 61% rename from cli/src/highlight.rs rename to crates/cli/src/highlight.rs index 60f73d49..a2cb01cf 100644 --- a/cli/src/highlight.rs +++ b/crates/cli/src/highlight.rs @@ -1,22 +1,24 @@ use std::{ - collections::HashMap, + collections::{BTreeMap, HashSet}, fmt::Write, fs, io::{self, Write as _}, - path, str, - sync::atomic::AtomicUsize, + path::{self, Path, PathBuf}, + str, + sync::{atomic::AtomicUsize, Arc}, time::Instant, }; +use ansi_colours::{ansi256_from_rgb, rgb_from_ansi256}; use anstyle::{Ansi256Color, AnsiColor, Color, Effects, RgbColor}; use anyhow::Result; -use lazy_static::lazy_static; +use log::{info, warn}; use serde::{ser::SerializeMap, Deserialize, Deserializer, Serialize, Serializer}; use serde_json::{json, Value}; use tree_sitter_highlight::{HighlightConfiguration, HighlightEvent, Highlighter, HtmlRenderer}; use tree_sitter_loader::Loader; -pub const HTML_HEADER: &str = " +pub const HTML_HEAD_HEADER: &str = " Tree-sitter Highlighting @@ -33,7 +35,9 @@ pub const HTML_HEADER: &str = " .line { white-space: pre; } - + "; + +pub const HTML_BODY_HEADER: &str = " "; @@ -42,11 +46,6 @@ pub const HTML_FOOTER: &str = " "; -lazy_static! { - static ref CSS_STYLES_BY_COLOR_ID: Vec = - serde_json::from_str(include_str!("../vendor/xterm-colors.json")).unwrap(); -} - #[derive(Debug, Default)] pub struct Style { pub ansi: anstyle::Style, @@ -84,9 +83,9 @@ impl<'de> Deserialize<'de> for Theme { { let mut styles = Vec::new(); let mut highlight_names = Vec::new(); - if let Ok(colors) = HashMap::::deserialize(deserializer) { - highlight_names.reserve(colors.len()); + if let Ok(colors) = BTreeMap::::deserialize(deserializer) { styles.reserve(colors.len()); + highlight_names.reserve(colors.len()); for (name, style_value) in colors { let mut style = Style::default(); parse_style(&mut style, style_value); @@ -129,7 +128,7 @@ impl Serialize for Theme { || effects.contains(Effects::ITALIC) || effects.contains(Effects::UNDERLINE) { - let mut style_json = HashMap::new(); + let mut style_json = BTreeMap::new(); if let Some(color) = color { style_json.insert("color", color); } @@ -156,28 +155,32 @@ impl Serialize for Theme { impl Default for Theme { fn default() -> Self { serde_json::from_value(json!({ - "attribute": {"color": 124, "italic": true}, - "comment": {"color": 245, "italic": true}, - "constant.builtin": {"color": 94, "bold": true}, - "constant": 94, - "constructor": 136, - "embedded": null, - "function.builtin": {"color": 26, "bold": true}, - "function": 26, - "keyword": 56, - "number": {"color": 94, "bold": true}, - "module": 136, - "property": 124, - "operator": {"color": 239, "bold": true}, - "punctuation.bracket": 239, - "punctuation.delimiter": 239, - "string.special": 30, - "string": 28, - "tag": 18, - "type": 23, - "type.builtin": {"color": 23, "bold": true}, - "variable.builtin": {"bold": true}, - "variable.parameter": {"underline": true} + "attribute": {"color": 124, "italic": true}, + "comment": {"color": 245, "italic": true}, + "constant": 94, + "constant.builtin": {"color": 94, "bold": true}, + "constructor": 136, + "embedded": null, + "function": 26, + "function.builtin": {"color": 26, "bold": true}, + "keyword": 56, + "module": 136, + "number": {"color": 94, "bold": true}, + "operator": {"color": 239, "bold": true}, + "property": 124, + "property.builtin": {"color": 124, "bold": true}, + "punctuation": 239, + "punctuation.bracket": 239, + "punctuation.delimiter": 239, + "punctuation.special": 239, + "string": 28, + "string.special": 30, + "tag": 18, + "type": 23, + "type.builtin": {"color": 23, "bold": true}, + "variable": 252, + "variable.builtin": {"color": 252, "bold": true}, + "variable.parameter": {"color": 252, "underline": true} })) .unwrap() } @@ -220,9 +223,8 @@ fn parse_style(style: &mut Style, json: Value) { if let Some(Color::Rgb(RgbColor(red, green, blue))) = style.ansi.get_fg_color() { if !terminal_supports_truecolor() { - style.ansi = style - .ansi - .fg_color(Some(closest_xterm_color(red, green, blue))); + let ansi256 = Color::Ansi256(Ansi256Color(ansi256_from_rgb((red, green, blue)))); + style.ansi = style.ansi.fg_color(Some(ansi256)); } } } @@ -268,7 +270,7 @@ fn hex_string_to_rgb(s: &str) -> Option<(u8, u8, u8)> { } fn style_to_css(style: anstyle::Style) -> String { - let mut result = "style='".to_string(); + let mut result = String::new(); let effects = style.get_effects(); if effects.contains(Effects::UNDERLINE) { write!(&mut result, "text-decoration: underline;").unwrap(); @@ -282,7 +284,6 @@ fn style_to_css(style: anstyle::Style) -> String { if let Some(color) = style.get_fg_color() { write_color(&mut result, color); } - result.push('\''); result } @@ -300,7 +301,8 @@ fn write_color(buffer: &mut String, color: Color) { _ => unreachable!(), }, Color::Ansi256(Ansi256Color(n)) => { - write!(buffer, "color: {}", CSS_STYLES_BY_COLOR_ID[n as usize]).unwrap(); + let (r, g, b) = rgb_from_ansi256(n); + write!(buffer, "color: #{r:02x}{g:02x}{b:02x}").unwrap(); } Color::Rgb(RgbColor(r, g, b)) => write!(buffer, "color: #{r:02x}{g:02x}{b:02x}").unwrap(), } @@ -311,115 +313,144 @@ fn terminal_supports_truecolor() -> bool { .is_ok_and(|truecolor| truecolor == "truecolor" || truecolor == "24bit") } -fn closest_xterm_color(red: u8, green: u8, blue: u8) -> Color { - use std::cmp::{max, min}; - - let colors = CSS_STYLES_BY_COLOR_ID - .iter() - .enumerate() - .map(|(color_id, hex)| (color_id as u8, hex_string_to_rgb(hex).unwrap())); - - // Get the xterm color with the minimum Euclidean distance to the target color - // i.e. distance = √ (r2 - r1)² + (g2 - g1)² + (b2 - b1)² - let distances = colors.map(|(color_id, (r, g, b))| { - let r_delta = (max(r, red) - min(r, red)) as u32; - let g_delta = (max(g, green) - min(g, green)) as u32; - let b_delta = (max(b, blue) - min(b, blue)) as u32; - let distance = r_delta.pow(2) + g_delta.pow(2) + b_delta.pow(2); - // don't need to actually take the square root for the sake of comparison - (color_id, distance) - }); - - Color::Ansi256(Ansi256Color( - distances.min_by(|(_, d1), (_, d2)| d1.cmp(d2)).unwrap().0, - )) +pub struct HighlightOptions { + pub theme: Theme, + pub check: bool, + pub captures_path: Option, + pub inline_styles: bool, + pub html: bool, + pub quiet: bool, + pub print_time: bool, + pub cancellation_flag: Arc, } -pub fn ansi( +pub fn highlight( loader: &Loader, - theme: &Theme, - source: &[u8], + path: &Path, + name: &str, config: &HighlightConfiguration, - print_time: bool, - cancellation_flag: Option<&AtomicUsize>, + print_name: bool, + opts: &HighlightOptions, ) -> Result<()> { + if opts.check { + let names = if let Some(path) = opts.captures_path.as_deref() { + let file = fs::read_to_string(path)?; + let capture_names = file + .lines() + .filter_map(|line| { + if line.trim().is_empty() || line.trim().starts_with(';') { + return None; + } + line.split(';').next().map(|s| s.trim().trim_matches('"')) + }) + .collect::>(); + config.nonconformant_capture_names(&capture_names) + } else { + config.nonconformant_capture_names(&HashSet::new()) + }; + if names.is_empty() { + info!("All highlight captures conform to standards."); + } else { + warn!( + "Non-standard highlight {} detected:\n* {}", + if names.len() > 1 { + "captures" + } else { + "capture" + }, + names.join("\n* ") + ); + } + } + + let source = fs::read(path)?; let stdout = io::stdout(); let mut stdout = stdout.lock(); let time = Instant::now(); let mut highlighter = Highlighter::new(); + let events = + highlighter.highlight(config, &source, Some(&opts.cancellation_flag), |string| { + loader.highlight_config_for_injection_string(string) + })?; + let theme = &opts.theme; - let events = highlighter.highlight(config, source, cancellation_flag, |string| { - loader.highlight_config_for_injection_string(string) - })?; + if !opts.quiet && print_name { + writeln!(&mut stdout, "{name}")?; + } - let mut style_stack = vec![theme.default_style().ansi]; - for event in events { - match event? { - HighlightEvent::HighlightStart(highlight) => { - style_stack.push(theme.styles[highlight.0].ansi); + if opts.html { + if !opts.quiet { + writeln!(&mut stdout, "{HTML_HEAD_HEADER}")?; + writeln!(&mut stdout, " ")?; + writeln!(&mut stdout, "{HTML_BODY_HEADER}")?; + } + + let mut renderer = HtmlRenderer::new(); + renderer.render(events, &source, &move |highlight, output| { + if opts.inline_styles { + output.extend(b"style='"); + output.extend( + theme.styles[highlight.0] + .css + .as_ref() + .map_or_else(|| "".as_bytes(), |css_style| css_style.as_bytes()), + ); + output.extend(b"'"); + } else { + output.extend(b"class='"); + let mut parts = theme.highlight_names[highlight.0].split('.').peekable(); + while let Some(part) = parts.next() { + output.extend(part.as_bytes()); + if parts.peek().is_some() { + output.extend(b" "); + } + } + output.extend(b"'"); } - HighlightEvent::Source { start, end } => { - let style = style_stack.last().unwrap(); - write!(&mut stdout, "{style}").unwrap(); - stdout.write_all(&source[start..end])?; - write!(&mut stdout, "{style:#}").unwrap(); + })?; + + if !opts.quiet { + writeln!(&mut stdout, "")?; + for (i, line) in renderer.lines().enumerate() { + writeln!( + &mut stdout, + "", + i + 1, + )?; + } + writeln!(&mut stdout, "
{}{line}
")?; + writeln!(&mut stdout, "{HTML_FOOTER}")?; + } + } else { + let mut style_stack = vec![theme.default_style().ansi]; + for event in events { + match event? { + HighlightEvent::HighlightStart(highlight) => { + style_stack.push(theme.styles[highlight.0].ansi); + } + HighlightEvent::HighlightEnd => { + style_stack.pop(); + } + HighlightEvent::Source { start, end } => { + let style = style_stack.last().unwrap(); + write!(&mut stdout, "{style}").unwrap(); + stdout.write_all(&source[start..end])?; + write!(&mut stdout, "{style:#}").unwrap(); + } } } } - if print_time { - eprintln!("Time: {}ms", time.elapsed().as_millis()); - } - - Ok(()) -} - -pub fn html( - loader: &Loader, - theme: &Theme, - source: &[u8], - config: &HighlightConfiguration, - quiet: bool, - print_time: bool, - cancellation_flag: Option<&AtomicUsize>, -) -> Result<()> { - use std::io::Write; - - let stdout = io::stdout(); - let mut stdout = stdout.lock(); - let time = Instant::now(); - let mut highlighter = Highlighter::new(); - - let events = highlighter.highlight(config, source, cancellation_flag, |string| { - loader.highlight_config_for_injection_string(string) - })?; - - let mut renderer = HtmlRenderer::new(); - renderer.render(events, source, &move |highlight| { - theme.styles[highlight.0] - .css - .as_ref() - .map_or_else(|| "".as_bytes(), |css_style| css_style.as_bytes()) - })?; - - if !quiet { - writeln!(&mut stdout, "")?; - for (i, line) in renderer.lines().enumerate() { - writeln!( - &mut stdout, - "", - i + 1, - )?; - } - - writeln!(&mut stdout, "
{}{line}
")?; - } - - if print_time { - eprintln!("Time: {}ms", time.elapsed().as_millis()); + if opts.print_time { + info!("Time: {}ms", time.elapsed().as_millis()); } Ok(()) @@ -449,7 +480,7 @@ mod tests { style.ansi.get_fg_color(), Some(Color::Ansi256(Ansi256Color(36))) ); - assert_eq!(style.css, Some("style=\'color: #00af87\'".to_string())); + assert_eq!(style.css, Some("color: #00af87".to_string())); // junglegreen is not an ANSI color and is preserved when the terminal supports it env::set_var("COLORTERM", "truecolor"); @@ -458,16 +489,16 @@ mod tests { style.ansi.get_fg_color(), Some(Color::Rgb(RgbColor(38, 166, 154))) ); - assert_eq!(style.css, Some("style=\'color: #26a69a\'".to_string())); + assert_eq!(style.css, Some("color: #26a69a".to_string())); - // junglegreen gets approximated as darkcyan when the terminal does not support it + // junglegreen gets approximated as cadetblue when the terminal does not support it env::set_var("COLORTERM", ""); parse_style(&mut style, Value::String(JUNGLE_GREEN.to_string())); assert_eq!( style.ansi.get_fg_color(), - Some(Color::Ansi256(Ansi256Color(36))) + Some(Color::Ansi256(Ansi256Color(72))) ); - assert_eq!(style.css, Some("style=\'color: #26a69a\'".to_string())); + assert_eq!(style.css, Some("color: #26a69a".to_string())); if let Ok(environment_variable) = original_environment_variable { env::set_var("COLORTERM", environment_variable); diff --git a/crates/cli/src/init.rs b/crates/cli/src/init.rs new file mode 100644 index 00000000..70ca25af --- /dev/null +++ b/crates/cli/src/init.rs @@ -0,0 +1,1548 @@ +use std::{ + fs, + path::{Path, PathBuf}, + str::{self, FromStr}, +}; + +use anyhow::{anyhow, Context, Result}; +use crc32fast::hash as crc32; +use heck::{ToKebabCase, ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase}; +use indoc::{formatdoc, indoc}; +use log::info; +use rand::{thread_rng, Rng}; +use semver::Version; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use tree_sitter_generate::write_file; +use tree_sitter_loader::{ + Author, Bindings, Grammar, Links, Metadata, PathsJSON, TreeSitterJSON, + DEFAULT_HIGHLIGHTS_QUERY_FILE_NAME, DEFAULT_INJECTIONS_QUERY_FILE_NAME, + DEFAULT_LOCALS_QUERY_FILE_NAME, DEFAULT_TAGS_QUERY_FILE_NAME, +}; + +const CLI_VERSION: &str = env!("CARGO_PKG_VERSION"); +const CLI_VERSION_PLACEHOLDER: &str = "CLI_VERSION"; + +const ABI_VERSION_MAX: usize = tree_sitter::LANGUAGE_VERSION; +const ABI_VERSION_MAX_PLACEHOLDER: &str = "ABI_VERSION_MAX"; + +const PARSER_NAME_PLACEHOLDER: &str = "PARSER_NAME"; +const CAMEL_PARSER_NAME_PLACEHOLDER: &str = "CAMEL_PARSER_NAME"; +const TITLE_PARSER_NAME_PLACEHOLDER: &str = "TITLE_PARSER_NAME"; +const UPPER_PARSER_NAME_PLACEHOLDER: &str = "UPPER_PARSER_NAME"; +const LOWER_PARSER_NAME_PLACEHOLDER: &str = "LOWER_PARSER_NAME"; +const KEBAB_PARSER_NAME_PLACEHOLDER: &str = "KEBAB_PARSER_NAME"; +const PARSER_CLASS_NAME_PLACEHOLDER: &str = "PARSER_CLASS_NAME"; + +const PARSER_DESCRIPTION_PLACEHOLDER: &str = "PARSER_DESCRIPTION"; +const PARSER_LICENSE_PLACEHOLDER: &str = "PARSER_LICENSE"; +const PARSER_NS_PLACEHOLDER: &str = "PARSER_NS"; +const PARSER_NS_CLEANED_PLACEHOLDER: &str = "PARSER_NS_CLEANED"; +const PARSER_URL_PLACEHOLDER: &str = "PARSER_URL"; +const PARSER_URL_STRIPPED_PLACEHOLDER: &str = "PARSER_URL_STRIPPED"; +const PARSER_VERSION_PLACEHOLDER: &str = "PARSER_VERSION"; +const PARSER_FINGERPRINT_PLACEHOLDER: &str = "PARSER_FINGERPRINT"; + +const AUTHOR_NAME_PLACEHOLDER: &str = "PARSER_AUTHOR_NAME"; +const AUTHOR_EMAIL_PLACEHOLDER: &str = "PARSER_AUTHOR_EMAIL"; +const AUTHOR_URL_PLACEHOLDER: &str = "PARSER_AUTHOR_URL"; + +const AUTHOR_BLOCK_JS: &str = "\n \"author\": {"; +const AUTHOR_NAME_PLACEHOLDER_JS: &str = "\n \"name\": \"PARSER_AUTHOR_NAME\","; +const AUTHOR_EMAIL_PLACEHOLDER_JS: &str = ",\n \"email\": \"PARSER_AUTHOR_EMAIL\""; +const AUTHOR_URL_PLACEHOLDER_JS: &str = ",\n \"url\": \"PARSER_AUTHOR_URL\""; + +const AUTHOR_BLOCK_PY: &str = "\nauthors = [{"; +const AUTHOR_NAME_PLACEHOLDER_PY: &str = "name = \"PARSER_AUTHOR_NAME\""; +const AUTHOR_EMAIL_PLACEHOLDER_PY: &str = ", email = \"PARSER_AUTHOR_EMAIL\""; + +const AUTHOR_BLOCK_RS: &str = "\nauthors = ["; +const AUTHOR_NAME_PLACEHOLDER_RS: &str = "PARSER_AUTHOR_NAME"; +const AUTHOR_EMAIL_PLACEHOLDER_RS: &str = " PARSER_AUTHOR_EMAIL"; + +const AUTHOR_BLOCK_JAVA: &str = "\n "; +const AUTHOR_NAME_PLACEHOLDER_JAVA: &str = "\n PARSER_AUTHOR_NAME"; +const AUTHOR_EMAIL_PLACEHOLDER_JAVA: &str = "\n PARSER_AUTHOR_EMAIL"; +const AUTHOR_URL_PLACEHOLDER_JAVA: &str = "\n PARSER_AUTHOR_URL"; + +const AUTHOR_BLOCK_GRAMMAR: &str = "\n * @author "; +const AUTHOR_NAME_PLACEHOLDER_GRAMMAR: &str = "PARSER_AUTHOR_NAME"; +const AUTHOR_EMAIL_PLACEHOLDER_GRAMMAR: &str = " PARSER_AUTHOR_EMAIL"; + +const FUNDING_URL_PLACEHOLDER: &str = "FUNDING_URL"; + +const HIGHLIGHTS_QUERY_PATH_PLACEHOLDER: &str = "HIGHLIGHTS_QUERY_PATH"; +const INJECTIONS_QUERY_PATH_PLACEHOLDER: &str = "INJECTIONS_QUERY_PATH"; +const LOCALS_QUERY_PATH_PLACEHOLDER: &str = "LOCALS_QUERY_PATH"; +const TAGS_QUERY_PATH_PLACEHOLDER: &str = "TAGS_QUERY_PATH"; + +const GRAMMAR_JS_TEMPLATE: &str = include_str!("./templates/grammar.js"); +const PACKAGE_JSON_TEMPLATE: &str = include_str!("./templates/package.json"); +const GITIGNORE_TEMPLATE: &str = include_str!("./templates/gitignore"); +const GITATTRIBUTES_TEMPLATE: &str = include_str!("./templates/gitattributes"); +const EDITORCONFIG_TEMPLATE: &str = include_str!("./templates/.editorconfig"); + +const RUST_BINDING_VERSION: &str = env!("CARGO_PKG_VERSION"); +const RUST_BINDING_VERSION_PLACEHOLDER: &str = "RUST_BINDING_VERSION"; + +const LIB_RS_TEMPLATE: &str = include_str!("./templates/lib.rs"); +const BUILD_RS_TEMPLATE: &str = include_str!("./templates/build.rs"); +const CARGO_TOML_TEMPLATE: &str = include_str!("./templates/_cargo.toml"); + +const INDEX_JS_TEMPLATE: &str = include_str!("./templates/index.js"); +const INDEX_D_TS_TEMPLATE: &str = include_str!("./templates/index.d.ts"); +const JS_BINDING_CC_TEMPLATE: &str = include_str!("./templates/js-binding.cc"); +const BINDING_GYP_TEMPLATE: &str = include_str!("./templates/binding.gyp"); +const BINDING_TEST_JS_TEMPLATE: &str = include_str!("./templates/binding_test.js"); + +const MAKEFILE_TEMPLATE: &str = include_str!("./templates/makefile"); +const CMAKELISTS_TXT_TEMPLATE: &str = include_str!("./templates/cmakelists.cmake"); +const PARSER_NAME_H_TEMPLATE: &str = include_str!("./templates/PARSER_NAME.h"); +const PARSER_NAME_PC_IN_TEMPLATE: &str = include_str!("./templates/PARSER_NAME.pc.in"); + +const GO_MOD_TEMPLATE: &str = include_str!("./templates/go.mod"); +const BINDING_GO_TEMPLATE: &str = include_str!("./templates/binding.go"); +const BINDING_TEST_GO_TEMPLATE: &str = include_str!("./templates/binding_test.go"); + +const SETUP_PY_TEMPLATE: &str = include_str!("./templates/setup.py"); +const INIT_PY_TEMPLATE: &str = include_str!("./templates/__init__.py"); +const INIT_PYI_TEMPLATE: &str = include_str!("./templates/__init__.pyi"); +const PYPROJECT_TOML_TEMPLATE: &str = include_str!("./templates/pyproject.toml"); +const PY_BINDING_C_TEMPLATE: &str = include_str!("./templates/py-binding.c"); +const TEST_BINDING_PY_TEMPLATE: &str = include_str!("./templates/test_binding.py"); + +const PACKAGE_SWIFT_TEMPLATE: &str = include_str!("./templates/package.swift"); +const TESTS_SWIFT_TEMPLATE: &str = include_str!("./templates/tests.swift"); + +const POM_XML_TEMPLATE: &str = include_str!("./templates/pom.xml"); +const BINDING_JAVA_TEMPLATE: &str = include_str!("./templates/binding.java"); +const TEST_JAVA_TEMPLATE: &str = include_str!("./templates/test.java"); + +const BUILD_ZIG_TEMPLATE: &str = include_str!("./templates/build.zig"); +const BUILD_ZIG_ZON_TEMPLATE: &str = include_str!("./templates/build.zig.zon"); +const ROOT_ZIG_TEMPLATE: &str = include_str!("./templates/root.zig"); +const TEST_ZIG_TEMPLATE: &str = include_str!("./templates/test.zig"); + +pub const TREE_SITTER_JSON_SCHEMA: &str = + "https://tree-sitter.github.io/tree-sitter/assets/schemas/config.schema.json"; + +#[derive(Serialize, Deserialize, Clone)] +pub struct JsonConfigOpts { + pub name: String, + pub camelcase: String, + pub title: String, + pub description: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub repository: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub funding: Option, + pub scope: String, + pub file_types: Vec, + pub version: Version, + pub license: String, + pub author: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub email: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub url: Option, + pub namespace: Option, + pub bindings: Bindings, +} + +impl JsonConfigOpts { + #[must_use] + pub fn to_tree_sitter_json(self) -> TreeSitterJSON { + TreeSitterJSON { + schema: Some(TREE_SITTER_JSON_SCHEMA.to_string()), + grammars: vec![Grammar { + name: self.name.clone(), + camelcase: Some(self.camelcase), + title: Some(self.title), + scope: self.scope, + path: None, + external_files: PathsJSON::Empty, + file_types: Some(self.file_types), + highlights: PathsJSON::Empty, + injections: PathsJSON::Empty, + locals: PathsJSON::Empty, + tags: PathsJSON::Empty, + injection_regex: Some(format!("^{}$", self.name)), + first_line_regex: None, + content_regex: None, + class_name: Some(format!("TreeSitter{}", self.name.to_upper_camel_case())), + }], + metadata: Metadata { + version: self.version, + license: Some(self.license), + description: Some(self.description), + authors: Some(vec![Author { + name: self.author, + email: self.email, + url: self.url, + }]), + links: Some(Links { + repository: self.repository.unwrap_or_else(|| { + format!("https://github.com/tree-sitter/tree-sitter-{}", self.name) + }), + funding: self.funding, + }), + namespace: self.namespace, + }, + bindings: self.bindings, + } + } +} + +impl Default for JsonConfigOpts { + fn default() -> Self { + Self { + name: String::new(), + camelcase: String::new(), + title: String::new(), + description: String::new(), + repository: None, + funding: None, + scope: String::new(), + file_types: vec![], + version: Version::from_str("0.1.0").unwrap(), + license: String::new(), + author: String::new(), + email: None, + url: None, + namespace: None, + bindings: Bindings::default(), + } + } +} + +struct GenerateOpts<'a> { + author_name: Option<&'a str>, + author_email: Option<&'a str>, + author_url: Option<&'a str>, + license: Option<&'a str>, + description: Option<&'a str>, + repository: Option<&'a str>, + funding: Option<&'a str>, + version: &'a Version, + camel_parser_name: &'a str, + title_parser_name: &'a str, + class_name: &'a str, + highlights_query_path: &'a str, + injections_query_path: &'a str, + locals_query_path: &'a str, + tags_query_path: &'a str, + namespace: Option<&'a str>, +} + +pub fn generate_grammar_files( + repo_path: &Path, + language_name: &str, + allow_update: bool, + opts: Option<&JsonConfigOpts>, +) -> Result<()> { + let dashed_language_name = language_name.to_kebab_case(); + + let tree_sitter_config = missing_path_else( + repo_path.join("tree-sitter.json"), + true, + |path| { + // invariant: opts is always Some when `tree-sitter.json` doesn't exist + let Some(opts) = opts else { unreachable!() }; + + let tree_sitter_json = opts.clone().to_tree_sitter_json(); + write_file(path, serde_json::to_string_pretty(&tree_sitter_json)?)?; + Ok(()) + }, + |path| { + // updating the config, if needed + if let Some(opts) = opts { + let tree_sitter_json = opts.clone().to_tree_sitter_json(); + write_file(path, serde_json::to_string_pretty(&tree_sitter_json)?)?; + } + Ok(()) + }, + )?; + + let tree_sitter_config = serde_json::from_str::( + &fs::read_to_string(tree_sitter_config.as_path()) + .with_context(|| "Failed to read tree-sitter.json")?, + )?; + + let authors = tree_sitter_config.metadata.authors.as_ref(); + let camel_name = tree_sitter_config.grammars[0] + .camelcase + .clone() + .unwrap_or_else(|| language_name.to_upper_camel_case()); + let title_name = tree_sitter_config.grammars[0] + .title + .clone() + .unwrap_or_else(|| language_name.to_upper_camel_case()); + let class_name = tree_sitter_config.grammars[0] + .class_name + .clone() + .unwrap_or_else(|| format!("TreeSitter{}", language_name.to_upper_camel_case())); + + let default_highlights_path = Path::new("queries").join(DEFAULT_HIGHLIGHTS_QUERY_FILE_NAME); + let default_injections_path = Path::new("queries").join(DEFAULT_INJECTIONS_QUERY_FILE_NAME); + let default_locals_path = Path::new("queries").join(DEFAULT_LOCALS_QUERY_FILE_NAME); + let default_tags_path = Path::new("queries").join(DEFAULT_TAGS_QUERY_FILE_NAME); + + let generate_opts = GenerateOpts { + author_name: authors + .map(|a| a.first().map(|a| a.name.as_str())) + .unwrap_or_default(), + author_email: authors + .map(|a| a.first().and_then(|a| a.email.as_deref())) + .unwrap_or_default(), + author_url: authors + .map(|a| a.first().and_then(|a| a.url.as_deref())) + .unwrap_or_default(), + license: tree_sitter_config.metadata.license.as_deref(), + description: tree_sitter_config.metadata.description.as_deref(), + repository: tree_sitter_config + .metadata + .links + .as_ref() + .map(|l| l.repository.as_str()), + funding: tree_sitter_config + .metadata + .links + .as_ref() + .and_then(|l| l.funding.as_deref()), + version: &tree_sitter_config.metadata.version, + camel_parser_name: &camel_name, + title_parser_name: &title_name, + class_name: &class_name, + highlights_query_path: tree_sitter_config.grammars[0] + .highlights + .to_variable_value(&default_highlights_path), + injections_query_path: tree_sitter_config.grammars[0] + .injections + .to_variable_value(&default_injections_path), + locals_query_path: tree_sitter_config.grammars[0] + .locals + .to_variable_value(&default_locals_path), + tags_query_path: tree_sitter_config.grammars[0] + .tags + .to_variable_value(&default_tags_path), + namespace: tree_sitter_config.metadata.namespace.as_deref(), + }; + + // Create package.json + missing_path_else( + repo_path.join("package.json"), + allow_update, + |path| { + generate_file( + path, + PACKAGE_JSON_TEMPLATE, + dashed_language_name.as_str(), + &generate_opts, + ) + }, + |path| { + let mut contents = fs::read_to_string(path)? + .replace( + r#""node-addon-api": "^8.3.1""#, + r#""node-addon-api": "^8.5.0""#, + ) + .replace( + indoc! {r#" + "prebuildify": "^6.0.1", + "tree-sitter-cli":"#}, + indoc! {r#" + "prebuildify": "^6.0.1", + "tree-sitter": "^0.25.0", + "tree-sitter-cli":"#}, + ); + if !contents.contains("module") { + info!("Migrating package.json to ESM"); + contents = contents.replace( + r#""repository":"#, + indoc! {r#" + "type": "module", + "repository":"#}, + ); + } + write_file(path, contents)?; + Ok(()) + }, + )?; + + // Do not create a grammar.js file in a repo with multiple language configs + if !tree_sitter_config.has_multiple_language_configs() { + missing_path_else( + repo_path.join("grammar.js"), + allow_update, + |path| generate_file(path, GRAMMAR_JS_TEMPLATE, language_name, &generate_opts), + |path| { + let mut contents = fs::read_to_string(path)?; + if contents.contains("module.exports") { + info!("Migrating grammars.js to ESM"); + contents = contents.replace("module.exports =", "export default"); + write_file(path, contents)?; + } + + Ok(()) + }, + )?; + } + + // Write .gitignore file + missing_path_else( + repo_path.join(".gitignore"), + allow_update, + |path| generate_file(path, GITIGNORE_TEMPLATE, language_name, &generate_opts), + |path| { + let mut contents = fs::read_to_string(path)?; + if !contents.contains("Zig artifacts") { + info!("Adding zig entries to .gitignore"); + contents.push('\n'); + contents.push_str(indoc! {" + # Zig artifacts + .zig-cache/ + zig-cache/ + zig-out/ + "}); + } + Ok(()) + }, + )?; + + // Write .gitattributes file + missing_path_else( + repo_path.join(".gitattributes"), + allow_update, + |path| generate_file(path, GITATTRIBUTES_TEMPLATE, language_name, &generate_opts), + |path| { + let mut contents = fs::read_to_string(path)?; + let c_bindings_entry = "bindings/c/* "; + if contents.contains(c_bindings_entry) { + info!("Updating c bindings entry in .gitattributes"); + contents = contents.replace(c_bindings_entry, "bindings/c/** "); + } + if !contents.contains("Zig bindings") { + info!("Adding zig entries to .gitattributes"); + contents.push('\n'); + contents.push_str(indoc! {" + # Zig bindings + build.zig linguist-generated + build.zig.zon linguist-generated + "}); + } + write_file(path, contents)?; + Ok(()) + }, + )?; + + // Write .editorconfig file + missing_path(repo_path.join(".editorconfig"), |path| { + generate_file(path, EDITORCONFIG_TEMPLATE, language_name, &generate_opts) + })?; + + let bindings_dir = repo_path.join("bindings"); + + // Generate Rust bindings + if tree_sitter_config.bindings.rust { + missing_path(bindings_dir.join("rust"), create_dir)?.apply(|path| { + missing_path_else(path.join("lib.rs"), allow_update, |path| { + generate_file(path, LIB_RS_TEMPLATE, language_name, &generate_opts) + }, |path| { + let mut contents = fs::read_to_string(path)?; + if !contents.contains("#[cfg(with_highlights_query)]") { + info!("Updating query constants in bindings/rust/lib.rs"); + let replacement = indoc! {r#" + #[cfg(with_highlights_query)] + /// The syntax highlighting query for this grammar. + pub const HIGHLIGHTS_QUERY: &str = include_str!("../../HIGHLIGHTS_QUERY_PATH"); + + #[cfg(with_injections_query)] + /// The language injection query for this grammar. + pub const INJECTIONS_QUERY: &str = include_str!("../../INJECTIONS_QUERY_PATH"); + + #[cfg(with_locals_query)] + /// The local variable query for this grammar. + pub const LOCALS_QUERY: &str = include_str!("../../LOCALS_QUERY_PATH"); + + #[cfg(with_tags_query)] + /// The symbol tagging query for this grammar. + pub const TAGS_QUERY: &str = include_str!("../../TAGS_QUERY_PATH"); + "#} + .replace("HIGHLIGHTS_QUERY_PATH", generate_opts.highlights_query_path) + .replace("INJECTIONS_QUERY_PATH", generate_opts.injections_query_path) + .replace("LOCALS_QUERY_PATH", generate_opts.locals_query_path) + .replace("TAGS_QUERY_PATH", generate_opts.tags_query_path); + contents = contents + .replace( + indoc! {r#" + // NOTE: uncomment these to include any queries that this grammar contains: + + // pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm"); + // pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm"); + // pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm"); + // pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm"); + "#}, + &replacement, + ); + } + write_file(path, contents)?; + Ok(()) + })?; + + missing_path_else( + path.join("build.rs"), + allow_update, + |path| generate_file(path, BUILD_RS_TEMPLATE, language_name, &generate_opts), + |path| { + let mut contents = fs::read_to_string(path)?; + if !contents.contains("wasm32-unknown-unknown") { + info!("Adding wasm32-unknown-unknown target to bindings/rust/build.rs"); + let replacement = indoc!{r#" + c_config.flag("-utf-8"); + + if std::env::var("TARGET").unwrap() == "wasm32-unknown-unknown" { + let Ok(wasm_headers) = std::env::var("DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS") else { + panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS must be set by the language crate"); + }; + let Ok(wasm_src) = + std::env::var("DEP_TREE_SITTER_LANGUAGE_WASM_SRC").map(std::path::PathBuf::from) + else { + panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_SRC must be set by the language crate"); + }; + + c_config.include(&wasm_headers); + c_config.files([ + wasm_src.join("stdio.c"), + wasm_src.join("stdlib.c"), + wasm_src.join("string.c"), + ]); + } + "#} + .lines() + .map(|line| if line.is_empty() { line.to_string() } else { format!(" {line}") }) + .collect::>() + .join("\n"); + + contents = contents.replace(r#" c_config.flag("-utf-8");"#, &replacement); + } + + // Introduce configuration variables for dynamic query inclusion + if !contents.contains("with_highlights_query") { + info!("Adding support for dynamic query inclusion to bindings/rust/build.rs"); + let replaced = indoc! {r#" + c_config.compile("tree-sitter-KEBAB_PARSER_NAME"); + }"#} + .replace("KEBAB_PARSER_NAME", &language_name.to_kebab_case()); + + let replacement = indoc! {r#" + c_config.compile("tree-sitter-KEBAB_PARSER_NAME"); + + println!("cargo:rustc-check-cfg=cfg(with_highlights_query)"); + if !"HIGHLIGHTS_QUERY_PATH".is_empty() && std::path::Path::new("HIGHLIGHTS_QUERY_PATH").exists() { + println!("cargo:rustc-cfg=with_highlights_query"); + } + println!("cargo:rustc-check-cfg=cfg(with_injections_query)"); + if !"INJECTIONS_QUERY_PATH".is_empty() && std::path::Path::new("INJECTIONS_QUERY_PATH").exists() { + println!("cargo:rustc-cfg=with_injections_query"); + } + println!("cargo:rustc-check-cfg=cfg(with_locals_query)"); + if !"LOCALS_QUERY_PATH".is_empty() && std::path::Path::new("LOCALS_QUERY_PATH").exists() { + println!("cargo:rustc-cfg=with_locals_query"); + } + println!("cargo:rustc-check-cfg=cfg(with_tags_query)"); + if !"TAGS_QUERY_PATH".is_empty() && std::path::Path::new("TAGS_QUERY_PATH").exists() { + println!("cargo:rustc-cfg=with_tags_query"); + } + }"#} + .replace("KEBAB_PARSER_NAME", &language_name.to_kebab_case()) + .replace("HIGHLIGHTS_QUERY_PATH", generate_opts.highlights_query_path) + .replace("INJECTIONS_QUERY_PATH", generate_opts.injections_query_path) + .replace("LOCALS_QUERY_PATH", generate_opts.locals_query_path) + .replace("TAGS_QUERY_PATH", generate_opts.tags_query_path); + + contents = contents.replace( + &replaced, + &replacement, + ); + } + + write_file(path, contents)?; + Ok(()) + }, + )?; + + missing_path_else( + repo_path.join("Cargo.toml"), + allow_update, + |path| { + generate_file( + path, + CARGO_TOML_TEMPLATE, + dashed_language_name.as_str(), + &generate_opts, + ) + }, + |path| { + let contents = fs::read_to_string(path)?; + if contents.contains("\"LICENSE\"") { + info!("Adding LICENSE entry to bindings/rust/Cargo.toml"); + write_file(path, contents.replace("\"LICENSE\"", "\"/LICENSE\""))?; + } + Ok(()) + }, + )?; + + Ok(()) + })?; + } + + // Generate Node bindings + if tree_sitter_config.bindings.node { + missing_path(bindings_dir.join("node"), create_dir)?.apply(|path| { + missing_path_else( + path.join("index.js"), + allow_update, + |path| generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts), + |path| { + let contents = fs::read_to_string(path)?; + if !contents.contains("Object.defineProperty") { + info!("Replacing index.js"); + generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts)?; + } + Ok(()) + }, + )?; + + missing_path_else( + path.join("index.d.ts"), + allow_update, + |path| generate_file(path, INDEX_D_TS_TEMPLATE, language_name, &generate_opts), + |path| { + let contents = fs::read_to_string(path)?; + if !contents.contains("export default binding") { + info!("Replacing index.d.ts"); + generate_file(path, INDEX_D_TS_TEMPLATE, language_name, &generate_opts)?; + } + Ok(()) + }, + )?; + + missing_path_else( + path.join("binding_test.js"), + allow_update, + |path| { + generate_file( + path, + BINDING_TEST_JS_TEMPLATE, + language_name, + &generate_opts, + ) + }, + |path| { + let contents = fs::read_to_string(path)?; + if !contents.contains("import") { + info!("Replacing binding_test.js"); + generate_file( + path, + BINDING_TEST_JS_TEMPLATE, + language_name, + &generate_opts, + )?; + } + Ok(()) + }, + )?; + + missing_path(path.join("binding.cc"), |path| { + generate_file(path, JS_BINDING_CC_TEMPLATE, language_name, &generate_opts) + })?; + + missing_path_else( + repo_path.join("binding.gyp"), + allow_update, + |path| generate_file(path, BINDING_GYP_TEMPLATE, language_name, &generate_opts), + |path| { + let contents = fs::read_to_string(path)?; + if contents.contains("fs.exists(") { + info!("Replacing `fs.exists` calls in binding.gyp"); + write_file(path, contents.replace("fs.exists(", "fs.existsSync("))?; + } + Ok(()) + }, + )?; + + Ok(()) + })?; + } + + // Generate C bindings + if tree_sitter_config.bindings.c { + let kebab_case_name = language_name.to_kebab_case(); + missing_path(bindings_dir.join("c"), create_dir)?.apply(|path| { + let header_name = format!("tree-sitter-{kebab_case_name}.h"); + let old_file = &path.join(&header_name); + if allow_update && fs::exists(old_file).unwrap_or(false) { + info!("Removing bindings/c/{header_name}"); + fs::remove_file(old_file)?; + } + missing_path(path.join("tree_sitter"), create_dir)?.apply(|include_path| { + missing_path( + include_path.join(&header_name), + |path| { + generate_file(path, PARSER_NAME_H_TEMPLATE, language_name, &generate_opts) + }, + )?; + Ok(()) + })?; + + missing_path( + path.join(format!("tree-sitter-{kebab_case_name}.pc.in")), + |path| { + generate_file( + path, + PARSER_NAME_PC_IN_TEMPLATE, + language_name, + &generate_opts, + ) + }, + )?; + + missing_path_else( + repo_path.join("Makefile"), + allow_update, + |path| { + generate_file(path, MAKEFILE_TEMPLATE, language_name, &generate_opts) + }, + |path| { + let mut contents = fs::read_to_string(path)?; + if !contents.contains("cd '$(DESTDIR)$(LIBDIR)' && ln -sf") { + info!("Replacing Makefile"); + generate_file(path, MAKEFILE_TEMPLATE, language_name, &generate_opts)?; + } else { + let replaced = indoc! {r" + $(PARSER): $(SRC_DIR)/grammar.json + $(TS) generate $^ + "}; + if contents.contains(replaced) { + info!("Adding --no-parser target to Makefile"); + contents = contents + .replace( + replaced, + indoc! {r" + $(SRC_DIR)/grammar.json: grammar.js + $(TS) generate --no-parser $^ + + $(PARSER): $(SRC_DIR)/grammar.json + $(TS) generate $^ + "} + ); + } + write_file(path, contents)?; + } + Ok(()) + }, + )?; + + missing_path_else( + repo_path.join("CMakeLists.txt"), + allow_update, + |path| generate_file(path, CMAKELISTS_TXT_TEMPLATE, language_name, &generate_opts), + |path| { + let contents = fs::read_to_string(path)?; + let replaced_contents = contents + .replace("add_custom_target(test", "add_custom_target(ts-test") + .replace( + &formatdoc! {r#" + install(FILES bindings/c/tree-sitter-{language_name}.h + DESTINATION "${{CMAKE_INSTALL_INCLUDEDIR}}/tree_sitter") + "#}, + indoc! {r#" + install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/bindings/c/tree_sitter" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" + FILES_MATCHING PATTERN "*.h") + "#} + ).replace( + &format!("target_include_directories(tree-sitter-{language_name} PRIVATE src)"), + &formatdoc! {" + target_include_directories(tree-sitter-{language_name} + PRIVATE src + INTERFACE $ + $) + "} + ).replace( + indoc! {r#" + add_custom_command(OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/src/parser.c" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/grammar.json" + COMMAND "${TREE_SITTER_CLI}" generate src/grammar.json + --abi=${TREE_SITTER_ABI_VERSION} + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + COMMENT "Generating parser.c") + "#}, + indoc! {r#" + add_custom_command(OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/src/grammar.json" + "${CMAKE_CURRENT_SOURCE_DIR}/src/node-types.json" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/grammar.js" + COMMAND "${TREE_SITTER_CLI}" generate grammar.js --no-parser + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + COMMENT "Generating grammar.json") + + add_custom_command(OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/src/parser.c" + BYPRODUCTS "${CMAKE_CURRENT_SOURCE_DIR}/src/tree_sitter/parser.h" + "${CMAKE_CURRENT_SOURCE_DIR}/src/tree_sitter/alloc.h" + "${CMAKE_CURRENT_SOURCE_DIR}/src/tree_sitter/array.h" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/grammar.json" + COMMAND "${TREE_SITTER_CLI}" generate src/grammar.json + --abi=${TREE_SITTER_ABI_VERSION} + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + COMMENT "Generating parser.c") + "#} + ); + if !replaced_contents.eq(&contents) { + info!("Updating CMakeLists.txt"); + write_file(path, replaced_contents)?; + } + Ok(()) + }, + )?; + + Ok(()) + })?; + } + + // Generate Go bindings + if tree_sitter_config.bindings.go { + missing_path(bindings_dir.join("go"), create_dir)?.apply(|path| { + missing_path(path.join("binding.go"), |path| { + generate_file(path, BINDING_GO_TEMPLATE, language_name, &generate_opts) + })?; + + missing_path(path.join("binding_test.go"), |path| { + generate_file( + path, + BINDING_TEST_GO_TEMPLATE, + language_name, + &generate_opts, + ) + })?; + + missing_path(repo_path.join("go.mod"), |path| { + generate_file(path, GO_MOD_TEMPLATE, language_name, &generate_opts) + })?; + + Ok(()) + })?; + } + + // Generate Python bindings + if tree_sitter_config.bindings.python { + missing_path(bindings_dir.join("python"), create_dir)?.apply(|path| { + let snake_case_grammar_name = format!("tree_sitter_{}", language_name.to_snake_case()); + let lang_path = path.join(&snake_case_grammar_name); + missing_path(&lang_path, create_dir)?; + + missing_path_else( + lang_path.join("binding.c"), + allow_update, + |path| generate_file(path, PY_BINDING_C_TEMPLATE, language_name, &generate_opts), + |path| { + let mut contents = fs::read_to_string(path)?; + if !contents.contains("PyModuleDef_Init") { + info!("Updating bindings/python/{snake_case_grammar_name}/binding.c"); + contents = contents + .replace("PyModule_Create", "PyModuleDef_Init") + .replace( + "static PyMethodDef methods[] = {\n", + indoc! {" + static struct PyModuleDef_Slot slots[] = { + #ifdef Py_GIL_DISABLED + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + #endif + {0, NULL} + }; + + static PyMethodDef methods[] = { + "}, + ) + .replace( + indoc! {" + .m_size = -1, + .m_methods = methods + "}, + indoc! {" + .m_size = 0, + .m_methods = methods, + .m_slots = slots, + "}, + ); + write_file(path, contents)?; + } + Ok(()) + }, + )?; + + missing_path_else( + lang_path.join("__init__.py"), + allow_update, + |path| { + generate_file(path, INIT_PY_TEMPLATE, language_name, &generate_opts) + }, + |path| { + let contents = fs::read_to_string(path)?; + if !contents.contains("uncomment these to include any queries") { + info!("Replacing __init__.py"); + generate_file(path, INIT_PY_TEMPLATE, language_name, &generate_opts)?; + } + Ok(()) + }, + )?; + + missing_path_else( + lang_path.join("__init__.pyi"), + allow_update, + |path| generate_file(path, INIT_PYI_TEMPLATE, language_name, &generate_opts), + |path| { + let mut contents = fs::read_to_string(path)?; + if contents.contains("uncomment these to include any queries") { + info!("Replacing __init__.pyi"); + generate_file(path, INIT_PYI_TEMPLATE, language_name, &generate_opts)?; + } else if !contents.contains("CapsuleType") { + info!("Updating __init__.pyi"); + contents = contents + .replace( + "from typing import Final", + "from typing import Final\nfrom typing_extensions import CapsuleType" + ) + .replace("-> object:", "-> CapsuleType:"); + write_file(path, contents)?; + } + Ok(()) + }, + )?; + + missing_path(lang_path.join("py.typed"), |path| { + generate_file(path, "", language_name, &generate_opts) // py.typed is empty + })?; + + missing_path(path.join("tests"), create_dir)?.apply(|path| { + missing_path_else( + path.join("test_binding.py"), + allow_update, + |path| { + generate_file( + path, + TEST_BINDING_PY_TEMPLATE, + language_name, + &generate_opts, + ) + }, + |path| { + let mut contents = fs::read_to_string(path)?; + if !contents.contains("Parser(Language(") { + info!("Updating Language function in bindings/python/tests/test_binding.py"); + contents = contents + .replace("tree_sitter.Language(", "Parser(Language(") + .replace(".language())\n", ".language()))\n") + .replace( + "import tree_sitter\n", + "from tree_sitter import Language, Parser\n", + ); + write_file(path, contents)?; + } + Ok(()) + }, + )?; + Ok(()) + })?; + + missing_path_else( + repo_path.join("setup.py"), + allow_update, + |path| generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts), + |path| { + let mut contents = fs::read_to_string(path)?; + if !contents.contains("build_ext") { + info!("Replacing setup.py"); + generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts)?; + } + if !contents.contains(" and not get_config_var") { + info!("Updating Python free-threading support in setup.py"); + contents = contents.replace( + r#"startswith("cp"):"#, + r#"startswith("cp") and not get_config_var("Py_GIL_DISABLED"):"# + ); + write_file(path, contents)?; + } + Ok(()) + }, + )?; + + missing_path_else( + repo_path.join("pyproject.toml"), + allow_update, + |path| { + generate_file( + path, + PYPROJECT_TOML_TEMPLATE, + dashed_language_name.as_str(), + &generate_opts, + ) + }, + |path| { + let mut contents = fs::read_to_string(path)?; + if !contents.contains("cp310-*") { + info!("Updating dependencies in pyproject.toml"); + contents = contents + .replace(r#"build = "cp39-*""#, r#"build = "cp310-*""#) + .replace(r#"python = ">=3.9""#, r#"python = ">=3.10""#) + .replace("tree-sitter~=0.22", "tree-sitter~=0.24"); + write_file(path, contents)?; + } + Ok(()) + }, + )?; + + Ok(()) + })?; + } + + // Generate Swift bindings + if tree_sitter_config.bindings.swift { + missing_path(bindings_dir.join("swift"), create_dir)?.apply(|path| { + let lang_path = path.join(&class_name); + missing_path(&lang_path, create_dir)?; + + missing_path(lang_path.join(format!("{language_name}.h")), |path| { + generate_file(path, PARSER_NAME_H_TEMPLATE, language_name, &generate_opts) + })?; + + missing_path(path.join(format!("{class_name}Tests")), create_dir)?.apply(|path| { + missing_path(path.join(format!("{class_name}Tests.swift")), |path| { + generate_file(path, TESTS_SWIFT_TEMPLATE, language_name, &generate_opts) + })?; + + Ok(()) + })?; + + missing_path_else( + repo_path.join("Package.swift"), + allow_update, + |path| generate_file(path, PACKAGE_SWIFT_TEMPLATE, language_name, &generate_opts), + |path| { + let contents = fs::read_to_string(path)?; + let replaced_contents = contents + .replace( + "https://github.com/ChimeHQ/SwiftTreeSitter", + "https://github.com/tree-sitter/swift-tree-sitter", + ) + .replace("version: \"0.8.0\")", "version: \"0.9.0\")") + .replace("(url:", "(name: \"SwiftTreeSitter\", url:"); + if !replaced_contents.eq(&contents) { + info!("Updating tree-sitter dependency in Package.swift"); + write_file(path, contents)?; + } + Ok(()) + }, + )?; + + Ok(()) + })?; + } + + // Generate Zig bindings + if tree_sitter_config.bindings.zig { + missing_path_else( + repo_path.join("build.zig"), + allow_update, + |path| generate_file(path, BUILD_ZIG_TEMPLATE, language_name, &generate_opts), + |path| { + let contents = fs::read_to_string(path)?; + if !contents.contains("b.pkg_hash.len") { + info!("Replacing build.zig"); + generate_file(path, BUILD_ZIG_TEMPLATE, language_name, &generate_opts) + } else { + Ok(()) + } + }, + )?; + + missing_path_else( + repo_path.join("build.zig.zon"), + allow_update, + |path| generate_file(path, BUILD_ZIG_ZON_TEMPLATE, language_name, &generate_opts), + |path| { + let contents = fs::read_to_string(path)?; + if !contents.contains(".name = .tree_sitter_") { + info!("Replacing build.zig.zon"); + generate_file(path, BUILD_ZIG_ZON_TEMPLATE, language_name, &generate_opts) + } else { + Ok(()) + } + }, + )?; + + missing_path(bindings_dir.join("zig"), create_dir)?.apply(|path| { + missing_path_else( + path.join("root.zig"), + allow_update, + |path| generate_file(path, ROOT_ZIG_TEMPLATE, language_name, &generate_opts), + |path| { + let contents = fs::read_to_string(path)?; + if contents.contains("ts.Language") { + info!("Replacing root.zig"); + generate_file(path, ROOT_ZIG_TEMPLATE, language_name, &generate_opts) + } else { + Ok(()) + } + }, + )?; + + missing_path(path.join("test.zig"), |path| { + generate_file(path, TEST_ZIG_TEMPLATE, language_name, &generate_opts) + })?; + + Ok(()) + })?; + } + + // Generate Java bindings + if tree_sitter_config.bindings.java { + missing_path(repo_path.join("pom.xml"), |path| { + generate_file(path, POM_XML_TEMPLATE, language_name, &generate_opts) + })?; + + missing_path(bindings_dir.join("java"), create_dir)?.apply(|path| { + missing_path(path.join("main"), create_dir)?.apply(|path| { + let package_path = generate_opts + .namespace + .unwrap_or("io.github.treesitter") + .replace(['-', '_'], "") + .split('.') + .fold(path.to_path_buf(), |path, dir| path.join(dir)) + .join("jtreesitter") + .join(language_name.to_lowercase().replace('_', "")); + missing_path(package_path, create_dir)?.apply(|path| { + missing_path(path.join(format!("{class_name}.java")), |path| { + generate_file(path, BINDING_JAVA_TEMPLATE, language_name, &generate_opts) + })?; + + Ok(()) + })?; + + Ok(()) + })?; + + missing_path(path.join("test"), create_dir)?.apply(|path| { + missing_path(path.join(format!("{class_name}Test.java")), |path| { + generate_file(path, TEST_JAVA_TEMPLATE, language_name, &generate_opts) + })?; + + Ok(()) + })?; + + Ok(()) + })?; + } + + Ok(()) +} + +pub fn get_root_path(path: &Path) -> Result { + let mut pathbuf = path.to_owned(); + let filename = path.file_name().unwrap().to_str().unwrap(); + let is_package_json = filename == "package.json"; + loop { + let json = pathbuf + .exists() + .then(|| { + let contents = fs::read_to_string(pathbuf.as_path()) + .with_context(|| format!("Failed to read {filename}"))?; + if is_package_json { + serde_json::from_str::>(&contents) + .context(format!("Failed to parse {filename}")) + .map(|v| v.contains_key("tree-sitter")) + } else { + serde_json::from_str::(&contents) + .context(format!("Failed to parse {filename}")) + .map(|_| true) + } + }) + .transpose()?; + if json == Some(true) { + return Ok(pathbuf.parent().unwrap().to_path_buf()); + } + pathbuf.pop(); // filename + if !pathbuf.pop() { + return Err(anyhow!(format!( + concat!( + "Failed to locate a {} file,", + " please ensure you have one, and if you don't then consult the docs", + ), + filename + ))); + } + pathbuf.push(filename); + } +} + +fn generate_file( + path: &Path, + template: &str, + language_name: &str, + generate_opts: &GenerateOpts, +) -> Result<()> { + let filename = path.file_name().unwrap().to_str().unwrap(); + + let lower_parser_name = if path + .extension() + .is_some_and(|e| e.eq_ignore_ascii_case("java")) + { + language_name.to_snake_case().replace('_', "") + } else { + language_name.to_snake_case() + }; + + let mut replacement = template + .replace( + CAMEL_PARSER_NAME_PLACEHOLDER, + generate_opts.camel_parser_name, + ) + .replace( + TITLE_PARSER_NAME_PLACEHOLDER, + generate_opts.title_parser_name, + ) + .replace( + UPPER_PARSER_NAME_PLACEHOLDER, + &language_name.to_shouty_snake_case(), + ) + .replace( + KEBAB_PARSER_NAME_PLACEHOLDER, + &language_name.to_kebab_case(), + ) + .replace(LOWER_PARSER_NAME_PLACEHOLDER, &lower_parser_name) + .replace(PARSER_NAME_PLACEHOLDER, language_name) + .replace(CLI_VERSION_PLACEHOLDER, CLI_VERSION) + .replace(RUST_BINDING_VERSION_PLACEHOLDER, RUST_BINDING_VERSION) + .replace(ABI_VERSION_MAX_PLACEHOLDER, &ABI_VERSION_MAX.to_string()) + .replace( + PARSER_VERSION_PLACEHOLDER, + &generate_opts.version.to_string(), + ) + .replace(PARSER_CLASS_NAME_PLACEHOLDER, generate_opts.class_name) + .replace( + HIGHLIGHTS_QUERY_PATH_PLACEHOLDER, + generate_opts.highlights_query_path, + ) + .replace( + INJECTIONS_QUERY_PATH_PLACEHOLDER, + generate_opts.injections_query_path, + ) + .replace( + LOCALS_QUERY_PATH_PLACEHOLDER, + generate_opts.locals_query_path, + ) + .replace(TAGS_QUERY_PATH_PLACEHOLDER, generate_opts.tags_query_path); + + if let Some(name) = generate_opts.author_name { + replacement = replacement.replace(AUTHOR_NAME_PLACEHOLDER, name); + } else { + match filename { + "package.json" => { + replacement = replacement.replace(AUTHOR_NAME_PLACEHOLDER_JS, ""); + } + "pyproject.toml" => { + replacement = replacement.replace(AUTHOR_NAME_PLACEHOLDER_PY, ""); + } + "grammar.js" => { + replacement = replacement.replace(AUTHOR_NAME_PLACEHOLDER_GRAMMAR, ""); + } + "Cargo.toml" => { + replacement = replacement.replace(AUTHOR_NAME_PLACEHOLDER_RS, ""); + } + "pom.xml" => { + replacement = replacement.replace(AUTHOR_NAME_PLACEHOLDER_JAVA, ""); + } + _ => {} + } + } + + if let Some(email) = generate_opts.author_email { + replacement = match filename { + "Cargo.toml" | "grammar.js" => { + replacement.replace(AUTHOR_EMAIL_PLACEHOLDER, &format!("<{email}>")) + } + _ => replacement.replace(AUTHOR_EMAIL_PLACEHOLDER, email), + } + } else { + match filename { + "package.json" => { + replacement = replacement.replace(AUTHOR_EMAIL_PLACEHOLDER_JS, ""); + } + "pyproject.toml" => { + replacement = replacement.replace(AUTHOR_EMAIL_PLACEHOLDER_PY, ""); + } + "grammar.js" => { + replacement = replacement.replace(AUTHOR_EMAIL_PLACEHOLDER_GRAMMAR, ""); + } + "Cargo.toml" => { + replacement = replacement.replace(AUTHOR_EMAIL_PLACEHOLDER_RS, ""); + } + "pom.xml" => { + replacement = replacement.replace(AUTHOR_EMAIL_PLACEHOLDER_JAVA, ""); + } + _ => {} + } + } + + match (generate_opts.author_url, filename) { + (Some(url), "package.json" | "pom.xml") => { + replacement = replacement.replace(AUTHOR_URL_PLACEHOLDER, url); + } + (None, "package.json") => { + replacement = replacement.replace(AUTHOR_URL_PLACEHOLDER_JS, ""); + } + (None, "pom.xml") => { + replacement = replacement.replace(AUTHOR_URL_PLACEHOLDER_JAVA, ""); + } + _ => {} + } + + if generate_opts.author_name.is_none() + && generate_opts.author_email.is_none() + && generate_opts.author_url.is_none() + { + match filename { + "package.json" => { + if let Some(start_idx) = replacement.find(AUTHOR_BLOCK_JS) { + if let Some(end_idx) = replacement[start_idx..] + .find("},") + .map(|i| i + start_idx + 2) + { + replacement.replace_range(start_idx..end_idx, ""); + } + } + } + "pom.xml" => { + if let Some(start_idx) = replacement.find(AUTHOR_BLOCK_JAVA) { + if let Some(end_idx) = replacement[start_idx..] + .find("") + .map(|i| i + start_idx + 12) + { + replacement.replace_range(start_idx..end_idx, ""); + } + } + } + _ => {} + } + } else if generate_opts.author_name.is_none() && generate_opts.author_email.is_none() { + match filename { + "pyproject.toml" => { + if let Some(start_idx) = replacement.find(AUTHOR_BLOCK_PY) { + if let Some(end_idx) = replacement[start_idx..] + .find("}]") + .map(|i| i + start_idx + 2) + { + replacement.replace_range(start_idx..end_idx, ""); + } + } + } + "grammar.js" => { + if let Some(start_idx) = replacement.find(AUTHOR_BLOCK_GRAMMAR) { + if let Some(end_idx) = replacement[start_idx..] + .find(" \n") + .map(|i| i + start_idx + 1) + { + replacement.replace_range(start_idx..end_idx, ""); + } + } + } + "Cargo.toml" => { + if let Some(start_idx) = replacement.find(AUTHOR_BLOCK_RS) { + if let Some(end_idx) = replacement[start_idx..] + .find("\"]") + .map(|i| i + start_idx + 2) + { + replacement.replace_range(start_idx..end_idx, ""); + } + } + } + _ => {} + } + } + + if let Some(license) = generate_opts.license { + replacement = replacement.replace(PARSER_LICENSE_PLACEHOLDER, license); + } else { + replacement = replacement.replace(PARSER_LICENSE_PLACEHOLDER, "MIT"); + } + + if let Some(description) = generate_opts.description { + replacement = replacement.replace(PARSER_DESCRIPTION_PLACEHOLDER, description); + } else { + replacement = replacement.replace( + PARSER_DESCRIPTION_PLACEHOLDER, + &format!( + "{} grammar for tree-sitter", + generate_opts.camel_parser_name, + ), + ); + } + + if let Some(repository) = generate_opts.repository { + replacement = replacement + .replace( + PARSER_URL_STRIPPED_PLACEHOLDER, + &repository.replace("https://", "").to_lowercase(), + ) + .replace(PARSER_URL_PLACEHOLDER, &repository.to_lowercase()); + } else { + replacement = replacement + .replace( + PARSER_URL_STRIPPED_PLACEHOLDER, + &format!( + "github.com/tree-sitter/tree-sitter-{}", + language_name.to_lowercase() + ), + ) + .replace( + PARSER_URL_PLACEHOLDER, + &format!( + "https://github.com/tree-sitter/tree-sitter-{}", + language_name.to_lowercase() + ), + ); + } + + if let Some(namespace) = generate_opts.namespace { + replacement = replacement + .replace( + PARSER_NS_CLEANED_PLACEHOLDER, + &namespace.replace(['-', '_'], ""), + ) + .replace(PARSER_NS_PLACEHOLDER, namespace); + } else { + replacement = replacement + .replace(PARSER_NS_CLEANED_PLACEHOLDER, "io.github.treesitter") + .replace(PARSER_NS_PLACEHOLDER, "io.github.tree-sitter"); + } + + if let Some(funding_url) = generate_opts.funding { + match filename { + "pyproject.toml" | "package.json" => { + replacement = replacement.replace(FUNDING_URL_PLACEHOLDER, funding_url); + } + _ => {} + } + } else { + match filename { + "package.json" => { + replacement = replacement.replace(" \"funding\": \"FUNDING_URL\",\n", ""); + } + "pyproject.toml" => { + replacement = replacement.replace("Funding = \"FUNDING_URL\"\n", ""); + } + _ => {} + } + } + + if filename == "build.zig.zon" { + let id = thread_rng().gen_range(1u32..0xFFFF_FFFFu32); + let checksum = crc32(format!("tree_sitter_{language_name}").as_bytes()); + replacement = replacement.replace( + PARSER_FINGERPRINT_PLACEHOLDER, + #[cfg(target_endian = "little")] + &format!("0x{checksum:x}{id:x}"), + #[cfg(target_endian = "big")] + &format!("0x{id:x}{checksum:x}"), + ); + } + + write_file(path, replacement)?; + Ok(()) +} + +fn create_dir(path: &Path) -> Result<()> { + fs::create_dir_all(path) + .with_context(|| format!("Failed to create {:?}", path.to_string_lossy())) +} + +#[derive(PartialEq, Eq, Debug)] +enum PathState

+where + P: AsRef, +{ + Exists(P), + Missing(P), +} + +#[allow(dead_code)] +impl

PathState

+where + P: AsRef, +{ + fn exists(&self, mut action: impl FnMut(&Path) -> Result<()>) -> Result<&Self> { + if let Self::Exists(path) = self { + action(path.as_ref())?; + } + Ok(self) + } + + fn missing(&self, mut action: impl FnMut(&Path) -> Result<()>) -> Result<&Self> { + if let Self::Missing(path) = self { + action(path.as_ref())?; + } + Ok(self) + } + + fn apply(&self, mut action: impl FnMut(&Path) -> Result<()>) -> Result<&Self> { + action(self.as_path())?; + Ok(self) + } + + fn apply_state(&self, mut action: impl FnMut(&Self) -> Result<()>) -> Result<&Self> { + action(self)?; + Ok(self) + } + + fn as_path(&self) -> &Path { + match self { + Self::Exists(path) | Self::Missing(path) => path.as_ref(), + } + } +} + +fn missing_path(path: P, mut action: F) -> Result> +where + P: AsRef, + F: FnMut(&Path) -> Result<()>, +{ + let path_ref = path.as_ref(); + if !path_ref.exists() { + action(path_ref)?; + Ok(PathState::Missing(path)) + } else { + Ok(PathState::Exists(path)) + } +} + +fn missing_path_else( + path: P, + allow_update: bool, + mut action: T, + mut else_action: F, +) -> Result> +where + P: AsRef, + T: FnMut(&Path) -> Result<()>, + F: FnMut(&Path) -> Result<()>, +{ + let path_ref = path.as_ref(); + if !path_ref.exists() { + action(path_ref)?; + Ok(PathState::Missing(path)) + } else { + if allow_update { + else_action(path_ref)?; + } + Ok(PathState::Exists(path)) + } +} diff --git a/crates/cli/src/input.rs b/crates/cli/src/input.rs new file mode 100644 index 00000000..5ab82087 --- /dev/null +++ b/crates/cli/src/input.rs @@ -0,0 +1,187 @@ +use std::{ + fs, + io::{Read, Write}, + path::{Path, PathBuf}, + sync::{ + atomic::{AtomicUsize, Ordering}, + mpsc, Arc, + }, +}; + +use anyhow::{anyhow, bail, Context, Result}; +use glob::glob; + +use crate::test::{parse_tests, TestEntry}; + +pub enum CliInput { + Paths(Vec), + Test { + name: String, + contents: Vec, + languages: Vec>, + }, + Stdin(Vec), +} + +pub fn get_input( + paths_file: Option<&Path>, + paths: Option>, + test_number: Option, + cancellation_flag: &Arc, +) -> Result { + if let Some(paths_file) = paths_file { + return Ok(CliInput::Paths( + fs::read_to_string(paths_file) + .with_context(|| format!("Failed to read paths file {}", paths_file.display()))? + .trim() + .lines() + .map(PathBuf::from) + .collect::>(), + )); + } + + if let Some(test_number) = test_number { + let current_dir = std::env::current_dir().unwrap(); + let test_dir = current_dir.join("test").join("corpus"); + + if !test_dir.exists() { + return Err(anyhow!( + "Test corpus directory not found in current directory, see https://tree-sitter.github.io/tree-sitter/creating-parsers/5-writing-tests" + )); + } + + let test_entry = parse_tests(&test_dir)?; + let mut test_num = 0; + let Some((name, contents, languages)) = + get_test_info(&test_entry, test_number.max(1) - 1, &mut test_num) + else { + return Err(anyhow!("Failed to fetch contents of test #{test_number}")); + }; + + return Ok(CliInput::Test { + name, + contents, + languages, + }); + } + + if let Some(paths) = paths { + let mut result = Vec::new(); + + let mut incorporate_path = |path: PathBuf, positive| { + if positive { + result.push(path); + } else if let Some(index) = result.iter().position(|p| *p == path) { + result.remove(index); + } + }; + + for mut path in paths { + let mut positive = true; + if path.starts_with("!") { + positive = false; + path = path.strip_prefix("!").unwrap().to_path_buf(); + } + + if path.exists() { + incorporate_path(path, positive); + } else { + let Some(path_str) = path.to_str() else { + bail!("Invalid path: {}", path.display()); + }; + let paths = glob(path_str) + .with_context(|| format!("Invalid glob pattern {}", path.display()))?; + for path in paths { + incorporate_path(path?, positive); + } + } + } + + if result.is_empty() { + return Err(anyhow!( + "No files were found at or matched by the provided pathname/glob" + )); + } + + return Ok(CliInput::Paths(result)); + } + + let reader_flag = cancellation_flag.clone(); + let (tx, rx) = mpsc::channel(); + + // Spawn a thread to read from stdin, until ctrl-c or EOF is received + std::thread::spawn(move || { + let mut input = Vec::new(); + let stdin = std::io::stdin(); + let mut handle = stdin.lock(); + + // Read in chunks, so we can check the ctrl-c flag + loop { + if reader_flag.load(Ordering::Relaxed) == 1 { + break; + } + let mut buffer = [0; 1024]; + match handle.read(&mut buffer) { + Ok(0) | Err(_) => break, + Ok(n) => input.extend_from_slice(&buffer[..n]), + } + } + + // Signal to the main thread that we're done + tx.send(input).ok(); + }); + + loop { + // If we've received a ctrl-c signal, exit + if cancellation_flag.load(Ordering::Relaxed) == 1 { + bail!("\n"); + } + + // If we're done receiving input from stdin, return it + if let Ok(input) = rx.try_recv() { + return Ok(CliInput::Stdin(input)); + } + + std::thread::sleep(std::time::Duration::from_millis(50)); + } +} + +#[allow(clippy::type_complexity)] +pub fn get_test_info( + test_entry: &TestEntry, + target_test: u32, + test_num: &mut u32, +) -> Option<(String, Vec, Vec>)> { + match test_entry { + TestEntry::Example { + name, + input, + attributes, + .. + } => { + if *test_num == target_test { + return Some((name.clone(), input.clone(), attributes.languages.clone())); + } + *test_num += 1; + } + TestEntry::Group { children, .. } => { + for child in children { + if let Some((name, input, languages)) = get_test_info(child, target_test, test_num) + { + return Some((name, input, languages)); + } + } + } + } + + None +} + +/// Writes `contents` to a temporary file and returns the path to that file. +pub fn get_tmp_source_file(contents: &[u8]) -> Result { + let parse_path = std::env::temp_dir().join(".tree-sitter-temp"); + let mut parse_file = std::fs::File::create(&parse_path)?; + parse_file.write_all(contents)?; + + Ok(parse_path) +} diff --git a/crates/cli/src/logger.rs b/crates/cli/src/logger.rs new file mode 100644 index 00000000..41b11906 --- /dev/null +++ b/crates/cli/src/logger.rs @@ -0,0 +1,55 @@ +use std::io::Write; + +use anstyle::{AnsiColor, Color, Style}; +use log::{Level, LevelFilter, Log, Metadata, Record}; + +pub fn paint(color: Option>, text: &str) -> String { + let style = Style::new().fg_color(color.map(Into::into)); + format!("{style}{text}{style:#}") +} + +struct Logger; + +impl Log for Logger { + fn enabled(&self, _: &Metadata) -> bool { + true + } + + fn log(&self, record: &Record) { + match record.level() { + Level::Error => eprintln!( + "{} {}", + paint(Some(AnsiColor::Red), "Error:"), + record.args() + ), + Level::Warn => eprintln!( + "{} {}", + paint(Some(AnsiColor::Yellow), "Warning:"), + record.args() + ), + Level::Info | Level::Debug => eprintln!("{}", record.args()), + Level::Trace => eprintln!( + "[{}] {}", + record + .module_path() + .unwrap_or_default() + .trim_start_matches("rust_tree_sitter_cli::"), + record.args() + ), + } + } + + fn flush(&self) { + let mut stderr = std::io::stderr().lock(); + let _ = stderr.flush(); + } +} + +pub fn init() { + log::set_boxed_logger(Box::new(Logger {})).unwrap(); + log::set_max_level(LevelFilter::Info); +} + +pub fn enable_debug() { + log::set_max_level(LevelFilter::Debug); +} diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs new file mode 100644 index 00000000..fbf75b8e --- /dev/null +++ b/crates/cli/src/main.rs @@ -0,0 +1,2162 @@ +use std::{ + collections::HashSet, + env, fs, + path::{Path, PathBuf}, +}; + +use anstyle::{AnsiColor, Color, Style}; +use anyhow::{anyhow, Context, Result}; +use clap::{crate_authors, Args, Command, FromArgMatches as _, Subcommand, ValueEnum}; +use clap_complete::generate; +use dialoguer::{theme::ColorfulTheme, Confirm, FuzzySelect, Input, MultiSelect}; +use heck::ToUpperCamelCase; +use log::{error, info, warn}; +use regex::Regex; +use semver::Version as SemverVersion; +use tree_sitter::{ffi, Parser, Point}; +use tree_sitter_cli::{ + fuzz::{ + fuzz_language_corpus, FuzzOptions, EDIT_COUNT, ITERATION_COUNT, LOG_ENABLED, + LOG_GRAPH_ENABLED, START_SEED, + }, + highlight::{self, HighlightOptions}, + init::{generate_grammar_files, JsonConfigOpts, TREE_SITTER_JSON_SCHEMA}, + input::{get_input, get_tmp_source_file, CliInput}, + logger, + parse::{self, ParseDebugType, ParseFileOptions, ParseOutput, ParseTheme}, + playground, + query::{self, QueryFileOptions}, + tags::{self, TagsOptions}, + test::{self, TestOptions, TestStats, TestSummary}, + test_highlight, test_tags, util, + version::{self, BumpLevel}, + wasm, +}; +use tree_sitter_config::Config; +use tree_sitter_generate::OptLevel; +use tree_sitter_highlight::Highlighter; +use tree_sitter_loader::{self as loader, Bindings, TreeSitterJSON}; +use tree_sitter_tags::TagsContext; + +const BUILD_VERSION: &str = env!("CARGO_PKG_VERSION"); +const BUILD_SHA: Option<&'static str> = option_env!("BUILD_SHA"); +const DEFAULT_GENERATE_ABI_VERSION: usize = 15; + +#[derive(Subcommand)] +#[command(about="Generates and tests parsers", author=crate_authors!("\n"), styles=get_styles())] +enum Commands { + /// Generate a default config file + InitConfig(InitConfig), + /// Initialize a grammar repository + Init(Init), + /// Generate a parser + Generate(Generate), + /// Compile a parser + Build(Build), + /// Parse files + Parse(Parse), + /// Run a parser's tests + Test(Test), + /// Display or increment the version of a grammar + Version(Version), + /// Fuzz a parser + Fuzz(Fuzz), + /// Search files using a syntax tree query + Query(Query), + /// Highlight a file + Highlight(Highlight), + /// Generate a list of tags + Tags(Tags), + /// Start local playground for a parser in the browser + Playground(Playground), + /// Print info about all known language parsers + DumpLanguages(DumpLanguages), + /// Generate shell completions + Complete(Complete), +} + +#[derive(Args)] +struct InitConfig; + +#[derive(Args)] +#[command(alias = "i")] +struct Init { + /// Update outdated files + #[arg(long, short)] + pub update: bool, + /// The path to the tree-sitter grammar directory + #[arg(long, short = 'p')] + pub grammar_path: Option, +} + +#[derive(Args)] +#[command(alias = "gen", alias = "g")] +struct Generate { + /// The path to the grammar file + #[arg(index = 1)] + pub grammar_path: Option, + /// Show debug log during generation + #[arg(long, short)] + pub log: bool, + #[arg( + long = "abi", + value_name = "VERSION", + env = "TREE_SITTER_ABI_VERSION", + help = format!(concat!( + "Select the language ABI version to generate (default {}).\n", + "Use --abi=latest to generate the newest supported version ({}).", + ), + DEFAULT_GENERATE_ABI_VERSION, + tree_sitter::LANGUAGE_VERSION, + ) + )] + pub abi_version: Option, + /// Only generate `grammar.json` and `node-types.json` + #[arg(long)] + pub no_parser: bool, + /// Deprecated: use the `build` command + #[arg(long, short = 'b')] + pub build: bool, + /// Deprecated: use the `build` command + #[arg(long, short = '0')] + pub debug_build: bool, + /// Deprecated: use the `build` command + #[arg(long, value_name = "PATH")] + pub libdir: Option, + /// The path to output the generated source files + #[arg(long, short, value_name = "DIRECTORY")] + pub output: Option, + /// Produce a report of the states for the given rule, use `-` to report every rule + #[arg(long, conflicts_with = "json", conflicts_with = "json_summary")] + pub report_states_for_rule: Option, + /// Deprecated: use --json-summary + #[arg( + long, + conflicts_with = "json_summary", + conflicts_with = "report_states_for_rule" + )] + pub json: bool, + /// Report conflicts in a JSON format + #[arg( + long, + conflicts_with = "json", + conflicts_with = "report_states_for_rule" + )] + pub json_summary: bool, + /// The name or path of the JavaScript runtime to use for generating parsers + #[cfg(not(feature = "qjs-rt"))] + #[arg( + long, + value_name = "EXECUTABLE", + env = "TREE_SITTER_JS_RUNTIME", + default_value = "node" + )] + pub js_runtime: Option, + + #[cfg(feature = "qjs-rt")] + #[arg( + long, + value_name = "EXECUTABLE", + env = "TREE_SITTER_JS_RUNTIME", + default_value = "node" + )] + /// The name or path of the JavaScript runtime to use for generating parsers, specify `native` + /// to use the native `QuickJS` runtime + pub js_runtime: Option, + + /// Disable optimizations when generating the parser. Currently, this only affects + /// the merging of compatible parse states. + #[arg(long)] + pub disable_optimizations: bool, +} + +#[derive(Args)] +#[command(alias = "b")] +struct Build { + /// Build a Wasm module instead of a dynamic library + #[arg(short, long)] + pub wasm: bool, + /// The path to output the compiled file + #[arg(short, long)] + pub output: Option, + /// The path to the grammar directory + #[arg(index = 1, num_args = 1)] + pub path: Option, + /// Make the parser reuse the same allocator as the library + #[arg(long)] + pub reuse_allocator: bool, + /// Compile a parser in debug mode + #[arg(long, short = '0')] + pub debug: bool, +} + +#[derive(Args)] +#[command(alias = "p")] +struct Parse { + /// The path to a file with paths to source file(s) + #[arg(long = "paths")] + pub paths_file: Option, + /// The source file(s) to use + #[arg(num_args=1..)] + pub paths: Option>, + /// The path to the tree-sitter grammar directory, implies --rebuild + #[arg(long, short = 'p', conflicts_with = "rebuild")] + pub grammar_path: Option, + /// The path to the parser's dynamic library + #[arg(long, short = 'l')] + pub lib_path: Option, + /// If `--lib-path` is used, the name of the language used to extract the + /// library's language function + #[arg(long)] + pub lang_name: Option, + /// Select a language by the scope instead of a file extension + #[arg(long)] + pub scope: Option, + /// Show parsing debug log + #[arg(long, short = 'd')] // TODO: Rework once clap adds `default_missing_value_t` + #[allow(clippy::option_option)] + pub debug: Option>, + /// Compile a parser in debug mode + #[arg(long, short = '0')] + pub debug_build: bool, + /// Produce the log.html file with debug graphs + #[arg(long, short = 'D')] + pub debug_graph: bool, + /// Compile parsers to Wasm instead of native dynamic libraries + #[arg(long, hide = cfg!(not(feature = "wasm")))] + pub wasm: bool, + /// Output the parse data with graphviz dot + #[arg(long = "dot")] + pub output_dot: bool, + /// Output the parse data in XML format + #[arg(long = "xml", short = 'x')] + pub output_xml: bool, + /// Output the parse data in a pretty-printed CST format + #[arg(long = "cst", short = 'c')] + pub output_cst: bool, + /// Show parsing statistic + #[arg(long, short, conflicts_with = "json", conflicts_with = "json_summary")] + pub stat: bool, + /// Interrupt the parsing process by timeout (µs) + #[arg(long)] + pub timeout: Option, + /// Measure execution time + #[arg(long, short)] + pub time: bool, + /// Suppress main output + #[arg(long, short)] + pub quiet: bool, + #[allow(clippy::doc_markdown)] + /// Apply edits in the format: \"row,col|position delcount insert_text\", can be supplied + /// multiple times + #[arg( + long, + num_args = 1.., + )] + pub edits: Option>, + /// The encoding of the input files + #[arg(long)] + pub encoding: Option, + /// Open `log.html` in the default browser, if `--debug-graph` is supplied + #[arg(long)] + pub open_log: bool, + /// Deprecated: use --json-summary + #[arg(long, conflicts_with = "json_summary", conflicts_with = "stat")] + pub json: bool, + /// Output parsing results in a JSON format + #[arg(long, short = 'j', conflicts_with = "json", conflicts_with = "stat")] + pub json_summary: bool, + /// The path to an alternative config.json file + #[arg(long)] + pub config_path: Option, + /// Parse the contents of a specific test + #[arg(long, short = 'n')] + #[clap(conflicts_with = "paths", conflicts_with = "paths_file")] + pub test_number: Option, + /// Force rebuild the parser + #[arg(short, long)] + pub rebuild: bool, + /// Omit ranges in the output + #[arg(long)] + pub no_ranges: bool, +} + +#[derive(ValueEnum, Clone)] +pub enum Encoding { + Utf8, + Utf16LE, + Utf16BE, +} + +#[derive(Args)] +#[command(alias = "t")] +struct Test { + /// Only run corpus test cases whose name matches the given regex + #[arg(long, short)] + pub include: Option, + /// Only run corpus test cases whose name does not match the given regex + #[arg(long, short)] + pub exclude: Option, + /// Only run corpus test cases from a given filename + #[arg(long)] + pub file_name: Option, + /// The path to the tree-sitter grammar directory, implies --rebuild + #[arg(long, short = 'p', conflicts_with = "rebuild")] + pub grammar_path: Option, + /// The path to the parser's dynamic library + #[arg(long, short = 'l')] + pub lib_path: Option, + /// If `--lib-path` is used, the name of the language used to extract the + /// library's language function + #[arg(long)] + pub lang_name: Option, + /// Update all syntax trees in corpus files with current parser output + #[arg(long, short)] + pub update: bool, + /// Show parsing debug log + #[arg(long, short = 'd')] + pub debug: bool, + /// Compile a parser in debug mode + #[arg(long, short = '0')] + pub debug_build: bool, + /// Produce the log.html file with debug graphs + #[arg(long, short = 'D')] + pub debug_graph: bool, + /// Compile parsers to Wasm instead of native dynamic libraries + #[arg(long, hide = cfg!(not(feature = "wasm")))] + pub wasm: bool, + /// Open `log.html` in the default browser, if `--debug-graph` is supplied + #[arg(long)] + pub open_log: bool, + /// The path to an alternative config.json file + #[arg(long)] + pub config_path: Option, + /// Force showing fields in test diffs + #[arg(long)] + pub show_fields: bool, + /// Show parsing statistics + #[arg(long)] + pub stat: Option, + /// Force rebuild the parser + #[arg(short, long)] + pub rebuild: bool, + /// Show only the pass-fail overview tree + #[arg(long)] + pub overview_only: bool, + /// Output the test summary in a JSON format + #[arg(long)] + pub json_summary: bool, +} + +#[derive(Args)] +#[command(alias = "publish")] +/// Display or increment the version of a grammar +struct Version { + /// The version to bump to + #[arg( + conflicts_with = "bump", + long_help = "\ + The version to bump to\n\ + \n\ + Examples:\n \ + tree-sitter version: display the current version\n \ + tree-sitter version : bump to specified version\n \ + tree-sitter version --bump : automatic bump" + )] + pub version: Option, + /// The path to the tree-sitter grammar directory + #[arg(long, short = 'p')] + pub grammar_path: Option, + /// Automatically bump from the current version + #[arg(long, value_enum, conflicts_with = "version")] + pub bump: Option, +} + +#[derive(Args)] +#[command(alias = "f")] +struct Fuzz { + /// List of test names to skip + #[arg(long, short)] + pub skip: Option>, + /// Subdirectory to the language + #[arg(long)] + pub subdir: Option, + /// The path to the tree-sitter grammar directory, implies --rebuild + #[arg(long, short = 'p', conflicts_with = "rebuild")] + pub grammar_path: Option, + /// The path to the parser's dynamic library + #[arg(long)] + pub lib_path: Option, + /// If `--lib-path` is used, the name of the language used to extract the + /// library's language function + #[arg(long)] + pub lang_name: Option, + /// Maximum number of edits to perform per fuzz test + #[arg(long)] + pub edits: Option, + /// Number of fuzzing iterations to run per test + #[arg(long)] + pub iterations: Option, + /// Only fuzz corpus test cases whose name matches the given regex + #[arg(long, short)] + pub include: Option, + /// Only fuzz corpus test cases whose name does not match the given regex + #[arg(long, short)] + pub exclude: Option, + /// Enable logging of graphs and input + #[arg(long)] + pub log_graphs: bool, + /// Enable parser logging + #[arg(long, short)] + pub log: bool, + /// Force rebuild the parser + #[arg(short, long)] + pub rebuild: bool, +} + +#[derive(Args)] +#[command(alias = "q")] +struct Query { + /// Path to a file with queries + #[arg(index = 1, required = true)] + query_path: PathBuf, + /// The path to the tree-sitter grammar directory, implies --rebuild + #[arg(long, short = 'p', conflicts_with = "rebuild")] + pub grammar_path: Option, + /// The path to the parser's dynamic library + #[arg(long, short = 'l')] + pub lib_path: Option, + /// If `--lib-path` is used, the name of the language used to extract the + /// library's language function + #[arg(long)] + pub lang_name: Option, + /// Measure execution time + #[arg(long, short)] + pub time: bool, + /// Suppress main output + #[arg(long, short)] + pub quiet: bool, + /// The path to a file with paths to source file(s) + #[arg(long = "paths")] + pub paths_file: Option, + /// The source file(s) to use + #[arg(index = 2, num_args=1..)] + pub paths: Option>, + /// The range of byte offsets in which the query will be executed + #[arg(long)] + pub byte_range: Option, + /// The range of rows in which the query will be executed + #[arg(long)] + pub row_range: Option, + /// The range of byte offsets in which the query will be executed. Only the matches that are fully contained within the provided + /// byte range will be returned. + #[arg(long)] + pub containing_byte_range: Option, + /// The range of rows in which the query will be executed. Only the matches that are fully contained within the provided row range + /// will be returned. + #[arg(long)] + pub containing_row_range: Option, + /// Select a language by the scope instead of a file extension + #[arg(long)] + pub scope: Option, + /// Order by captures instead of matches + #[arg(long, short)] + pub captures: bool, + /// Whether to run query tests or not + #[arg(long)] + pub test: bool, + /// The path to an alternative config.json file + #[arg(long)] + pub config_path: Option, + /// Query the contents of a specific test + #[arg(long, short = 'n')] + #[clap(conflicts_with = "paths", conflicts_with = "paths_file")] + pub test_number: Option, + /// Force rebuild the parser + #[arg(short, long)] + pub rebuild: bool, +} + +#[derive(Args)] +#[command(alias = "hi")] +struct Highlight { + /// Generate highlighting as an HTML document + #[arg(long, short = 'H')] + pub html: bool, + /// When generating HTML, use css classes rather than inline styles + #[arg(long)] + pub css_classes: bool, + /// Check that highlighting captures conform strictly to standards + #[arg(long)] + pub check: bool, + /// The path to a file with captures + #[arg(long)] + pub captures_path: Option, + /// The paths to files with queries + #[arg(long, num_args = 1..)] + pub query_paths: Option>, + /// Select a language by the scope instead of a file extension + #[arg(long)] + pub scope: Option, + /// Measure execution time + #[arg(long, short)] + pub time: bool, + /// Suppress main output + #[arg(long, short)] + pub quiet: bool, + /// The path to a file with paths to source file(s) + #[arg(long = "paths")] + pub paths_file: Option, + /// The source file(s) to use + #[arg(num_args = 1..)] + pub paths: Option>, + /// The path to the tree-sitter grammar directory, implies --rebuild + #[arg(long, short = 'p', conflicts_with = "rebuild")] + pub grammar_path: Option, + /// The path to an alternative config.json file + #[arg(long)] + pub config_path: Option, + /// Highlight the contents of a specific test + #[arg(long, short = 'n')] + #[clap(conflicts_with = "paths", conflicts_with = "paths_file")] + pub test_number: Option, + /// Force rebuild the parser + #[arg(short, long)] + pub rebuild: bool, +} + +#[derive(Args)] +struct Tags { + /// Select a language by the scope instead of a file extension + #[arg(long)] + pub scope: Option, + /// Measure execution time + #[arg(long, short)] + pub time: bool, + /// Suppress main output + #[arg(long, short)] + pub quiet: bool, + /// The path to a file with paths to source file(s) + #[arg(long = "paths")] + pub paths_file: Option, + /// The source file(s) to use + #[arg(num_args = 1..)] + pub paths: Option>, + /// The path to the tree-sitter grammar directory, implies --rebuild + #[arg(long, short = 'p', conflicts_with = "rebuild")] + pub grammar_path: Option, + /// The path to an alternative config.json file + #[arg(long)] + pub config_path: Option, + /// Generate tags from the contents of a specific test + #[arg(long, short = 'n')] + #[clap(conflicts_with = "paths", conflicts_with = "paths_file")] + pub test_number: Option, + /// Force rebuild the parser + #[arg(short, long)] + pub rebuild: bool, +} + +#[derive(Args)] +#[command(alias = "play", alias = "pg", alias = "web-ui")] +struct Playground { + /// Don't open in default browser + #[arg(long, short)] + pub quiet: bool, + /// Path to the directory containing the grammar and Wasm files + #[arg(long)] + pub grammar_path: Option, + /// Export playground files to specified directory instead of serving them + #[arg(long, short)] + pub export: Option, +} + +#[derive(Args)] +#[command(alias = "langs")] +struct DumpLanguages { + /// The path to an alternative config.json file + #[arg(long)] + pub config_path: Option, +} + +#[derive(Args)] +#[command(alias = "comp")] +struct Complete { + /// The shell to generate completions for + #[arg(long, short, value_enum)] + pub shell: Shell, +} + +#[derive(ValueEnum, Clone)] +pub enum Shell { + Bash, + Elvish, + Fish, + PowerShell, + Zsh, + Nushell, +} + +/// Complete `action` if the wasm feature is enabled, otherwise return an error +macro_rules! checked_wasm { + ($action:block) => { + #[cfg(feature = "wasm")] + { + $action + } + #[cfg(not(feature = "wasm"))] + { + Err(anyhow!("--wasm flag specified, but this build of tree-sitter-cli does not include the wasm feature"))?; + } + }; +} + +impl InitConfig { + fn run() -> Result<()> { + if let Ok(Some(config_path)) = Config::find_config_file() { + return Err(anyhow!( + "Remove your existing config file first: {}", + config_path.to_string_lossy() + )); + } + let mut config = Config::initial()?; + config.add(tree_sitter_loader::Config::initial())?; + config.add(tree_sitter_cli::highlight::ThemeConfig::default())?; + config.save()?; + info!( + "Saved initial configuration to {}", + config.location.display() + ); + Ok(()) + } +} + +impl Init { + fn run(self, current_dir: &Path) -> Result<()> { + let configure_json = !current_dir.join("tree-sitter.json").exists(); + + let (language_name, json_config_opts) = if configure_json { + let mut opts = JsonConfigOpts::default(); + + let name = || { + Input::::with_theme(&ColorfulTheme::default()) + .with_prompt("Parser name") + .validate_with(|input: &String| { + if input.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_') { + Ok(()) + } else { + Err("The name must be lowercase and contain only letters, digits, and underscores") + } + }) + .interact_text() + }; + + let camelcase_name = |name: &str| { + Input::::with_theme(&ColorfulTheme::default()) + .with_prompt("CamelCase name") + .default(name.to_upper_camel_case()) + .validate_with(|input: &String| { + if input + .chars() + .all(|c| c.is_ascii_alphabetic() || c.is_ascii_digit() || c == '_') + { + Ok(()) + } else { + Err("The name must contain only letters, digits, and underscores") + } + }) + .interact_text() + }; + + let title = |name: &str| { + Input::::with_theme(&ColorfulTheme::default()) + .with_prompt("Title (human-readable name)") + .default(name.to_upper_camel_case()) + .interact_text() + }; + + let description = |name: &str| { + Input::::with_theme(&ColorfulTheme::default()) + .with_prompt("Description") + .default(format!( + "{} grammar for tree-sitter", + name.to_upper_camel_case() + )) + .show_default(false) + .allow_empty(true) + .interact_text() + }; + + let repository = |name: &str| { + Input::::with_theme(&ColorfulTheme::default()) + .with_prompt("Repository URL") + .allow_empty(true) + .default(format!("https://github.com/tree-sitter/tree-sitter-{name}")) + .show_default(false) + .interact_text() + }; + + let funding = || { + Input::::with_theme(&ColorfulTheme::default()) + .with_prompt("Funding URL") + .allow_empty(true) + .interact_text() + .map(|e| Some(e.trim().to_string())) + }; + + let scope = |name: &str| { + Input::::with_theme(&ColorfulTheme::default()) + .with_prompt("TextMate scope") + .default(format!("source.{name}")) + .validate_with(|input: &String| { + if input.starts_with("source.") || input.starts_with("text.") { + Ok(()) + } else { + Err("The scope must start with 'source.' or 'text.'") + } + }) + .interact_text() + }; + + let file_types = |name: &str| { + Input::::with_theme(&ColorfulTheme::default()) + .with_prompt("File types (space-separated)") + .default(name.to_string()) + .interact_text() + .map(|ft| { + let mut set = HashSet::new(); + for ext in ft.split(' ') { + let ext = ext.trim(); + if !ext.is_empty() { + set.insert(ext.to_string()); + } + } + set.into_iter().collect::>() + }) + }; + + let initial_version = || { + Input::::with_theme(&ColorfulTheme::default()) + .with_prompt("Version") + .default(SemverVersion::new(0, 1, 0)) + .interact_text() + }; + + let license = || { + Input::::with_theme(&ColorfulTheme::default()) + .with_prompt("License") + .default("MIT".to_string()) + .allow_empty(true) + .interact() + }; + + let author = || { + Input::::with_theme(&ColorfulTheme::default()) + .with_prompt("Author name") + .interact_text() + }; + + let email = || { + Input::::with_theme(&ColorfulTheme::default()) + .with_prompt("Author email") + .allow_empty(true) + .interact_text() + .map(|e| (!e.trim().is_empty()).then_some(e)) + }; + + let url = || { + Input::::with_theme(&ColorfulTheme::default()) + .with_prompt("Author URL") + .allow_empty(true) + .interact_text() + .map(|e| Some(e.trim().to_string())) + }; + + let namespace = || { + Input::::with_theme(&ColorfulTheme::default()) + .with_prompt("Package namespace") + .default("io.github.tree-sitter".to_string()) + .allow_empty(true) + .interact() + }; + + let bindings = || { + let languages = Bindings::default().languages(); + + let enabled = MultiSelect::new() + .with_prompt("Bindings") + .items_checked(&languages) + .interact()? + .into_iter() + .map(|i| languages[i].0); + + let out = Bindings::with_enabled_languages(enabled) + .expect("unexpected unsupported language"); + anyhow::Ok(out) + }; + + let choices = [ + "name", + "camelcase", + "title", + "description", + "repository", + "funding", + "scope", + "file_types", + "version", + "license", + "author", + "email", + "url", + "namespace", + "bindings", + "exit", + ]; + + macro_rules! set_choice { + ($choice:expr) => { + match $choice { + "name" => opts.name = name()?, + "camelcase" => opts.camelcase = camelcase_name(&opts.name)?, + "title" => opts.title = title(&opts.name)?, + "description" => opts.description = description(&opts.name)?, + "repository" => opts.repository = Some(repository(&opts.name)?), + "funding" => opts.funding = funding()?, + "scope" => opts.scope = scope(&opts.name)?, + "file_types" => opts.file_types = file_types(&opts.name)?, + "version" => opts.version = initial_version()?, + "license" => opts.license = license()?, + "author" => opts.author = author()?, + "email" => opts.email = email()?, + "url" => opts.url = url()?, + "namespace" => opts.namespace = Some(namespace()?), + "bindings" => opts.bindings = bindings()?, + "exit" => break, + _ => unreachable!(), + } + }; + } + + // Initial configuration + for choice in choices.iter().take(choices.len() - 1) { + set_choice!(*choice); + } + + // Loop for editing the configuration + loop { + info!( + "Your current configuration:\n{}", + serde_json::to_string_pretty(&opts)? + ); + + if Confirm::with_theme(&ColorfulTheme::default()) + .with_prompt("Does the config above look correct?") + .interact()? + { + break; + } + + let idx = FuzzySelect::with_theme(&ColorfulTheme::default()) + .with_prompt("Which field would you like to change?") + .items(&choices) + .interact()?; + + set_choice!(choices[idx]); + } + + (opts.name.clone(), Some(opts)) + } else { + let old_config = fs::read_to_string(current_dir.join("tree-sitter.json")) + .with_context(|| "Failed to read tree-sitter.json")?; + + let mut json = serde_json::from_str::(&old_config)?; + if json.schema.is_none() { + json.schema = Some(TREE_SITTER_JSON_SCHEMA.to_string()); + } + + let new_config = format!("{}\n", serde_json::to_string_pretty(&json)?); + // Write the re-serialized config back, as newly added optional boolean fields + // will be included with explicit `false`s rather than implict `null`s + if self.update && !old_config.trim().eq(new_config.trim()) { + info!("Updating tree-sitter.json"); + fs::write( + current_dir.join("tree-sitter.json"), + serde_json::to_string_pretty(&json)?, + ) + .with_context(|| "Failed to write tree-sitter.json")?; + } + + (json.grammars.swap_remove(0).name, None) + }; + + generate_grammar_files( + current_dir, + &language_name, + self.update, + json_config_opts.as_ref(), + )?; + + Ok(()) + } +} + +impl Generate { + fn run(self, mut loader: loader::Loader, current_dir: &Path) -> Result<()> { + if self.log { + logger::enable_debug(); + } + let abi_version = + self.abi_version + .as_ref() + .map_or(DEFAULT_GENERATE_ABI_VERSION, |version| { + if version == "latest" { + tree_sitter::LANGUAGE_VERSION + } else { + version.parse().expect("invalid abi version flag") + } + }); + + let json_summary = if self.json { + warn!("--json is deprecated, use --json-summary instead"); + true + } else { + self.json_summary + }; + + if let Err(err) = tree_sitter_generate::generate_parser_in_directory( + current_dir, + self.output.as_deref(), + self.grammar_path.as_deref(), + abi_version, + self.report_states_for_rule.as_deref(), + self.js_runtime.as_deref(), + !self.no_parser, + if self.disable_optimizations { + OptLevel::empty() + } else { + OptLevel::default() + }, + ) { + if json_summary { + eprintln!("{}", serde_json::to_string_pretty(&err)?); + // Exit early to prevent errors from being printed a second time in the caller + std::process::exit(1); + } else { + // Removes extra context associated with the error + Err(anyhow!(err.to_string())).with_context(|| "Error when generating parser")?; + } + } + if self.build { + warn!("--build is deprecated, use the `build` command"); + if let Some(path) = self.libdir { + loader = loader::Loader::with_parser_lib_path(path); + } + loader.debug_build(self.debug_build); + loader.languages_at_path(current_dir)?; + } + Ok(()) + } +} + +impl Build { + fn run(self, mut loader: loader::Loader, current_dir: &Path) -> Result<()> { + let grammar_path = current_dir.join(self.path.unwrap_or_default()); + + loader.debug_build(self.debug); + + if self.wasm { + let output_path = self.output.map(|path| current_dir.join(path)); + wasm::compile_language_to_wasm(&loader, &grammar_path, current_dir, output_path)?; + } else { + let output_path = if let Some(ref path) = self.output { + let path = Path::new(path); + let full_path = if path.is_absolute() { + path.to_path_buf() + } else { + current_dir.join(path) + }; + let parent_path = full_path + .parent() + .context("Output path must have a parent")?; + let name = full_path + .file_name() + .context("Ouput path must have a filename")?; + fs::create_dir_all(parent_path).context("Failed to create output path")?; + let mut canon_path = parent_path.canonicalize().context("Invalid output path")?; + canon_path.push(name); + canon_path + } else { + let file_name = grammar_path + .file_stem() + .unwrap() + .to_str() + .unwrap() + .strip_prefix("tree-sitter-") + .unwrap_or("parser"); + current_dir + .join(file_name) + .with_extension(env::consts::DLL_EXTENSION) + }; + + let flags: &[&str] = match (self.reuse_allocator, self.debug) { + (true, true) => &["TREE_SITTER_REUSE_ALLOCATOR", "TREE_SITTER_DEBUG"], + (true, false) => &["TREE_SITTER_REUSE_ALLOCATOR"], + (false, true) => &["TREE_SITTER_DEBUG"], + (false, false) => &[], + }; + + loader.force_rebuild(true); + + loader + .compile_parser_at_path(&grammar_path, output_path, flags) + .context("Failed to compile parser")?; + } + Ok(()) + } +} + +impl Parse { + fn run(self, mut loader: loader::Loader, current_dir: &Path) -> Result<()> { + let config = Config::load(self.config_path)?; + let color = env::var("NO_COLOR").map_or(true, |v| v != "1"); + let json_summary = if self.json { + warn!("--json is deprecated, use --json-summary instead"); + true + } else { + self.json_summary + }; + let output = if self.output_dot { + ParseOutput::Dot + } else if self.output_xml { + ParseOutput::Xml + } else if self.output_cst { + ParseOutput::Cst + } else if self.quiet || json_summary { + ParseOutput::Quiet + } else { + ParseOutput::Normal + }; + + let parse_theme = if color { + config + .get::() + .with_context(|| "Failed to parse CST theme")? + .parse_theme + .unwrap_or_default() + .into() + } else { + ParseTheme::empty() + }; + + let encoding = self.encoding.map(|e| match e { + Encoding::Utf8 => ffi::TSInputEncodingUTF8, + Encoding::Utf16LE => ffi::TSInputEncodingUTF16LE, + Encoding::Utf16BE => ffi::TSInputEncodingUTF16BE, + }); + + let time = self.time; + let edits = self.edits.unwrap_or_default(); + let cancellation_flag = util::cancel_on_signal(); + let mut parser = Parser::new(); + + loader.debug_build(self.debug_build); + loader.force_rebuild(self.rebuild || self.grammar_path.is_some()); + + if self.wasm { + checked_wasm!({ + let engine = tree_sitter::wasmtime::Engine::default(); + parser + .set_wasm_store(tree_sitter::WasmStore::new(&engine).unwrap()) + .unwrap(); + loader.use_wasm(&engine); + }); + } + + let timeout = self.timeout.unwrap_or_default(); + + let mut has_error = false; + let loader_config = config.get()?; + loader.find_all_languages(&loader_config)?; + + let should_track_stats = self.stat; + let mut stats = parse::ParseStats::default(); + let debug: ParseDebugType = match self.debug { + None => ParseDebugType::Quiet, + Some(None) => ParseDebugType::Normal, + Some(Some(specifier)) => specifier, + }; + + let mut options = ParseFileOptions { + edits: &edits + .iter() + .map(std::string::String::as_str) + .collect::>(), + output, + print_time: time, + timeout, + stats: &mut stats, + debug, + debug_graph: self.debug_graph, + cancellation_flag: Some(&cancellation_flag), + encoding, + open_log: self.open_log, + no_ranges: self.no_ranges, + parse_theme: &parse_theme, + }; + + let mut update_stats = |stats: &mut parse::ParseStats| { + let parse_result = stats.parse_summaries.last().unwrap(); + if should_track_stats || json_summary { + stats.cumulative_stats.total_parses += 1; + if parse_result.successful { + stats.cumulative_stats.successful_parses += 1; + } + if let (Some(duration), Some(bytes)) = (parse_result.duration, parse_result.bytes) { + stats.cumulative_stats.total_bytes += bytes; + stats.cumulative_stats.total_duration += duration; + } + } + + has_error |= !parse_result.successful; + }; + + if self.lib_path.is_none() && self.lang_name.is_some() { + warn!("--lang-name` specified without --lib-path. This argument will be ignored."); + } + let lib_info = get_lib_info(self.lib_path.as_ref(), self.lang_name.as_ref(), current_dir); + + let input = get_input( + self.paths_file.as_deref(), + self.paths, + self.test_number, + &cancellation_flag, + )?; + match input { + CliInput::Paths(paths) => { + let max_path_length = paths + .iter() + .map(|p| p.to_string_lossy().chars().count()) + .max() + .unwrap_or(0); + options.stats.source_count = paths.len(); + + for path in &paths { + let path = Path::new(&path); + let language = loader + .select_language( + Some(path), + current_dir, + self.scope.as_deref(), + lib_info.as_ref(), + ) + .with_context(|| { + anyhow!("Failed to load language for path \"{}\"", path.display()) + })?; + + parse::parse_file_at_path( + &mut parser, + &language, + path, + &path.display().to_string(), + max_path_length, + &mut options, + )?; + update_stats(options.stats); + } + } + + CliInput::Test { + name, + contents, + languages: language_names, + } => { + let path = get_tmp_source_file(&contents)?; + let languages = loader.languages_at_path(current_dir)?; + + let language = if let Some(ref lib_path) = self.lib_path { + &loader + .select_language( + None, + current_dir, + self.scope.as_deref(), + lib_info.as_ref(), + ) + .with_context(|| { + anyhow!( + "Failed to load language for path \"{}\"", + lib_path.display() + ) + })? + } else { + &languages + .iter() + .find(|(_, n)| language_names.contains(&Box::from(n.as_str()))) + .or_else(|| languages.first()) + .map(|(l, _)| l.clone()) + .ok_or_else(|| anyhow!("No language found"))? + }; + + parse::parse_file_at_path( + &mut parser, + language, + &path, + &name, + name.chars().count(), + &mut options, + )?; + update_stats(&mut stats); + fs::remove_file(path)?; + } + + CliInput::Stdin(contents) => { + // Place user input and parser output on separate lines + println!(); + + let path = get_tmp_source_file(&contents)?; + let name = "stdin"; + let language = loader.select_language( + None, + current_dir, + self.scope.as_deref(), + lib_info.as_ref(), + )?; + + parse::parse_file_at_path( + &mut parser, + &language, + &path, + name, + name.chars().count(), + &mut options, + )?; + update_stats(&mut stats); + fs::remove_file(path)?; + } + } + + if should_track_stats { + println!("\n{}", stats.cumulative_stats); + } + if json_summary { + println!("{}", serde_json::to_string_pretty(&stats)?); + } + + if has_error { + return Err(anyhow!("")); + } + + Ok(()) + } +} + +/// In case an error is encountered, prints out the contents of `test_summary` and +/// propagates the error +fn check_test( + test_result: Result<()>, + test_summary: &TestSummary, + json_summary: bool, +) -> Result<()> { + if let Err(e) = test_result { + if json_summary { + let json_summary = serde_json::to_string_pretty(test_summary) + .expect("Failed to encode summary to JSON"); + println!("{json_summary}"); + } else { + println!("{test_summary}"); + } + + Err(e)?; + } + + Ok(()) +} + +impl Test { + fn run(self, mut loader: loader::Loader, current_dir: &Path) -> Result<()> { + let config = Config::load(self.config_path)?; + let color = env::var("NO_COLOR").map_or(true, |v| v != "1"); + let stat = self.stat.unwrap_or_default(); + + loader.debug_build(self.debug_build); + loader.force_rebuild(self.rebuild || self.grammar_path.is_some()); + + let mut parser = Parser::new(); + + if self.wasm { + checked_wasm!({ + let engine = tree_sitter::wasmtime::Engine::default(); + parser + .set_wasm_store(tree_sitter::WasmStore::new(&engine).unwrap()) + .unwrap(); + loader.use_wasm(&engine); + }); + } + + if self.lib_path.is_none() && self.lang_name.is_some() { + warn!("--lang-name` specified without --lib-path. This argument will be ignored."); + } + let languages = loader.languages_at_path(current_dir)?; + let language = if let Some(ref lib_path) = self.lib_path { + let lib_info = + get_lib_info(self.lib_path.as_ref(), self.lang_name.as_ref(), current_dir); + &loader + .select_language(None, current_dir, None, lib_info.as_ref()) + .with_context(|| { + anyhow!( + "Failed to load language for path \"{}\"", + lib_path.display() + ) + })? + } else { + &languages + .first() + .ok_or_else(|| anyhow!("No language found"))? + .0 + }; + parser.set_language(language)?; + + let test_dir = current_dir.join("test"); + let mut test_summary = TestSummary::new( + color, + stat, + self.update, + self.overview_only, + self.json_summary, + ); + + // Run the corpus tests. Look for them in `test/corpus`. + let test_corpus_dir = test_dir.join("corpus"); + if test_corpus_dir.is_dir() { + let opts = TestOptions { + path: test_corpus_dir, + debug: self.debug, + debug_graph: self.debug_graph, + include: self.include, + exclude: self.exclude, + file_name: self.file_name, + update: self.update, + open_log: self.open_log, + languages: languages.iter().map(|(l, n)| (n.as_str(), l)).collect(), + color, + show_fields: self.show_fields, + overview_only: self.overview_only, + }; + + check_test( + test::run_tests_at_path(&mut parser, &opts, &mut test_summary), + &test_summary, + self.json_summary, + )?; + test_summary.test_num = 1; + } + + // Check that all of the queries are valid. + let query_dir = current_dir.join("queries"); + check_test( + test::check_queries_at_path(language, &query_dir), + &test_summary, + self.json_summary, + )?; + test_summary.test_num = 1; + + // Run the syntax highlighting tests. + let test_highlight_dir = test_dir.join("highlight"); + if test_highlight_dir.is_dir() { + let mut highlighter = Highlighter::new(); + highlighter.parser = parser; + check_test( + test_highlight::test_highlights( + &loader, + &config.get()?, + &mut highlighter, + &test_highlight_dir, + &mut test_summary, + ), + &test_summary, + self.json_summary, + )?; + parser = highlighter.parser; + test_summary.test_num = 1; + } + + let test_tag_dir = test_dir.join("tags"); + if test_tag_dir.is_dir() { + let mut tags_context = TagsContext::new(); + tags_context.parser = parser; + check_test( + test_tags::test_tags( + &loader, + &config.get()?, + &mut tags_context, + &test_tag_dir, + &mut test_summary, + ), + &test_summary, + self.json_summary, + )?; + test_summary.test_num = 1; + } + + // For the rest of the queries, find their tests and run them + for entry in walkdir::WalkDir::new(&query_dir) + .into_iter() + .filter_map(|e| e.ok()) + .filter(|e| e.file_type().is_file()) + { + let stem = entry + .path() + .file_stem() + .map(|s| s.to_str().unwrap_or_default()) + .unwrap_or_default(); + if stem != "highlights" && stem != "tags" { + let entries = walkdir::WalkDir::new(test_dir.join(stem)) + .into_iter() + .filter_map(|e| { + let entry = e.ok()?; + if entry.file_type().is_file() { + Some(entry) + } else { + None + } + }) + .collect::>(); + if !entries.is_empty() { + test_summary.query_results.add_group(stem); + } + + test_summary.test_num = 1; + let opts = QueryFileOptions::default(); + for entry in &entries { + let path = entry.path(); + check_test( + query::query_file_at_path( + language, + path, + &path.display().to_string(), + path, + &opts, + Some(&mut test_summary), + ), + &test_summary, + self.json_summary, + )?; + } + if !entries.is_empty() { + test_summary.query_results.pop_traversal(); + } + } + } + test_summary.test_num = 1; + + if self.json_summary { + let json_summary = serde_json::to_string_pretty(&test_summary) + .expect("Failed to encode test summary to JSON"); + println!("{json_summary}"); + } else { + println!("{test_summary}"); + } + + Ok(()) + } +} + +impl Version { + fn run(self, current_dir: PathBuf) -> Result<()> { + Ok(version::Version::new(self.version, current_dir, self.bump).run()?) + } +} + +impl Fuzz { + fn run(self, mut loader: loader::Loader, current_dir: &Path) -> Result<()> { + loader.sanitize_build(true); + loader.force_rebuild(self.rebuild || self.grammar_path.is_some()); + + if self.lib_path.is_none() && self.lang_name.is_some() { + warn!("--lang-name` specified without --lib-path. This argument will be ignored."); + } + let languages = loader.languages_at_path(current_dir)?; + let (language, language_name) = if let Some(ref lib_path) = self.lib_path { + let lib_info = get_lib_info(Some(lib_path), self.lang_name.as_ref(), current_dir) + .with_context(|| anyhow!("No language name found for {}", lib_path.display()))?; + let lang_name = lib_info.1.to_string(); + &( + loader + .select_language(None, current_dir, None, Some(&lib_info)) + .with_context(|| { + anyhow!( + "Failed to load language for path \"{}\"", + lib_path.display() + ) + })?, + lang_name, + ) + } else { + languages + .first() + .ok_or_else(|| anyhow!("No language found"))? + }; + + let mut fuzz_options = FuzzOptions { + skipped: self.skip, + subdir: self.subdir, + edits: self.edits.unwrap_or(*EDIT_COUNT), + iterations: self.iterations.unwrap_or(*ITERATION_COUNT), + include: self.include, + exclude: self.exclude, + log_graphs: self.log_graphs || *LOG_GRAPH_ENABLED, + log: self.log || *LOG_ENABLED, + }; + + fuzz_language_corpus( + language, + language_name, + *START_SEED, + current_dir, + &mut fuzz_options, + ); + Ok(()) + } +} + +impl Query { + fn run(self, mut loader: loader::Loader, current_dir: &Path) -> Result<()> { + let config = Config::load(self.config_path)?; + let loader_config = config.get()?; + loader.force_rebuild(self.rebuild || self.grammar_path.is_some()); + loader.find_all_languages(&loader_config)?; + let query_path = Path::new(&self.query_path); + + let byte_range = parse_range(&self.byte_range, |x| x)?; + let point_range = parse_range(&self.row_range, |row| Point::new(row, 0))?; + let containing_byte_range = parse_range(&self.containing_byte_range, |x| x)?; + let containing_point_range = + parse_range(&self.containing_row_range, |row| Point::new(row, 0))?; + + let cancellation_flag = util::cancel_on_signal(); + + if self.lib_path.is_none() && self.lang_name.is_some() { + warn!("--lang-name specified without --lib-path. This argument will be ignored."); + } + let lib_info = get_lib_info(self.lib_path.as_ref(), self.lang_name.as_ref(), current_dir); + + let input = get_input( + self.paths_file.as_deref(), + self.paths, + self.test_number, + &cancellation_flag, + )?; + + match input { + CliInput::Paths(paths) => { + let language = loader.select_language( + Some(Path::new(&paths[0])), + current_dir, + self.scope.as_deref(), + lib_info.as_ref(), + )?; + + let opts = QueryFileOptions { + ordered_captures: self.captures, + byte_range, + point_range, + containing_byte_range, + containing_point_range, + quiet: self.quiet, + print_time: self.time, + stdin: false, + }; + for path in paths { + query::query_file_at_path( + &language, + &path, + &path.display().to_string(), + query_path, + &opts, + None, + )?; + } + } + CliInput::Test { + name, + contents, + languages: language_names, + } => { + let path = get_tmp_source_file(&contents)?; + let languages = loader.languages_at_path(current_dir)?; + let language = if let Some(ref lib_path) = self.lib_path { + &loader + .select_language(None, current_dir, None, lib_info.as_ref()) + .with_context(|| { + anyhow!( + "Failed to load language for path \"{}\"", + lib_path.display() + ) + })? + } else { + &languages + .iter() + .find(|(_, n)| language_names.contains(&Box::from(n.as_str()))) + .or_else(|| languages.first()) + .map(|(l, _)| l.clone()) + .ok_or_else(|| anyhow!("No language found"))? + }; + let opts = QueryFileOptions { + ordered_captures: self.captures, + byte_range, + point_range, + containing_byte_range, + containing_point_range, + quiet: self.quiet, + print_time: self.time, + stdin: true, + }; + query::query_file_at_path(language, &path, &name, query_path, &opts, None)?; + fs::remove_file(path)?; + } + CliInput::Stdin(contents) => { + // Place user input and query output on separate lines + println!(); + + let path = get_tmp_source_file(&contents)?; + let language = + loader.select_language(None, current_dir, None, lib_info.as_ref())?; + let opts = QueryFileOptions { + ordered_captures: self.captures, + byte_range, + point_range, + containing_byte_range, + containing_point_range, + quiet: self.quiet, + print_time: self.time, + stdin: true, + }; + query::query_file_at_path(&language, &path, "stdin", query_path, &opts, None)?; + fs::remove_file(path)?; + } + } + + Ok(()) + } +} + +impl Highlight { + fn run(self, mut loader: loader::Loader, current_dir: &Path) -> Result<()> { + let config = Config::load(self.config_path)?; + let theme_config: tree_sitter_cli::highlight::ThemeConfig = config.get()?; + loader.configure_highlights(&theme_config.theme.highlight_names); + let loader_config = config.get()?; + loader.find_all_languages(&loader_config)?; + loader.force_rebuild(self.rebuild || self.grammar_path.is_some()); + let languages = loader.languages_at_path(current_dir)?; + + let cancellation_flag = util::cancel_on_signal(); + + let (mut language, mut language_configuration) = (None, None); + if let Some(scope) = self.scope.as_deref() { + if let Some((lang, lang_config)) = loader.language_configuration_for_scope(scope)? { + language = Some(lang); + language_configuration = Some(lang_config); + } + if language.is_none() { + return Err(anyhow!("Unknown scope '{scope}'")); + } + } + + let options = HighlightOptions { + theme: theme_config.theme, + check: self.check, + captures_path: self.captures_path, + inline_styles: !self.css_classes, + html: self.html, + quiet: self.quiet, + print_time: self.time, + cancellation_flag: cancellation_flag.clone(), + }; + + let input = get_input( + self.paths_file.as_deref(), + self.paths, + self.test_number, + &cancellation_flag, + )?; + match input { + CliInput::Paths(paths) => { + let print_name = paths.len() > 1; + for path in paths { + let (language, language_config) = + match (language.clone(), language_configuration) { + (Some(l), Some(lc)) => (l, lc), + _ => { + if let Some((lang, lang_config)) = + loader.language_configuration_for_file_name(&path)? + { + (lang, lang_config) + } else { + warn!( + "{}", + util::lang_not_found_for_path(&path, &loader_config) + ); + continue; + } + } + }; + + if let Some(highlight_config) = + language_config.highlight_config(language, self.query_paths.as_deref())? + { + highlight::highlight( + &loader, + &path, + &path.display().to_string(), + highlight_config, + print_name, + &options, + )?; + } else { + warn!( + "No syntax highlighting config found for path {}", + path.display() + ); + } + } + } + + CliInput::Test { + name, + contents, + languages: language_names, + } => { + let path = get_tmp_source_file(&contents)?; + + let language = languages + .iter() + .find(|(_, n)| language_names.contains(&Box::from(n.as_str()))) + .or_else(|| languages.first()) + .map(|(l, _)| l.clone()) + .ok_or_else(|| anyhow!("No language found in current path"))?; + let language_config = loader + .get_language_configuration_in_current_path() + .ok_or_else(|| anyhow!("No language configuration found in current path"))?; + + if let Some(highlight_config) = + language_config.highlight_config(language, self.query_paths.as_deref())? + { + highlight::highlight(&loader, &path, &name, highlight_config, false, &options)?; + } else { + warn!("No syntax highlighting config found for test {name}"); + } + fs::remove_file(path)?; + } + + CliInput::Stdin(contents) => { + // Place user input and highlight output on separate lines + println!(); + + let path = get_tmp_source_file(&contents)?; + + let (language, language_config) = + if let (Some(l), Some(lc)) = (language.clone(), language_configuration) { + (l, lc) + } else { + let language = languages + .first() + .map(|(l, _)| l.clone()) + .ok_or_else(|| anyhow!("No language found in current path"))?; + let language_configuration = loader + .get_language_configuration_in_current_path() + .ok_or_else(|| { + anyhow!("No language configuration found in current path") + })?; + (language, language_configuration) + }; + + if let Some(highlight_config) = + language_config.highlight_config(language, self.query_paths.as_deref())? + { + highlight::highlight( + &loader, + &path, + "stdin", + highlight_config, + false, + &options, + )?; + } else { + warn!( + "No syntax highlighting config found for path {}", + current_dir.display() + ); + } + fs::remove_file(path)?; + } + } + + Ok(()) + } +} + +impl Tags { + fn run(self, mut loader: loader::Loader, current_dir: &Path) -> Result<()> { + let config = Config::load(self.config_path)?; + let loader_config = config.get()?; + loader.find_all_languages(&loader_config)?; + loader.force_rebuild(self.rebuild || self.grammar_path.is_some()); + + let cancellation_flag = util::cancel_on_signal(); + + let (mut language, mut language_configuration) = (None, None); + if let Some(scope) = self.scope.as_deref() { + if let Some((lang, lang_config)) = loader.language_configuration_for_scope(scope)? { + language = Some(lang); + language_configuration = Some(lang_config); + } + if language.is_none() { + return Err(anyhow!("Unknown scope '{scope}'")); + } + } + + let options = TagsOptions { + scope: self.scope, + quiet: self.quiet, + print_time: self.time, + cancellation_flag: cancellation_flag.clone(), + }; + + let input = get_input( + self.paths_file.as_deref(), + self.paths, + self.test_number, + &cancellation_flag, + )?; + match input { + CliInput::Paths(paths) => { + let indent = paths.len() > 1; + for path in paths { + let (language, language_config) = + match (language.clone(), language_configuration) { + (Some(l), Some(lc)) => (l, lc), + _ => { + if let Some((lang, lang_config)) = + loader.language_configuration_for_file_name(&path)? + { + (lang, lang_config) + } else { + warn!( + "{}", + util::lang_not_found_for_path(&path, &loader_config) + ); + continue; + } + } + }; + + if let Some(tags_config) = language_config.tags_config(language)? { + tags::generate_tags( + &path, + &path.display().to_string(), + tags_config, + indent, + &options, + )?; + } else { + warn!("No tags config found for path {}", path.display()); + } + } + } + + CliInput::Test { + name, + contents, + languages: language_names, + } => { + let path = get_tmp_source_file(&contents)?; + + let languages = loader.languages_at_path(current_dir)?; + let language = languages + .iter() + .find(|(_, n)| language_names.contains(&Box::from(n.as_str()))) + .or_else(|| languages.first()) + .map(|(l, _)| l.clone()) + .ok_or_else(|| anyhow!("No language found in current path"))?; + let language_config = loader + .get_language_configuration_in_current_path() + .ok_or_else(|| anyhow!("No language configuration found in current path"))?; + + if let Some(tags_config) = language_config.tags_config(language)? { + tags::generate_tags(&path, &name, tags_config, false, &options)?; + } else { + warn!("No tags config found for test {name}"); + } + fs::remove_file(path)?; + } + + CliInput::Stdin(contents) => { + // Place user input and tags output on separate lines + println!(); + + let path = get_tmp_source_file(&contents)?; + + let (language, language_config) = + if let (Some(l), Some(lc)) = (language.clone(), language_configuration) { + (l, lc) + } else { + let languages = loader.languages_at_path(current_dir)?; + let language = languages + .first() + .map(|(l, _)| l.clone()) + .ok_or_else(|| anyhow!("No language found in current path"))?; + let language_configuration = loader + .get_language_configuration_in_current_path() + .ok_or_else(|| { + anyhow!("No language configuration found in current path") + })?; + (language, language_configuration) + }; + + if let Some(tags_config) = language_config.tags_config(language)? { + tags::generate_tags(&path, "stdin", tags_config, false, &options)?; + } else { + warn!("No tags config found for path {}", current_dir.display()); + } + fs::remove_file(path)?; + } + } + + Ok(()) + } +} + +impl Playground { + fn run(self, current_dir: &Path) -> Result<()> { + let grammar_path = self.grammar_path.as_deref().map_or(current_dir, Path::new); + + if let Some(export_path) = self.export { + playground::export(grammar_path, &export_path)?; + } else { + let open_in_browser = !self.quiet; + playground::serve(grammar_path, open_in_browser)?; + } + + Ok(()) + } +} + +impl DumpLanguages { + fn run(self, mut loader: loader::Loader) -> Result<()> { + let config = Config::load(self.config_path)?; + let loader_config = config.get()?; + loader.find_all_languages(&loader_config)?; + for (configuration, language_path) in loader.get_all_language_configurations() { + info!( + concat!( + "name: {}\n", + "scope: {}\n", + "parser: {:?}\n", + "highlights: {:?}\n", + "file_types: {:?}\n", + "content_regex: {:?}\n", + "injection_regex: {:?}\n", + ), + configuration.language_name, + configuration.scope.as_ref().unwrap_or(&String::new()), + language_path, + configuration.highlights_filenames, + configuration.file_types, + configuration.content_regex, + configuration.injection_regex, + ); + } + Ok(()) + } +} + +impl Complete { + fn run(self, cli: &mut Command) { + let name = cli.get_name().to_string(); + let mut stdout = std::io::stdout(); + + match self.shell { + Shell::Bash => generate(clap_complete::shells::Bash, cli, &name, &mut stdout), + Shell::Elvish => generate(clap_complete::shells::Elvish, cli, &name, &mut stdout), + Shell::Fish => generate(clap_complete::shells::Fish, cli, &name, &mut stdout), + Shell::PowerShell => { + generate(clap_complete::shells::PowerShell, cli, &name, &mut stdout); + } + Shell::Zsh => generate(clap_complete::shells::Zsh, cli, &name, &mut stdout), + Shell::Nushell => generate(clap_complete_nushell::Nushell, cli, &name, &mut stdout), + } + } +} + +fn main() { + let result = run(); + if let Err(err) = &result { + // Ignore BrokenPipe errors + if let Some(error) = err.downcast_ref::() { + if error.kind() == std::io::ErrorKind::BrokenPipe { + return; + } + } + if !err.to_string().is_empty() { + error!("{err:?}"); + } + std::process::exit(1); + } +} + +fn run() -> Result<()> { + logger::init(); + + let version = BUILD_SHA.map_or_else( + || BUILD_VERSION.to_string(), + |build_sha| format!("{BUILD_VERSION} ({build_sha})"), + ); + + let cli = Command::new("tree-sitter") + .help_template(concat!( + "\n", + "{before-help}{name} {version}\n", + "{author-with-newline}{about-with-newline}\n", + "{usage-heading} {usage}\n", + "\n", + "{all-args}{after-help}\n", + "\n" + )) + .version(version) + .subcommand_required(true) + .arg_required_else_help(true) + .disable_help_subcommand(true) + .disable_colored_help(false); + let mut cli = Commands::augment_subcommands(cli); + + let command = Commands::from_arg_matches(&cli.clone().get_matches())?; + + let current_dir = match &command { + Commands::Init(Init { grammar_path, .. }) + | Commands::Parse(Parse { grammar_path, .. }) + | Commands::Test(Test { grammar_path, .. }) + | Commands::Version(Version { grammar_path, .. }) + | Commands::Fuzz(Fuzz { grammar_path, .. }) + | Commands::Query(Query { grammar_path, .. }) + | Commands::Highlight(Highlight { grammar_path, .. }) + | Commands::Tags(Tags { grammar_path, .. }) + | Commands::Playground(Playground { grammar_path, .. }) => grammar_path, + Commands::Build(_) + | Commands::Generate(_) + | Commands::InitConfig(_) + | Commands::DumpLanguages(_) + | Commands::Complete(_) => &None, + } + .as_ref() + .map_or_else(|| env::current_dir().unwrap(), |p| p.clone()); + + let loader = loader::Loader::new()?; + + match command { + Commands::InitConfig(_) => InitConfig::run()?, + Commands::Init(init_options) => init_options.run(¤t_dir)?, + Commands::Generate(generate_options) => generate_options.run(loader, ¤t_dir)?, + Commands::Build(build_options) => build_options.run(loader, ¤t_dir)?, + Commands::Parse(parse_options) => parse_options.run(loader, ¤t_dir)?, + Commands::Test(test_options) => test_options.run(loader, ¤t_dir)?, + Commands::Version(version_options) => version_options.run(current_dir)?, + Commands::Fuzz(fuzz_options) => fuzz_options.run(loader, ¤t_dir)?, + Commands::Query(query_options) => query_options.run(loader, ¤t_dir)?, + Commands::Highlight(highlight_options) => highlight_options.run(loader, ¤t_dir)?, + Commands::Tags(tags_options) => tags_options.run(loader, ¤t_dir)?, + Commands::Playground(playground_options) => playground_options.run(¤t_dir)?, + Commands::DumpLanguages(dump_options) => dump_options.run(loader)?, + Commands::Complete(complete_options) => complete_options.run(&mut cli), + } + + Ok(()) +} + +#[must_use] +const fn get_styles() -> clap::builder::Styles { + clap::builder::Styles::styled() + .usage( + Style::new() + .bold() + .fg_color(Some(Color::Ansi(AnsiColor::Yellow))), + ) + .header( + Style::new() + .bold() + .fg_color(Some(Color::Ansi(AnsiColor::Yellow))), + ) + .literal(Style::new().fg_color(Some(Color::Ansi(AnsiColor::Green)))) + .invalid( + Style::new() + .bold() + .fg_color(Some(Color::Ansi(AnsiColor::Red))), + ) + .error( + Style::new() + .bold() + .fg_color(Some(Color::Ansi(AnsiColor::Red))), + ) + .valid( + Style::new() + .bold() + .fg_color(Some(Color::Ansi(AnsiColor::Green))), + ) + .placeholder(Style::new().fg_color(Some(Color::Ansi(AnsiColor::White)))) +} + +/// Utility to extract the shared library path and language function name from user-provided +/// arguments if present. +fn get_lib_info<'a>( + lib_path: Option<&'a PathBuf>, + language_name: Option<&'a String>, + current_dir: &Path, +) -> Option<(PathBuf, &'a str)> { + if let Some(lib_path) = lib_path { + let absolute_lib_path = if lib_path.is_absolute() { + lib_path.clone() + } else { + current_dir.join(lib_path) + }; + // Use the user-specified name if present, otherwise try to derive it from + // the lib path + match ( + language_name.map(|s| s.as_str()), + lib_path.file_stem().and_then(|s| s.to_str()), + ) { + (Some(name), _) | (None, Some(name)) => Some((absolute_lib_path, name)), + _ => None, + } + } else { + None + } +} + +/// Parse a range string of the form "start:end" into an optional Range. +fn parse_range( + range_str: &Option, + make: impl Fn(usize) -> T, +) -> Result>> { + if let Some(range) = range_str.as_ref() { + let err_msg = format!("Invalid range '{range}', expected 'start:end'"); + let mut parts = range.split(':'); + + let Some(part) = parts.next() else { + Err(anyhow!(err_msg))? + }; + let Ok(start) = part.parse::() else { + Err(anyhow!(err_msg))? + }; + + let Some(part) = parts.next() else { + Err(anyhow!(err_msg))? + }; + let Ok(end) = part.parse::() else { + Err(anyhow!(err_msg))? + }; + + Ok(Some(make(start)..make(end))) + } else { + Ok(None) + } +} diff --git a/cli/src/parse.rs b/crates/cli/src/parse.rs similarity index 68% rename from cli/src/parse.rs rename to crates/cli/src/parse.rs index 3eb86299..3551f556 100644 --- a/cli/src/parse.rs +++ b/crates/cli/src/parse.rs @@ -1,23 +1,26 @@ use std::{ fmt, fs, - io::{self, StdoutLock, Write}, - path::Path, + io::{self, Write}, + ops::ControlFlow, + path::{Path, PathBuf}, sync::atomic::{AtomicUsize, Ordering}, time::{Duration, Instant}, }; use anstyle::{AnsiColor, Color, RgbColor}; use anyhow::{anyhow, Context, Result}; +use clap::ValueEnum; +use log::info; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use tree_sitter::{ ffi, InputEdit, Language, LogType, ParseOptions, ParseState, Parser, Point, Range, Tree, TreeCursor, }; -use super::util; -use crate::{fuzz::edits::Edit, test::paint}; +use crate::{fuzz::edits::Edit, logger::paint, util}; -#[derive(Debug, Default)] +#[derive(Debug, Default, Serialize, JsonSchema)] pub struct Stats { pub successful_parses: usize, pub total_parses: usize, @@ -28,18 +31,28 @@ pub struct Stats { impl fmt::Display for Stats { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let duration_us = self.total_duration.as_micros(); + let success_rate = if self.total_parses > 0 { + format!( + "{:.2}%", + ((self.successful_parses as f64) / (self.total_parses as f64)) * 100.0, + ) + } else { + "N/A".to_string() + }; + let duration_str = match (self.total_parses, duration_us) { + (0, _) => "N/A".to_string(), + (_, 0) => "0 bytes/ms".to_string(), + (_, _) => format!( + "{} bytes/ms", + ((self.total_bytes as u128) * 1_000) / duration_us + ), + }; writeln!( f, - "Total parses: {}; successful parses: {}; failed parses: {}; success percentage: {:.2}%; average speed: {} bytes/ms", + "Total parses: {}; successful parses: {}; failed parses: {}; success percentage: {success_rate}; average speed: {duration_str}", self.total_parses, self.successful_parses, self.total_parses - self.successful_parses, - ((self.successful_parses as f64) / (self.total_parses as f64)) * 100.0, - if duration_us != 0 { - ((self.total_bytes as u128) * 1_000) / duration_us - } else { - 0 - } ) } } @@ -177,15 +190,79 @@ pub enum ParseOutput { Dot, } +/// A position in a multi-line text document, in terms of rows and columns. +/// +/// Rows and columns are zero-based. +/// +/// This serves as a serializable wrapper for `Point` +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize)] +pub struct ParsePoint { + pub row: usize, + pub column: usize, +} + +impl From for ParsePoint { + fn from(value: Point) -> Self { + Self { + row: value.row, + column: value.column, + } + } +} + +#[derive(Serialize, Default, Debug, Clone)] +pub struct ParseSummary { + pub file: PathBuf, + pub successful: bool, + pub start: Option, + pub end: Option, + pub duration: Option, + pub bytes: Option, +} + +impl ParseSummary { + #[must_use] + pub fn new(path: &Path) -> Self { + Self { + file: path.to_path_buf(), + successful: false, + ..Default::default() + } + } +} + +#[derive(Serialize, Debug)] +pub struct ParseStats { + pub parse_summaries: Vec, + pub cumulative_stats: Stats, + pub source_count: usize, +} + +impl Default for ParseStats { + fn default() -> Self { + Self { + parse_summaries: Vec::new(), + cumulative_stats: Stats::default(), + source_count: 1, + } + } +} + +#[derive(Serialize, ValueEnum, Debug, Copy, Clone, Default, Eq, PartialEq)] +pub enum ParseDebugType { + #[default] + Quiet, + Normal, + Pretty, +} + pub struct ParseFileOptions<'a> { - pub language: Language, - pub path: &'a Path, pub edits: &'a [&'a str], - pub max_path_length: usize, pub output: ParseOutput, + pub stats: &'a mut ParseStats, pub print_time: bool, pub timeout: u64, - pub debug: bool, + pub debug: ParseDebugType, pub debug_graph: bool, pub cancellation_flag: Option<&'a AtomicUsize>, pub encoding: Option, @@ -201,27 +278,65 @@ pub struct ParseResult { pub duration: Option, } -pub fn parse_file_at_path(parser: &mut Parser, opts: &ParseFileOptions) -> Result { +pub fn parse_file_at_path( + parser: &mut Parser, + language: &Language, + path: &Path, + name: &str, + max_path_length: usize, + opts: &mut ParseFileOptions, +) -> Result<()> { let mut _log_session = None; - parser.set_language(&opts.language)?; - let mut source_code = fs::read(opts.path) - .with_context(|| format!("Error reading source file {:?}", opts.path))?; + parser.set_language(language)?; + let mut source_code = fs::read(path).with_context(|| format!("Error reading {name:?}"))?; // Render an HTML graph if `--debug-graph` was passed if opts.debug_graph { _log_session = Some(util::log_graphs(parser, "log.html", opts.open_log)?); } // Log to stderr if `--debug` was passed - else if opts.debug { - parser.set_logger(Some(Box::new(|log_type, message| { - if log_type == LogType::Lex { - io::stderr().write_all(b" ").unwrap(); + else if opts.debug != ParseDebugType::Quiet { + let mut curr_version: usize = 0; + let use_color = std::env::var("NO_COLOR").map_or(true, |v| v != "1"); + let debug = opts.debug; + parser.set_logger(Some(Box::new(move |log_type, message| { + if debug == ParseDebugType::Normal { + if log_type == LogType::Lex { + write!(&mut io::stderr(), " ").unwrap(); + } + writeln!(&mut io::stderr(), "{message}").unwrap(); + } else { + let colors = &[ + AnsiColor::White, + AnsiColor::Red, + AnsiColor::Blue, + AnsiColor::Green, + AnsiColor::Cyan, + AnsiColor::Yellow, + ]; + if message.starts_with("process version:") { + let comma_idx = message.find(',').unwrap(); + curr_version = message["process version:".len()..comma_idx] + .parse() + .unwrap(); + } + let color = if use_color { + Some(colors[curr_version]) + } else { + None + }; + let mut out = if log_type == LogType::Lex { + " ".to_string() + } else { + String::new() + }; + out += &paint(color, message); + writeln!(&mut io::stderr(), "{out}").unwrap(); } - writeln!(&mut io::stderr(), "{message}").unwrap(); }))); } - let time = Instant::now(); + let parse_time = Instant::now(); #[inline(always)] fn is_utf16_le_bom(bom_bytes: &[u8]) -> bool { @@ -255,15 +370,15 @@ pub fn parse_file_at_path(parser: &mut Parser, opts: &ParseFileOptions) -> Resul let progress_callback = &mut |_: &ParseState| { if let Some(cancellation_flag) = opts.cancellation_flag { if cancellation_flag.load(Ordering::SeqCst) != 0 { - return true; + return ControlFlow::Break(()); } } if opts.timeout > 0 && start_time.elapsed().as_micros() > opts.timeout as u128 { - return true; + return ControlFlow::Break(()); } - false + ControlFlow::Continue(()) }; let parse_opts = ParseOptions::new().progress_callback(progress_callback); @@ -315,29 +430,32 @@ pub fn parse_file_at_path(parser: &mut Parser, opts: &ParseFileOptions) -> Resul Some(parse_opts), ), }; + let parse_duration = parse_time.elapsed(); let stdout = io::stdout(); let mut stdout = stdout.lock(); if let Some(mut tree) = tree { if opts.debug_graph && !opts.edits.is_empty() { - println!("BEFORE:\n{}", String::from_utf8_lossy(&source_code)); + info!("BEFORE:\n{}", String::from_utf8_lossy(&source_code)); } + let edit_time = Instant::now(); for (i, edit) in opts.edits.iter().enumerate() { let edit = parse_edit_flag(&source_code, edit)?; perform_edit(&mut tree, &mut source_code, &edit)?; tree = parser.parse(&source_code, Some(&tree)).unwrap(); if opts.debug_graph { - println!("AFTER {i}:\n{}", String::from_utf8_lossy(&source_code)); + info!("AFTER {i}:\n{}", String::from_utf8_lossy(&source_code)); } } + let edit_duration = edit_time.elapsed(); parser.stop_printing_dot_graphs(); - let duration = time.elapsed(); - let duration_ms = duration.as_micros() as f64 / 1e3; + let parse_duration_ms = parse_duration.as_micros() as f64 / 1e3; + let edit_duration_ms = edit_duration.as_micros() as f64 / 1e3; let mut cursor = tree.walk(); if opts.output == ParseOutput::Normal { @@ -396,55 +514,23 @@ pub fn parse_file_at_path(parser: &mut Parser, opts: &ParseFileOptions) -> Resul } if opts.output == ParseOutput::Cst { - let lossy_source_code = String::from_utf8_lossy(&source_code); - let total_width = lossy_source_code - .lines() - .enumerate() - .map(|(row, col)| { - (row as f64).log10() as usize + (col.len() as f64).log10() as usize + 1 - }) - .max() - .unwrap_or(1); - let mut indent_level = 1; - let mut did_visit_children = false; - loop { - if did_visit_children { - if cursor.goto_next_sibling() { - did_visit_children = false; - } else if cursor.goto_parent() { - did_visit_children = true; - indent_level -= 1; - } else { - break; - } - } else { - cst_render_node( - opts, - &mut cursor, - &source_code, - &mut stdout, - total_width, - indent_level, - )?; - if cursor.goto_first_child() { - did_visit_children = false; - indent_level += 1; - } else { - did_visit_children = true; - } - } - } - cursor.reset(tree.root_node()); - println!(); + render_cst(&source_code, &tree, &mut cursor, opts, &mut stdout)?; } if opts.output == ParseOutput::Xml { let mut needs_newline = false; - let mut indent_level = 0; + let mut indent_level = 2; let mut did_visit_children = false; let mut had_named_children = false; let mut tags = Vec::<&str>::new(); - writeln!(&mut stdout, "")?; + + // If we're parsing the first file, write the header + if opts.stats.parse_summaries.is_empty() { + writeln!(&mut stdout, "")?; + writeln!(&mut stdout, "")?; + } + writeln!(&mut stdout, " ", path.display())?; + loop { let node = cursor.node(); let is_named = node.is_named(); @@ -459,7 +545,7 @@ pub fn parse_file_at_path(parser: &mut Parser, opts: &ParseFileOptions) -> Resul write!(&mut stdout, "", tag.expect("there is a tag"))?; // we only write a line in the case where it's the last sibling if let Some(parent) = node.parent() { - if parent.child(parent.child_count() - 1).unwrap() == node { + if parent.child(parent.child_count() as u32 - 1).unwrap() == node { stdout.write_all(b"\n")?; } } @@ -523,8 +609,14 @@ pub fn parse_file_at_path(parser: &mut Parser, opts: &ParseFileOptions) -> Resul } } } + writeln!(&mut stdout)?; + writeln!(&mut stdout, " ")?; + + // If we parsed the last file, write the closing tag for the `sources` header + if opts.stats.parse_summaries.len() == opts.stats.source_count - 1 { + writeln!(&mut stdout, "")?; + } cursor.reset(tree.root_node()); - println!(); } if opts.output == ParseOutput::Dot { @@ -532,14 +624,39 @@ pub fn parse_file_at_path(parser: &mut Parser, opts: &ParseFileOptions) -> Resul } let mut first_error = None; - loop { + let mut earliest_node_with_error = None; + 'outer: loop { let node = cursor.node(); if node.has_error() { + if earliest_node_with_error.is_none() { + earliest_node_with_error = Some(node); + } if node.is_error() || node.is_missing() { first_error = Some(node); break; } + + // If there's no more children, even though some outer node has an error, + // then that means that the first error is hidden, but the later error could be + // visible. So, we walk back up to the child of the first node with an error, + // and then check its siblings for errors. if !cursor.goto_first_child() { + let earliest = earliest_node_with_error.unwrap(); + while cursor.goto_parent() { + if cursor.node().parent().is_some_and(|p| p == earliest) { + while cursor.goto_next_sibling() { + let sibling = cursor.node(); + if sibling.is_error() || sibling.is_missing() { + first_error = Some(sibling); + break 'outer; + } + if sibling.has_error() && cursor.goto_first_child() { + continue 'outer; + } + } + break; + } + } break; } } else if !cursor.goto_next_sibling() { @@ -548,63 +665,88 @@ pub fn parse_file_at_path(parser: &mut Parser, opts: &ParseFileOptions) -> Resul } if first_error.is_some() || opts.print_time { + let path = path.to_string_lossy(); write!( &mut stdout, - "{:width$}\t{duration_ms:>7.2} ms\t{:>6} bytes/ms", - opts.path.to_str().unwrap(), - (source_code.len() as u128 * 1_000_000) / duration.as_nanos(), - width = opts.max_path_length + "{:width$}\tParse: {parse_duration_ms:>7.2} ms\t{:>6} bytes/ms", + name, + (source_code.len() as u128 * 1_000_000) / parse_duration.as_nanos(), + width = max_path_length )?; if let Some(node) = first_error { - let start = node.start_position(); - let end = node.end_position(); + let node_kind = node.kind(); + let mut node_text = String::with_capacity(node_kind.len()); + for c in node_kind.chars() { + if let Some(escaped) = escape_invisible(c) { + node_text += escaped; + } else { + node_text.push(c); + } + } write!(&mut stdout, "\t(")?; if node.is_missing() { if node.is_named() { - write!(&mut stdout, "MISSING {}", node.kind())?; + write!(&mut stdout, "MISSING {node_text}")?; } else { - write!( - &mut stdout, - "MISSING \"{}\"", - node.kind().replace('\n', "\\n") - )?; + write!(&mut stdout, "MISSING \"{node_text}\"")?; } } else { - write!(&mut stdout, "{}", node.kind())?; + write!(&mut stdout, "{node_text}")?; } + + let start = node.start_position(); + let end = node.end_position(); write!( &mut stdout, " [{}, {}] - [{}, {}])", start.row, start.column, end.row, end.column )?; } + if !opts.edits.is_empty() { + write!( + &mut stdout, + "\n{:width$}\tEdit: {edit_duration_ms:>7.2} ms", + " ".repeat(path.len()), + width = max_path_length, + )?; + } writeln!(&mut stdout)?; } - return Ok(ParseResult { + opts.stats.parse_summaries.push(ParseSummary { + file: path.to_path_buf(), successful: first_error.is_none(), - bytes: source_code.len(), - duration: Some(duration), + start: Some(tree.root_node().start_position().into()), + end: Some(tree.root_node().end_position().into()), + duration: Some(parse_duration), + bytes: Some(source_code.len()), }); + + return Ok(()); } parser.stop_printing_dot_graphs(); if opts.print_time { - let duration = time.elapsed(); + let duration = parse_time.elapsed(); let duration_ms = duration.as_micros() as f64 / 1e3; writeln!( &mut stdout, - "{:width$}\t{duration_ms:>7.2} ms\t(timed out)", - opts.path.to_str().unwrap(), - width = opts.max_path_length + "{:width$}\tParse: {duration_ms:>7.2} ms\t(timed out)", + path.to_str().unwrap(), + width = max_path_length )?; } - Ok(ParseResult { + opts.stats.parse_summaries.push(ParseSummary { + file: path.to_path_buf(), successful: false, - bytes: source_code.len(), + start: None, + end: None, duration: None, - }) + bytes: Some(source_code.len()), + }); + + Ok(()) } const fn escape_invisible(c: char) -> Option<&'static str> { @@ -620,12 +762,77 @@ const fn escape_invisible(c: char) -> Option<&'static str> { }) } +const fn escape_delimiter(c: char) -> Option<&'static str> { + Some(match c { + '`' => "\\`", + '\"' => "\\\"", + _ => return None, + }) +} + +pub fn render_cst<'a, 'b: 'a>( + source_code: &[u8], + tree: &'b Tree, + cursor: &mut TreeCursor<'a>, + opts: &ParseFileOptions, + out: &mut impl Write, +) -> Result<()> { + let lossy_source_code = String::from_utf8_lossy(source_code); + let total_width = lossy_source_code + .lines() + .enumerate() + .map(|(row, col)| (row as f64).log10() as usize + (col.len() as f64).log10() as usize + 1) + .max() + .unwrap_or(1); + let mut indent_level = usize::from(!opts.no_ranges); + let mut did_visit_children = false; + let mut in_error = false; + loop { + if did_visit_children { + if cursor.goto_next_sibling() { + did_visit_children = false; + } else if cursor.goto_parent() { + did_visit_children = true; + indent_level -= 1; + if !cursor.node().has_error() { + in_error = false; + } + } else { + break; + } + } else { + cst_render_node( + opts, + cursor, + source_code, + out, + total_width, + indent_level, + in_error, + )?; + if cursor.goto_first_child() { + did_visit_children = false; + indent_level += 1; + if cursor.node().has_error() { + in_error = true; + } + } else { + did_visit_children = true; + } + } + } + cursor.reset(tree.root_node()); + Ok(()) +} + fn render_node_text(source: &str) -> String { source .chars() .fold(String::with_capacity(source.len()), |mut acc, c| { if let Some(esc) = escape_invisible(c) { acc.push_str(esc); + } else if let Some(esc) = escape_delimiter(c) { + acc.push_str(esc); } else { acc.push(c); } @@ -635,7 +842,7 @@ fn render_node_text(source: &str) -> String { fn write_node_text( opts: &ParseFileOptions, - stdout: &mut StdoutLock<'static>, + out: &mut impl Write, cursor: &TreeCursor, is_named: bool, source: &str, @@ -651,13 +858,14 @@ fn write_node_text( if !is_named { write!( - stdout, + out, "{}{}{}", paint(quote_color, &String::from(quote)), paint(color, &render_node_text(source)), paint(quote_color, &String::from(quote)), )?; } else { + let multiline = source.contains('\n'); for (i, line) in source.split_inclusive('\n').enumerate() { if line.is_empty() { break; @@ -667,28 +875,31 @@ fn write_node_text( // and adjust the column by setting it to the length of *this* line. node_range.start_point.row += i; node_range.end_point.row = node_range.start_point.row; - node_range.end_point.column = line.len(); + node_range.end_point.column = line.len() + + if i == 0 { + node_range.start_point.column + } else { + 0 + }; let formatted_line = render_line_feed(line, opts); - if !opts.no_ranges { - write!( - stdout, - "\n{}{}{}{}{}", - render_node_range(opts, cursor, is_named, true, total_width, node_range), - " ".repeat(indent_level + 1), - paint(quote_color, &String::from(quote)), - &paint(color, &render_node_text(&formatted_line)), - paint(quote_color, &String::from(quote)), - )?; - } else { - write!( - stdout, - "\n{}{}{}{}", - " ".repeat(indent_level + 1), - paint(quote_color, &String::from(quote)), - &paint(color, &render_node_text(&formatted_line)), - paint(quote_color, &String::from(quote)), - )?; - } + write!( + out, + "{}{}{}{}{}{}", + if multiline { "\n" } else { " " }, + if multiline && !opts.no_ranges { + render_node_range(opts, cursor, is_named, true, total_width, node_range) + } else { + String::new() + }, + if multiline { + " ".repeat(indent_level + 1) + } else { + String::new() + }, + paint(quote_color, &String::from(quote)), + paint(color, &render_node_text(&formatted_line)), + paint(quote_color, &String::from(quote)), + )?; } } @@ -742,46 +953,59 @@ fn render_node_range( fn cst_render_node( opts: &ParseFileOptions, - cursor: &mut TreeCursor, + cursor: &TreeCursor, source_code: &[u8], - stdout: &mut StdoutLock<'static>, + out: &mut impl Write, total_width: usize, indent_level: usize, + in_error: bool, ) -> Result<()> { let node = cursor.node(); let is_named = node.is_named(); if !opts.no_ranges { write!( - stdout, + out, "{}", render_node_range(opts, cursor, is_named, false, total_width, node.range()) )?; } - write!(stdout, "{}", " ".repeat(indent_level))?; + write!( + out, + "{}{}", + " ".repeat(indent_level), + if in_error && !node.has_error() { + " " + } else { + "" + } + )?; if is_named { if let Some(field_name) = cursor.field_name() { write!( - stdout, + out, "{}", paint(opts.parse_theme.field, &format!("{field_name}: ")) )?; } - let kind_color = if node.has_error() { - write!(stdout, "{}", paint(opts.parse_theme.error, "•"))?; + if node.has_error() || node.is_error() { + write!(out, "{}", paint(opts.parse_theme.error, "•"))?; + } + + let kind_color = if node.is_error() { opts.parse_theme.error - } else if node.is_extra() || node.parent().is_some_and(|p| p.is_extra()) { + } else if node.is_extra() || node.parent().is_some_and(|p| p.is_extra() && !p.is_error()) { opts.parse_theme.extra } else { opts.parse_theme.node_kind }; - write!(stdout, "{} ", paint(kind_color, node.kind()))?; + write!(out, "{}", paint(kind_color, node.kind()))?; if node.child_count() == 0 { // Node text from a pattern or external scanner write_node_text( opts, - stdout, + out, cursor, is_named, &String::from_utf8_lossy(&source_code[node.start_byte()..node.end_byte()]), @@ -790,17 +1014,13 @@ fn cst_render_node( )?; } } else if node.is_missing() { - write!(stdout, "{}: ", paint(opts.parse_theme.missing, "MISSING"))?; - write!( - stdout, - "\"{}\"", - paint(opts.parse_theme.missing, node.kind()) - )?; + write!(out, "{}: ", paint(opts.parse_theme.missing, "MISSING"))?; + write!(out, "\"{}\"", paint(opts.parse_theme.missing, node.kind()))?; } else { // Terminal literals, like "fn" write_node_text( opts, - stdout, + out, cursor, is_named, node.kind(), @@ -808,7 +1028,7 @@ fn cst_render_node( (total_width, indent_level), )?; } - writeln!(stdout)?; + writeln!(out)?; Ok(()) } @@ -895,7 +1115,7 @@ pub fn offset_for_position(input: &[u8], position: Point) -> Result { if let Some(pos) = iter.next() { if (pos - offset < position.column) || (input[offset] == b'\n' && position.column > 0) { return Err(anyhow!("Failed to address a column: {}", position.column)); - }; + } } else if input.len() - offset < position.column { return Err(anyhow!("Failed to address a column over the end")); } diff --git a/crates/cli/src/playground.html b/crates/cli/src/playground.html new file mode 100644 index 00000000..147516ac --- /dev/null +++ b/crates/cli/src/playground.html @@ -0,0 +1,481 @@ + + + + + tree-sitter THE_LANGUAGE_NAME + + + + + + + + +

+ + + + + + + + + diff --git a/cli/src/playground.rs b/crates/cli/src/playground.rs similarity index 59% rename from cli/src/playground.rs rename to crates/cli/src/playground.rs index 12348b40..f2c04fed 100644 --- a/cli/src/playground.rs +++ b/crates/cli/src/playground.rs @@ -7,6 +7,7 @@ use std::{ }; use anyhow::{anyhow, Context, Result}; +use log::{error, info}; use tiny_http::{Header, Response, Server}; use super::wasm; @@ -18,7 +19,7 @@ macro_rules! optional_resource { if let Some(tree_sitter_dir) = tree_sitter_dir { Cow::Owned(fs::read(tree_sitter_dir.join($path)).unwrap()) } else { - Cow::Borrowed(include_bytes!(concat!("../../", $path))) + Cow::Borrowed(include_bytes!(concat!("../../../", $path))) } } @@ -33,26 +34,92 @@ macro_rules! optional_resource { }; } -optional_resource!(get_playground_js, "docs/assets/js/playground.js"); -optional_resource!(get_lib_js, "lib/binding_web/tree-sitter.js"); -optional_resource!(get_lib_wasm, "lib/binding_web/tree-sitter.wasm"); +optional_resource!(get_playground_js, "docs/src/assets/js/playground.js"); +optional_resource!(get_lib_js, "lib/binding_web/web-tree-sitter.js"); +optional_resource!(get_lib_wasm, "lib/binding_web/web-tree-sitter.wasm"); fn get_main_html(tree_sitter_dir: Option<&Path>) -> Cow<'static, [u8]> { tree_sitter_dir.map_or( Cow::Borrowed(include_bytes!("playground.html")), |tree_sitter_dir| { - Cow::Owned(fs::read(tree_sitter_dir.join("cli/src/playground.html")).unwrap()) + Cow::Owned(fs::read(tree_sitter_dir.join("crates/cli/src/playground.html")).unwrap()) }, ) } +pub fn export(grammar_path: &Path, export_path: &Path) -> Result<()> { + let (grammar_name, language_wasm) = wasm::load_language_wasm_file(grammar_path)?; + + fs::create_dir_all(export_path).with_context(|| { + format!( + "Failed to create export directory: {}", + export_path.display() + ) + })?; + + let tree_sitter_dir = env::var("TREE_SITTER_BASE_DIR").map(PathBuf::from).ok(); + + let playground_js = get_playground_js(tree_sitter_dir.as_deref()); + let lib_js = get_lib_js(tree_sitter_dir.as_deref()); + let lib_wasm = get_lib_wasm(tree_sitter_dir.as_deref()); + + let has_local_playground_js = !playground_js.is_empty(); + let has_local_lib_js = !lib_js.is_empty(); + let has_local_lib_wasm = !lib_wasm.is_empty(); + + let mut main_html = str::from_utf8(&get_main_html(tree_sitter_dir.as_deref())) + .unwrap() + .replace("THE_LANGUAGE_NAME", &grammar_name); + + if !has_local_playground_js { + main_html = main_html.replace( + r#""#, + r#""# + ); + } + if !has_local_lib_js { + main_html = main_html.replace( + "import * as TreeSitter from './web-tree-sitter.js';", + "import * as TreeSitter from 'https://tree-sitter.github.io/web-tree-sitter.js';", + ); + } + + fs::write(export_path.join("index.html"), main_html.as_bytes()) + .with_context(|| "Failed to write index.html")?; + + fs::write(export_path.join("tree-sitter-parser.wasm"), language_wasm) + .with_context(|| "Failed to write parser wasm file")?; + + if has_local_playground_js { + fs::write(export_path.join("playground.js"), playground_js) + .with_context(|| "Failed to write playground.js")?; + } + + if has_local_lib_js { + fs::write(export_path.join("web-tree-sitter.js"), lib_js) + .with_context(|| "Failed to write web-tree-sitter.js")?; + } + + if has_local_lib_wasm { + fs::write(export_path.join("web-tree-sitter.wasm"), lib_wasm) + .with_context(|| "Failed to write web-tree-sitter.wasm")?; + } + + println!( + "Exported playground to {}", + export_path.canonicalize()?.display() + ); + + Ok(()) +} + pub fn serve(grammar_path: &Path, open_in_browser: bool) -> Result<()> { let server = get_server()?; let (grammar_name, language_wasm) = wasm::load_language_wasm_file(grammar_path)?; let url = format!("http://{}", server.server_addr()); - println!("Started playground on: {url}"); + info!("Started playground on: {url}"); if open_in_browser && webbrowser::open(&url).is_err() { - eprintln!("Failed to open '{url}' in a web browser"); + error!("Failed to open '{url}' in a web browser"); } let tree_sitter_dir = env::var("TREE_SITTER_BASE_DIR").map(PathBuf::from).ok(); @@ -79,16 +146,16 @@ pub fn serve(grammar_path: &Path, open_in_browser: bool) -> Result<()> { response(&playground_js, &js_header) } } - "/tree-sitter.js" => { + "/web-tree-sitter.js" => { if lib_js.is_empty() { - redirect("https://tree-sitter.github.io/tree-sitter.js") + redirect("https://tree-sitter.github.io/web-tree-sitter.js") } else { response(&lib_js, &js_header) } } - "/tree-sitter.wasm" => { + "/web-tree-sitter.wasm" => { if lib_wasm.is_empty() { - redirect("https://tree-sitter.github.io/tree-sitter.wasm") + redirect("https://tree-sitter.github.io/web-tree-sitter.wasm") } else { response(&lib_wasm, &wasm_header) } diff --git a/crates/cli/src/query.rs b/crates/cli/src/query.rs new file mode 100644 index 00000000..54674115 --- /dev/null +++ b/crates/cli/src/query.rs @@ -0,0 +1,174 @@ +use std::{ + fs, + io::{self, Write}, + ops::Range, + path::Path, + time::Instant, +}; + +use anyhow::{Context, Result}; +use log::warn; +use streaming_iterator::StreamingIterator; +use tree_sitter::{Language, Parser, Point, Query, QueryCursor}; + +use crate::{ + query_testing::{self, to_utf8_point}, + test::{TestInfo, TestOutcome, TestResult, TestSummary}, +}; + +#[derive(Default)] +pub struct QueryFileOptions { + pub ordered_captures: bool, + pub byte_range: Option>, + pub point_range: Option>, + pub containing_byte_range: Option>, + pub containing_point_range: Option>, + pub quiet: bool, + pub print_time: bool, + pub stdin: bool, +} + +pub fn query_file_at_path( + language: &Language, + path: &Path, + name: &str, + query_path: &Path, + opts: &QueryFileOptions, + test_summary: Option<&mut TestSummary>, +) -> Result<()> { + let stdout = io::stdout(); + let mut stdout = stdout.lock(); + + let query_source = fs::read_to_string(query_path) + .with_context(|| format!("Error reading query file {}", query_path.display()))?; + let query = Query::new(language, &query_source).with_context(|| "Query compilation failed")?; + + let mut query_cursor = QueryCursor::new(); + if let Some(ref range) = opts.byte_range { + query_cursor.set_byte_range(range.clone()); + } + if let Some(ref range) = opts.point_range { + query_cursor.set_point_range(range.clone()); + } + if let Some(ref range) = opts.containing_byte_range { + query_cursor.set_containing_byte_range(range.clone()); + } + if let Some(ref range) = opts.containing_point_range { + query_cursor.set_containing_point_range(range.clone()); + } + + let mut parser = Parser::new(); + parser.set_language(language)?; + + let mut results = Vec::new(); + let should_test = test_summary.is_some(); + + if !should_test && !opts.stdin { + writeln!(&mut stdout, "{name}")?; + } + + let source_code = + fs::read(path).with_context(|| format!("Error reading source file {}", path.display()))?; + let tree = parser.parse(&source_code, None).unwrap(); + + let start = Instant::now(); + if opts.ordered_captures { + let mut captures = query_cursor.captures(&query, tree.root_node(), source_code.as_slice()); + while let Some((mat, capture_index)) = captures.next() { + let capture = mat.captures[*capture_index]; + let capture_name = &query.capture_names()[capture.index as usize]; + if !opts.quiet && !should_test { + writeln!( + &mut stdout, + " pattern: {:>2}, capture: {} - {capture_name}, start: {}, end: {}, text: `{}`", + mat.pattern_index, + capture.index, + capture.node.start_position(), + capture.node.end_position(), + capture.node.utf8_text(&source_code).unwrap_or("") + )?; + } + if should_test { + results.push(query_testing::CaptureInfo { + name: (*capture_name).to_string(), + start: to_utf8_point(capture.node.start_position(), source_code.as_slice()), + end: to_utf8_point(capture.node.end_position(), source_code.as_slice()), + }); + } + } + } else { + let mut matches = query_cursor.matches(&query, tree.root_node(), source_code.as_slice()); + while let Some(m) = matches.next() { + if !opts.quiet && !should_test { + writeln!(&mut stdout, " pattern: {}", m.pattern_index)?; + } + for capture in m.captures { + let start = capture.node.start_position(); + let end = capture.node.end_position(); + let capture_name = &query.capture_names()[capture.index as usize]; + if !opts.quiet && !should_test { + if end.row == start.row { + writeln!( + &mut stdout, + " capture: {} - {capture_name}, start: {start}, end: {end}, text: `{}`", + capture.index, + capture.node.utf8_text(&source_code).unwrap_or("") + )?; + } else { + writeln!( + &mut stdout, + " capture: {capture_name}, start: {start}, end: {end}", + )?; + } + } + if should_test { + results.push(query_testing::CaptureInfo { + name: (*capture_name).to_string(), + start: to_utf8_point(capture.node.start_position(), source_code.as_slice()), + end: to_utf8_point(capture.node.end_position(), source_code.as_slice()), + }); + } + } + } + } + if query_cursor.did_exceed_match_limit() { + warn!("Query exceeded maximum number of in-progress captures!"); + } + if should_test { + let path_name = if opts.stdin { + "stdin" + } else { + Path::new(&path).file_name().unwrap().to_str().unwrap() + }; + // Invariant: `test_summary` will always be `Some` when `should_test` is true + let test_summary = test_summary.unwrap(); + match query_testing::assert_expected_captures(&results, path, &mut parser, language) { + Ok(assertion_count) => { + test_summary.query_results.add_case(TestResult { + name: path_name.to_string(), + info: TestInfo::AssertionTest { + outcome: TestOutcome::AssertionPassed { assertion_count }, + test_num: test_summary.test_num, + }, + }); + } + Err(e) => { + test_summary.query_results.add_case(TestResult { + name: path_name.to_string(), + info: TestInfo::AssertionTest { + outcome: TestOutcome::AssertionFailed { + error: e.to_string(), + }, + test_num: test_summary.test_num, + }, + }); + return Err(e); + } + } + } + if opts.print_time { + writeln!(&mut stdout, "{:?}", start.elapsed())?; + } + + Ok(()) +} diff --git a/cli/src/query_testing.rs b/crates/cli/src/query_testing.rs similarity index 81% rename from cli/src/query_testing.rs rename to crates/cli/src/query_testing.rs index 81ea18fb..e4923d65 100644 --- a/cli/src/query_testing.rs +++ b/crates/cli/src/query_testing.rs @@ -1,14 +1,11 @@ -use std::fs; +use std::{fs, path::Path, sync::LazyLock}; use anyhow::{anyhow, Result}; use bstr::{BStr, ByteSlice}; -use lazy_static::lazy_static; use regex::Regex; use tree_sitter::{Language, Parser, Point}; -lazy_static! { - static ref CAPTURE_NAME_REGEX: Regex = Regex::new("[\\w_\\-.]+").unwrap(); -} +static CAPTURE_NAME_REGEX: LazyLock = LazyLock::new(|| Regex::new("[\\w_\\-.]+").unwrap()); #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct Utf8Point { @@ -62,6 +59,7 @@ pub struct CaptureInfo { #[derive(Debug, PartialEq, Eq)] pub struct Assertion { pub position: Utf8Point, + pub length: usize, pub negative: bool, pub expected_capture_name: String, } @@ -71,11 +69,13 @@ impl Assertion { pub const fn new( row: usize, col: usize, + length: usize, negative: bool, expected_capture_name: String, ) -> Self { Self { position: Utf8Point::new(row, col), + length, negative, expected_capture_name, } @@ -117,6 +117,7 @@ pub fn parse_position_comments( let mut has_arrow = false; let mut negative = false; let mut arrow_end = 0; + let mut arrow_count = 1; for (i, c) in text.char_indices() { arrow_end = i + 1; if c == '-' && has_left_caret { @@ -126,6 +127,14 @@ pub fn parse_position_comments( if c == '^' { has_arrow = true; position.column += i; + // Continue counting remaining arrows and update their end column + for (_, c) in text[arrow_end..].char_indices() { + if c != '^' { + arrow_end += arrow_count - 1; + break; + } + arrow_count += 1; + } break; } has_left_caret = c == '<'; @@ -152,6 +161,7 @@ pub fn parse_position_comments( assertion_ranges.push((node.start_position(), node.end_position())); result.push(Assertion { position: to_utf8_point(position, source), + length: arrow_count, negative, expected_capture_name: mat.as_str().to_string(), }); @@ -177,13 +187,21 @@ pub fn parse_position_comments( let mut i = 0; let lines = source.lines_with_terminator().collect::>(); for assertion in &mut result { + let original_position = assertion.position; loop { let on_assertion_line = assertion_ranges[i..] .iter() .any(|(start, _)| start.row == assertion.position.row); let on_empty_line = lines[assertion.position.row].len() <= assertion.position.column; if on_assertion_line || on_empty_line { - assertion.position.row -= 1; + if assertion.position.row > 0 { + assertion.position.row -= 1; + } else { + return Err(anyhow!( + "Error: could not find a line that corresponds to the assertion `{}` located at {original_position}", + assertion.expected_capture_name + )); + } } else { while i < assertion_ranges.len() && assertion_ranges[i].0.row < assertion.position.row @@ -203,30 +221,32 @@ pub fn parse_position_comments( pub fn assert_expected_captures( infos: &[CaptureInfo], - path: &str, + path: &Path, parser: &mut Parser, language: &Language, ) -> Result { let contents = fs::read_to_string(path)?; let pairs = parse_position_comments(parser, language, contents.as_bytes())?; for assertion in &pairs { - if let Some(found) = &infos - .iter() - .find(|p| assertion.position >= p.start && assertion.position < p.end) - { + if let Some(found) = &infos.iter().find(|p| { + assertion.position >= p.start + && (assertion.position.row < p.end.row + || assertion.position.column + assertion.length - 1 < p.end.column) + }) { if assertion.expected_capture_name != found.name && found.name != "name" { return Err(anyhow!( "Assertion failed: at {}, found {}, expected {}", found.start, + found.name, assertion.expected_capture_name, - found.name )); } } else { return Err(anyhow!( - "Assertion failed: could not match {} at {}", + "Assertion failed: could not match {} at row {}, column {}", assertion.expected_capture_name, - assertion.position + assertion.position.row, + assertion.position.column + assertion.length - 1, )); } } diff --git a/crates/cli/src/tags.rs b/crates/cli/src/tags.rs new file mode 100644 index 00000000..2d205c04 --- /dev/null +++ b/crates/cli/src/tags.rs @@ -0,0 +1,78 @@ +use std::{ + fs, + io::{self, Write}, + path::Path, + str, + sync::{atomic::AtomicUsize, Arc}, + time::Instant, +}; + +use anyhow::Result; +use tree_sitter_tags::{TagsConfiguration, TagsContext}; + +pub struct TagsOptions { + pub scope: Option, + pub quiet: bool, + pub print_time: bool, + pub cancellation_flag: Arc, +} + +pub fn generate_tags( + path: &Path, + name: &str, + config: &TagsConfiguration, + indent: bool, + opts: &TagsOptions, +) -> Result<()> { + let mut context = TagsContext::new(); + let stdout = io::stdout(); + let mut stdout = stdout.lock(); + + let indent_str = if indent { + if !opts.quiet { + writeln!(&mut stdout, "{name}")?; + } + "\t" + } else { + "" + }; + + let source = fs::read(path)?; + let start = Instant::now(); + for tag in context + .generate_tags(config, &source, Some(&opts.cancellation_flag))? + .0 + { + let tag = tag?; + if !opts.quiet { + write!( + &mut stdout, + "{indent_str}{:<10}\t | {:<8}\t{} {} - {} `{}`", + str::from_utf8(&source[tag.name_range]).unwrap_or(""), + &config.syntax_type_name(tag.syntax_type_id), + if tag.is_definition { "def" } else { "ref" }, + tag.span.start, + tag.span.end, + str::from_utf8(&source[tag.line_range]).unwrap_or(""), + )?; + if let Some(docs) = tag.docs { + if docs.len() > 120 { + write!(&mut stdout, "\t{:?}...", docs.get(0..120).unwrap_or(""))?; + } else { + write!(&mut stdout, "\t{:?}", &docs)?; + } + } + writeln!(&mut stdout)?; + } + } + + if opts.print_time { + writeln!( + &mut stdout, + "{indent_str}time: {}ms", + start.elapsed().as_millis(), + )?; + } + + Ok(()) +} diff --git a/cli/src/templates/.editorconfig b/crates/cli/src/templates/.editorconfig similarity index 85% rename from cli/src/templates/.editorconfig rename to crates/cli/src/templates/.editorconfig index 65330c40..ff17b12f 100644 --- a/cli/src/templates/.editorconfig +++ b/crates/cli/src/templates/.editorconfig @@ -3,11 +3,11 @@ root = true [*] charset = utf-8 -[*.{json,toml,yml,gyp}] +[*.{json,toml,yml,gyp,xml}] indent_style = space indent_size = 2 -[*.js] +[*.{js,ts}] indent_style = space indent_size = 2 @@ -31,6 +31,10 @@ indent_size = 4 indent_style = space indent_size = 4 +[*.java] +indent_style = space +indent_size = 4 + [*.go] indent_style = tab indent_size = 8 diff --git a/cli/src/templates/PARSER_NAME.h b/crates/cli/src/templates/PARSER_NAME.h similarity index 100% rename from cli/src/templates/PARSER_NAME.h rename to crates/cli/src/templates/PARSER_NAME.h diff --git a/cli/src/templates/PARSER_NAME.pc.in b/crates/cli/src/templates/PARSER_NAME.pc.in similarity index 100% rename from cli/src/templates/PARSER_NAME.pc.in rename to crates/cli/src/templates/PARSER_NAME.pc.in diff --git a/crates/cli/src/templates/__init__.py b/crates/cli/src/templates/__init__.py new file mode 100644 index 00000000..784887a7 --- /dev/null +++ b/crates/cli/src/templates/__init__.py @@ -0,0 +1,43 @@ +"""PARSER_DESCRIPTION""" + +from importlib.resources import files as _files + +from ._binding import language + + +def _get_query(name, file): + try: + query = _files(f"{__package__}") / file + globals()[name] = query.read_text() + except FileNotFoundError: + globals()[name] = None + return globals()[name] + + +def __getattr__(name): + if name == "HIGHLIGHTS_QUERY": + return _get_query("HIGHLIGHTS_QUERY", "HIGHLIGHTS_QUERY_PATH") + if name == "INJECTIONS_QUERY": + return _get_query("INJECTIONS_QUERY", "INJECTIONS_QUERY_PATH") + if name == "LOCALS_QUERY": + return _get_query("LOCALS_QUERY", "LOCALS_QUERY_PATH") + if name == "TAGS_QUERY": + return _get_query("TAGS_QUERY", "TAGS_QUERY_PATH") + + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +__all__ = [ + "language", + "HIGHLIGHTS_QUERY", + "INJECTIONS_QUERY", + "LOCALS_QUERY", + "TAGS_QUERY", +] + + +def __dir__(): + return sorted(__all__ + [ + "__all__", "__builtins__", "__cached__", "__doc__", "__file__", + "__loader__", "__name__", "__package__", "__path__", "__spec__", + ]) diff --git a/crates/cli/src/templates/__init__.pyi b/crates/cli/src/templates/__init__.pyi new file mode 100644 index 00000000..5c88ff6c --- /dev/null +++ b/crates/cli/src/templates/__init__.pyi @@ -0,0 +1,17 @@ +from typing import Final +from typing_extensions import CapsuleType + +HIGHLIGHTS_QUERY: Final[str] | None +"""The syntax highlighting query for this grammar.""" + +INJECTIONS_QUERY: Final[str] | None +"""The language injection query for this grammar.""" + +LOCALS_QUERY: Final[str] | None +"""The local variable query for this grammar.""" + +TAGS_QUERY: Final[str] | None +"""The symbol tagging query for this grammar.""" + +def language() -> CapsuleType: + """The tree-sitter language function for this grammar.""" diff --git a/cli/src/templates/_cargo.toml b/crates/cli/src/templates/_cargo.toml similarity index 82% rename from cli/src/templates/_cargo.toml rename to crates/cli/src/templates/_cargo.toml index 0dbe2f80..74712694 100644 --- a/cli/src/templates/_cargo.toml +++ b/crates/cli/src/templates/_cargo.toml @@ -12,7 +12,14 @@ edition = "2021" autoexamples = false build = "bindings/rust/build.rs" -include = ["bindings/rust/*", "grammar.js", "queries/*", "src/*", "tree-sitter.json"] +include = [ + "bindings/rust/*", + "grammar.js", + "queries/*", + "src/*", + "tree-sitter.json", + "/LICENSE", +] [lib] path = "bindings/rust/lib.rs" @@ -21,7 +28,7 @@ path = "bindings/rust/lib.rs" tree-sitter-language = "0.1" [build-dependencies] -cc = "1.1.22" +cc = "1.2" [dev-dependencies] tree-sitter = "RUST_BINDING_VERSION" diff --git a/cli/src/templates/binding.go b/crates/cli/src/templates/binding.go similarity index 100% rename from cli/src/templates/binding.go rename to crates/cli/src/templates/binding.go diff --git a/cli/src/templates/binding.gyp b/crates/cli/src/templates/binding.gyp similarity index 100% rename from cli/src/templates/binding.gyp rename to crates/cli/src/templates/binding.gyp diff --git a/crates/cli/src/templates/binding.java b/crates/cli/src/templates/binding.java new file mode 100644 index 00000000..704064a0 --- /dev/null +++ b/crates/cli/src/templates/binding.java @@ -0,0 +1,65 @@ +package PARSER_NS_CLEANED.jtreesitter.LOWER_PARSER_NAME; + +import java.lang.foreign.*; + +public final class PARSER_CLASS_NAME { + private static final ValueLayout VOID_PTR = + ValueLayout.ADDRESS.withTargetLayout(MemoryLayout.sequenceLayout(Long.MAX_VALUE, ValueLayout.JAVA_BYTE)); + private static final FunctionDescriptor FUNC_DESC = FunctionDescriptor.of(VOID_PTR); + private static final Linker LINKER = Linker.nativeLinker(); + private static final PARSER_CLASS_NAME INSTANCE = new PARSER_CLASS_NAME(); + + private final Arena arena = Arena.ofAuto(); + private volatile SymbolLookup lookup = null; + + private PARSER_CLASS_NAME() {} + + /** + * Get the tree-sitter language for this grammar. + */ + public static MemorySegment language() { + if (INSTANCE.lookup == null) + INSTANCE.lookup = INSTANCE.findLibrary(); + return language(INSTANCE.lookup); + } + + /** + * Get the tree-sitter language for this grammar. + * + * The {@linkplain Arena} used in the {@code lookup} + * must not be closed while the language is being used. + */ + public static MemorySegment language(SymbolLookup lookup) { + return call(lookup, "tree_sitter_PARSER_NAME"); + } + + private SymbolLookup findLibrary() { + try { + var library = System.mapLibraryName("tree-sitter-KEBAB_PARSER_NAME"); + return SymbolLookup.libraryLookup(library, arena); + } catch (IllegalArgumentException ex1) { + try { + System.loadLibrary("tree-sitter-KEBAB_PARSER_NAME"); + return SymbolLookup.loaderLookup(); + } catch (UnsatisfiedLinkError ex2) { + ex1.addSuppressed(ex2); + throw ex1; + } + } + } + + private static UnsatisfiedLinkError unresolved(String name) { + return new UnsatisfiedLinkError("Unresolved symbol: %s".formatted(name)); + } + + @SuppressWarnings("SameParameterValue") + private static MemorySegment call(SymbolLookup lookup, String name) throws UnsatisfiedLinkError { + var address = lookup.find(name).orElseThrow(() -> unresolved(name)); + try { + var function = LINKER.downcallHandle(address, FUNC_DESC); + return (MemorySegment) function.invokeExact(); + } catch (Throwable e) { + throw new RuntimeException("Call to %s failed".formatted(name), e); + } + } +} diff --git a/cli/src/templates/binding_test.go b/crates/cli/src/templates/binding_test.go similarity index 86% rename from cli/src/templates/binding_test.go rename to crates/cli/src/templates/binding_test.go index ef9ae219..7e74e8ab 100644 --- a/cli/src/templates/binding_test.go +++ b/crates/cli/src/templates/binding_test.go @@ -10,6 +10,6 @@ import ( func TestCanLoadGrammar(t *testing.T) { language := tree_sitter.NewLanguage(tree_sitter_LOWER_PARSER_NAME.Language()) if language == nil { - t.Errorf("Error loading CAMEL_PARSER_NAME grammar") + t.Errorf("Error loading TITLE_PARSER_NAME grammar") } } diff --git a/crates/cli/src/templates/binding_test.js b/crates/cli/src/templates/binding_test.js new file mode 100644 index 00000000..7a91a84d --- /dev/null +++ b/crates/cli/src/templates/binding_test.js @@ -0,0 +1,11 @@ +import assert from "node:assert"; +import { test } from "node:test"; +import Parser from "tree-sitter"; + +test("can load grammar", () => { + const parser = new Parser(); + assert.doesNotReject(async () => { + const { default: language } = await import("./index.js"); + parser.setLanguage(language); + }); +}); diff --git a/crates/cli/src/templates/build.rs b/crates/cli/src/templates/build.rs new file mode 100644 index 00000000..e3fffe4b --- /dev/null +++ b/crates/cli/src/templates/build.rs @@ -0,0 +1,56 @@ +fn main() { + let src_dir = std::path::Path::new("src"); + + let mut c_config = cc::Build::new(); + c_config.std("c11").include(src_dir); + + #[cfg(target_env = "msvc")] + c_config.flag("-utf-8"); + + if std::env::var("TARGET").unwrap() == "wasm32-unknown-unknown" { + let Ok(wasm_headers) = std::env::var("DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS") else { + panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS must be set by the language crate"); + }; + let Ok(wasm_src) = + std::env::var("DEP_TREE_SITTER_LANGUAGE_WASM_SRC").map(std::path::PathBuf::from) + else { + panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_SRC must be set by the language crate"); + }; + + c_config.include(&wasm_headers); + c_config.files([ + wasm_src.join("stdio.c"), + wasm_src.join("stdlib.c"), + wasm_src.join("string.c"), + ]); + } + + let parser_path = src_dir.join("parser.c"); + c_config.file(&parser_path); + println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap()); + + let scanner_path = src_dir.join("scanner.c"); + if scanner_path.exists() { + c_config.file(&scanner_path); + println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap()); + } + + c_config.compile("tree-sitter-KEBAB_PARSER_NAME"); + + println!("cargo:rustc-check-cfg=cfg(with_highlights_query)"); + if !"HIGHLIGHTS_QUERY_PATH".is_empty() && std::path::Path::new("HIGHLIGHTS_QUERY_PATH").exists() { + println!("cargo:rustc-cfg=with_highlights_query"); + } + println!("cargo:rustc-check-cfg=cfg(with_injections_query)"); + if !"INJECTIONS_QUERY_PATH".is_empty() && std::path::Path::new("INJECTIONS_QUERY_PATH").exists() { + println!("cargo:rustc-cfg=with_injections_query"); + } + println!("cargo:rustc-check-cfg=cfg(with_locals_query)"); + if !"LOCALS_QUERY_PATH".is_empty() && std::path::Path::new("LOCALS_QUERY_PATH").exists() { + println!("cargo:rustc-cfg=with_locals_query"); + } + println!("cargo:rustc-check-cfg=cfg(with_tags_query)"); + if !"TAGS_QUERY_PATH".is_empty() && std::path::Path::new("TAGS_QUERY_PATH").exists() { + println!("cargo:rustc-cfg=with_tags_query"); + } +} diff --git a/crates/cli/src/templates/build.zig b/crates/cli/src/templates/build.zig new file mode 100644 index 00000000..c2428289 --- /dev/null +++ b/crates/cli/src/templates/build.zig @@ -0,0 +1,93 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) !void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + const shared = b.option(bool, "build-shared", "Build a shared library") orelse true; + const reuse_alloc = b.option(bool, "reuse-allocator", "Reuse the library allocator") orelse false; + + const library_name = "tree-sitter-PARSER_NAME"; + + const lib: *std.Build.Step.Compile = b.addLibrary(.{ + .name = library_name, + .linkage = if (shared) .dynamic else .static, + .root_module = b.createModule(.{ + .target = target, + .optimize = optimize, + .link_libc = true, + .pic = if (shared) true else null, + }), + }); + + lib.addCSourceFile(.{ + .file = b.path("src/parser.c"), + .flags = &.{"-std=c11"}, + }); + if (fileExists(b, "src/scanner.c")) { + lib.addCSourceFile(.{ + .file = b.path("src/scanner.c"), + .flags = &.{"-std=c11"}, + }); + } + + if (reuse_alloc) { + lib.root_module.addCMacro("TREE_SITTER_REUSE_ALLOCATOR", ""); + } + if (optimize == .Debug) { + lib.root_module.addCMacro("TREE_SITTER_DEBUG", ""); + } + + lib.addIncludePath(b.path("src")); + + b.installArtifact(lib); + b.installFile("src/node-types.json", "node-types.json"); + + if (fileExists(b, "queries")) { + b.installDirectory(.{ + .source_dir = b.path("queries"), + .install_dir = .prefix, + .install_subdir = "queries", + .include_extensions = &.{"scm"}, + }); + } + + const module = b.addModule(library_name, .{ + .root_source_file = b.path("bindings/zig/root.zig"), + .target = target, + .optimize = optimize, + }); + module.linkLibrary(lib); + + const tests = b.addTest(.{ + .root_module = b.createModule(.{ + .root_source_file = b.path("bindings/zig/test.zig"), + .target = target, + .optimize = optimize, + }), + }); + tests.root_module.addImport(library_name, module); + + // HACK: fetch tree-sitter dependency only when testing this module + if (b.pkg_hash.len == 0) { + var args = try std.process.argsWithAllocator(b.allocator); + defer args.deinit(); + while (args.next()) |a| { + if (std.mem.eql(u8, a, "test")) { + const ts_dep = b.lazyDependency("tree_sitter", .{}) orelse continue; + tests.root_module.addImport("tree-sitter", ts_dep.module("tree-sitter")); + break; + } + } + } + + const run_tests = b.addRunArtifact(tests); + const test_step = b.step("test", "Run unit tests"); + test_step.dependOn(&run_tests.step); +} + +inline fn fileExists(b: *std.Build, filename: []const u8) bool { + const dir = b.build_root.handle; + dir.access(filename, .{}) catch return false; + return true; +} diff --git a/crates/cli/src/templates/build.zig.zon b/crates/cli/src/templates/build.zig.zon new file mode 100644 index 00000000..0d542675 --- /dev/null +++ b/crates/cli/src/templates/build.zig.zon @@ -0,0 +1,21 @@ +.{ + .name = .tree_sitter_PARSER_NAME, + .fingerprint = PARSER_FINGERPRINT, + .version = "PARSER_VERSION", + .dependencies = .{ + .tree_sitter = .{ + .url = "git+https://github.com/tree-sitter/zig-tree-sitter#b4b72c903e69998fc88e27e154a5e3cc9166551b", + .hash = "tree_sitter-0.25.0-8heIf51vAQConvVIgvm-9mVIbqh7yabZYqPXfOpS3YoG", + .lazy = true, + }, + }, + .paths = .{ + "build.zig", + "build.zig.zon", + "bindings/zig", + "src", + "queries", + "LICENSE", + "README.md", + }, +} diff --git a/crates/cli/src/templates/cmakelists.cmake b/crates/cli/src/templates/cmakelists.cmake new file mode 100644 index 00000000..06acbc8f --- /dev/null +++ b/crates/cli/src/templates/cmakelists.cmake @@ -0,0 +1,76 @@ +cmake_minimum_required(VERSION 3.13) + +project(tree-sitter-KEBAB_PARSER_NAME + VERSION "PARSER_VERSION" + DESCRIPTION "PARSER_DESCRIPTION" + HOMEPAGE_URL "PARSER_URL" + LANGUAGES C) + +option(BUILD_SHARED_LIBS "Build using shared libraries" ON) +option(TREE_SITTER_REUSE_ALLOCATOR "Reuse the library allocator" OFF) + +set(TREE_SITTER_ABI_VERSION ABI_VERSION_MAX CACHE STRING "Tree-sitter ABI version") +if(NOT ${TREE_SITTER_ABI_VERSION} MATCHES "^[0-9]+$") + unset(TREE_SITTER_ABI_VERSION CACHE) + message(FATAL_ERROR "TREE_SITTER_ABI_VERSION must be an integer") +endif() + +include(GNUInstallDirs) + +find_program(TREE_SITTER_CLI tree-sitter DOC "Tree-sitter CLI") + +add_custom_command(OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/src/grammar.json" + "${CMAKE_CURRENT_SOURCE_DIR}/src/node-types.json" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/grammar.js" + COMMAND "${TREE_SITTER_CLI}" generate grammar.js --no-parser + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + COMMENT "Generating grammar.json") + +add_custom_command(OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/src/parser.c" + BYPRODUCTS "${CMAKE_CURRENT_SOURCE_DIR}/src/tree_sitter/parser.h" + "${CMAKE_CURRENT_SOURCE_DIR}/src/tree_sitter/alloc.h" + "${CMAKE_CURRENT_SOURCE_DIR}/src/tree_sitter/array.h" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/grammar.json" + COMMAND "${TREE_SITTER_CLI}" generate src/grammar.json + --abi=${TREE_SITTER_ABI_VERSION} + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + COMMENT "Generating parser.c") + +add_library(tree-sitter-KEBAB_PARSER_NAME src/parser.c) +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/src/scanner.c) + target_sources(tree-sitter-KEBAB_PARSER_NAME PRIVATE src/scanner.c) +endif() +target_include_directories(tree-sitter-KEBAB_PARSER_NAME + PRIVATE src + INTERFACE $ + $) + +target_compile_definitions(tree-sitter-KEBAB_PARSER_NAME PRIVATE + $<$:TREE_SITTER_REUSE_ALLOCATOR> + $<$:TREE_SITTER_DEBUG>) + +set_target_properties(tree-sitter-KEBAB_PARSER_NAME + PROPERTIES + C_STANDARD 11 + POSITION_INDEPENDENT_CODE ON + SOVERSION "${TREE_SITTER_ABI_VERSION}.${PROJECT_VERSION_MAJOR}" + DEFINE_SYMBOL "") + +configure_file(bindings/c/tree-sitter-KEBAB_PARSER_NAME.pc.in + "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter-KEBAB_PARSER_NAME.pc" @ONLY) + +install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/bindings/c/tree_sitter" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" + FILES_MATCHING PATTERN "*.h") +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter-KEBAB_PARSER_NAME.pc" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") +install(TARGETS tree-sitter-KEBAB_PARSER_NAME + LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}") + +file(GLOB QUERIES queries/*.scm) +install(FILES ${QUERIES} + DESTINATION "${CMAKE_INSTALL_DATADIR}/tree-sitter/queries/KEBAB_PARSER_NAME") + +add_custom_target(ts-test "${TREE_SITTER_CLI}" test + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + COMMENT "tree-sitter test") diff --git a/cli/src/templates/gitattributes b/crates/cli/src/templates/gitattributes similarity index 78% rename from cli/src/templates/gitattributes rename to crates/cli/src/templates/gitattributes index 7e2cae0c..027ac707 100644 --- a/cli/src/templates/gitattributes +++ b/crates/cli/src/templates/gitattributes @@ -6,7 +6,7 @@ src/parser.c linguist-generated src/tree_sitter/* linguist-generated # C bindings -bindings/c/* linguist-generated +bindings/c/** linguist-generated CMakeLists.txt linguist-generated Makefile linguist-generated @@ -35,3 +35,12 @@ go.sum linguist-generated bindings/swift/** linguist-generated Package.swift linguist-generated Package.resolved linguist-generated + +# Zig bindings +bindings/zig/* linguist-generated +build.zig linguist-generated +build.zig.zon linguist-generated + +# Java bindings +pom.xml linguist-generated +bindings/java/** linguist-generated diff --git a/cli/src/templates/gitignore b/crates/cli/src/templates/gitignore similarity index 82% rename from cli/src/templates/gitignore rename to crates/cli/src/templates/gitignore index 308fcab2..7c0cb7f5 100644 --- a/cli/src/templates/gitignore +++ b/crates/cli/src/templates/gitignore @@ -25,6 +25,13 @@ dist/ *.dylib *.dll *.pc +*.exp +*.lib + +# Zig artifacts +.zig-cache/ +zig-cache/ +zig-out/ # Example dirs /examples/*/ @@ -38,3 +45,4 @@ dist/ *.tar.gz *.tgz *.zip +*.jar diff --git a/cli/src/templates/go.mod b/crates/cli/src/templates/go.mod similarity index 100% rename from cli/src/templates/go.mod rename to crates/cli/src/templates/go.mod diff --git a/cli/src/templates/grammar.js b/crates/cli/src/templates/grammar.js similarity index 91% rename from cli/src/templates/grammar.js rename to crates/cli/src/templates/grammar.js index 01586557..edee3cbc 100644 --- a/cli/src/templates/grammar.js +++ b/crates/cli/src/templates/grammar.js @@ -7,7 +7,7 @@ /// // @ts-check -module.exports = grammar({ +export default grammar({ name: "LOWER_PARSER_NAME", rules: { diff --git a/crates/cli/src/templates/index.d.ts b/crates/cli/src/templates/index.d.ts new file mode 100644 index 00000000..24576d32 --- /dev/null +++ b/crates/cli/src/templates/index.d.ts @@ -0,0 +1,60 @@ +type BaseNode = { + type: string; + named: boolean; +}; + +type ChildNode = { + multiple: boolean; + required: boolean; + types: BaseNode[]; +}; + +type NodeInfo = + | (BaseNode & { + subtypes: BaseNode[]; + }) + | (BaseNode & { + fields: { [name: string]: ChildNode }; + children: ChildNode[]; + }); + +/** + * The tree-sitter language object for this grammar. + * + * @see {@linkcode https://tree-sitter.github.io/node-tree-sitter/interfaces/Parser.Language.html Parser.Language} + * + * @example + * import Parser from "tree-sitter"; + * import CAMEL_PARSER_NAME from "tree-sitter-KEBAB_PARSER_NAME"; + * + * const parser = new Parser(); + * parser.setLanguage(CAMEL_PARSER_NAME); + */ +declare const binding: { + /** + * The inner language object. + * @private + */ + language: unknown; + + /** + * The content of the `node-types.json` file for this grammar. + * + * @see {@linkplain https://tree-sitter.github.io/tree-sitter/using-parsers/6-static-node-types Static Node Types} + */ + nodeTypeInfo: NodeInfo[]; + + /** The syntax highlighting query for this grammar. */ + HIGHLIGHTS_QUERY?: string; + + /** The language injection query for this grammar. */ + INJECTIONS_QUERY?: string; + + /** The local variable query for this grammar. */ + LOCALS_QUERY?: string; + + /** The symbol tagging query for this grammar. */ + TAGS_QUERY?: string; +}; + +export default binding; diff --git a/crates/cli/src/templates/index.js b/crates/cli/src/templates/index.js new file mode 100644 index 00000000..b3edc2e3 --- /dev/null +++ b/crates/cli/src/templates/index.js @@ -0,0 +1,37 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +const root = fileURLToPath(new URL("../..", import.meta.url)); + +const binding = typeof process.versions.bun === "string" + // Support `bun build --compile` by being statically analyzable enough to find the .node file at build-time + ? await import(`${root}/prebuilds/${process.platform}-${process.arch}/tree-sitter-KEBAB_PARSER_NAME.node`) + : (await import("node-gyp-build")).default(root); + +try { + const nodeTypes = await import(`${root}/src/node-types.json`, { with: { type: "json" } }); + binding.nodeTypeInfo = nodeTypes.default; +} catch { } + +const queries = [ + ["HIGHLIGHTS_QUERY", `${root}/HIGHLIGHTS_QUERY_PATH`], + ["INJECTIONS_QUERY", `${root}/INJECTIONS_QUERY_PATH`], + ["LOCALS_QUERY", `${root}/LOCALS_QUERY_PATH`], + ["TAGS_QUERY", `${root}/TAGS_QUERY_PATH`], +]; + +for (const [prop, path] of queries) { + Object.defineProperty(binding, prop, { + configurable: true, + enumerable: true, + get() { + delete binding[prop]; + try { + binding[prop] = readFileSync(path, "utf8"); + } catch { } + return binding[prop]; + } + }); +} + +export default binding; diff --git a/cli/src/templates/js-binding.cc b/crates/cli/src/templates/js-binding.cc similarity index 100% rename from cli/src/templates/js-binding.cc rename to crates/cli/src/templates/js-binding.cc diff --git a/crates/cli/src/templates/lib.rs b/crates/cli/src/templates/lib.rs new file mode 100644 index 00000000..1e8c9ca3 --- /dev/null +++ b/crates/cli/src/templates/lib.rs @@ -0,0 +1,60 @@ +//! This crate provides TITLE_PARSER_NAME language support for the [tree-sitter] parsing library. +//! +//! Typically, you will use the [`LANGUAGE`] constant to add this language to a +//! tree-sitter [`Parser`], and then use the parser to parse some code: +//! +//! ``` +//! let code = r#" +//! "#; +//! let mut parser = tree_sitter::Parser::new(); +//! let language = tree_sitter_PARSER_NAME::LANGUAGE; +//! parser +//! .set_language(&language.into()) +//! .expect("Error loading TITLE_PARSER_NAME parser"); +//! let tree = parser.parse(code, None).unwrap(); +//! assert!(!tree.root_node().has_error()); +//! ``` +//! +//! [`Parser`]: https://docs.rs/tree-sitter/RUST_BINDING_VERSION/tree_sitter/struct.Parser.html +//! [tree-sitter]: https://tree-sitter.github.io/ + +use tree_sitter_language::LanguageFn; + +extern "C" { + fn tree_sitter_PARSER_NAME() -> *const (); +} + +/// The tree-sitter [`LanguageFn`] for this grammar. +pub const LANGUAGE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_PARSER_NAME) }; + +/// The content of the [`node-types.json`] file for this grammar. +/// +/// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers/6-static-node-types +pub const NODE_TYPES: &str = include_str!("../../src/node-types.json"); + +#[cfg(with_highlights_query)] +/// The syntax highlighting query for this grammar. +pub const HIGHLIGHTS_QUERY: &str = include_str!("../../HIGHLIGHTS_QUERY_PATH"); + +#[cfg(with_injections_query)] +/// The language injection query for this grammar. +pub const INJECTIONS_QUERY: &str = include_str!("../../INJECTIONS_QUERY_PATH"); + +#[cfg(with_locals_query)] +/// The local variable query for this grammar. +pub const LOCALS_QUERY: &str = include_str!("../../LOCALS_QUERY_PATH"); + +#[cfg(with_tags_query)] +/// The symbol tagging query for this grammar. +pub const TAGS_QUERY: &str = include_str!("../../TAGS_QUERY_PATH"); + +#[cfg(test)] +mod tests { + #[test] + fn test_can_load_grammar() { + let mut parser = tree_sitter::Parser::new(); + parser + .set_language(&super::LANGUAGE.into()) + .expect("Error loading TITLE_PARSER_NAME parser"); + } +} diff --git a/cli/src/templates/makefile b/crates/cli/src/templates/makefile similarity index 62% rename from cli/src/templates/makefile rename to crates/cli/src/templates/makefile index 8691b286..b42dab97 100644 --- a/cli/src/templates/makefile +++ b/crates/cli/src/templates/makefile @@ -1,8 +1,4 @@ -ifeq ($(OS),Windows_NT) -$(error Windows is not supported) -endif - -LANGUAGE_NAME := tree-sitter-PARSER_NAME +LANGUAGE_NAME := tree-sitter-KEBAB_PARSER_NAME HOMEPAGE_URL := PARSER_URL VERSION := PARSER_VERSION @@ -13,8 +9,10 @@ TS ?= tree-sitter # install directory layout PREFIX ?= /usr/local +DATADIR ?= $(PREFIX)/share INCLUDEDIR ?= $(PREFIX)/include LIBDIR ?= $(PREFIX)/lib +BINDIR ?= $(PREFIX)/bin PCLIBDIR ?= $(LIBDIR)/pkgconfig # source/object files @@ -31,20 +29,25 @@ SONAME_MAJOR = $(shell sed -n 's/\#define LANGUAGE_VERSION //p' $(PARSER)) SONAME_MINOR = $(word 1,$(subst ., ,$(VERSION))) # OS-specific bits -ifeq ($(shell uname),Darwin) +MACHINE := $(shell $(CC) -dumpmachine) + +ifneq ($(findstring darwin,$(MACHINE)),) SOEXT = dylib SOEXTVER_MAJOR = $(SONAME_MAJOR).$(SOEXT) SOEXTVER = $(SONAME_MAJOR).$(SONAME_MINOR).$(SOEXT) LINKSHARED = -dynamiclib -Wl,-install_name,$(LIBDIR)/lib$(LANGUAGE_NAME).$(SOEXTVER),-rpath,@executable_path/../Frameworks +else ifneq ($(findstring mingw32,$(MACHINE)),) + SOEXT = dll + LINKSHARED += -s -shared -Wl,--out-implib,lib$(LANGUAGE_NAME).dll.a else SOEXT = so SOEXTVER_MAJOR = $(SOEXT).$(SONAME_MAJOR) SOEXTVER = $(SOEXT).$(SONAME_MAJOR).$(SONAME_MINOR) LINKSHARED = -shared -Wl,-soname,lib$(LANGUAGE_NAME).$(SOEXTVER) -endif ifneq ($(filter $(shell uname),FreeBSD NetBSD DragonFly),) PCLIBDIR := $(PREFIX)/libdata/pkgconfig endif +endif all: lib$(LANGUAGE_NAME).a lib$(LANGUAGE_NAME).$(SOEXT) $(LANGUAGE_NAME).pc @@ -57,6 +60,10 @@ ifneq ($(STRIP),) $(STRIP) $@ endif +ifneq ($(findstring mingw32,$(MACHINE)),) +lib$(LANGUAGE_NAME).dll.a: lib$(LANGUAGE_NAME).$(SOEXT) +endif + $(LANGUAGE_NAME).pc: bindings/c/$(LANGUAGE_NAME).pc.in sed -e 's|@PROJECT_VERSION@|$(VERSION)|' \ -e 's|@CMAKE_INSTALL_LIBDIR@|$(LIBDIR:$(PREFIX)/%=%)|' \ @@ -65,17 +72,30 @@ $(LANGUAGE_NAME).pc: bindings/c/$(LANGUAGE_NAME).pc.in -e 's|@PROJECT_HOMEPAGE_URL@|$(HOMEPAGE_URL)|' \ -e 's|@CMAKE_INSTALL_PREFIX@|$(PREFIX)|' $< > $@ +$(SRC_DIR)/grammar.json: grammar.js + $(TS) generate --no-parser $^ + $(PARSER): $(SRC_DIR)/grammar.json $(TS) generate $^ install: all - install -d '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter '$(DESTDIR)$(PCLIBDIR)' '$(DESTDIR)$(LIBDIR)' - install -m644 bindings/c/$(LANGUAGE_NAME).h '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/$(LANGUAGE_NAME).h + install -d '$(DESTDIR)$(DATADIR)'/tree-sitter/queries/KEBAB_PARSER_NAME '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter '$(DESTDIR)$(PCLIBDIR)' '$(DESTDIR)$(LIBDIR)' + install -m644 bindings/c/tree_sitter/$(LANGUAGE_NAME).h '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/$(LANGUAGE_NAME).h install -m644 $(LANGUAGE_NAME).pc '$(DESTDIR)$(PCLIBDIR)'/$(LANGUAGE_NAME).pc install -m644 lib$(LANGUAGE_NAME).a '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).a install -m755 lib$(LANGUAGE_NAME).$(SOEXT) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER) - ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) - ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXT) +ifneq ($(findstring mingw32,$(MACHINE)),) + install -d '$(DESTDIR)$(BINDIR)' + install -m755 lib$(LANGUAGE_NAME).dll '$(DESTDIR)$(BINDIR)'/lib$(LANGUAGE_NAME).dll + install -m755 lib$(LANGUAGE_NAME).dll.a '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).dll.a +else + install -m755 lib$(LANGUAGE_NAME).$(SOEXT) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER) + cd '$(DESTDIR)$(LIBDIR)' && ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER) lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) + cd '$(DESTDIR)$(LIBDIR)' && ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) lib$(LANGUAGE_NAME).$(SOEXT) +endif +ifneq ($(wildcard queries/*.scm),) + install -m644 queries/*.scm '$(DESTDIR)$(DATADIR)'/tree-sitter/queries/KEBAB_PARSER_NAME +endif uninstall: $(RM) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).a \ @@ -84,9 +104,10 @@ uninstall: '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXT) \ '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/$(LANGUAGE_NAME).h \ '$(DESTDIR)$(PCLIBDIR)'/$(LANGUAGE_NAME).pc + $(RM) -r '$(DESTDIR)$(DATADIR)'/tree-sitter/queries/KEBAB_PARSER_NAME clean: - $(RM) $(OBJS) $(LANGUAGE_NAME).pc lib$(LANGUAGE_NAME).a lib$(LANGUAGE_NAME).$(SOEXT) + $(RM) $(OBJS) $(LANGUAGE_NAME).pc lib$(LANGUAGE_NAME).a lib$(LANGUAGE_NAME).$(SOEXT) lib$(LANGUAGE_NAME).dll.a test: $(TS) test diff --git a/cli/src/templates/package.json b/crates/cli/src/templates/package.json similarity index 80% rename from cli/src/templates/package.json rename to crates/cli/src/templates/package.json index 1db11b2d..d9dbbd33 100644 --- a/cli/src/templates/package.json +++ b/crates/cli/src/templates/package.json @@ -2,7 +2,12 @@ "name": "tree-sitter-PARSER_NAME", "version": "PARSER_VERSION", "description": "PARSER_DESCRIPTION", - "repository": "PARSER_URL", + "type": "module", + "repository": { + "type": "git", + "url": "git+PARSER_URL.git" + }, + "funding": "FUNDING_URL", "license": "PARSER_LICENSE", "author": { "name": "PARSER_AUTHOR_NAME", @@ -28,15 +33,16 @@ "*.wasm" ], "dependencies": { - "node-addon-api": "^8.2.1", - "node-gyp-build": "^4.8.2" + "node-addon-api": "^8.5.0", + "node-gyp-build": "^4.8.4" }, "devDependencies": { "prebuildify": "^6.0.1", + "tree-sitter": "^0.25.0", "tree-sitter-cli": "^CLI_VERSION" }, "peerDependencies": { - "tree-sitter": "^0.21.1" + "tree-sitter": "^0.25.0" }, "peerDependenciesMeta": { "tree-sitter": { diff --git a/cli/src/templates/package.swift b/crates/cli/src/templates/package.swift similarity index 62% rename from cli/src/templates/package.swift rename to crates/cli/src/templates/package.swift index b42ffc5c..f5562891 100644 --- a/cli/src/templates/package.swift +++ b/crates/cli/src/templates/package.swift @@ -9,16 +9,16 @@ if FileManager.default.fileExists(atPath: "src/scanner.c") { } let package = Package( - name: "TreeSitterCAMEL_PARSER_NAME", + name: "PARSER_CLASS_NAME", products: [ - .library(name: "TreeSitterCAMEL_PARSER_NAME", targets: ["TreeSitterCAMEL_PARSER_NAME"]), + .library(name: "PARSER_CLASS_NAME", targets: ["PARSER_CLASS_NAME"]), ], dependencies: [ - .package(url: "https://github.com/ChimeHQ/SwiftTreeSitter", from: "0.8.0"), + .package(name: "SwiftTreeSitter", url: "https://github.com/tree-sitter/swift-tree-sitter", from: "0.9.0"), ], targets: [ .target( - name: "TreeSitterCAMEL_PARSER_NAME", + name: "PARSER_CLASS_NAME", dependencies: [], path: ".", sources: sources, @@ -29,12 +29,12 @@ let package = Package( cSettings: [.headerSearchPath("src")] ), .testTarget( - name: "TreeSitterCAMEL_PARSER_NAMETests", + name: "PARSER_CLASS_NAMETests", dependencies: [ "SwiftTreeSitter", - "TreeSitterCAMEL_PARSER_NAME", + "PARSER_CLASS_NAME", ], - path: "bindings/swift/TreeSitterCAMEL_PARSER_NAMETests" + path: "bindings/swift/PARSER_CLASS_NAMETests" ) ], cLanguageStandard: .c11 diff --git a/crates/cli/src/templates/pom.xml b/crates/cli/src/templates/pom.xml new file mode 100644 index 00000000..661fe42b --- /dev/null +++ b/crates/cli/src/templates/pom.xml @@ -0,0 +1,154 @@ + + + 4.0.0 + PARSER_NS + jtreesitter-KEBAB_PARSER_NAME + JTreeSitter CAMEL_PARSER_NAME + PARSER_VERSION + PARSER_DESCRIPTION + PARSER_URL + + + PARSER_LICENSE + https://spdx.org/licenses/PARSER_LICENSE.html + + + + + PARSER_AUTHOR_NAME + PARSER_AUTHOR_EMAIL + PARSER_AUTHOR_URL + + + + PARSER_URL + scm:git:git://PARSER_URL_STRIPPED.git + scm:git:ssh://PARSER_URL_STRIPPED.git + + + 23 + UTF-8 + true + true + false + true + + + + io.github.tree-sitter + jtreesitter + 0.26.0 + true + + + org.junit.jupiter + junit-jupiter-api + 6.0.1 + test + + + + bindings/java/main + bindings/java/test + + + maven-surefire-plugin + 3.5.4 + + + ${project.build.directory}/reports/surefire + + --enable-native-access=ALL-UNNAMED + + + + maven-javadoc-plugin + 3.12.0 + + + + jar + + + + + public + true + true + all,-missing + + + + maven-source-plugin + 3.3.1 + + + + jar-no-fork + + + + + + maven-gpg-plugin + 3.2.8 + + + verify + + sign + + + true + + --no-tty + --pinentry-mode + loopback + + + + + + + io.github.mavenplugins + central-publishing-maven-plugin + 1.1.1 + + + deploy + + publish + + + validated + ${publish.auto} + ${publish.skip} + ${project.artifactId}-${project.version}.zip + ${project.artifactId}-${project.version}.zip + + + + true + + + + + + ci + + + env.CI + true + + + + false + true + false + + + + diff --git a/cli/src/templates/py-binding.c b/crates/cli/src/templates/py-binding.c similarity index 72% rename from cli/src/templates/py-binding.c rename to crates/cli/src/templates/py-binding.c index 74309fa8..c74db112 100644 --- a/cli/src/templates/py-binding.c +++ b/crates/cli/src/templates/py-binding.c @@ -8,6 +8,13 @@ static PyObject* _binding_language(PyObject *Py_UNUSED(self), PyObject *Py_UNUSE return PyCapsule_New(tree_sitter_LOWER_PARSER_NAME(), "tree_sitter.Language", NULL); } +static struct PyModuleDef_Slot slots[] = { +#ifdef Py_GIL_DISABLED + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, +#endif + {0, NULL} +}; + static PyMethodDef methods[] = { {"language", _binding_language, METH_NOARGS, "Get the tree-sitter language for this grammar."}, @@ -18,10 +25,11 @@ static struct PyModuleDef module = { .m_base = PyModuleDef_HEAD_INIT, .m_name = "_binding", .m_doc = NULL, - .m_size = -1, - .m_methods = methods + .m_size = 0, + .m_methods = methods, + .m_slots = slots, }; PyMODINIT_FUNC PyInit__binding(void) { - return PyModule_Create(&module); + return PyModuleDef_Init(&module); } diff --git a/cli/src/templates/pyproject.toml b/crates/cli/src/templates/pyproject.toml similarity index 82% rename from cli/src/templates/pyproject.toml rename to crates/cli/src/templates/pyproject.toml index e0db4e21..0f47e0f4 100644 --- a/cli/src/templates/pyproject.toml +++ b/crates/cli/src/templates/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=42", "wheel"] +requires = ["setuptools>=62.4.0", "wheel"] build-backend = "setuptools.build_meta" [project] @@ -14,16 +14,17 @@ classifiers = [ "Typing :: Typed", ] authors = [{ name = "PARSER_AUTHOR_NAME", email = "PARSER_AUTHOR_EMAIL" }] -requires-python = ">=3.9" +requires-python = ">=3.10" license.text = "PARSER_LICENSE" readme = "README.md" [project.urls] Homepage = "PARSER_URL" +Funding = "FUNDING_URL" [project.optional-dependencies] -core = ["tree-sitter~=0.22"] +core = ["tree-sitter~=0.24"] [tool.cibuildwheel] -build = "cp39-*" +build = "cp310-*" build-frontend = "build" diff --git a/crates/cli/src/templates/root.zig b/crates/cli/src/templates/root.zig new file mode 100644 index 00000000..0e8f24bd --- /dev/null +++ b/crates/cli/src/templates/root.zig @@ -0,0 +1,5 @@ +extern fn tree_sitter_PARSER_NAME() callconv(.c) *const anyopaque; + +pub fn language() *const anyopaque { + return tree_sitter_PARSER_NAME(); +} diff --git a/crates/cli/src/templates/setup.py b/crates/cli/src/templates/setup.py new file mode 100644 index 00000000..bcf184b7 --- /dev/null +++ b/crates/cli/src/templates/setup.py @@ -0,0 +1,77 @@ +from os import path +from sysconfig import get_config_var + +from setuptools import Extension, find_packages, setup +from setuptools.command.build import build +from setuptools.command.build_ext import build_ext +from setuptools.command.egg_info import egg_info +from wheel.bdist_wheel import bdist_wheel + + +class Build(build): + def run(self): + if path.isdir("queries"): + dest = path.join(self.build_lib, "tree_sitter_PARSER_NAME", "queries") + self.copy_tree("queries", dest) + super().run() + + +class BuildExt(build_ext): + def build_extension(self, ext: Extension): + if self.compiler.compiler_type != "msvc": + ext.extra_compile_args = ["-std=c11", "-fvisibility=hidden"] + else: + ext.extra_compile_args = ["/std:c11", "/utf-8"] + if path.exists("src/scanner.c"): + ext.sources.append("src/scanner.c") + if ext.py_limited_api: + ext.define_macros.append(("Py_LIMITED_API", "0x030A0000")) + super().build_extension(ext) + + +class BdistWheel(bdist_wheel): + def get_tag(self): + python, abi, platform = super().get_tag() + if python.startswith("cp") and not get_config_var("Py_GIL_DISABLED"): + python, abi = "cp310", "abi3" + return python, abi, platform + + +class EggInfo(egg_info): + def find_sources(self): + super().find_sources() + self.filelist.recursive_include("queries", "*.scm") + self.filelist.include("src/tree_sitter/*.h") + + +setup( + packages=find_packages("bindings/python"), + package_dir={"": "bindings/python"}, + package_data={ + "tree_sitter_LOWER_PARSER_NAME": ["*.pyi", "py.typed"], + "tree_sitter_LOWER_PARSER_NAME.queries": ["*.scm"], + }, + ext_package="tree_sitter_LOWER_PARSER_NAME", + ext_modules=[ + Extension( + name="_binding", + sources=[ + "bindings/python/tree_sitter_LOWER_PARSER_NAME/binding.c", + "src/parser.c", + ], + define_macros=[ + ("PY_SSIZE_T_CLEAN", None), + ("TREE_SITTER_HIDE_SYMBOLS", None), + ], + include_dirs=["src"], + py_limited_api=not get_config_var("Py_GIL_DISABLED"), + ) + ], + cmdclass={ + "build": Build, + "build_ext": BuildExt, + "bdist_wheel": BdistWheel, + "egg_info": EggInfo, + }, + zip_safe=False +) diff --git a/crates/cli/src/templates/test.java b/crates/cli/src/templates/test.java new file mode 100644 index 00000000..8bf81ea0 --- /dev/null +++ b/crates/cli/src/templates/test.java @@ -0,0 +1,12 @@ +import io.github.treesitter.jtreesitter.Language; +import PARSER_NS_CLEANED.jtreesitter.LOWER_PARSER_NAME.PARSER_CLASS_NAME; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; + +public class PARSER_CLASS_NAMETest { + @Test + public void testCanLoadLanguage() { + assertDoesNotThrow(() -> new Language(PARSER_CLASS_NAME.language())); + } +} diff --git a/crates/cli/src/templates/test.zig b/crates/cli/src/templates/test.zig new file mode 100644 index 00000000..af84322f --- /dev/null +++ b/crates/cli/src/templates/test.zig @@ -0,0 +1,17 @@ +const testing = @import("std").testing; + +const ts = @import("tree-sitter"); +const root = @import("tree-sitter-PARSER_NAME"); +const Language = ts.Language; +const Parser = ts.Parser; + +test "can load grammar" { + const parser = Parser.create(); + defer parser.destroy(); + + const lang: *const ts.Language = Language.fromRaw(root.language()); + defer lang.destroy(); + + try testing.expectEqual(void{}, parser.setLanguage(lang)); + try testing.expectEqual(lang, parser.getLanguage()); +} diff --git a/crates/cli/src/templates/test_binding.py b/crates/cli/src/templates/test_binding.py new file mode 100644 index 00000000..a832c368 --- /dev/null +++ b/crates/cli/src/templates/test_binding.py @@ -0,0 +1,12 @@ +from unittest import TestCase + +from tree_sitter import Language, Parser +import tree_sitter_LOWER_PARSER_NAME + + +class TestLanguage(TestCase): + def test_can_load_grammar(self): + try: + Parser(Language(tree_sitter_LOWER_PARSER_NAME.language())) + except Exception: + self.fail("Error loading TITLE_PARSER_NAME grammar") diff --git a/cli/src/templates/tests.swift b/crates/cli/src/templates/tests.swift similarity index 61% rename from cli/src/templates/tests.swift rename to crates/cli/src/templates/tests.swift index 3d0b8ce0..8b3c071b 100644 --- a/cli/src/templates/tests.swift +++ b/crates/cli/src/templates/tests.swift @@ -1,12 +1,12 @@ import XCTest import SwiftTreeSitter -import TreeSitterCAMEL_PARSER_NAME +import PARSER_CLASS_NAME -final class TreeSitterCAMEL_PARSER_NAMETests: XCTestCase { +final class PARSER_CLASS_NAMETests: XCTestCase { func testCanLoadGrammar() throws { let parser = Parser() let language = Language(language: tree_sitter_LOWER_PARSER_NAME()) XCTAssertNoThrow(try parser.setLanguage(language), - "Error loading CAMEL_PARSER_NAME grammar") + "Error loading TITLE_PARSER_NAME grammar") } } diff --git a/crates/cli/src/test.rs b/crates/cli/src/test.rs new file mode 100644 index 00000000..b568e4a3 --- /dev/null +++ b/crates/cli/src/test.rs @@ -0,0 +1,2427 @@ +use std::{ + collections::BTreeMap, + ffi::OsStr, + fmt::Display as _, + fs, + io::{self, Write}, + path::{Path, PathBuf}, + str, + sync::LazyLock, + time::Duration, +}; + +use anstyle::AnsiColor; +use anyhow::{anyhow, Context, Result}; +use clap::ValueEnum; +use indoc::indoc; +use regex::{ + bytes::{Regex as ByteRegex, RegexBuilder as ByteRegexBuilder}, + Regex, +}; +use schemars::{JsonSchema, Schema, SchemaGenerator}; +use serde::Serialize; +use similar::{ChangeTag, TextDiff}; +use tree_sitter::{format_sexp, Language, LogType, Parser, Query, Tree}; +use walkdir::WalkDir; + +use super::util; +use crate::{ + logger::paint, + parse::{ + render_cst, ParseDebugType, ParseFileOptions, ParseOutput, ParseStats, ParseTheme, Stats, + }, +}; + +static HEADER_REGEX: LazyLock = LazyLock::new(|| { + ByteRegexBuilder::new( + r"^(?x) + (?P(?:=+){3,}) + (?P[^=\r\n][^\r\n]*)? + \r?\n + (?P(?:([^=\r\n]|\s+:)[^\r\n]*\r?\n)+) + ===+ + (?P[^=\r\n][^\r\n]*)?\r?\n", + ) + .multi_line(true) + .build() + .unwrap() +}); + +static DIVIDER_REGEX: LazyLock = LazyLock::new(|| { + ByteRegexBuilder::new(r"^(?P(?:-+){3,})(?P[^-\r\n][^\r\n]*)?\r?\n") + .multi_line(true) + .build() + .unwrap() +}); + +static COMMENT_REGEX: LazyLock = LazyLock::new(|| Regex::new(r"(?m)^\s*;.*$").unwrap()); + +static WHITESPACE_REGEX: LazyLock = LazyLock::new(|| Regex::new(r"\s+").unwrap()); + +static SEXP_FIELD_REGEX: LazyLock = LazyLock::new(|| Regex::new(r" \w+: \(").unwrap()); + +static POINT_REGEX: LazyLock = + LazyLock::new(|| Regex::new(r"\s*\[\s*\d+\s*,\s*\d+\s*\]\s*").unwrap()); + +#[derive(Debug, PartialEq, Eq)] +pub enum TestEntry { + Group { + name: String, + children: Vec, + file_path: Option, + }, + Example { + name: String, + input: Vec, + output: String, + header_delim_len: usize, + divider_delim_len: usize, + has_fields: bool, + attributes_str: String, + attributes: TestAttributes, + file_name: Option, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TestAttributes { + pub skip: bool, + pub platform: bool, + pub fail_fast: bool, + pub error: bool, + pub cst: bool, + pub languages: Vec>, +} + +impl Default for TestEntry { + fn default() -> Self { + Self::Group { + name: String::new(), + children: Vec::new(), + file_path: None, + } + } +} + +impl Default for TestAttributes { + fn default() -> Self { + Self { + skip: false, + platform: true, + fail_fast: false, + error: false, + cst: false, + languages: vec!["".into()], + } + } +} + +#[derive(ValueEnum, Default, Debug, Copy, Clone, PartialEq, Eq, Serialize)] +pub enum TestStats { + All, + #[default] + OutliersAndTotal, + TotalOnly, +} + +pub struct TestOptions<'a> { + pub path: PathBuf, + pub debug: bool, + pub debug_graph: bool, + pub include: Option, + pub exclude: Option, + pub file_name: Option, + pub update: bool, + pub open_log: bool, + pub languages: BTreeMap<&'a str, &'a Language>, + pub color: bool, + pub show_fields: bool, + pub overview_only: bool, +} + +/// A stateful object used to collect results from running a grammar's test suite +#[derive(Debug, Default, Serialize, JsonSchema)] +pub struct TestSummary { + // Parse test results and associated data + #[schemars(schema_with = "schema_as_array")] + #[serde(serialize_with = "serialize_as_array")] + pub parse_results: TestResultHierarchy, + pub parse_failures: Vec, + pub parse_stats: Stats, + #[schemars(skip)] + #[serde(skip)] + pub has_parse_errors: bool, + #[schemars(skip)] + #[serde(skip)] + pub parse_stat_display: TestStats, + + // Other test results + #[schemars(schema_with = "schema_as_array")] + #[serde(serialize_with = "serialize_as_array")] + pub highlight_results: TestResultHierarchy, + #[schemars(schema_with = "schema_as_array")] + #[serde(serialize_with = "serialize_as_array")] + pub tag_results: TestResultHierarchy, + #[schemars(schema_with = "schema_as_array")] + #[serde(serialize_with = "serialize_as_array")] + pub query_results: TestResultHierarchy, + + // Data used during construction + #[schemars(skip)] + #[serde(skip)] + pub test_num: usize, + // Options passed in from the CLI which control how the summary is displayed + #[schemars(skip)] + #[serde(skip)] + pub color: bool, + #[schemars(skip)] + #[serde(skip)] + pub overview_only: bool, + #[schemars(skip)] + #[serde(skip)] + pub update: bool, + #[schemars(skip)] + #[serde(skip)] + pub json: bool, +} + +impl TestSummary { + #[must_use] + pub fn new( + color: bool, + stat_display: TestStats, + parse_update: bool, + overview_only: bool, + json_summary: bool, + ) -> Self { + Self { + color, + parse_stat_display: stat_display, + update: parse_update, + overview_only, + json: json_summary, + test_num: 1, + ..Default::default() + } + } +} + +#[derive(Debug, Default, JsonSchema)] +pub struct TestResultHierarchy { + root_group: Vec, + traversal_idxs: Vec, +} + +fn serialize_as_array(results: &TestResultHierarchy, serializer: S) -> Result +where + S: serde::Serializer, +{ + results.root_group.serialize(serializer) +} + +fn schema_as_array(gen: &mut SchemaGenerator) -> Schema { + gen.subschema_for::>() +} + +/// Stores arbitrarily nested parent test groups and child cases. Supports creation +/// in DFS traversal order +impl TestResultHierarchy { + /// Signifies the start of a new group's traversal during construction. + fn push_traversal(&mut self, idx: usize) { + self.traversal_idxs.push(idx); + } + + /// Signifies the end of the current group's traversal during construction. + /// Must be paired with a prior call to [`TestResultHierarchy::add_group`]. + pub fn pop_traversal(&mut self) { + self.traversal_idxs.pop(); + } + + /// Adds a new group as a child of the current group. Caller is responsible + /// for calling [`TestResultHierarchy::pop_traversal`] once the group is done + /// being traversed. + pub fn add_group(&mut self, group_name: &str) { + let new_group_idx = self.curr_group_len(); + self.push(TestResult { + name: group_name.to_string(), + info: TestInfo::Group { + children: Vec::new(), + }, + }); + self.push_traversal(new_group_idx); + } + + /// Adds a new test example as a child of the current group. + /// Asserts that `test_case.info` is not [`TestInfo::Group`]. + pub fn add_case(&mut self, test_case: TestResult) { + assert!(!matches!(test_case.info, TestInfo::Group { .. })); + self.push(test_case); + } + + /// Adds a new `TestResult` to the current group. + fn push(&mut self, result: TestResult) { + // If there are no traversal steps, we're adding to the root + if self.traversal_idxs.is_empty() { + self.root_group.push(result); + return; + } + + #[allow(clippy::manual_let_else)] + let mut curr_group = match self.root_group[self.traversal_idxs[0]].info { + TestInfo::Group { ref mut children } => children, + _ => unreachable!(), + }; + for idx in self.traversal_idxs.iter().skip(1) { + curr_group = match curr_group[*idx].info { + TestInfo::Group { ref mut children } => children, + _ => unreachable!(), + }; + } + + curr_group.push(result); + } + + fn curr_group_len(&self) -> usize { + if self.traversal_idxs.is_empty() { + return self.root_group.len(); + } + + #[allow(clippy::manual_let_else)] + let mut curr_group = match self.root_group[self.traversal_idxs[0]].info { + TestInfo::Group { ref children } => children, + _ => unreachable!(), + }; + for idx in self.traversal_idxs.iter().skip(1) { + curr_group = match curr_group[*idx].info { + TestInfo::Group { ref children } => children, + _ => unreachable!(), + }; + } + curr_group.len() + } + + #[allow(clippy::iter_without_into_iter)] + #[must_use] + pub fn iter(&self) -> TestResultIterWithDepth<'_> { + let mut stack = Vec::with_capacity(self.root_group.len()); + for child in self.root_group.iter().rev() { + stack.push((0, child)); + } + TestResultIterWithDepth { stack } + } +} + +pub struct TestResultIterWithDepth<'a> { + stack: Vec<(usize, &'a TestResult)>, +} + +impl<'a> Iterator for TestResultIterWithDepth<'a> { + type Item = (usize, &'a TestResult); + + fn next(&mut self) -> Option { + self.stack.pop().inspect(|(depth, result)| { + if let TestInfo::Group { children } = &result.info { + for child in children.iter().rev() { + self.stack.push((depth + 1, child)); + } + } + }) + } +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct TestResult { + pub name: String, + #[schemars(flatten)] + #[serde(flatten)] + pub info: TestInfo, +} + +#[derive(Debug, Serialize, JsonSchema)] +#[schemars(untagged)] +#[serde(untagged)] +pub enum TestInfo { + Group { + children: Vec, + }, + ParseTest { + outcome: TestOutcome, + // True parse rate, adjusted parse rate + #[schemars(schema_with = "parse_rate_schema")] + #[serde(serialize_with = "serialize_parse_rates")] + parse_rate: Option<(f64, f64)>, + test_num: usize, + }, + AssertionTest { + outcome: TestOutcome, + test_num: usize, + }, +} + +fn serialize_parse_rates( + parse_rate: &Option<(f64, f64)>, + serializer: S, +) -> Result +where + S: serde::Serializer, +{ + match parse_rate { + None => serializer.serialize_none(), + Some((first, _)) => serializer.serialize_some(first), + } +} + +fn parse_rate_schema(gen: &mut SchemaGenerator) -> Schema { + gen.subschema_for::>() +} + +#[derive(Debug, Clone, Eq, PartialEq, Serialize, JsonSchema)] +pub enum TestOutcome { + // Parse outcomes + Passed, + Failed, + Updated, + Skipped, + Platform, + + // Highlight/Tag/Query outcomes + AssertionPassed { assertion_count: usize }, + AssertionFailed { error: String }, +} + +impl TestSummary { + fn fmt_parse_results(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let (count, total_adj_parse_time) = self + .parse_results + .iter() + .filter_map(|(_, result)| match result.info { + TestInfo::Group { .. } => None, + TestInfo::ParseTest { parse_rate, .. } => parse_rate, + _ => unreachable!(), + }) + .fold((0usize, 0.0f64), |(count, rate_accum), (_, adj_rate)| { + (count + 1, rate_accum + adj_rate) + }); + + let avg = total_adj_parse_time / count as f64; + let std_dev = { + let variance = self + .parse_results + .iter() + .filter_map(|(_, result)| match result.info { + TestInfo::Group { .. } => None, + TestInfo::ParseTest { parse_rate, .. } => parse_rate, + _ => unreachable!(), + }) + .map(|(_, rate_i)| (rate_i - avg).powi(2)) + .sum::() + / count as f64; + variance.sqrt() + }; + + for (depth, entry) in self.parse_results.iter() { + write!(f, "{}", " ".repeat(depth + 1))?; + match &entry.info { + TestInfo::Group { .. } => writeln!(f, "{}:", entry.name)?, + TestInfo::ParseTest { + outcome, + parse_rate, + test_num, + } => { + let (color, result_char) = match outcome { + TestOutcome::Passed => (AnsiColor::Green, "✓"), + TestOutcome::Failed => (AnsiColor::Red, "✗"), + TestOutcome::Updated => (AnsiColor::Blue, "✓"), + TestOutcome::Skipped => (AnsiColor::Yellow, "⌀"), + TestOutcome::Platform => (AnsiColor::Magenta, "⌀"), + _ => unreachable!(), + }; + let stat_display = match (self.parse_stat_display, parse_rate) { + (TestStats::TotalOnly, _) | (_, None) => String::new(), + (display, Some((true_rate, adj_rate))) => { + let mut stats = if display == TestStats::All { + format!(" ({true_rate:.3} bytes/ms)") + } else { + String::new() + }; + // 3 standard deviations below the mean, aka the "Empirical Rule" + if *adj_rate < 3.0f64.mul_add(-std_dev, avg) { + stats += &paint( + self.color.then_some(AnsiColor::Yellow), + &format!( + " -- Warning: Slow parse rate ({true_rate:.3} bytes/ms)" + ), + ); + } + stats + } + }; + writeln!( + f, + "{test_num:>3}. {result_char} {}{stat_display}", + paint(self.color.then_some(color), &entry.name), + )?; + } + TestInfo::AssertionTest { .. } => unreachable!(), + } + } + + // Parse failure info + if !self.parse_failures.is_empty() && self.update && !self.has_parse_errors { + writeln!( + f, + "\n{} update{}:\n", + self.parse_failures.len(), + if self.parse_failures.len() == 1 { + "" + } else { + "s" + } + )?; + + for (i, TestFailure { name, .. }) in self.parse_failures.iter().enumerate() { + writeln!(f, " {}. {name}", i + 1)?; + } + } else if !self.parse_failures.is_empty() && !self.overview_only { + if !self.has_parse_errors { + writeln!( + f, + "\n{} failure{}:", + self.parse_failures.len(), + if self.parse_failures.len() == 1 { + "" + } else { + "s" + } + )?; + } + + if self.color { + DiffKey.fmt(f)?; + } + for ( + i, + TestFailure { + name, + actual, + expected, + is_cst, + }, + ) in self.parse_failures.iter().enumerate() + { + if expected == "NO ERROR" { + writeln!(f, "\n {}. {name}:\n", i + 1)?; + writeln!(f, " Expected an ERROR node, but got:")?; + let actual = if *is_cst { + actual + } else { + &format_sexp(actual, 2) + }; + writeln!( + f, + " {}", + paint(self.color.then_some(AnsiColor::Red), actual) + )?; + } else { + writeln!(f, "\n {}. {name}:", i + 1)?; + if *is_cst { + writeln!( + f, + "{}", + TestDiff::new(actual, expected).with_color(self.color) + )?; + } else { + writeln!( + f, + "{}", + TestDiff::new(&format_sexp(actual, 2), &format_sexp(expected, 2)) + .with_color(self.color,) + )?; + } + } + } + } else { + writeln!(f)?; + } + + Ok(()) + } +} + +impl std::fmt::Display for TestSummary { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.fmt_parse_results(f)?; + + let mut render_assertion_results = + |name: &str, results: &TestResultHierarchy| -> std::fmt::Result { + writeln!(f, "{name}:")?; + for (depth, entry) in results.iter() { + write!(f, "{}", " ".repeat(depth + 2))?; + match &entry.info { + TestInfo::Group { .. } => writeln!(f, "{}", entry.name)?, + TestInfo::AssertionTest { outcome, test_num } => match outcome { + TestOutcome::AssertionPassed { assertion_count } => writeln!( + f, + "{:>3}. ✓ {} ({assertion_count} assertions)", + test_num, + paint(self.color.then_some(AnsiColor::Green), &entry.name) + )?, + TestOutcome::AssertionFailed { error } => { + writeln!( + f, + "{:>3}. ✗ {}", + test_num, + paint(self.color.then_some(AnsiColor::Red), &entry.name) + )?; + writeln!(f, "{} {error}", " ".repeat(depth + 1))?; + } + _ => unreachable!(), + }, + TestInfo::ParseTest { .. } => unreachable!(), + } + } + Ok(()) + }; + + if !self.highlight_results.root_group.is_empty() { + render_assertion_results("syntax highlighting", &self.highlight_results)?; + } + + if !self.tag_results.root_group.is_empty() { + render_assertion_results("tags", &self.tag_results)?; + } + + if !self.query_results.root_group.is_empty() { + render_assertion_results("queries", &self.query_results)?; + } + + write!(f, "{}", self.parse_stats)?; + + Ok(()) + } +} + +pub fn run_tests_at_path( + parser: &mut Parser, + opts: &TestOptions, + test_summary: &mut TestSummary, +) -> Result<()> { + let test_entry = parse_tests(&opts.path)?; + let mut _log_session = None; + + if opts.debug_graph { + _log_session = Some(util::log_graphs(parser, "log.html", opts.open_log)?); + } else if opts.debug { + parser.set_logger(Some(Box::new(|log_type, message| { + if log_type == LogType::Lex { + io::stderr().write_all(b" ").unwrap(); + } + writeln!(&mut io::stderr(), "{message}").unwrap(); + }))); + } + + let mut corrected_entries = Vec::new(); + run_tests( + parser, + test_entry, + opts, + test_summary, + &mut corrected_entries, + true, + )?; + + parser.stop_printing_dot_graphs(); + + if test_summary.parse_failures.is_empty() || (opts.update && !test_summary.has_parse_errors) { + Ok(()) + } else if opts.update && test_summary.has_parse_errors { + Err(anyhow!(indoc! {" + Some tests failed to parse with unexpected `ERROR` or `MISSING` nodes, as shown above, and cannot be updated automatically. + Either fix the grammar or manually update the tests if this is expected."})) + } else { + Err(anyhow!("")) + } +} + +pub fn check_queries_at_path(language: &Language, path: &Path) -> Result<()> { + if path.exists() { + for entry in WalkDir::new(path) + .into_iter() + .filter_map(std::result::Result::ok) + .filter(|e| { + e.file_type().is_file() + && e.path().extension().and_then(OsStr::to_str) == Some("scm") + && !e.path().starts_with(".") + }) + { + let filepath = entry.file_name().to_str().unwrap_or(""); + let content = fs::read_to_string(entry.path()) + .with_context(|| format!("Error reading query file {filepath:?}"))?; + Query::new(language, &content) + .with_context(|| format!("Error in query file {filepath:?}"))?; + } + } + Ok(()) +} + +pub struct DiffKey; + +impl std::fmt::Display for DiffKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "\ncorrect / {} / {}", + paint(Some(AnsiColor::Green), "expected"), + paint(Some(AnsiColor::Red), "unexpected") + )?; + Ok(()) + } +} + +impl DiffKey { + /// Writes [`DiffKey`] to stdout + pub fn print() { + println!("{Self}"); + } +} + +pub struct TestDiff<'a> { + pub actual: &'a str, + pub expected: &'a str, + pub color: bool, +} + +impl<'a> TestDiff<'a> { + #[must_use] + pub const fn new(actual: &'a str, expected: &'a str) -> Self { + Self { + actual, + expected, + color: true, + } + } + + #[must_use] + pub const fn with_color(mut self, color: bool) -> Self { + self.color = color; + self + } +} + +impl std::fmt::Display for TestDiff<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let diff = TextDiff::from_lines(self.actual, self.expected); + for diff in diff.iter_all_changes() { + match diff.tag() { + ChangeTag::Equal => { + if self.color { + write!(f, "{diff}")?; + } else { + write!(f, " {diff}")?; + } + } + ChangeTag::Insert => { + if self.color { + write!( + f, + "{}", + paint(Some(AnsiColor::Green), diff.as_str().unwrap()) + )?; + } else { + write!(f, "+{diff}")?; + } + if diff.missing_newline() { + writeln!(f)?; + } + } + ChangeTag::Delete => { + if self.color { + write!(f, "{}", paint(Some(AnsiColor::Red), diff.as_str().unwrap()))?; + } else { + write!(f, "-{diff}")?; + } + if diff.missing_newline() { + writeln!(f)?; + } + } + } + } + + Ok(()) + } +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct TestFailure { + name: String, + actual: String, + expected: String, + is_cst: bool, +} + +impl TestFailure { + fn new(name: T, actual: U, expected: V, is_cst: bool) -> Self + where + T: Into, + U: Into, + V: Into, + { + Self { + name: name.into(), + actual: actual.into(), + expected: expected.into(), + is_cst, + } + } +} + +struct TestCorrection { + name: String, + input: String, + output: String, + attributes_str: String, + header_delim_len: usize, + divider_delim_len: usize, +} + +impl TestCorrection { + fn new( + name: T, + input: U, + output: V, + attributes_str: W, + header_delim_len: usize, + divider_delim_len: usize, + ) -> Self + where + T: Into, + U: Into, + V: Into, + W: Into, + { + Self { + name: name.into(), + input: input.into(), + output: output.into(), + attributes_str: attributes_str.into(), + header_delim_len, + divider_delim_len, + } + } +} + +/// This will return false if we want to "fail fast". It will bail and not parse any more tests. +fn run_tests( + parser: &mut Parser, + test_entry: TestEntry, + opts: &TestOptions, + test_summary: &mut TestSummary, + corrected_entries: &mut Vec, + is_root: bool, +) -> Result { + match test_entry { + TestEntry::Example { + name, + input, + output, + header_delim_len, + divider_delim_len, + has_fields, + attributes_str, + attributes, + .. + } => { + if attributes.skip { + test_summary.parse_results.add_case(TestResult { + name: name.clone(), + info: TestInfo::ParseTest { + outcome: TestOutcome::Skipped, + parse_rate: None, + test_num: test_summary.test_num, + }, + }); + test_summary.test_num += 1; + return Ok(true); + } + + if !attributes.platform { + test_summary.parse_results.add_case(TestResult { + name: name.clone(), + info: TestInfo::ParseTest { + outcome: TestOutcome::Platform, + parse_rate: None, + test_num: test_summary.test_num, + }, + }); + test_summary.test_num += 1; + return Ok(true); + } + + for (i, language_name) in attributes.languages.iter().enumerate() { + if !language_name.is_empty() { + let language = opts + .languages + .get(language_name.as_ref()) + .ok_or_else(|| anyhow!("Language not found: {language_name}"))?; + parser.set_language(language)?; + } + let start = std::time::Instant::now(); + let tree = parser.parse(&input, None).unwrap(); + let parse_rate = { + let parse_time = start.elapsed(); + let true_parse_rate = tree.root_node().byte_range().len() as f64 + / (parse_time.as_nanos() as f64 / 1_000_000.0); + let adj_parse_rate = adjusted_parse_rate(&tree, parse_time); + + test_summary.parse_stats.total_parses += 1; + test_summary.parse_stats.total_duration += parse_time; + test_summary.parse_stats.total_bytes += tree.root_node().byte_range().len(); + + Some((true_parse_rate, adj_parse_rate)) + }; + + if attributes.error { + if tree.root_node().has_error() { + test_summary.parse_results.add_case(TestResult { + name: name.clone(), + info: TestInfo::ParseTest { + outcome: TestOutcome::Passed, + parse_rate, + test_num: test_summary.test_num, + }, + }); + test_summary.parse_stats.successful_parses += 1; + if opts.update { + let input = String::from_utf8(input.clone()).unwrap(); + let output = if attributes.cst { + output.clone() + } else { + format_sexp(&output, 0) + }; + corrected_entries.push(TestCorrection::new( + &name, + input, + output, + &attributes_str, + header_delim_len, + divider_delim_len, + )); + } + } else { + if opts.update { + let input = String::from_utf8(input.clone()).unwrap(); + // Keep the original `expected` output if the actual output has no error + let output = if attributes.cst { + output.clone() + } else { + format_sexp(&output, 0) + }; + corrected_entries.push(TestCorrection::new( + &name, + input, + output, + &attributes_str, + header_delim_len, + divider_delim_len, + )); + } + test_summary.parse_results.add_case(TestResult { + name: name.clone(), + info: TestInfo::ParseTest { + outcome: TestOutcome::Failed, + parse_rate, + test_num: test_summary.test_num, + }, + }); + let actual = if attributes.cst { + render_test_cst(&input, &tree)? + } else { + tree.root_node().to_sexp() + }; + test_summary.parse_failures.push(TestFailure::new( + &name, + actual, + "NO ERROR", + attributes.cst, + )); + } + + if attributes.fail_fast { + return Ok(false); + } + } else { + let mut actual = if attributes.cst { + render_test_cst(&input, &tree)? + } else { + tree.root_node().to_sexp() + }; + if !(attributes.cst || opts.show_fields || has_fields) { + actual = strip_sexp_fields(&actual); + } + + if actual == output { + test_summary.parse_results.add_case(TestResult { + name: name.clone(), + info: TestInfo::ParseTest { + outcome: TestOutcome::Passed, + parse_rate, + test_num: test_summary.test_num, + }, + }); + test_summary.parse_stats.successful_parses += 1; + if opts.update { + let input = String::from_utf8(input.clone()).unwrap(); + let output = if attributes.cst { + actual + } else { + format_sexp(&output, 0) + }; + corrected_entries.push(TestCorrection::new( + &name, + input, + output, + &attributes_str, + header_delim_len, + divider_delim_len, + )); + } + } else { + if opts.update { + let input = String::from_utf8(input.clone()).unwrap(); + let (expected_output, actual_output) = if attributes.cst { + (output.clone(), actual.clone()) + } else { + (format_sexp(&output, 0), format_sexp(&actual, 0)) + }; + + // Only bail early before updating if the actual is not the output, + // sometimes users want to test cases that + // are intended to have errors, hence why this + // check isn't shown above + if actual.contains("ERROR") || actual.contains("MISSING") { + test_summary.has_parse_errors = true; + + // keep the original `expected` output if the actual output has an + // error + corrected_entries.push(TestCorrection::new( + &name, + input, + expected_output, + &attributes_str, + header_delim_len, + divider_delim_len, + )); + } else { + corrected_entries.push(TestCorrection::new( + &name, + input, + actual_output, + &attributes_str, + header_delim_len, + divider_delim_len, + )); + test_summary.parse_results.add_case(TestResult { + name: name.clone(), + info: TestInfo::ParseTest { + outcome: TestOutcome::Updated, + parse_rate, + test_num: test_summary.test_num, + }, + }); + } + } else { + test_summary.parse_results.add_case(TestResult { + name: name.clone(), + info: TestInfo::ParseTest { + outcome: TestOutcome::Failed, + parse_rate, + test_num: test_summary.test_num, + }, + }); + } + test_summary.parse_failures.push(TestFailure::new( + &name, + actual, + &output, + attributes.cst, + )); + + if attributes.fail_fast { + return Ok(false); + } + } + } + + if i == attributes.languages.len() - 1 { + // reset to the first language + parser.set_language(opts.languages.values().next().unwrap())?; + } + } + test_summary.test_num += 1; + } + TestEntry::Group { + name, + children, + file_path, + } => { + if children.is_empty() { + return Ok(true); + } + + let mut ran_test_in_group = false; + + let matches_filter = |name: &str, file_name: &Option, opts: &TestOptions| { + if let (Some(test_file_path), Some(filter_file_name)) = (file_name, &opts.file_name) + { + if !filter_file_name.eq(test_file_path) { + return false; + } + } + if let Some(include) = &opts.include { + include.is_match(name) + } else if let Some(exclude) = &opts.exclude { + !exclude.is_match(name) + } else { + true + } + }; + + for child in children { + if let TestEntry::Example { + ref name, + ref file_name, + ref input, + ref output, + ref attributes_str, + header_delim_len, + divider_delim_len, + .. + } = child + { + if !matches_filter(name, file_name, opts) { + if opts.update { + let input = String::from_utf8(input.clone()).unwrap(); + let output = format_sexp(output, 0); + corrected_entries.push(TestCorrection::new( + name, + input, + output, + attributes_str, + header_delim_len, + divider_delim_len, + )); + } + + test_summary.test_num += 1; + continue; + } + } + + if !ran_test_in_group && !is_root { + test_summary.parse_results.add_group(&name); + ran_test_in_group = true; + } + if !run_tests(parser, child, opts, test_summary, corrected_entries, false)? { + // fail fast + return Ok(false); + } + } + // Now that we're done traversing the children of the current group, pop + // the index + test_summary.parse_results.pop_traversal(); + + if let Some(file_path) = file_path { + if opts.update { + write_tests(&file_path, corrected_entries)?; + } + corrected_entries.clear(); + } + } + } + Ok(true) +} + +/// Convenience wrapper to render a CST for a test entry. +fn render_test_cst(input: &[u8], tree: &Tree) -> Result { + let mut rendered_cst: Vec = Vec::new(); + let mut cursor = tree.walk(); + let opts = ParseFileOptions { + edits: &[], + output: ParseOutput::Cst, + stats: &mut ParseStats::default(), + print_time: false, + timeout: 0, + debug: ParseDebugType::Quiet, + debug_graph: false, + cancellation_flag: None, + encoding: None, + open_log: false, + no_ranges: false, + parse_theme: &ParseTheme::empty(), + }; + render_cst(input, tree, &mut cursor, &opts, &mut rendered_cst)?; + Ok(String::from_utf8_lossy(&rendered_cst).trim().to_string()) +} + +// Parse time is interpreted in ns before converting to ms to avoid truncation issues +// Parse rates often have several outliers, leading to a large standard deviation. Taking +// the log of these rates serves to "flatten" out the distribution, yielding a more +// usable standard deviation for finding statistically significant slow parse rates +// NOTE: This is just a heuristic +#[must_use] +pub fn adjusted_parse_rate(tree: &Tree, parse_time: Duration) -> f64 { + f64::ln( + tree.root_node().byte_range().len() as f64 / (parse_time.as_nanos() as f64 / 1_000_000.0), + ) +} + +fn write_tests(file_path: &Path, corrected_entries: &[TestCorrection]) -> Result<()> { + let mut buffer = fs::File::create(file_path)?; + write_tests_to_buffer(&mut buffer, corrected_entries) +} + +fn write_tests_to_buffer( + buffer: &mut impl Write, + corrected_entries: &[TestCorrection], +) -> Result<()> { + for ( + i, + TestCorrection { + name, + input, + output, + attributes_str, + header_delim_len, + divider_delim_len, + }, + ) in corrected_entries.iter().enumerate() + { + if i > 0 { + writeln!(buffer)?; + } + writeln!( + buffer, + "{}\n{name}\n{}{}\n{input}\n{}\n\n{}", + "=".repeat(*header_delim_len), + if attributes_str.is_empty() { + attributes_str.clone() + } else { + format!("{attributes_str}\n") + }, + "=".repeat(*header_delim_len), + "-".repeat(*divider_delim_len), + output.trim() + )?; + } + Ok(()) +} + +pub fn parse_tests(path: &Path) -> io::Result { + let name = path + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("") + .to_string(); + if path.is_dir() { + let mut children = Vec::new(); + for entry in fs::read_dir(path)? { + let entry = entry?; + let hidden = entry.file_name().to_str().unwrap_or("").starts_with('.'); + if !hidden { + children.push(entry.path()); + } + } + children.sort_by(|a, b| { + a.file_name() + .unwrap_or_default() + .cmp(b.file_name().unwrap_or_default()) + }); + let children = children + .iter() + .map(|path| parse_tests(path)) + .collect::>>()?; + Ok(TestEntry::Group { + name, + children, + file_path: None, + }) + } else { + let content = fs::read_to_string(path)?; + Ok(parse_test_content(name, &content, Some(path.to_path_buf()))) + } +} + +#[must_use] +pub fn strip_sexp_fields(sexp: &str) -> String { + SEXP_FIELD_REGEX.replace_all(sexp, " (").to_string() +} + +#[must_use] +pub fn strip_points(sexp: &str) -> String { + POINT_REGEX.replace_all(sexp, "").to_string() +} + +fn parse_test_content(name: String, content: &str, file_path: Option) -> TestEntry { + let mut children = Vec::new(); + let bytes = content.as_bytes(); + let mut prev_name = String::new(); + let mut prev_attributes_str = String::new(); + let mut prev_header_end = 0; + + // Find the first test header in the file, and determine if it has a + // custom suffix. If so, then this suffix will be used to identify + // all subsequent headers and divider lines in the file. + let first_suffix = HEADER_REGEX + .captures(bytes) + .and_then(|c| c.name("suffix1")) + .map(|m| String::from_utf8_lossy(m.as_bytes())); + + // Find all of the `===` test headers, which contain the test names. + // Ignore any matches whose suffix does not match the first header + // suffix in the file. + let header_matches = HEADER_REGEX.captures_iter(bytes).filter_map(|c| { + let header_delim_len = c.name("equals").map_or(80, |m| m.as_bytes().len()); + let suffix1 = c + .name("suffix1") + .map(|m| String::from_utf8_lossy(m.as_bytes())); + let suffix2 = c + .name("suffix2") + .map(|m| String::from_utf8_lossy(m.as_bytes())); + + let (mut skip, mut platform, mut fail_fast, mut error, mut cst, mut languages) = + (false, None, false, false, false, vec![]); + + let test_name_and_markers = c + .name("test_name_and_markers") + .map_or("".as_bytes(), |m| m.as_bytes()); + + let mut test_name = String::new(); + let mut attributes_str = String::new(); + + let mut seen_marker = false; + + let test_name_and_markers = str::from_utf8(test_name_and_markers).unwrap(); + for line in test_name_and_markers + .split_inclusive('\n') + .filter(|s| !s.is_empty()) + { + let trimmed_line = line.trim(); + match trimmed_line.split('(').next().unwrap() { + ":skip" => (seen_marker, skip) = (true, true), + ":platform" => { + if let Some(platforms) = trimmed_line.strip_prefix(':').and_then(|s| { + s.strip_prefix("platform(") + .and_then(|s| s.strip_suffix(')')) + }) { + seen_marker = true; + platform = Some( + platform.unwrap_or(false) || platforms.trim() == std::env::consts::OS, + ); + } + } + ":fail-fast" => (seen_marker, fail_fast) = (true, true), + ":error" => (seen_marker, error) = (true, true), + ":language" => { + if let Some(lang) = trimmed_line.strip_prefix(':').and_then(|s| { + s.strip_prefix("language(") + .and_then(|s| s.strip_suffix(')')) + }) { + seen_marker = true; + languages.push(lang.into()); + } + } + ":cst" => (seen_marker, cst) = (true, true), + _ if !seen_marker => { + test_name.push_str(line); + } + _ => {} + } + } + attributes_str.push_str(test_name_and_markers.strip_prefix(&test_name).unwrap()); + + // prefer skip over error, both shouldn't be set + if skip { + error = false; + } + + // add a default language if none are specified, will defer to the first language + if languages.is_empty() { + languages.push("".into()); + } + + if suffix1 == first_suffix && suffix2 == first_suffix { + let header_range = c.get(0).unwrap().range(); + let test_name = if test_name.is_empty() { + None + } else { + Some(test_name.trim_end().to_string()) + }; + let attributes_str = if attributes_str.is_empty() { + None + } else { + Some(attributes_str.trim_end().to_string()) + }; + Some(( + header_delim_len, + header_range, + test_name, + attributes_str, + TestAttributes { + skip, + platform: platform.unwrap_or(true), + fail_fast, + error, + cst, + languages, + }, + )) + } else { + None + } + }); + + let (mut prev_header_len, mut prev_attributes) = (80, TestAttributes::default()); + for (header_delim_len, header_range, test_name, attributes_str, attributes) in header_matches + .chain(Some(( + 80, + bytes.len()..bytes.len(), + None, + None, + TestAttributes::default(), + ))) + { + // Find the longest line of dashes following each test description. That line + // separates the input from the expected output. Ignore any matches whose suffix + // does not match the first suffix in the file. + if prev_header_end > 0 { + let divider_range = DIVIDER_REGEX + .captures_iter(&bytes[prev_header_end..header_range.start]) + .filter_map(|m| { + let divider_delim_len = m.name("hyphens").map_or(80, |m| m.as_bytes().len()); + let suffix = m + .name("suffix") + .map(|m| String::from_utf8_lossy(m.as_bytes())); + if suffix == first_suffix { + let range = m.get(0).unwrap().range(); + Some(( + divider_delim_len, + (prev_header_end + range.start)..(prev_header_end + range.end), + )) + } else { + None + } + }) + .max_by_key(|(_, range)| range.len()); + + if let Some((divider_delim_len, divider_range)) = divider_range { + if let Ok(output) = str::from_utf8(&bytes[divider_range.end..header_range.start]) { + let mut input = bytes[prev_header_end..divider_range.start].to_vec(); + + // Remove trailing newline from the input. + input.pop(); + if input.last() == Some(&b'\r') { + input.pop(); + } + + let (output, has_fields) = if prev_attributes.cst { + (output.trim().to_string(), false) + } else { + // Remove all comments + let output = COMMENT_REGEX.replace_all(output, "").to_string(); + + // Normalize the whitespace in the expected output. + let output = WHITESPACE_REGEX.replace_all(output.trim(), " "); + let output = output.replace(" )", ")"); + + // Identify if the expected output has fields indicated. If not, then + // fields will not be checked. + let has_fields = SEXP_FIELD_REGEX.is_match(&output); + + (output, has_fields) + }; + + let file_name = if let Some(ref path) = file_path { + path.file_name().map(|n| n.to_string_lossy().to_string()) + } else { + None + }; + + let t = TestEntry::Example { + name: prev_name, + input, + output, + header_delim_len: prev_header_len, + divider_delim_len, + has_fields, + attributes_str: prev_attributes_str, + attributes: prev_attributes, + file_name, + }; + + children.push(t); + } + } + } + prev_attributes = attributes; + prev_name = test_name.unwrap_or_default(); + prev_attributes_str = attributes_str.unwrap_or_default(); + prev_header_len = header_delim_len; + prev_header_end = header_range.end; + } + TestEntry::Group { + name, + children, + file_path, + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use crate::tests::get_language; + + use super::*; + + #[test] + fn test_parse_test_content_simple() { + let entry = parse_test_content( + "the-filename".to_string(), + r" +=============== +The first test +=============== + +a b c + +--- + +(a + (b c)) + +================ +The second test +================ +d +--- +(d) + " + .trim(), + None, + ); + + assert_eq!( + entry, + TestEntry::Group { + name: "the-filename".to_string(), + children: vec![ + TestEntry::Example { + name: "The first test".to_string(), + input: b"\na b c\n".to_vec(), + output: "(a (b c))".to_string(), + header_delim_len: 15, + divider_delim_len: 3, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + }, + TestEntry::Example { + name: "The second test".to_string(), + input: b"d".to_vec(), + output: "(d)".to_string(), + header_delim_len: 16, + divider_delim_len: 3, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + }, + ], + file_path: None, + } + ); + } + + #[test] + fn test_parse_test_content_with_dashes_in_source_code() { + let entry = parse_test_content( + "the-filename".to_string(), + r" +================== +Code with dashes +================== +abc +--- +defg +---- +hijkl +------- + +(a (b)) + +========================= +Code ending with dashes +========================= +abc +----------- +------------------- + +(c (d)) + " + .trim(), + None, + ); + + assert_eq!( + entry, + TestEntry::Group { + name: "the-filename".to_string(), + children: vec![ + TestEntry::Example { + name: "Code with dashes".to_string(), + input: b"abc\n---\ndefg\n----\nhijkl".to_vec(), + output: "(a (b))".to_string(), + header_delim_len: 18, + divider_delim_len: 7, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + }, + TestEntry::Example { + name: "Code ending with dashes".to_string(), + input: b"abc\n-----------".to_vec(), + output: "(c (d))".to_string(), + header_delim_len: 25, + divider_delim_len: 19, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + }, + ], + file_path: None, + } + ); + } + + #[test] + fn test_format_sexp() { + assert_eq!(format_sexp("", 0), ""); + assert_eq!( + format_sexp("(a b: (c) (d) e: (f (g (h (MISSING i)))))", 0), + r" +(a + b: (c) + (d) + e: (f + (g + (h + (MISSING i))))) +" + .trim() + ); + assert_eq!( + format_sexp("(program (ERROR (UNEXPECTED ' ')) (identifier))", 0), + r" +(program + (ERROR + (UNEXPECTED ' ')) + (identifier)) +" + .trim() + ); + assert_eq!( + format_sexp(r#"(source_file (MISSING ")"))"#, 0), + r#" +(source_file + (MISSING ")")) + "# + .trim() + ); + assert_eq!( + format_sexp( + r"(source_file (ERROR (UNEXPECTED 'f') (UNEXPECTED '+')))", + 0 + ), + r" +(source_file + (ERROR + (UNEXPECTED 'f') + (UNEXPECTED '+'))) +" + .trim() + ); + } + + #[test] + fn test_write_tests_to_buffer() { + let mut buffer = Vec::new(); + let corrected_entries = vec![ + TestCorrection::new( + "title 1".to_string(), + "input 1".to_string(), + "output 1".to_string(), + String::new(), + 80, + 80, + ), + TestCorrection::new( + "title 2".to_string(), + "input 2".to_string(), + "output 2".to_string(), + String::new(), + 80, + 80, + ), + ]; + write_tests_to_buffer(&mut buffer, &corrected_entries).unwrap(); + assert_eq!( + String::from_utf8(buffer).unwrap(), + r" +================================================================================ +title 1 +================================================================================ +input 1 +-------------------------------------------------------------------------------- + +output 1 + +================================================================================ +title 2 +================================================================================ +input 2 +-------------------------------------------------------------------------------- + +output 2 +" + .trim_start() + .to_string() + ); + } + + #[test] + fn test_parse_test_content_with_comments_in_sexp() { + let entry = parse_test_content( + "the-filename".to_string(), + r#" +================== +sexp with comment +================== +code +--- + +; Line start comment +(a (b)) + +================== +sexp with comment between +================== +code +--- + +; Line start comment +(a +; ignore this + (b) + ; also ignore this +) + +========================= +sexp with ';' +========================= +code +--- + +(MISSING ";") + "# + .trim(), + None, + ); + + assert_eq!( + entry, + TestEntry::Group { + name: "the-filename".to_string(), + children: vec![ + TestEntry::Example { + name: "sexp with comment".to_string(), + input: b"code".to_vec(), + output: "(a (b))".to_string(), + header_delim_len: 18, + divider_delim_len: 3, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + }, + TestEntry::Example { + name: "sexp with comment between".to_string(), + input: b"code".to_vec(), + output: "(a (b))".to_string(), + header_delim_len: 18, + divider_delim_len: 3, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + }, + TestEntry::Example { + name: "sexp with ';'".to_string(), + input: b"code".to_vec(), + output: "(MISSING \";\")".to_string(), + header_delim_len: 25, + divider_delim_len: 3, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + } + ], + file_path: None, + } + ); + } + + #[test] + fn test_parse_test_content_with_suffixes() { + let entry = parse_test_content( + "the-filename".to_string(), + r" +==================asdf\()[]|{}*+?^$.- +First test +==================asdf\()[]|{}*+?^$.- + +========================= +NOT A TEST HEADER +========================= +------------------------- + +---asdf\()[]|{}*+?^$.- + +(a) + +==================asdf\()[]|{}*+?^$.- +Second test +==================asdf\()[]|{}*+?^$.- + +========================= +NOT A TEST HEADER +========================= +------------------------- + +---asdf\()[]|{}*+?^$.- + +(a) + +=========================asdf\()[]|{}*+?^$.- +Test name with = symbol +=========================asdf\()[]|{}*+?^$.- + +========================= +NOT A TEST HEADER +========================= +------------------------- + +---asdf\()[]|{}*+?^$.- + +(a) + +==============================asdf\()[]|{}*+?^$.- +Test containing equals +==============================asdf\()[]|{}*+?^$.- + +=== + +------------------------------asdf\()[]|{}*+?^$.- + +(a) + +==============================asdf\()[]|{}*+?^$.- +Subsequent test containing equals +==============================asdf\()[]|{}*+?^$.- + +=== + +------------------------------asdf\()[]|{}*+?^$.- + +(a) +" + .trim(), + None, + ); + + let expected_input = b"\n=========================\n\ + NOT A TEST HEADER\n\ + =========================\n\ + -------------------------\n" + .to_vec(); + pretty_assertions::assert_eq!( + entry, + TestEntry::Group { + name: "the-filename".to_string(), + children: vec![ + TestEntry::Example { + name: "First test".to_string(), + input: expected_input.clone(), + output: "(a)".to_string(), + header_delim_len: 18, + divider_delim_len: 3, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + }, + TestEntry::Example { + name: "Second test".to_string(), + input: expected_input.clone(), + output: "(a)".to_string(), + header_delim_len: 18, + divider_delim_len: 3, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + }, + TestEntry::Example { + name: "Test name with = symbol".to_string(), + input: expected_input, + output: "(a)".to_string(), + header_delim_len: 25, + divider_delim_len: 3, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + }, + TestEntry::Example { + name: "Test containing equals".to_string(), + input: "\n===\n".into(), + output: "(a)".into(), + header_delim_len: 30, + divider_delim_len: 30, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + }, + TestEntry::Example { + name: "Subsequent test containing equals".to_string(), + input: "\n===\n".into(), + output: "(a)".into(), + header_delim_len: 30, + divider_delim_len: 30, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + } + ], + file_path: None, + } + ); + } + + #[test] + fn test_parse_test_content_with_newlines_in_test_names() { + let entry = parse_test_content( + "the-filename".to_string(), + r" +=============== +name +with +newlines +=============== +a +--- +(b) + +==================== +name with === signs +==================== +code with ---- +--- +(d) +", + None, + ); + + assert_eq!( + entry, + TestEntry::Group { + name: "the-filename".to_string(), + file_path: None, + children: vec![ + TestEntry::Example { + name: "name\nwith\nnewlines".to_string(), + input: b"a".to_vec(), + output: "(b)".to_string(), + header_delim_len: 15, + divider_delim_len: 3, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + }, + TestEntry::Example { + name: "name with === signs".to_string(), + input: b"code with ----".to_vec(), + output: "(d)".to_string(), + header_delim_len: 20, + divider_delim_len: 3, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + } + ] + } + ); + } + + #[test] + fn test_parse_test_with_markers() { + // do one with :skip, we should not see it in the entry output + + let entry = parse_test_content( + "the-filename".to_string(), + r" +===================== +Test with skip marker +:skip +===================== +a +--- +(b) +", + None, + ); + + assert_eq!( + entry, + TestEntry::Group { + name: "the-filename".to_string(), + file_path: None, + children: vec![TestEntry::Example { + name: "Test with skip marker".to_string(), + input: b"a".to_vec(), + output: "(b)".to_string(), + header_delim_len: 21, + divider_delim_len: 3, + has_fields: false, + attributes_str: ":skip".to_string(), + attributes: TestAttributes { + skip: true, + platform: true, + fail_fast: false, + error: false, + cst: false, + languages: vec!["".into()] + }, + file_name: None, + }] + } + ); + + let entry = parse_test_content( + "the-filename".to_string(), + &format!( + r" +========================= +Test with platform marker +:platform({}) +:fail-fast +========================= +a +--- +(b) + +============================= +Test with bad platform marker +:platform({}) + +:language(foo) +============================= +a +--- +(b) + +==================== +Test with cst marker +:cst +==================== +1 +--- +0:0 - 1:0 source_file +0:0 - 0:1 expression +0:0 - 0:1 number_literal `1` +", + std::env::consts::OS, + if std::env::consts::OS == "linux" { + "macos" + } else { + "linux" + } + ), + None, + ); + + assert_eq!( + entry, + TestEntry::Group { + name: "the-filename".to_string(), + file_path: None, + children: vec![ + TestEntry::Example { + name: "Test with platform marker".to_string(), + input: b"a".to_vec(), + output: "(b)".to_string(), + header_delim_len: 25, + divider_delim_len: 3, + has_fields: false, + attributes_str: format!(":platform({})\n:fail-fast", std::env::consts::OS), + attributes: TestAttributes { + skip: false, + platform: true, + fail_fast: true, + error: false, + cst: false, + languages: vec!["".into()] + }, + file_name: None, + }, + TestEntry::Example { + name: "Test with bad platform marker".to_string(), + input: b"a".to_vec(), + output: "(b)".to_string(), + header_delim_len: 29, + divider_delim_len: 3, + has_fields: false, + attributes_str: if std::env::consts::OS == "linux" { + ":platform(macos)\n\n:language(foo)".to_string() + } else { + ":platform(linux)\n\n:language(foo)".to_string() + }, + attributes: TestAttributes { + skip: false, + platform: false, + fail_fast: false, + error: false, + cst: false, + languages: vec!["foo".into()] + }, + file_name: None, + }, + TestEntry::Example { + name: "Test with cst marker".to_string(), + input: b"1".to_vec(), + output: "0:0 - 1:0 source_file +0:0 - 0:1 expression +0:0 - 0:1 number_literal `1`" + .to_string(), + header_delim_len: 20, + divider_delim_len: 3, + has_fields: false, + attributes_str: ":cst".to_string(), + attributes: TestAttributes { + skip: false, + platform: true, + fail_fast: false, + error: false, + cst: true, + languages: vec!["".into()] + }, + file_name: None, + } + ] + } + ); + } + + fn clear_parse_rate(result: &mut TestResult) { + let test_case_info = &mut result.info; + match test_case_info { + TestInfo::ParseTest { + ref mut parse_rate, .. + } => { + assert!(parse_rate.is_some()); + *parse_rate = None; + } + TestInfo::Group { .. } | TestInfo::AssertionTest { .. } => { + panic!("Unexpected test result") + } + } + } + + #[test] + fn run_tests_simple() { + let mut parser = Parser::new(); + let language = get_language("c"); + parser + .set_language(&language) + .expect("Failed to set language"); + let mut languages = BTreeMap::new(); + languages.insert("c", &language); + let opts = TestOptions { + path: PathBuf::from("foo"), + debug: true, + debug_graph: false, + include: None, + exclude: None, + file_name: None, + update: false, + open_log: false, + languages, + color: true, + show_fields: false, + overview_only: false, + }; + + // NOTE: The following test cases are combined to work around a race condition + // in the loader + { + let test_entry = TestEntry::Group { + name: "foo".to_string(), + file_path: None, + children: vec![TestEntry::Example { + name: "C Test 1".to_string(), + input: b"1;\n".to_vec(), + output: "(translation_unit (expression_statement (number_literal)))" + .to_string(), + header_delim_len: 25, + divider_delim_len: 3, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + }], + }; + + let mut test_summary = TestSummary::new(true, TestStats::All, false, false, false); + let mut corrected_entries = Vec::new(); + run_tests( + &mut parser, + test_entry, + &opts, + &mut test_summary, + &mut corrected_entries, + true, + ) + .expect("Failed to run tests"); + + // parse rates will always be different, so we need to clear out these + // fields to reliably assert equality below + clear_parse_rate(&mut test_summary.parse_results.root_group[0]); + test_summary.parse_stats.total_duration = Duration::from_secs(0); + + let json_results = serde_json::to_string(&test_summary).unwrap(); + + assert_eq!( + json_results, + json!({ + "parse_results": [ + { + "name": "C Test 1", + "outcome": "Passed", + "parse_rate": null, + "test_num": 1 + } + ], + "parse_failures": [], + "parse_stats": { + "successful_parses": 1, + "total_parses": 1, + "total_bytes": 3, + "total_duration": { + "secs": 0, + "nanos": 0, + } + }, + "highlight_results": [], + "tag_results": [], + "query_results": [] + }) + .to_string() + ); + } + { + let test_entry = TestEntry::Group { + name: "corpus".to_string(), + file_path: None, + children: vec![ + TestEntry::Group { + name: "group1".to_string(), + // This test passes + children: vec![TestEntry::Example { + name: "C Test 1".to_string(), + input: b"1;\n".to_vec(), + output: "(translation_unit (expression_statement (number_literal)))" + .to_string(), + header_delim_len: 25, + divider_delim_len: 3, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + }], + file_path: None, + }, + TestEntry::Group { + name: "group2".to_string(), + children: vec![ + // This test passes + TestEntry::Example { + name: "C Test 2".to_string(), + input: b"1;\n".to_vec(), + output: + "(translation_unit (expression_statement (number_literal)))" + .to_string(), + header_delim_len: 25, + divider_delim_len: 3, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + }, + // This test fails, and is marked with fail-fast + TestEntry::Example { + name: "C Test 3".to_string(), + input: b"1;\n".to_vec(), + output: + "(translation_unit (expression_statement (string_literal)))" + .to_string(), + header_delim_len: 25, + divider_delim_len: 3, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes { + fail_fast: true, + ..Default::default() + }, + file_name: None, + }, + ], + file_path: None, + }, + // This group never runs because of the previous failure + TestEntry::Group { + name: "group3".to_string(), + // This test fails, and is marked with fail-fast + children: vec![TestEntry::Example { + name: "C Test 4".to_string(), + input: b"1;\n".to_vec(), + output: "(translation_unit (expression_statement (number_literal)))" + .to_string(), + header_delim_len: 25, + divider_delim_len: 3, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + file_name: None, + }], + file_path: None, + }, + ], + }; + + let mut test_summary = TestSummary::new(true, TestStats::All, false, false, false); + let mut corrected_entries = Vec::new(); + run_tests( + &mut parser, + test_entry, + &opts, + &mut test_summary, + &mut corrected_entries, + true, + ) + .expect("Failed to run tests"); + + // parse rates will always be different, so we need to clear out these + // fields to reliably assert equality below + { + let test_group_1_info = &mut test_summary.parse_results.root_group[0].info; + match test_group_1_info { + TestInfo::Group { + ref mut children, .. + } => clear_parse_rate(&mut children[0]), + TestInfo::ParseTest { .. } | TestInfo::AssertionTest { .. } => { + panic!("Unexpected test result"); + } + } + let test_group_2_info = &mut test_summary.parse_results.root_group[1].info; + match test_group_2_info { + TestInfo::Group { + ref mut children, .. + } => { + clear_parse_rate(&mut children[0]); + clear_parse_rate(&mut children[1]); + } + TestInfo::ParseTest { .. } | TestInfo::AssertionTest { .. } => { + panic!("Unexpected test result"); + } + } + test_summary.parse_stats.total_duration = Duration::from_secs(0); + } + + let json_results = serde_json::to_string(&test_summary).unwrap(); + + assert_eq!( + json_results, + json!({ + "parse_results": [ + { + "name": "group1", + "children": [ + { + "name": "C Test 1", + "outcome": "Passed", + "parse_rate": null, + "test_num": 1 + } + ] + }, + { + "name": "group2", + "children": [ + { + "name": "C Test 2", + "outcome": "Passed", + "parse_rate": null, + "test_num": 2 + }, + { + "name": "C Test 3", + "outcome": "Failed", + "parse_rate": null, + "test_num": 3 + } + ] + } + ], + "parse_failures": [ + { + "name": "C Test 3", + "actual": "(translation_unit (expression_statement (number_literal)))", + "expected": "(translation_unit (expression_statement (string_literal)))", + "is_cst": false, + } + ], + "parse_stats": { + "successful_parses": 2, + "total_parses": 3, + "total_bytes": 9, + "total_duration": { + "secs": 0, + "nanos": 0, + } + }, + "highlight_results": [], + "tag_results": [], + "query_results": [] + }) + .to_string() + ); + } + } +} diff --git a/cli/src/test_highlight.rs b/crates/cli/src/test_highlight.rs similarity index 81% rename from cli/src/test_highlight.rs rename to crates/cli/src/test_highlight.rs index 34be438f..d96f90c2 100644 --- a/cli/src/test_highlight.rs +++ b/crates/cli/src/test_highlight.rs @@ -1,14 +1,13 @@ use std::{fs, path::Path}; -use anstyle::AnsiColor; use anyhow::{anyhow, Result}; use tree_sitter::Point; use tree_sitter_highlight::{Highlight, HighlightConfiguration, HighlightEvent, Highlighter}; use tree_sitter_loader::{Config, Loader}; -use super::{ +use crate::{ query_testing::{parse_position_comments, to_utf8_point, Assertion, Utf8Point}, - test::paint, + test::{TestInfo, TestOutcome, TestResult, TestSummary}, util, }; @@ -48,19 +47,7 @@ pub fn test_highlights( loader_config: &Config, highlighter: &mut Highlighter, directory: &Path, - use_color: bool, -) -> Result<()> { - println!("syntax highlighting:"); - test_highlights_indented(loader, loader_config, highlighter, directory, use_color, 2) -} - -fn test_highlights_indented( - loader: &Loader, - loader_config: &Config, - highlighter: &mut Highlighter, - directory: &Path, - use_color: bool, - indent_level: usize, + test_summary: &mut TestSummary, ) -> Result<()> { let mut failed = false; @@ -68,25 +55,22 @@ fn test_highlights_indented( let highlight_test_file = highlight_test_file?; let test_file_path = highlight_test_file.path(); let test_file_name = highlight_test_file.file_name(); - print!( - "{indent:indent_level$}", - indent = "", - indent_level = indent_level * 2 - ); if test_file_path.is_dir() && test_file_path.read_dir()?.next().is_some() { - println!("{}:", test_file_name.into_string().unwrap()); - if test_highlights_indented( + test_summary + .highlight_results + .add_group(test_file_name.to_string_lossy().as_ref()); + if test_highlights( loader, loader_config, highlighter, &test_file_path, - use_color, - indent_level + 1, + test_summary, ) .is_err() { failed = true; } + test_summary.highlight_results.pop_traversal(); } else { let (language, language_config) = loader .language_configuration_for_file_name(&test_file_path)? @@ -98,7 +82,12 @@ fn test_highlights_indented( })?; let highlight_config = language_config .highlight_config(language, None)? - .ok_or_else(|| anyhow!("No highlighting config found for {test_file_path:?}"))?; + .ok_or_else(|| { + anyhow!( + "No highlighting config found for {}", + test_file_path.display() + ) + })?; match test_highlight( loader, highlighter, @@ -106,30 +95,28 @@ fn test_highlights_indented( fs::read(&test_file_path)?.as_slice(), ) { Ok(assertion_count) => { - println!( - "✓ {} ({assertion_count} assertions)", - paint( - use_color.then_some(AnsiColor::Green), - test_file_name.to_string_lossy().as_ref() - ), - ); + test_summary.highlight_results.add_case(TestResult { + name: test_file_name.to_string_lossy().to_string(), + info: TestInfo::AssertionTest { + outcome: TestOutcome::AssertionPassed { assertion_count }, + test_num: test_summary.test_num, + }, + }); } Err(e) => { - println!( - "✗ {}", - paint( - use_color.then_some(AnsiColor::Red), - test_file_name.to_string_lossy().as_ref() - ) - ); - println!( - "{indent:indent_level$} {e}", - indent = "", - indent_level = indent_level * 2 - ); + test_summary.highlight_results.add_case(TestResult { + name: test_file_name.to_string_lossy().to_string(), + info: TestInfo::AssertionTest { + outcome: TestOutcome::AssertionFailed { + error: e.to_string(), + }, + test_num: test_summary.test_num, + }, + }); failed = true; } } + test_summary.test_num += 1; } } @@ -150,11 +137,13 @@ pub fn iterate_assertions( let mut actual_highlights = Vec::new(); for Assertion { position, + length, negative, expected_capture_name: expected_highlight, } in assertions { let mut passed = false; + let mut end_column = position.column + length - 1; actual_highlights.clear(); // The assertions are ordered by position, so skip past all of the highlights that @@ -165,11 +154,12 @@ pub fn iterate_assertions( continue; } - // Iterate through all of the highlights that start at or before this assertion's, + // Iterate through all of the highlights that start at or before this assertion's // position, looking for one that matches the assertion. let mut j = i; while let (false, Some(highlight)) = (passed, highlights.get(j)) { - if highlight.0 > *position { + end_column = position.column + length - 1; + if highlight.0.row >= position.row && highlight.0.column > end_column { break 'highlight_loop; } @@ -193,7 +183,7 @@ pub fn iterate_assertions( if !passed { return Err(Failure { row: position.row, - column: position.column, + column: end_column, expected_highlight: expected_highlight.clone(), actual_highlights: actual_highlights.into_iter().cloned().collect(), } diff --git a/cli/src/test_tags.rs b/crates/cli/src/test_tags.rs similarity index 62% rename from cli/src/test_tags.rs rename to crates/cli/src/test_tags.rs index 5b290bda..882718e5 100644 --- a/cli/src/test_tags.rs +++ b/crates/cli/src/test_tags.rs @@ -1,13 +1,12 @@ use std::{fs, path::Path}; -use anstyle::AnsiColor; use anyhow::{anyhow, Result}; use tree_sitter_loader::{Config, Loader}; use tree_sitter_tags::{TagsConfiguration, TagsContext}; -use super::{ +use crate::{ query_testing::{parse_position_comments, to_utf8_point, Assertion, Utf8Point}, - test::paint, + test::{TestInfo, TestOutcome, TestResult, TestSummary}, util, }; @@ -47,50 +46,70 @@ pub fn test_tags( loader_config: &Config, tags_context: &mut TagsContext, directory: &Path, - use_color: bool, + test_summary: &mut TestSummary, ) -> Result<()> { let mut failed = false; - println!("tags:"); + for tag_test_file in fs::read_dir(directory)? { let tag_test_file = tag_test_file?; let test_file_path = tag_test_file.path(); let test_file_name = tag_test_file.file_name(); - let (language, language_config) = loader - .language_configuration_for_file_name(&test_file_path)? - .ok_or_else(|| { - anyhow!( - "{}", - util::lang_not_found_for_path(test_file_path.as_path(), loader_config) - ) - })?; - let tags_config = language_config - .tags_config(language)? - .ok_or_else(|| anyhow!("No tags config found for {:?}", test_file_path))?; - match test_tag( - tags_context, - tags_config, - fs::read(&test_file_path)?.as_slice(), - ) { - Ok(assertion_count) => { - println!( - " ✓ {} ({assertion_count} assertions)", - paint( - use_color.then_some(AnsiColor::Green), - test_file_name.to_string_lossy().as_ref() - ), - ); - } - Err(e) => { - println!( - " ✗ {}", - paint( - use_color.then_some(AnsiColor::Red), - test_file_name.to_string_lossy().as_ref() - ) - ); - println!(" {e}"); + if test_file_path.is_dir() && test_file_path.read_dir()?.next().is_some() { + test_summary + .tag_results + .add_group(test_file_name.to_string_lossy().as_ref()); + if test_tags( + loader, + loader_config, + tags_context, + &test_file_path, + test_summary, + ) + .is_err() + { failed = true; } + test_summary.tag_results.pop_traversal(); + } else { + let (language, language_config) = loader + .language_configuration_for_file_name(&test_file_path)? + .ok_or_else(|| { + anyhow!( + "{}", + util::lang_not_found_for_path(test_file_path.as_path(), loader_config) + ) + })?; + let tags_config = language_config + .tags_config(language)? + .ok_or_else(|| anyhow!("No tags config found for {}", test_file_path.display()))?; + match test_tag( + tags_context, + tags_config, + fs::read(&test_file_path)?.as_slice(), + ) { + Ok(assertion_count) => { + test_summary.tag_results.add_case(TestResult { + name: test_file_name.to_string_lossy().to_string(), + info: TestInfo::AssertionTest { + outcome: TestOutcome::AssertionPassed { assertion_count }, + test_num: test_summary.test_num, + }, + }); + } + Err(e) => { + test_summary.tag_results.add_case(TestResult { + name: test_file_name.to_string_lossy().to_string(), + info: TestInfo::AssertionTest { + outcome: TestOutcome::AssertionFailed { + error: e.to_string(), + }, + test_num: test_summary.test_num, + }, + }); + failed = true; + } + } + test_summary.test_num += 1; } } @@ -114,11 +133,13 @@ pub fn test_tag( let mut actual_tags = Vec::<&String>::new(); for Assertion { position, + length, negative, expected_capture_name: expected_tag, } in &assertions { let mut passed = false; + let mut end_column = position.column + length - 1; 'tag_loop: while let Some(tag) = tags.get(i) { if tag.1 <= *position { @@ -130,7 +151,8 @@ pub fn test_tag( // position, looking for one that matches the assertion let mut j = i; while let (false, Some(tag)) = (passed, tags.get(j)) { - if tag.0 > *position { + end_column = position.column + length - 1; + if tag.0.column > end_column { break 'tag_loop; } @@ -152,7 +174,7 @@ pub fn test_tag( if !passed { return Err(Failure { row: position.row, - column: position.column, + column: end_column, expected_tag: expected_tag.clone(), actual_tags: actual_tags.into_iter().cloned().collect(), } diff --git a/crates/cli/src/tests.rs b/crates/cli/src/tests.rs new file mode 100644 index 00000000..2439be38 --- /dev/null +++ b/crates/cli/src/tests.rs @@ -0,0 +1,35 @@ +mod async_boundary_test; +mod corpus_test; +mod detect_language; +mod helpers; +mod highlight_test; +mod language_test; +mod node_test; +mod parser_test; +mod pathological_test; +mod query_test; +mod tags_test; +mod test_highlight_test; +mod test_tags_test; +mod text_provider_test; +mod tree_test; + +#[cfg(feature = "wasm")] +mod wasm_language_test; + +use tree_sitter_generate::GenerateResult; + +pub use crate::fuzz::{ + allocations, + edits::{get_random_edit, invert_edit}, + random::Rand, + ITERATION_COUNT, +}; + +pub use helpers::fixtures::get_language; + +/// This is a simple wrapper around [`tree_sitter_generate::generate_parser_for_grammar`], because +/// our tests do not need to pass in a version number, only the grammar JSON. +fn generate_parser(grammar_json: &str) -> GenerateResult<(String, String)> { + tree_sitter_generate::generate_parser_for_grammar(grammar_json, Some((0, 0, 0))) +} diff --git a/crates/cli/src/tests/async_boundary_test.rs b/crates/cli/src/tests/async_boundary_test.rs new file mode 100644 index 00000000..254ed931 --- /dev/null +++ b/crates/cli/src/tests/async_boundary_test.rs @@ -0,0 +1,150 @@ +use std::{ + future::Future, + pin::Pin, + ptr, + task::{Context, Poll, RawWaker, RawWakerVTable, Waker}, +}; + +use tree_sitter::Parser; + +use super::helpers::fixtures::get_language; + +#[test] +fn test_node_across_async_boundaries() { + let mut parser = Parser::new(); + let language = get_language("bash"); + parser.set_language(&language).unwrap(); + let tree = parser.parse("#", None).unwrap(); + let root = tree.root_node(); + + let (result, yields) = simple_async_executor(async { + let root_ref = &root; + + // Test node captured by value + let fut_by_value = async { + yield_once().await; + root.child(0).unwrap().kind() + }; + + // Test node captured by reference + let fut_by_ref = async { + yield_once().await; + root_ref.child(0).unwrap().kind() + }; + + let result1 = fut_by_value.await; + let result2 = fut_by_ref.await; + + assert_eq!(result1, result2); + result1 + }); + + assert_eq!(result, "comment"); + assert_eq!(yields, 2); +} + +#[test] +fn test_cursor_across_async_boundaries() { + let mut parser = Parser::new(); + let language = get_language("c"); + parser.set_language(&language).unwrap(); + let tree = parser.parse("#", None).unwrap(); + let mut cursor = tree.walk(); + + let ((), yields) = simple_async_executor(async { + cursor.goto_first_child(); + + // Test cursor usage across yield point + yield_once().await; + cursor.goto_first_child(); + + // Test cursor in async block + let cursor_ref = &mut cursor; + let fut = async { + yield_once().await; + cursor_ref.goto_first_child(); + }; + fut.await; + }); + + assert_eq!(yields, 2); +} + +#[test] +fn test_node_and_cursor_together() { + let mut parser = Parser::new(); + let language = get_language("javascript"); + parser.set_language(&language).unwrap(); + let tree = parser.parse("#", None).unwrap(); + let root = tree.root_node(); + let mut cursor = tree.walk(); + + let ((), yields) = simple_async_executor(async { + cursor.goto_first_child(); + + let fut = async { + yield_once().await; + let _ = root.to_sexp(); + cursor.goto_first_child(); + }; + + yield_once().await; + fut.await; + }); + + assert_eq!(yields, 2); +} + +fn simple_async_executor(future: F) -> (F::Output, u32) +where + F: Future, +{ + let waker = noop_waker(); + let mut cx = Context::from_waker(&waker); + let mut yields = 0; + let mut future = Box::pin(future); + + loop { + match future.as_mut().poll(&mut cx) { + Poll::Ready(result) => return (result, yields), + Poll::Pending => yields += 1, + } + } +} + +async fn yield_once() { + struct YieldOnce { + yielded: bool, + } + + impl Future for YieldOnce { + type Output = (); + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { + if self.yielded { + Poll::Ready(()) + } else { + self.yielded = true; + cx.waker().wake_by_ref(); + Poll::Pending + } + } + } + + YieldOnce { yielded: false }.await; +} + +const fn noop_waker() -> Waker { + const VTABLE: RawWakerVTable = RawWakerVTable::new( + // Cloning just returns a new no-op raw waker + |_| RAW, + // `wake` does nothing + |_| {}, + // `wake_by_ref` does nothing + |_| {}, + // Dropping does nothing as we don't allocate anything + |_| {}, + ); + const RAW: RawWaker = RawWaker::new(ptr::null(), &VTABLE); + unsafe { Waker::from_raw(RAW) } +} diff --git a/cli/src/tests/corpus_test.rs b/crates/cli/src/tests/corpus_test.rs similarity index 87% rename from cli/src/tests/corpus_test.rs rename to crates/cli/src/tests/corpus_test.rs index 83760467..ba3fd68e 100644 --- a/cli/src/tests/corpus_test.rs +++ b/crates/cli/src/tests/corpus_test.rs @@ -1,5 +1,6 @@ use std::{collections::HashMap, env, fs}; +use anyhow::Context; use tree_sitter::Parser; use tree_sitter_proc_macro::test_with_seed; @@ -15,7 +16,7 @@ use crate::{ LOG_GRAPH_ENABLED, START_SEED, }, parse::perform_edit, - test::{parse_tests, print_diff, print_diff_key, strip_sexp_fields}, + test::{parse_tests, strip_sexp_fields, DiffKey, TestDiff}, tests::{ allocations, helpers::fixtures::{fixtures_dir, get_language, get_test_language, SCRATCH_BASE_DIR}, @@ -23,7 +24,7 @@ use crate::{ }; #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_bash(seed: usize) { +fn test_corpus_for_bash_language(seed: usize) { test_language_corpus( "bash", seed, @@ -39,73 +40,77 @@ fn test_corpus_for_bash(seed: usize) { } #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_c(seed: usize) { +fn test_corpus_for_c_language(seed: usize) { test_language_corpus("c", seed, None, None); } #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_cpp(seed: usize) { +fn test_corpus_for_cpp_language(seed: usize) { test_language_corpus("cpp", seed, None, None); } #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_embedded_template(seed: usize) { +fn test_corpus_for_embedded_template_language(seed: usize) { test_language_corpus("embedded-template", seed, None, None); } #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_go(seed: usize) { +fn test_corpus_for_go_language(seed: usize) { test_language_corpus("go", seed, None, None); } #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_html(seed: usize) { +fn test_corpus_for_html_language(seed: usize) { test_language_corpus("html", seed, None, None); } #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_java(seed: usize) { - test_language_corpus("java", seed, None, None); +fn test_corpus_for_java_language(seed: usize) { + test_language_corpus( + "java", + seed, + Some(&["java - corpus - expressions - switch with unnamed pattern variable"]), + None, + ); } #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_javascript(seed: usize) { +fn test_corpus_for_javascript_language(seed: usize) { test_language_corpus("javascript", seed, None, None); } #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_json(seed: usize) { +fn test_corpus_for_json_language(seed: usize) { test_language_corpus("json", seed, None, None); } -#[ignore] #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_php(seed: usize) { - test_language_corpus("php", seed, None, None); +fn test_corpus_for_php_language(seed: usize) { + test_language_corpus("php", seed, None, Some("php")); } #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_python(seed: usize) { +fn test_corpus_for_python_language(seed: usize) { test_language_corpus("python", seed, None, None); } #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_ruby(seed: usize) { +fn test_corpus_for_ruby_language(seed: usize) { test_language_corpus("ruby", seed, None, None); } #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_rust(seed: usize) { +fn test_corpus_for_rust_language(seed: usize) { test_language_corpus("rust", seed, None, None); } #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_typescript(seed: usize) { +fn test_corpus_for_typescript_language(seed: usize) { test_language_corpus("typescript", seed, None, Some("typescript")); } #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_tsx(seed: usize) { +fn test_corpus_for_tsx_language(seed: usize) { test_language_corpus("typescript", seed, None, Some("tsx")); } @@ -204,15 +209,14 @@ pub fn test_language_corpus( if actual_output != test.output { println!("Incorrect initial parse for {test_name}"); - print_diff_key(); - print_diff(&actual_output, &test.output, true); + DiffKey::print(); + println!("{}", TestDiff::new(&actual_output, &test.output)); println!(); return false; } true - }) - .unwrap(); + }); if !passed { failure_count += 1; @@ -239,8 +243,9 @@ pub fn test_language_corpus( } // Perform a random series of edits and reparse. - let mut undo_stack = Vec::new(); - for _ in 0..=rand.unsigned(*EDIT_COUNT) { + let edit_count = rand.unsigned(*EDIT_COUNT); + let mut undo_stack = Vec::with_capacity(edit_count); + for _ in 0..=edit_count { let edit = get_random_edit(&mut rand, &input); undo_stack.push(invert_edit(&input, &edit)); perform_edit(&mut tree, &mut input, &edit).unwrap(); @@ -292,8 +297,8 @@ pub fn test_language_corpus( if actual_output != test.output { println!("Incorrect parse for {test_name} - seed {seed}"); - print_diff_key(); - print_diff(&actual_output, &test.output, true); + DiffKey::print(); + println!("{}", TestDiff::new(&actual_output, &test.output)); println!(); return false; } @@ -306,7 +311,7 @@ pub fn test_language_corpus( } true - }).unwrap(); + }); if !passed { failure_count += 1; @@ -358,8 +363,16 @@ fn test_feature_corpus_files() { grammar_path = test_path.join("grammar.json"); } let error_message_path = test_path.join("expected_error.txt"); - let grammar_json = tree_sitter_generate::load_grammar_file(&grammar_path, None).unwrap(); - let generate_result = tree_sitter_generate::generate_parser_for_grammar(&grammar_json); + let grammar_json = tree_sitter_generate::load_grammar_file(&grammar_path, None) + .with_context(|| { + format!( + "Could not load grammar file for test language '{language_name}' at {}", + grammar_path.display() + ) + }) + .unwrap(); + let generate_result = + tree_sitter_generate::generate_parser_for_grammar(&grammar_json, Some((0, 0, 0))); if error_message_path.exists() { if EXAMPLE_INCLUDE.is_some() || EXAMPLE_EXCLUDE.is_some() { @@ -375,7 +388,7 @@ fn test_feature_corpus_files() { let actual_message = e.to_string().replace("\r\n", "\n"); if expected_message != actual_message { eprintln!( - "Unexpected error message.\n\nExpected:\n\n{expected_message}\nActual:\n\n{actual_message}\n", + "Unexpected error message.\n\nExpected:\n\n`{expected_message}`\nActual:\n\n`{actual_message}`\n", ); failure_count += 1; } @@ -415,17 +428,15 @@ fn test_feature_corpus_files() { if actual_output == test.output { true } else { - print_diff_key(); - print_diff(&actual_output, &test.output, true); + DiffKey::print(); + print!("{}", TestDiff::new(&actual_output, &test.output)); println!(); false } - }) - .unwrap(); + }); if !passed { failure_count += 1; - continue; } } } diff --git a/cli/src/tests/detect_language.rs b/crates/cli/src/tests/detect_language.rs similarity index 98% rename from cli/src/tests/detect_language.rs rename to crates/cli/src/tests/detect_language.rs index 50bb891e..c543c31e 100644 --- a/cli/src/tests/detect_language.rs +++ b/crates/cli/src/tests/detect_language.rs @@ -90,7 +90,7 @@ fn detect_language_by_first_line_regex() { } #[test] -fn detect_langauge_by_double_barrel_file_extension() { +fn detect_language_by_double_barrel_file_extension() { let blade_dir = tree_sitter_dir( r#"{ "grammars": [ @@ -229,7 +229,7 @@ fn tree_sitter_dir(tree_sitter_json: &str, name: &str) -> tempfile::TempDir { .unwrap(); fs::write( temp_dir.path().join("src/tree_sitter/parser.h"), - include_str!("../../../lib/src/parser.h"), + include_str!("../../../../lib/src/parser.h"), ) .unwrap(); temp_dir diff --git a/cli/src/tests/helpers/mod.rs b/crates/cli/src/tests/helpers.rs similarity index 67% rename from cli/src/tests/helpers/mod.rs rename to crates/cli/src/tests/helpers.rs index 298179c7..4d2e6128 100644 --- a/cli/src/tests/helpers/mod.rs +++ b/crates/cli/src/tests/helpers.rs @@ -1,4 +1,4 @@ -pub mod allocations; +pub use crate::fuzz::allocations; pub mod edits; pub(super) mod fixtures; pub(super) mod query_helpers; diff --git a/crates/cli/src/tests/helpers/dirs.rs b/crates/cli/src/tests/helpers/dirs.rs new file mode 100644 index 00000000..d63790cd --- /dev/null +++ b/crates/cli/src/tests/helpers/dirs.rs @@ -0,0 +1,65 @@ +pub static ROOT_DIR: LazyLock = LazyLock::new(|| { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .to_owned() +}); + +pub static FIXTURES_DIR: LazyLock = + LazyLock::new(|| ROOT_DIR.join("test").join("fixtures")); + +pub static HEADER_DIR: LazyLock = LazyLock::new(|| ROOT_DIR.join("lib").join("include")); + +pub static GRAMMARS_DIR: LazyLock = + LazyLock::new(|| ROOT_DIR.join("test").join("fixtures").join("grammars")); + +pub static SCRATCH_BASE_DIR: LazyLock = LazyLock::new(|| { + let result = ROOT_DIR.join("target").join("scratch"); + fs::create_dir_all(&result).unwrap(); + result +}); + +#[cfg(feature = "wasm")] +pub static WASM_DIR: LazyLock = LazyLock::new(|| ROOT_DIR.join("target").join("release")); + +pub static SCRATCH_DIR: LazyLock = LazyLock::new(|| { + // https://doc.rust-lang.org/reference/conditional-compilation.html + let vendor = if cfg!(target_vendor = "apple") { + "apple" + } else if cfg!(target_vendor = "fortanix") { + "fortanix" + } else if cfg!(target_vendor = "pc") { + "pc" + } else { + "unknown" + }; + let env = if cfg!(target_env = "gnu") { + "gnu" + } else if cfg!(target_env = "msvc") { + "msvc" + } else if cfg!(target_env = "musl") { + "musl" + } else if cfg!(target_env = "sgx") { + "sgx" + } else { + "unknown" + }; + let endian = if cfg!(target_endian = "little") { + "little" + } else if cfg!(target_endian = "big") { + "big" + } else { + "unknown" + }; + + let machine = format!( + "{}-{}-{vendor}-{env}-{endian}", + std::env::consts::ARCH, + std::env::consts::OS + ); + let result = SCRATCH_BASE_DIR.join(machine); + fs::create_dir_all(&result).unwrap(); + result +}); diff --git a/cli/src/tests/helpers/edits.rs b/crates/cli/src/tests/helpers/edits.rs similarity index 100% rename from cli/src/tests/helpers/edits.rs rename to crates/cli/src/tests/helpers/edits.rs diff --git a/cli/src/tests/helpers/fixtures.rs b/crates/cli/src/tests/helpers/fixtures.rs similarity index 67% rename from cli/src/tests/helpers/fixtures.rs rename to crates/cli/src/tests/helpers/fixtures.rs index 943d7cf3..0e0ff69d 100644 --- a/cli/src/tests/helpers/fixtures.rs +++ b/crates/cli/src/tests/helpers/fixtures.rs @@ -1,27 +1,30 @@ use std::{ env, fs, path::{Path, PathBuf}, + sync::LazyLock, }; use anyhow::Context; -use lazy_static::lazy_static; use tree_sitter::Language; -use tree_sitter_generate::{ALLOC_HEADER, ARRAY_HEADER}; +use tree_sitter_generate::{load_grammar_file, ALLOC_HEADER, ARRAY_HEADER}; use tree_sitter_highlight::HighlightConfiguration; use tree_sitter_loader::{CompileConfig, Loader}; use tree_sitter_tags::TagsConfiguration; +use crate::tests::generate_parser; + include!("./dirs.rs"); -lazy_static! { - static ref TEST_LOADER: Loader = { - let mut loader = Loader::with_parser_lib_path(SCRATCH_DIR.clone()); - if env::var("TREE_SITTER_GRAMMAR_DEBUG").is_ok() { - loader.debug_build(true); - } - loader - }; -} +static TEST_LOADER: LazyLock = LazyLock::new(|| { + let mut loader = Loader::with_parser_lib_path(SCRATCH_DIR.clone()); + if env::var("TREE_SITTER_GRAMMAR_DEBUG").is_ok() { + loader.debug_build(true); + } + loader +}); + +#[cfg(feature = "wasm")] +pub static ENGINE: LazyLock = LazyLock::new(Default::default); pub fn test_loader() -> &'static Loader { &TEST_LOADER @@ -42,6 +45,22 @@ pub fn get_language(name: &str) -> Language { TEST_LOADER.load_language_at_path(config).unwrap() } +pub fn get_test_fixture_language(name: &str) -> Language { + get_test_fixture_language_internal(name, false) +} + +#[cfg(feature = "wasm")] +pub fn get_test_fixture_language_wasm(name: &str) -> Language { + get_test_fixture_language_internal(name, true) +} + +fn get_test_fixture_language_internal(name: &str, wasm: bool) -> Language { + let grammar_dir_path = fixtures_dir().join("test_grammars").join(name); + let grammar_json = load_grammar_file(&grammar_dir_path.join("grammar.js"), None).unwrap(); + let (parser_name, parser_code) = generate_parser(&grammar_json).unwrap(); + get_test_language_internal(&parser_name, &parser_code, Some(&grammar_dir_path), wasm) +} + pub fn get_language_queries_path(language_name: &str) -> PathBuf { GRAMMARS_DIR.join(language_name).join("queries") } @@ -80,6 +99,15 @@ pub fn get_tags_config(language_name: &str) -> TagsConfiguration { } pub fn get_test_language(name: &str, parser_code: &str, path: Option<&Path>) -> Language { + get_test_language_internal(name, parser_code, path, false) +} + +fn get_test_language_internal( + name: &str, + parser_code: &str, + path: Option<&Path>, + wasm: bool, +) -> Language { let src_dir = scratch_dir().join("src").join(name); fs::create_dir_all(&src_dir).unwrap(); @@ -129,5 +157,21 @@ pub fn get_test_language(name: &str, parser_code: &str, path: Option<&Path>) -> config.header_paths = vec![&HEADER_DIR]; config.name = name.to_string(); - TEST_LOADER.load_language_at_path_with_name(config).unwrap() + if wasm { + #[cfg(feature = "wasm")] + { + let mut loader = Loader::with_parser_lib_path(SCRATCH_DIR.clone()); + loader.use_wasm(&ENGINE); + if env::var("TREE_SITTER_GRAMMAR_DEBUG").is_ok() { + loader.debug_build(true); + } + loader.load_language_at_path_with_name(config).unwrap() + } + #[cfg(not(feature = "wasm"))] + { + unimplemented!("Wasm feature is not enabled") + } + } else { + TEST_LOADER.load_language_at_path_with_name(config).unwrap() + } } diff --git a/cli/src/tests/helpers/query_helpers.rs b/crates/cli/src/tests/helpers/query_helpers.rs similarity index 96% rename from cli/src/tests/helpers/query_helpers.rs rename to crates/cli/src/tests/helpers/query_helpers.rs index e7e0f969..e2c68f17 100644 --- a/cli/src/tests/helpers/query_helpers.rs +++ b/crates/cli/src/tests/helpers/query_helpers.rs @@ -12,7 +12,7 @@ pub struct Pattern { named: bool, field: Option<&'static str>, capture: Option, - children: Vec, + children: Vec, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -131,11 +131,12 @@ impl Pattern { if self.named { string.push('('); - let mut has_contents = false; - if let Some(kind) = &self.kind { + let mut has_contents = if let Some(kind) = &self.kind { write!(string, "{kind}").unwrap(); - has_contents = true; - } + true + } else { + false + }; for child in &self.children { let indent = indent + 2; if has_contents { @@ -181,7 +182,9 @@ impl Pattern { } matches.sort_unstable(); - matches.iter_mut().for_each(|m| m.last_node = None); + for m in &mut matches { + m.last_node = None; + } matches.dedup(); matches } @@ -222,7 +225,7 @@ impl Pattern { } // Find every matching combination of child patterns and child nodes. - let mut finished_matches = Vec::::new(); + let mut finished_matches = Vec::>::new(); if cursor.goto_first_child() { let mut match_states = vec![(0, mat)]; loop { @@ -320,8 +323,8 @@ pub fn assert_query_matches( let tree = parser.parse(source, None).unwrap(); let mut cursor = QueryCursor::new(); let matches = cursor.matches(query, tree.root_node(), source.as_bytes()); - pretty_assertions::assert_eq!(collect_matches(matches, query, source), expected); - pretty_assertions::assert_eq!(cursor.did_exceed_match_limit(), false); + pretty_assertions::assert_eq!(expected, collect_matches(matches, query, source)); + pretty_assertions::assert_eq!(false, cursor.did_exceed_match_limit()); } pub fn collect_matches<'a>( diff --git a/cli/src/tests/highlight_test.rs b/crates/cli/src/tests/highlight_test.rs similarity index 92% rename from cli/src/tests/highlight_test.rs rename to crates/cli/src/tests/highlight_test.rs index ab9dfcbe..88209bfe 100644 --- a/cli/src/tests/highlight_test.rs +++ b/crates/cli/src/tests/highlight_test.rs @@ -3,65 +3,81 @@ use std::{ fs, os::raw::c_char, ptr, slice, str, - sync::atomic::{AtomicUsize, Ordering}, + sync::{ + atomic::{AtomicUsize, Ordering}, + LazyLock, + }, }; -use lazy_static::lazy_static; use tree_sitter_highlight::{ c, Error, Highlight, HighlightConfiguration, HighlightEvent, Highlighter, HtmlRenderer, }; use super::helpers::fixtures::{get_highlight_config, get_language, get_language_queries_path}; -lazy_static! { - static ref JS_HIGHLIGHT: HighlightConfiguration = - get_highlight_config("javascript", Some("injections.scm"), &HIGHLIGHT_NAMES); - static ref JSDOC_HIGHLIGHT: HighlightConfiguration = - get_highlight_config("jsdoc", None, &HIGHLIGHT_NAMES); - static ref HTML_HIGHLIGHT: HighlightConfiguration = - get_highlight_config("html", Some("injections.scm"), &HIGHLIGHT_NAMES); - static ref EJS_HIGHLIGHT: HighlightConfiguration = get_highlight_config( +static JS_HIGHLIGHT: LazyLock = + LazyLock::new(|| get_highlight_config("javascript", Some("injections.scm"), &HIGHLIGHT_NAMES)); + +static JSDOC_HIGHLIGHT: LazyLock = + LazyLock::new(|| get_highlight_config("jsdoc", None, &HIGHLIGHT_NAMES)); + +static HTML_HIGHLIGHT: LazyLock = + LazyLock::new(|| get_highlight_config("html", Some("injections.scm"), &HIGHLIGHT_NAMES)); + +static EJS_HIGHLIGHT: LazyLock = LazyLock::new(|| { + get_highlight_config( "embedded-template", Some("injections-ejs.scm"), - &HIGHLIGHT_NAMES - ); - static ref RUST_HIGHLIGHT: HighlightConfiguration = - get_highlight_config("rust", Some("injections.scm"), &HIGHLIGHT_NAMES); - static ref HIGHLIGHT_NAMES: Vec = [ + &HIGHLIGHT_NAMES, + ) +}); + +static RUST_HIGHLIGHT: LazyLock = + LazyLock::new(|| get_highlight_config("rust", Some("injections.scm"), &HIGHLIGHT_NAMES)); + +static HIGHLIGHT_NAMES: LazyLock> = LazyLock::new(|| { + [ "attribute", "boolean", "carriage-return", "comment", "constant", + "constant.builtin", "constructor", - "function.builtin", - "function", "embedded", + "function", + "function.builtin", "keyword", + "module", + "number", "operator", - "property.builtin", "property", + "property.builtin", "punctuation", "punctuation.bracket", "punctuation.delimiter", "punctuation.special", "string", + "string.special", "tag", - "type.builtin", "type", + "type.builtin", + "variable", "variable.builtin", "variable.parameter", - "variable", ] .iter() .copied() .map(String::from) - .collect(); - static ref HTML_ATTRS: Vec = HIGHLIGHT_NAMES + .collect() +}); + +static HTML_ATTRS: LazyLock> = LazyLock::new(|| { + HIGHLIGHT_NAMES .iter() .map(|s| format!("class={s}")) - .collect(); -} + .collect() +}); #[test] fn test_highlighting_javascript() { @@ -334,12 +350,11 @@ fn test_highlighting_empty_lines() { fn test_highlighting_carriage_returns() { let source = "a = \"a\rb\"\r\nb\r"; - // FIXME(amaanq): figure why this changed w/ JS's grammar changes assert_eq!( &to_html(source, &JS_HIGHLIGHT).unwrap(), &[ - "a = "ab"\n", - "b\n", + "a = "ab"\n", + "b\n", ], ); } @@ -466,7 +481,7 @@ fn test_highlighting_cancellation() { // The initial `highlight` call, which eagerly parses the outer document, should not fail. let mut highlighter = Highlighter::new(); - let events = highlighter + let mut events = highlighter .highlight( &HTML_HIGHLIGHT, source.as_bytes(), @@ -477,14 +492,18 @@ fn test_highlighting_cancellation() { // Iterating the scopes should not panic. It should return an error once the // cancellation is detected. - for event in events { - if let Err(e) = event { - assert_eq!(e, Error::Cancelled); - return; + let found_cancellation_error = events.any(|event| match event { + Ok(_) => false, + Err(Error::Cancelled) => true, + Err(Error::InvalidLanguage | Error::Unknown) => { + unreachable!("Unexpected error type while iterating events") } - } + }); - panic!("Expected an error while iterating highlighter"); + assert!( + found_cancellation_error, + "Expected a cancellation error while iterating events" + ); } #[test] @@ -582,7 +601,7 @@ fn test_highlighting_via_c_api() { let output_line_offsets = unsafe { slice::from_raw_parts(output_line_offsets, output_line_count as usize) }; - let mut lines = Vec::new(); + let mut lines = Vec::with_capacity(output_line_count as usize); for i in 0..(output_line_count as usize) { let line_start = output_line_offsets[i] as usize; let line_end = output_line_offsets @@ -718,7 +737,9 @@ fn to_html<'a>( .map(Highlight), ); renderer - .render(events, src, &|highlight| HTML_ATTRS[highlight.0].as_bytes()) + .render(events, src, &|highlight, output| { + output.extend(HTML_ATTRS[highlight.0].as_bytes()); + }) .unwrap(); Ok(renderer .lines() diff --git a/crates/cli/src/tests/language_test.rs b/crates/cli/src/tests/language_test.rs new file mode 100644 index 00000000..9f51f503 --- /dev/null +++ b/crates/cli/src/tests/language_test.rs @@ -0,0 +1,199 @@ +use tree_sitter::{self, Parser}; + +use super::helpers::fixtures::get_language; + +#[test] +fn test_lookahead_iterator() { + let mut parser = Parser::new(); + let language = get_language("rust"); + parser.set_language(&language).unwrap(); + + let tree = parser.parse("struct Stuff {}", None).unwrap(); + + let mut cursor = tree.walk(); + + assert!(cursor.goto_first_child()); // struct + assert!(cursor.goto_first_child()); // struct keyword + + let next_state = cursor.node().next_parse_state(); + assert_ne!(next_state, 0); + assert_eq!( + next_state, + language.next_state(cursor.node().parse_state(), cursor.node().grammar_id()) + ); + assert!((next_state as usize) < language.parse_state_count()); + assert!(cursor.goto_next_sibling()); // type_identifier + assert_eq!(next_state, cursor.node().parse_state()); + assert_eq!(cursor.node().grammar_name(), "identifier"); + assert_ne!(cursor.node().grammar_id(), cursor.node().kind_id()); + + let expected_symbols = ["//", "/*", "identifier", "line_comment", "block_comment"]; + let mut lookahead = language.lookahead_iterator(next_state).unwrap(); + assert_eq!(*lookahead.language(), language); + assert!(lookahead.iter_names().eq(expected_symbols)); + + lookahead.reset_state(next_state); + assert!(lookahead.iter_names().eq(expected_symbols)); + + lookahead.reset(&language, next_state); + assert!(lookahead + .map(|s| language.node_kind_for_id(s).unwrap()) + .eq(expected_symbols)); +} + +#[test] +fn test_lookahead_iterator_modifiable_only_by_mut() { + let mut parser = Parser::new(); + let language = get_language("rust"); + parser.set_language(&language).unwrap(); + + let tree = parser.parse("struct Stuff {}", None).unwrap(); + + let mut cursor = tree.walk(); + + assert!(cursor.goto_first_child()); // struct + assert!(cursor.goto_first_child()); // struct keyword + + let next_state = cursor.node().next_parse_state(); + assert_ne!(next_state, 0); + + let mut lookahead = language.lookahead_iterator(next_state).unwrap(); + let _ = lookahead.next(); + + let mut names = lookahead.iter_names(); + let _ = names.next(); +} + +#[test] +fn test_symbol_metadata_checks() { + let language = get_language("rust"); + for i in 0..language.node_kind_count() { + let sym = i as u16; + let name = language.node_kind_for_id(sym).unwrap(); + match name { + "_type" + | "_expression" + | "_pattern" + | "_literal" + | "_literal_pattern" + | "_declaration_statement" => assert!(language.node_kind_is_supertype(sym)), + + "_raw_string_literal_start" + | "_raw_string_literal_end" + | "_line_doc_comment" + | "_error_sentinel" => assert!(!language.node_kind_is_supertype(sym)), + + "enum_item" | "struct_item" | "type_item" => { + assert!(language.node_kind_is_named(sym)); + } + + "=>" | "[" | "]" | "(" | ")" | "{" | "}" => { + assert!(language.node_kind_is_visible(sym)); + } + + _ => {} + } + } +} + +#[test] +fn test_supertypes() { + let language = get_language("rust"); + let supertypes = language.supertypes(); + + if language.abi_version() < 15 { + return; + } + + assert_eq!(supertypes.len(), 5); + assert_eq!( + supertypes + .iter() + .filter_map(|&s| language.node_kind_for_id(s)) + .map(|s| s.to_string()) + .collect::>(), + vec![ + "_expression", + "_literal", + "_literal_pattern", + "_pattern", + "_type" + ] + ); + + for &supertype in supertypes { + let mut subtypes = language + .subtypes_for_supertype(supertype) + .iter() + .filter_map(|symbol| language.node_kind_for_id(*symbol)) + .collect::>(); + subtypes.sort_unstable(); + subtypes.dedup(); + + match language.node_kind_for_id(supertype) { + Some("_literal") => { + assert_eq!( + subtypes, + &[ + "boolean_literal", + "char_literal", + "float_literal", + "integer_literal", + "raw_string_literal", + "string_literal" + ] + ); + } + Some("_pattern") => { + assert_eq!( + subtypes, + &[ + "_", + "_literal_pattern", + "captured_pattern", + "const_block", + "generic_pattern", + "identifier", + "macro_invocation", + "mut_pattern", + "or_pattern", + "range_pattern", + "ref_pattern", + "reference_pattern", + "remaining_field_pattern", + "scoped_identifier", + "slice_pattern", + "struct_pattern", + "tuple_pattern", + "tuple_struct_pattern", + ] + ); + } + Some("_type") => { + assert_eq!( + subtypes, + &[ + "abstract_type", + "array_type", + "bounded_type", + "dynamic_type", + "function_type", + "generic_type", + "macro_invocation", + "metavariable", + "never_type", + "pointer_type", + "primitive_type", + "reference_type", + "removed_trait_bound", + "scoped_type_identifier", + "tuple_type", + "type_identifier", + "unit_type" + ] + ); + } + _ => {} + } + } +} diff --git a/cli/src/tests/node_test.rs b/crates/cli/src/tests/node_test.rs similarity index 88% rename from cli/src/tests/node_test.rs rename to crates/cli/src/tests/node_test.rs index 7217ee27..614bfdb9 100644 --- a/cli/src/tests/node_test.rs +++ b/crates/cli/src/tests/node_test.rs @@ -1,12 +1,15 @@ -use tree_sitter::{Node, Parser, Point, Tree}; -use tree_sitter_generate::{generate_parser_for_grammar, load_grammar_file}; +use tree_sitter::{InputEdit, Node, Parser, Point, Tree}; +use tree_sitter_generate::load_grammar_file; use super::{ get_random_edit, helpers::fixtures::{fixtures_dir, get_language, get_test_language}, Rand, }; -use crate::parse::perform_edit; +use crate::{ + parse::perform_edit, + tests::{generate_parser, helpers::fixtures::get_test_fixture_language}, +}; const JSON_EXAMPLE: &str = r#" @@ -308,19 +311,8 @@ fn test_parent_of_zero_width_node() { #[test] fn test_next_sibling_of_zero_width_node() { - let grammar_json = load_grammar_file( - &fixtures_dir() - .join("test_grammars") - .join("next_sibling_from_zwt") - .join("grammar.js"), - None, - ) - .unwrap(); - - let (parser_name, parser_code) = generate_parser_for_grammar(&grammar_json).unwrap(); - let mut parser = Parser::new(); - let language = get_test_language(&parser_name, &parser_code, None); + let language = get_test_fixture_language("next_sibling_from_zwt"); parser.set_language(&language).unwrap(); let tree = parser.parse("abdef", None).unwrap(); @@ -331,6 +323,49 @@ fn test_next_sibling_of_zero_width_node() { assert_eq!(missing_c.kind(), "c"); let node_d = root_node.child(3).unwrap(); assert_eq!(missing_c.next_sibling().unwrap(), node_d); + + let prev_sibling = node_d.prev_sibling().unwrap(); + assert_eq!(prev_sibling, missing_c); +} + +#[test] +fn test_first_child_for_offset() { + let mut parser = Parser::new(); + parser.set_language(&get_language("javascript")).unwrap(); + let tree = parser.parse("x10 + 100", None).unwrap(); + let sum_node = tree.root_node().child(0).unwrap().child(0).unwrap(); + + assert_eq!( + sum_node.first_child_for_byte(0).unwrap().kind(), + "identifier" + ); + assert_eq!( + sum_node.first_child_for_byte(1).unwrap().kind(), + "identifier" + ); + assert_eq!(sum_node.first_child_for_byte(3).unwrap().kind(), "+"); + assert_eq!(sum_node.first_child_for_byte(5).unwrap().kind(), "number"); +} + +#[test] +fn test_first_named_child_for_offset() { + let mut parser = Parser::new(); + parser.set_language(&get_language("javascript")).unwrap(); + let tree = parser.parse("x10 + 100", None).unwrap(); + let sum_node = tree.root_node().child(0).unwrap().child(0).unwrap(); + + assert_eq!( + sum_node.first_named_child_for_byte(0).unwrap().kind(), + "identifier" + ); + assert_eq!( + sum_node.first_named_child_for_byte(1).unwrap().kind(), + "identifier" + ); + assert_eq!( + sum_node.first_named_child_for_byte(3).unwrap().kind(), + "number" + ); } #[test] @@ -520,8 +555,7 @@ fn test_node_named_child() { #[test] fn test_node_named_child_with_aliases_and_extras() { - let (parser_name, parser_code) = - generate_parser_for_grammar(GRAMMAR_WITH_ALIASES_AND_EXTRAS).unwrap(); + let (parser_name, parser_code) = generate_parser(GRAMMAR_WITH_ALIASES_AND_EXTRAS).unwrap(); let mut parser = Parser::new(); parser @@ -711,6 +745,13 @@ fn test_node_descendant_for_range() { assert_eq!(child, child2); } + + // Negative test, start > end + assert_eq!(array_node.descendant_for_byte_range(1, 0), None); + assert_eq!( + array_node.descendant_for_point_range(Point::new(6, 8), Point::new(6, 7)), + None + ); } #[test] @@ -788,6 +829,106 @@ fn test_node_is_extra() { assert!(comment_node.is_extra()); } +#[test] +fn test_node_is_error() { + let mut parser = Parser::new(); + parser.set_language(&get_language("javascript")).unwrap(); + let tree = parser.parse("foo(", None).unwrap(); + let root_node = tree.root_node(); + assert_eq!(root_node.kind(), "program"); + assert!(root_node.has_error()); + + let child = root_node.child(0).unwrap(); + assert_eq!(child.kind(), "ERROR"); + assert!(child.is_error()); +} + +#[test] +fn test_edit_point() { + let edit = InputEdit { + start_byte: 5, + old_end_byte: 5, + new_end_byte: 10, + start_position: Point::new(0, 5), + old_end_position: Point::new(0, 5), + new_end_position: Point::new(0, 10), + }; + + // Point after edit + let mut point = Point::new(0, 8); + let mut byte = 8; + edit.edit_point(&mut point, &mut byte); + assert_eq!(point, Point::new(0, 13)); + assert_eq!(byte, 13); + + // Point before edit + let mut point = Point::new(0, 2); + let mut byte = 2; + edit.edit_point(&mut point, &mut byte); + assert_eq!(point, Point::new(0, 2)); + assert_eq!(byte, 2); + + // Point at edit start + let mut point = Point::new(0, 5); + let mut byte = 5; + edit.edit_point(&mut point, &mut byte); + assert_eq!(point, Point::new(0, 10)); + assert_eq!(byte, 10); +} + +#[test] +fn test_edit_range() { + use tree_sitter::{InputEdit, Point, Range}; + + let edit = InputEdit { + start_byte: 10, + old_end_byte: 15, + new_end_byte: 20, + start_position: Point::new(1, 0), + old_end_position: Point::new(1, 5), + new_end_position: Point::new(2, 0), + }; + + // Range after edit + let mut range = Range { + start_byte: 20, + end_byte: 25, + start_point: Point::new(2, 0), + end_point: Point::new(2, 5), + }; + edit.edit_range(&mut range); + assert_eq!(range.start_byte, 25); + assert_eq!(range.end_byte, 30); + assert_eq!(range.start_point, Point::new(3, 0)); + assert_eq!(range.end_point, Point::new(3, 5)); + + // Range before edit + let mut range = Range { + start_byte: 5, + end_byte: 8, + start_point: Point::new(0, 5), + end_point: Point::new(0, 8), + }; + edit.edit_range(&mut range); + assert_eq!(range.start_byte, 5); + assert_eq!(range.end_byte, 8); + assert_eq!(range.start_point, Point::new(0, 5)); + assert_eq!(range.end_point, Point::new(0, 8)); + + // Range overlapping edit + let mut range = Range { + start_byte: 8, + end_byte: 12, + start_point: Point::new(0, 8), + end_point: Point::new(1, 2), + }; + edit.edit_range(&mut range); + assert_eq!(range.start_byte, 8); + assert_eq!(range.end_byte, 10); + assert_eq!(range.start_point, Point::new(0, 8)); + assert_eq!(range.end_point, Point::new(1, 0)); +} + #[test] fn test_node_sexp() { let mut parser = Parser::new(); @@ -807,7 +948,7 @@ fn test_node_sexp() { #[test] fn test_node_field_names() { - let (parser_name, parser_code) = generate_parser_for_grammar( + let (parser_name, parser_code) = generate_parser( r#" { "name": "test_grammar_with_fields", @@ -917,7 +1058,7 @@ fn test_node_field_names() { #[test] fn test_node_field_calls_in_language_without_fields() { - let (parser_name, parser_code) = generate_parser_for_grammar( + let (parser_name, parser_code) = generate_parser( r#" { "name": "test_grammar_with_no_fields", @@ -975,7 +1116,7 @@ fn test_node_is_named_but_aliased_as_anonymous() { ) .unwrap(); - let (parser_name, parser_code) = generate_parser_for_grammar(&grammar_json).unwrap(); + let (parser_name, parser_code) = generate_parser(&grammar_json).unwrap(); let mut parser = Parser::new(); let language = get_test_language(&parser_name, &parser_code, None); diff --git a/cli/src/tests/parser_test.rs b/crates/cli/src/tests/parser_test.rs similarity index 84% rename from cli/src/tests/parser_test.rs rename to crates/cli/src/tests/parser_test.rs index e5dc15d9..f1d50319 100644 --- a/cli/src/tests/parser_test.rs +++ b/crates/cli/src/tests/parser_test.rs @@ -1,12 +1,17 @@ use std::{ - sync::atomic::{AtomicUsize, Ordering}, - thread, time, + ops::ControlFlow, + sync::{ + atomic::{AtomicUsize, Ordering}, + mpsc, + }, + thread, + time::{self, Duration}, }; use tree_sitter::{ Decode, IncludedRangesError, InputEdit, LogType, ParseOptions, ParseState, Parser, Point, Range, }; -use tree_sitter_generate::{generate_parser_for_grammar, load_grammar_file}; +use tree_sitter_generate::load_grammar_file; use tree_sitter_proc_macro::retry; use super::helpers::{ @@ -17,7 +22,11 @@ use super::helpers::{ use crate::{ fuzz::edits::Edit, parse::perform_edit, - tests::{helpers::fixtures::fixtures_dir, invert_edit}, + tests::{ + generate_parser, + helpers::fixtures::{fixtures_dir, get_test_fixture_language}, + invert_edit, + }, }; #[test] @@ -88,7 +97,6 @@ fn test_parsing_with_logging() { } #[test] -#[cfg(unix)] fn test_parsing_with_debug_graph_enabled() { use std::io::{BufRead, BufReader, Seek}; @@ -482,15 +490,9 @@ fn test_parsing_empty_file_with_reused_tree() { #[test] fn test_parsing_after_editing_tree_that_depends_on_column_values() { - let dir = fixtures_dir() - .join("test_grammars") - .join("uses_current_column"); - let grammar_json = load_grammar_file(&dir.join("grammar.js"), None).unwrap(); - let (grammar_name, parser_code) = generate_parser_for_grammar(&grammar_json).unwrap(); - let mut parser = Parser::new(); parser - .set_language(&get_test_language(&grammar_name, &parser_code, Some(&dir))) + .set_language(&get_test_fixture_language("uses_current_column")) .unwrap(); let mut code = b" @@ -559,16 +561,9 @@ h + i #[test] fn test_parsing_after_editing_tree_that_depends_on_column_position() { - let dir = fixtures_dir() - .join("test_grammars") - .join("depends_on_column"); - - let grammar_json = load_grammar_file(&dir.join("grammar.js"), None).unwrap(); - let (grammar_name, parser_code) = generate_parser_for_grammar(grammar_json.as_str()).unwrap(); - let mut parser = Parser::new(); parser - .set_language(&get_test_language(&grammar_name, &parser_code, Some(&dir))) + .set_language(&get_test_fixture_language("depends_on_column")) .unwrap(); let mut code = b"\n x".to_vec(); @@ -713,7 +708,13 @@ fn test_parsing_on_multiple_threads() { fn test_parsing_cancelled_by_another_thread() { let cancellation_flag = std::sync::Arc::new(AtomicUsize::new(0)); let flag = cancellation_flag.clone(); - let callback = &mut |_: &ParseState| cancellation_flag.load(Ordering::SeqCst) != 0; + let callback = &mut |_: &ParseState| { + if cancellation_flag.load(Ordering::SeqCst) != 0 { + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } + }; let mut parser = Parser::new(); parser.set_language(&get_language("javascript")).unwrap(); @@ -778,9 +779,13 @@ fn test_parsing_with_a_timeout() { } }, None, - Some( - ParseOptions::new().progress_callback(&mut |_| start_time.elapsed().as_micros() > 1000), - ), + Some(ParseOptions::new().progress_callback(&mut |_| { + if start_time.elapsed().as_micros() > 1000 { + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } + })), ); assert!(tree.is_none()); assert!(start_time.elapsed().as_micros() < 2000); @@ -796,9 +801,13 @@ fn test_parsing_with_a_timeout() { } }, None, - Some( - ParseOptions::new().progress_callback(&mut |_| start_time.elapsed().as_micros() > 5000), - ), + Some(ParseOptions::new().progress_callback(&mut |_| { + if start_time.elapsed().as_micros() > 5000 { + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } + })), ); assert!(tree.is_none()); assert!(start_time.elapsed().as_micros() > 100); @@ -836,7 +845,13 @@ fn test_parsing_with_a_timeout_and_a_reset() { } }, None, - Some(ParseOptions::new().progress_callback(&mut |_| start_time.elapsed().as_micros() > 5)), + Some(ParseOptions::new().progress_callback(&mut |_| { + if start_time.elapsed().as_micros() > 5 { + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } + })), ); assert!(tree.is_none()); @@ -867,7 +882,13 @@ fn test_parsing_with_a_timeout_and_a_reset() { } }, None, - Some(ParseOptions::new().progress_callback(&mut |_| start_time.elapsed().as_micros() > 5)), + Some(ParseOptions::new().progress_callback(&mut |_| { + if start_time.elapsed().as_micros() > 5 { + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } + })), ); assert!(tree.is_none()); @@ -907,10 +928,13 @@ fn test_parsing_with_a_timeout_and_implicit_reset() { } }, None, - Some( - ParseOptions::new() - .progress_callback(&mut |_| start_time.elapsed().as_micros() > 5), - ), + Some(ParseOptions::new().progress_callback(&mut |_| { + if start_time.elapsed().as_micros() > 5 { + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } + })), ); assert!(tree.is_none()); @@ -951,10 +975,13 @@ fn test_parsing_with_timeout_and_no_completion() { } }, None, - Some( - ParseOptions::new() - .progress_callback(&mut |_| start_time.elapsed().as_micros() > 5), - ), + Some(ParseOptions::new().progress_callback(&mut |_| { + if start_time.elapsed().as_micros() > 5 { + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } + })), ); assert!(tree.is_none()); @@ -962,6 +989,131 @@ fn test_parsing_with_timeout_and_no_completion() { }); } +#[test] +fn test_parsing_with_timeout_during_balancing() { + allocations::record(|| { + let mut parser = Parser::new(); + parser.set_language(&get_language("javascript")).unwrap(); + + let function_count = 100; + + let code = "function() {}\n".repeat(function_count); + let mut current_byte_offset = 0; + let mut in_balancing = false; + let tree = parser.parse_with_options( + &mut |offset, _| { + if offset >= code.len() { + &[] + } else { + &code.as_bytes()[offset..] + } + }, + None, + Some(ParseOptions::new().progress_callback(&mut |state| { + // The parser will call the progress_callback during parsing, and at the very end + // during tree-balancing. For very large trees, this balancing act can take quite + // some time, so we want to verify that timing out during this operation is + // possible. + // + // We verify this by checking the current byte offset, as this number will *not* be + // updated during tree balancing. If we see the same offset twice, we know that we + // are in the balancing phase. + if state.current_byte_offset() != current_byte_offset { + current_byte_offset = state.current_byte_offset(); + ControlFlow::Continue(()) + } else { + in_balancing = true; + ControlFlow::Break(()) + } + })), + ); + + assert!(tree.is_none()); + assert!(in_balancing); + + // This should not cause an assertion failure. + parser.reset(); + let tree = parser.parse_with_options( + &mut |offset, _| { + if offset >= code.len() { + &[] + } else { + &code.as_bytes()[offset..] + } + }, + None, + Some(ParseOptions::new().progress_callback(&mut |state| { + if state.current_byte_offset() != current_byte_offset { + current_byte_offset = state.current_byte_offset(); + ControlFlow::Continue(()) + } else { + in_balancing = true; + ControlFlow::Break(()) + } + })), + ); + + assert!(tree.is_none()); + assert!(in_balancing); + + // If we resume parsing (implying we didn't call `parser.reset()`), we should be able to + // finish parsing the tree, continuing from where we left off. + let tree = parser + .parse_with_options( + &mut |offset, _| { + if offset >= code.len() { + &[] + } else { + &code.as_bytes()[offset..] + } + }, + None, + Some(ParseOptions::new().progress_callback(&mut |state| { + // Because we've already finished parsing, we should only be resuming the + // balancing phase. + assert!(state.current_byte_offset() == current_byte_offset); + ControlFlow::Continue(()) + })), + ) + .unwrap(); + assert!(!tree.root_node().has_error()); + assert_eq!(tree.root_node().child_count(), function_count); + }); +} + +#[test] +fn test_parsing_with_timeout_when_error_detected() { + let mut parser = Parser::new(); + parser.set_language(&get_language("json")).unwrap(); + + // Parse an infinitely-long array, but insert an error after 1000 characters. + let mut offset = 0; + let erroneous_code = "!,"; + let tree = parser.parse_with_options( + &mut |i, _| match i { + 0 => "[", + 1..=1000 => "0,", + _ => erroneous_code, + }, + None, + Some(ParseOptions::new().progress_callback(&mut |state| { + offset = state.current_byte_offset(); + if state.has_error() { + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } + })), + ); + + // The callback is called at the end of parsing, however, what we're asserting here is that + // parsing ends immediately as the error is detected. This is verified by checking the offset + // of the last byte processed is the length of the erroneous code we inserted, aka, 1002, or + // 1000 + the length of the erroneous code. + assert_eq!(offset, 1000 + erroneous_code.len()); + assert!(tree.is_none()); +} + // Included Ranges #[test] @@ -1408,7 +1560,7 @@ fn test_parsing_with_a_newly_included_range() { #[test] fn test_parsing_with_included_ranges_and_missing_tokens() { - let (parser_name, parser_code) = generate_parser_for_grammar( + let (parser_name, parser_code) = generate_parser( r#"{ "name": "test_leading_missing_token", "rules": { @@ -1469,7 +1621,7 @@ fn test_parsing_with_included_ranges_and_missing_tokens() { #[test] fn test_grammars_that_can_hang_on_eof() { - let (parser_name, parser_code) = generate_parser_for_grammar( + let (parser_name, parser_code) = generate_parser( r#" { "name": "test_single_null_char_regex", @@ -1495,7 +1647,7 @@ fn test_grammars_that_can_hang_on_eof() { .unwrap(); parser.parse("\"", None).unwrap(); - let (parser_name, parser_code) = generate_parser_for_grammar( + let (parser_name, parser_code) = generate_parser( r#" { "name": "test_null_char_with_next_char_regex", @@ -1520,7 +1672,7 @@ fn test_grammars_that_can_hang_on_eof() { .unwrap(); parser.parse("\"", None).unwrap(); - let (parser_name, parser_code) = generate_parser_for_grammar( + let (parser_name, parser_code) = generate_parser( r#" { "name": "test_null_char_with_range_regex", @@ -1581,13 +1733,9 @@ if foo && bar || baz {} #[test] fn test_parsing_with_scanner_logging() { - let dir = fixtures_dir().join("test_grammars").join("external_tokens"); - let grammar_json = load_grammar_file(&dir.join("grammar.js"), None).unwrap(); - let (grammar_name, parser_code) = generate_parser_for_grammar(&grammar_json).unwrap(); - let mut parser = Parser::new(); parser - .set_language(&get_test_language(&grammar_name, &parser_code, Some(&dir))) + .set_language(&get_test_fixture_language("external_tokens")) .unwrap(); let mut found = false; @@ -1605,13 +1753,9 @@ fn test_parsing_with_scanner_logging() { #[test] fn test_parsing_get_column_at_eof() { - let dir = fixtures_dir().join("test_grammars").join("get_col_eof"); - let grammar_json = load_grammar_file(&dir.join("grammar.js"), None).unwrap(); - let (grammar_name, parser_code) = generate_parser_for_grammar(&grammar_json).unwrap(); - let mut parser = Parser::new(); parser - .set_language(&get_test_language(&grammar_name, &parser_code, Some(&dir))) + .set_language(&get_test_fixture_language("get_col_eof")) .unwrap(); parser.parse("a", None).unwrap(); @@ -1638,7 +1782,7 @@ fn test_parsing_by_halting_at_offset() { None, Some(ParseOptions::new().progress_callback(&mut |p| { seen_byte_offsets.push(p.current_byte_offset()); - false + ControlFlow::Continue(()) })), ) .unwrap(); @@ -1817,7 +1961,7 @@ fn test_decode_utf24le() { #[test] fn test_grammars_that_should_not_compile() { - assert!(generate_parser_for_grammar( + assert!(generate_parser( r#" { "name": "issue_1111", @@ -1829,7 +1973,7 @@ fn test_grammars_that_should_not_compile() { ) .is_err()); - assert!(generate_parser_for_grammar( + assert!(generate_parser( r#" { "name": "issue_1271", @@ -1844,11 +1988,11 @@ fn test_grammars_that_should_not_compile() { } }, } - "#, + "# ) .is_err()); - assert!(generate_parser_for_grammar( + assert!(generate_parser( r#" { "name": "issue_1156_expl_1", @@ -1862,11 +2006,11 @@ fn test_grammars_that_should_not_compile() { } }, } - "# + "# ) .is_err()); - assert!(generate_parser_for_grammar( + assert!(generate_parser( r#" { "name": "issue_1156_expl_2", @@ -1883,11 +2027,11 @@ fn test_grammars_that_should_not_compile() { } }, } - "# + "# ) .is_err()); - assert!(generate_parser_for_grammar( + assert!(generate_parser( r#" { "name": "issue_1156_expl_3", @@ -1901,11 +2045,11 @@ fn test_grammars_that_should_not_compile() { } }, } - "# + "# ) .is_err()); - assert!(generate_parser_for_grammar( + assert!(generate_parser( r#" { "name": "issue_1156_expl_4", @@ -1922,7 +2066,7 @@ fn test_grammars_that_should_not_compile() { } }, } - "# + "# ) .is_err()); } @@ -1937,5 +2081,99 @@ const fn simple_range(start: usize, end: usize) -> Range { } fn chunked_input<'a>(text: &'a str, size: usize) -> impl FnMut(usize, Point) -> &'a [u8] { - move |offset, _| text[offset..text.len().min(offset + size)].as_bytes() + move |offset, _| &text.as_bytes()[offset..text.len().min(offset + size)] +} + +#[test] +fn test_parse_options_reborrow() { + let mut parser = Parser::new(); + parser.set_language(&get_language("rust")).unwrap(); + + let parse_count = AtomicUsize::new(0); + + let mut callback = |_: &ParseState| { + parse_count.fetch_add(1, Ordering::SeqCst); + ControlFlow::Continue(()) + }; + let mut options = ParseOptions::new().progress_callback(&mut callback); + + let text1 = "fn first() {}".repeat(20); + let text2 = "fn second() {}".repeat(20); + + let tree1 = parser + .parse_with_options( + &mut |offset, _| { + if offset >= text1.len() { + &[] + } else { + &text1.as_bytes()[offset..] + } + }, + None, + Some(options.reborrow()), + ) + .unwrap(); + + assert_eq!(tree1.root_node().child(0).unwrap().kind(), "function_item"); + + let tree2 = parser + .parse_with_options( + &mut |offset, _| { + if offset >= text2.len() { + &[] + } else { + &text2.as_bytes()[offset..] + } + }, + None, + Some(options.reborrow()), + ) + .unwrap(); + + assert_eq!(tree2.root_node().child(0).unwrap().kind(), "function_item"); + + assert!(parse_count.load(Ordering::SeqCst) > 0); +} + +#[test] +fn test_grammar_that_should_hang_and_not_segfault() { + fn hang_test() { + let test_grammar_dir = fixtures_dir() + .join("test_grammars") + .join("get_col_should_hang_not_crash"); + + let grammar_json = load_grammar_file(&test_grammar_dir.join("grammar.js"), None) + .expect("Failed to load grammar file"); + + let (parser_name, parser_code) = + generate_parser(grammar_json.as_str()).expect("Failed to generate parser"); + + let language = + get_test_language(&parser_name, &parser_code, Some(test_grammar_dir.as_path())); + + let mut parser = Parser::new(); + parser + .set_language(&language) + .expect("Failed to set parser language"); + + let code_that_should_hang = "\nHello"; + + parser + .parse(code_that_should_hang, None) + .expect("Parse operation completed unexpectedly"); + } + + let timeout = Duration::from_millis(500); + let (tx, rx) = mpsc::channel(); + + thread::spawn(move || tx.send(std::panic::catch_unwind(hang_test))); + + match rx.recv_timeout(timeout) { + Ok(Ok(())) => panic!("The test completed rather than hanging"), + Ok(Err(panic_info)) => panic!("The test panicked unexpectedly: {panic_info:?}"), + Err(mpsc::RecvTimeoutError::Timeout) => {} // Expected + Err(mpsc::RecvTimeoutError::Disconnected) => { + panic!("The test thread disconnected unexpectedly") + } + } } diff --git a/cli/src/tests/pathological_test.rs b/crates/cli/src/tests/pathological_test.rs similarity index 100% rename from cli/src/tests/pathological_test.rs rename to crates/cli/src/tests/pathological_test.rs diff --git a/cli/src/tests/proc_macro/Cargo.toml b/crates/cli/src/tests/proc_macro/Cargo.toml similarity index 66% rename from cli/src/tests/proc_macro/Cargo.toml rename to crates/cli/src/tests/proc_macro/Cargo.toml index ef991956..915bd172 100644 --- a/cli/src/tests/proc_macro/Cargo.toml +++ b/crates/cli/src/tests/proc_macro/Cargo.toml @@ -12,7 +12,6 @@ workspace = true proc-macro = true [dependencies] -proc-macro2 = "1.0.86" -quote = "1.0.37" -rand = "0.8.5" -syn = { version = "2.0.79", features = ["full"] } +proc-macro2 = "1.0.93" +quote = "1.0.38" +syn = { version = "2.0.96", features = ["full"] } diff --git a/cli/src/tests/proc_macro/src/lib.rs b/crates/cli/src/tests/proc_macro/src/lib.rs similarity index 100% rename from cli/src/tests/proc_macro/src/lib.rs rename to crates/cli/src/tests/proc_macro/src/lib.rs diff --git a/cli/src/tests/query_test.rs b/crates/cli/src/tests/query_test.rs similarity index 87% rename from cli/src/tests/query_test.rs rename to crates/cli/src/tests/query_test.rs index e1433c97..3f1467e5 100644 --- a/cli/src/tests/query_test.rs +++ b/crates/cli/src/tests/query_test.rs @@ -1,14 +1,14 @@ -use std::{env, fmt::Write}; +use std::{env, fmt::Write, ops::ControlFlow, sync::LazyLock}; use indoc::indoc; -use lazy_static::lazy_static; use rand::{prelude::StdRng, SeedableRng}; use streaming_iterator::StreamingIterator; use tree_sitter::{ - CaptureQuantifier, Language, Node, Parser, Point, Query, QueryCursor, QueryCursorOptions, - QueryError, QueryErrorKind, QueryPredicate, QueryPredicateArg, QueryProperty, + CaptureQuantifier, InputEdit, Language, Node, Parser, Point, Query, QueryCursor, + QueryCursorOptions, QueryError, QueryErrorKind, QueryPredicate, QueryPredicateArg, + QueryProperty, Range, }; -use tree_sitter_generate::generate_parser_for_grammar; +use tree_sitter_generate::load_grammar_file; use unindent::Unindent; use super::helpers::{ @@ -17,13 +17,16 @@ use super::helpers::{ query_helpers::{assert_query_matches, Match, Pattern}, }; use crate::tests::{ - helpers::query_helpers::{collect_captures, collect_matches}, + generate_parser, + helpers::{ + fixtures::get_test_fixture_language, + query_helpers::{collect_captures, collect_matches}, + }, ITERATION_COUNT, }; -lazy_static! { - static ref EXAMPLE_FILTER: Option = env::var("TREE_SITTER_TEST_EXAMPLE_FILTER").ok(); -} +static EXAMPLE_FILTER: LazyLock> = + LazyLock::new(|| env::var("TREE_SITTER_TEST_EXAMPLE_FILTER").ok()); #[test] fn test_query_errors_on_invalid_syntax() { @@ -118,12 +121,24 @@ fn test_query_errors_on_invalid_syntax() { // Unclosed sibling expression with predicate assert_eq!( - Query::new(&language, r"((identifier) (#a)") + Query::new(&language, r"((identifier) (#a?)") .unwrap_err() .message, [ - "((identifier) (#a)", // - " ^", + "((identifier) (#a?)", // + " ^", + ] + .join("\n") + ); + + // Predicate not ending in `?` or `!` + assert_eq!( + Query::new(&language, r"((identifier) (#a))") + .unwrap_err() + .message, + [ + "((identifier) (#a))", // + " ^", ] .join("\n") ); @@ -193,6 +208,50 @@ fn test_query_errors_on_invalid_syntax() { ] .join("\n") ); + + // MISSING keyword with full pattern + assert_eq!( + Query::new( + &get_language("c"), + r"(MISSING (function_declarator (identifier))) " + ) + .unwrap_err() + .message, + [ + r"(MISSING (function_declarator (identifier))) ", + r" ^", + ] + .join("\n") + ); + + // MISSING keyword with multiple identifiers + assert_eq!( + Query::new( + &get_language("c"), + r"(MISSING function_declarator function_declarator) " + ) + .unwrap_err() + .message, + [ + r"(MISSING function_declarator function_declarator) ", + r" ^", + ] + .join("\n") + ); + assert_eq!( + Query::new(&language, "(statement / export_statement)").unwrap_err(), + QueryError { + row: 0, + offset: 11, + column: 11, + kind: QueryErrorKind::Syntax, + message: [ + "(statement / export_statement)", // + " ^" + ] + .join("\n") + } + ); }); } @@ -208,7 +267,7 @@ fn test_query_errors_on_invalid_symbols() { offset: 1, column: 1, kind: QueryErrorKind::NodeType, - message: ">>>>".to_string() + message: "\">>>>\"".to_string() } ); assert_eq!( @@ -218,7 +277,7 @@ fn test_query_errors_on_invalid_symbols() { offset: 1, column: 1, kind: QueryErrorKind::NodeType, - message: "te\\\"st".to_string() + message: "\"te\\\"st\"".to_string() } ); assert_eq!( @@ -228,7 +287,7 @@ fn test_query_errors_on_invalid_symbols() { offset: 1, column: 1, kind: QueryErrorKind::NodeType, - message: "\\\\".to_string() + message: "\"\\\\\"".to_string() } ); assert_eq!( @@ -238,7 +297,7 @@ fn test_query_errors_on_invalid_symbols() { offset: 1, column: 1, kind: QueryErrorKind::NodeType, - message: "clas".to_string() + message: "\"clas\"".to_string() } ); assert_eq!( @@ -248,7 +307,7 @@ fn test_query_errors_on_invalid_symbols() { offset: 15, column: 15, kind: QueryErrorKind::NodeType, - message: "arrayyyyy".to_string() + message: "\"arrayyyyy\"".to_string() }, ); assert_eq!( @@ -258,7 +317,7 @@ fn test_query_errors_on_invalid_symbols() { offset: 26, column: 26, kind: QueryErrorKind::NodeType, - message: "non_existent3".to_string() + message: "\"non_existent3\"".to_string() }, ); assert_eq!( @@ -268,7 +327,7 @@ fn test_query_errors_on_invalid_symbols() { offset: 14, column: 14, kind: QueryErrorKind::Field, - message: "condit".to_string() + message: "\"condit\"".to_string() }, ); assert_eq!( @@ -278,7 +337,7 @@ fn test_query_errors_on_invalid_symbols() { offset: 14, column: 14, kind: QueryErrorKind::Field, - message: "conditioning".to_string() + message: "\"conditioning\"".to_string() } ); assert_eq!( @@ -288,7 +347,7 @@ fn test_query_errors_on_invalid_symbols() { offset: 15, column: 15, kind: QueryErrorKind::Field, - message: "alternativ".to_string() + message: "\"alternativ\"".to_string() } ); assert_eq!( @@ -298,7 +357,17 @@ fn test_query_errors_on_invalid_symbols() { offset: 15, column: 15, kind: QueryErrorKind::Field, - message: "alternatives".to_string() + message: "\"alternatives\"".to_string() + } + ); + assert_eq!( + Query::new(&language, "fakefield: (identifier)").unwrap_err(), + QueryError { + row: 0, + offset: 0, + column: 0, + kind: QueryErrorKind::Field, + message: "\"fakefield\"".to_string() } ); }); @@ -341,7 +410,7 @@ fn test_query_errors_on_invalid_predicates() { row: 0, column: 29, offset: 29, - message: "ok".to_string(), + message: "\"ok\"".to_string(), } ); }); @@ -361,11 +430,11 @@ fn test_query_errors_on_impossible_patterns() { Err(QueryError { kind: QueryErrorKind::Structure, row: 0, - offset: 51, - column: 51, + offset: 37, + column: 37, message: [ "(binary_expression left: (expression (identifier)) left: (expression (identifier)))", - " ^", + " ^", ] .join("\n"), }) @@ -488,6 +557,51 @@ fn test_query_errors_on_impossible_patterns() { .join("\n") }) ); + assert_eq!( + Query::new(&js_lang, "(identifier/identifier)").unwrap_err(), + QueryError { + row: 0, + offset: 0, + column: 0, + kind: QueryErrorKind::Structure, + message: [ + "(identifier/identifier)", // + "^" + ] + .join("\n") + } + ); + + if js_lang.abi_version() >= 15 { + assert_eq!( + Query::new(&js_lang, "(statement/identifier)").unwrap_err(), + QueryError { + row: 0, + offset: 0, + column: 0, + kind: QueryErrorKind::Structure, + message: [ + "(statement/identifier)", // + "^" + ] + .join("\n") + } + ); + assert_eq!( + Query::new(&js_lang, "(statement/pattern)").unwrap_err(), + QueryError { + row: 0, + offset: 0, + column: 0, + kind: QueryErrorKind::Structure, + message: [ + "(statement/pattern)", // + "^" + ] + .join("\n") + } + ); + } }); } @@ -725,8 +839,7 @@ fn test_query_matches_with_many_overlapping_results() { // .foo(bar(BAZ)) // .foo(bar(BAZ)) // ... - let mut source = "a".to_string(); - source += &"\n .foo(bar(BAZ))".repeat(count); + let source = format!("a{}", "\n .foo(bar(BAZ))".repeat(count)); assert_query_matches( &language, @@ -767,6 +880,74 @@ fn test_query_matches_capturing_error_nodes() { }); } +#[test] +fn test_query_matches_capturing_missing_nodes() { + allocations::record(|| { + let language = get_language("javascript"); + let query = Query::new( + &language, + r#" + (MISSING + ; Comments should be valid + ) @missing + (MISSING + ; Comments should be valid + ";" + ; Comments should be valid + ) @missing-semicolon + "#, + ) + .unwrap(); + + // Missing anonymous nodes + assert_query_matches( + &language, + &query, + " + x = function(a) { b; } function(c) { d; } + // ^ MISSING semicolon here + ", + &[ + (0, vec![("missing", "")]), + (1, vec![("missing-semicolon", "")]), + ], + ); + + let language = get_language("c"); + let query = Query::new( + &language, + "(MISSING field_identifier) @missing-field-ident + (MISSING identifier) @missing-ident + (MISSING) @missing-anything", + ) + .unwrap(); + + // Missing named nodes + assert_query_matches( + &language, + &query, + " + int main() { + if (a.) { + // ^ MISSING field_identifier here + b(); + c(); + + if (*) d(); + // ^ MISSING identifier here + } + } + ", + &[ + (0, vec![("missing-field-ident", "")]), + (2, vec![("missing-anything", "")]), + (1, vec![("missing-ident", "")]), + (2, vec![("missing-anything", "")]), + ], + ); + }); +} + #[test] fn test_query_matches_with_extra_children() { allocations::record(|| { @@ -2488,6 +2669,64 @@ fn test_query_matches_within_range_of_long_repetition() { }); } +#[test] +fn test_query_matches_contained_within_range() { + allocations::record(|| { + let language = get_language("json"); + let query = Query::new( + &language, + r#" + ("[" @l_bracket "]" @r_bracket) + ("{" @l_brace "}" @r_brace) + "#, + ) + .unwrap(); + + let source = r#" + [ + {"key1": "value1"}, + {"key2": "value2"}, + {"key3": "value3"}, + {"key4": "value4"}, + {"key5": "value5"}, + {"key6": "value6"}, + {"key7": "value7"}, + {"key8": "value8"}, + {"key9": "value9"}, + {"key10": "value10"}, + {"key11": "value11"}, + {"key12": "value12"}, + ] + "# + .unindent(); + + let mut parser = Parser::new(); + parser.set_language(&language).unwrap(); + let tree = parser.parse(&source, None).unwrap(); + + let expected_matches = [ + (1, vec![("l_brace", "{"), ("r_brace", "}")]), + (1, vec![("l_brace", "{"), ("r_brace", "}")]), + ]; + { + let mut cursor = QueryCursor::new(); + let matches = cursor + .set_containing_point_range(Point::new(5, 0)..Point::new(7, 0)) + .matches(&query, tree.root_node(), source.as_bytes()); + assert_eq!(collect_matches(matches, &query, &source), &expected_matches); + } + { + let mut cursor = QueryCursor::new(); + let matches = cursor.set_containing_byte_range(78..120).matches( + &query, + tree.root_node(), + source.as_bytes(), + ); + assert_eq!(collect_matches(matches, &query, &source), &expected_matches); + } + }); +} + #[test] fn test_query_matches_different_queries_same_cursor() { allocations::record(|| { @@ -2836,6 +3075,61 @@ fn test_query_matches_with_deeply_nested_patterns_with_fields() { }); } +#[test] +fn test_query_matches_with_alternations_and_predicates() { + allocations::record(|| { + let language = get_language("java"); + let query = Query::new( + &language, + " + (block + [ + (local_variable_declaration + (variable_declarator + (identifier) @def.a + (string_literal) @lit.a + ) + ) + (local_variable_declaration + (variable_declarator + (identifier) @def.b + (null_literal) @lit.b + ) + ) + ] + (expression_statement + (method_invocation [ + (argument_list + (identifier) @ref.a + (string_literal) + ) + (argument_list + (null_literal) + (identifier) @ref.b + ) + ]) + ) + (#eq? @def.a @ref.a ) + (#eq? @def.b @ref.b ) + ) + ", + ) + .unwrap(); + + assert_query_matches( + &language, + &query, + r#" + void test() { + int a = "foo"; + f(null, b); + } + "#, + &[], + ); + }); +} + #[test] fn test_query_matches_with_indefinite_step_containing_no_captures() { allocations::record(|| { @@ -3965,12 +4259,9 @@ fn test_query_random() { let pattern = pattern_ast.to_string(); let expected_matches = pattern_ast.matches_in_tree(&test_tree); - let query = match Query::new(&language, &pattern) { - Ok(query) => query, - Err(e) => { - panic!("failed to build query for pattern {pattern} - {e}. seed: {seed}"); - } - }; + let query = Query::new(&language, &pattern).unwrap_or_else(|e| { + panic!("failed to build query for pattern {pattern}. seed: {seed}\n{e}") + }); let mut actual_matches = Vec::new(); let mut match_iter = cursor.matches( &query, @@ -4799,6 +5090,26 @@ fn test_query_quantified_captures() { ("comment.documentation", "// quuz"), ], }, + Row { + description: "multiple quantifiers should not hang query parsing", + language: get_language("c"), + code: indoc! {" + // foo + // bar + // baz + "}, + pattern: r" + ((comment) ?+ @comment) + ", + // This should be identical to the `*` quantifier. + captures: &[ + ("comment", "// foo"), + ("comment", "// foo"), + ("comment", "// foo"), + ("comment", "// bar"), + ("comment", "// baz"), + ], + }, ]; allocations::record(|| { @@ -4938,7 +5249,7 @@ fn test_query_error_does_not_oob() { offset: 1, column: 1, kind: QueryErrorKind::NodeType, - message: "clas".to_string() + message: "\"clas\"".to_string() } ); } @@ -5074,7 +5385,7 @@ fn test_grammar_with_aliased_literal_query() { // expansion: $ => seq('}'), // }, // }); - let (parser_name, parser_code) = generate_parser_for_grammar( + let (parser_name, parser_code) = generate_parser( r#" { "name": "test", @@ -5225,8 +5536,13 @@ fn test_query_execution_with_timeout() { &query, tree.root_node(), source_code.as_bytes(), - QueryCursorOptions::new() - .progress_callback(&mut |_| start_time.elapsed().as_micros() > 1000), + QueryCursorOptions::new().progress_callback(&mut |_| { + if start_time.elapsed().as_micros() > 1000 { + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } + }), ) .count(); assert!(matches < 1000); @@ -5236,3 +5552,412 @@ fn test_query_execution_with_timeout() { .count(); assert_eq!(matches, 1000); } + +#[test] +fn test_query_execution_with_points_causing_underflow() { + let language = get_language("rust"); + let mut parser = Parser::new(); + parser.set_language(&language).unwrap(); + + #[allow(clippy::literal_string_with_formatting_args)] + let code = r#"fn main() { + println!("{:?}", foo()); +}"#; + parser + .set_included_ranges(&[Range { + start_byte: 24, + end_byte: 39, + start_point: Point::new(0, 0), // 5, 12 + end_point: Point::new(0, 0), // 5, 27 + }]) + .unwrap(); + + let query = Query::new(&language, "(call_expression) @cap").unwrap(); + let mut cursor = QueryCursor::new(); + + let mut tree = parser.parse(code, None).unwrap(); + + let matches = { + let root_node = tree.root_node(); + let matches = cursor.matches(&query, root_node, code.as_bytes()); + collect_matches(matches, &query, code) + .into_iter() + .map(|(i, m)| { + ( + i, + m.into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect::>(), + ) + }) + .collect::>() + }; + + tree.edit(&InputEdit { + start_byte: 40, + old_end_byte: 40, + new_end_byte: 41, + start_position: Point::new(1, 28), + old_end_position: Point::new(1, 28), + new_end_position: Point::new(2, 0), + }); + + let tree2 = parser.parse(code, Some(&tree)).unwrap(); + + let matches2 = { + let root_node = tree2.root_node(); + let matches = cursor.matches(&query, root_node, code.as_bytes()); + collect_matches(matches, &query, code) + .into_iter() + .map(|(i, m)| { + ( + i, + m.into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect::>(), + ) + }) + .collect::>() + }; + + assert_eq!(matches, matches2); +} + +#[test] +fn test_wildcard_behavior_before_anchor() { + let language = get_language("python"); + let mut parser = Parser::new(); + parser.set_language(&language).unwrap(); + + let source = " + (a, b) + (c, d,) + "; + + // In this query, we're targeting any *named* node immediately before a closing parenthesis. + let query = Query::new(&language, r#"(tuple (_) @last . ")" .) @match"#).unwrap(); + assert_query_matches( + &language, + &query, + source, + &[ + (0, vec![("match", "(a, b)"), ("last", "b")]), + (0, vec![("match", "(c, d,)"), ("last", "d")]), + ], + ); + + // In this query, we're targeting *any* node immediately before a closing + // parenthesis. + let query = Query::new(&language, r#"(tuple _ @last . ")" .) @match"#).unwrap(); + assert_query_matches( + &language, + &query, + source, + &[ + (0, vec![("match", "(a, b)"), ("last", "b")]), + (0, vec![("match", "(c, d,)"), ("last", ",")]), + ], + ); +} + +#[test] +fn test_pattern_alternatives_follow_last_child_constraint() { + let language = get_language("rust"); + let mut parser = Parser::new(); + parser.set_language(&language).unwrap(); + + let code = " +fn f() { + if a {} // <- should NOT match + if b {} +}"; + + let tree = parser.parse(code, None).unwrap(); + let mut cursor = QueryCursor::new(); + + let query = Query::new( + &language, + "(block + [ + (type_cast_expression) + (expression_statement) + ] @last + . + )", + ) + .unwrap(); + + let matches = { + let root_node = tree.root_node(); + let matches = cursor.matches(&query, root_node, code.as_bytes()); + collect_matches(matches, &query, code) + .into_iter() + .map(|(i, m)| { + ( + i, + m.into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect::>(), + ) + }) + .collect::>() + }; + + let flipped_query = Query::new( + &language, + "(block + [ + (expression_statement) + (type_cast_expression) + ] @last + . + )", + ) + .unwrap(); + + let flipped_matches = { + let root_node = tree.root_node(); + let matches = cursor.matches(&flipped_query, root_node, code.as_bytes()); + collect_matches(matches, &flipped_query, code) + .into_iter() + .map(|(i, m)| { + ( + i, + m.into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect::>(), + ) + }) + .collect::>() + }; + + assert_eq!( + matches, + vec![(0, vec![(String::from("last"), String::from("if b {}"))])] + ); + assert_eq!(matches, flipped_matches); +} + +#[test] +fn test_wildcard_parent_allows_fallible_child_patterns() { + let language = get_language("javascript"); + let mut parser = Parser::new(); + parser.set_language(&language).unwrap(); + + let source_code = r#" +function foo() { + "bar" +} + "#; + + let query = Query::new( + &language, + "(function_declaration + (_ + (expression_statement) + ) + ) @part", + ) + .unwrap(); + + assert_query_matches( + &language, + &query, + source_code, + &[(0, vec![("part", "function foo() {\n \"bar\"\n}")])], + ); +} + +#[test] +fn test_unfinished_captures_are_not_definite_with_pending_anchors() { + let language = get_language("javascript"); + let mut parser = Parser::new(); + parser.set_language(&language).unwrap(); + + let source_code = " +const foo = [ + 1, 2, 3 +] +"; + + let tree = parser.parse(source_code, None).unwrap(); + let query = Query::new(&language, r#"(array (_) @foo . "]")"#).unwrap(); + let mut matches_cursor = QueryCursor::new(); + let mut captures_cursor = QueryCursor::new(); + + let captures = captures_cursor.captures(&query, tree.root_node(), source_code.as_bytes()); + let captures = collect_captures(captures, &query, source_code); + + let matches = matches_cursor.matches(&query, tree.root_node(), source_code.as_bytes()); + let matches = collect_matches(matches, &query, source_code); + + assert_eq!(captures, vec![("foo", "3")]); + assert_eq!(matches.len(), 1); + assert_eq!(matches[0].1, captures); +} + +#[test] +fn test_query_with_predicate_causing_oob_access() { + let language = get_language("rust"); + + let query = "(call_expression + function: (scoped_identifier + path: (scoped_identifier (identifier) @_regex (#any-of? @_regex \"Regex\" \"RegexBuilder\") .)) + (#set! injection.language \"regex\"))"; + Query::new(&language, query).unwrap(); +} + +#[test] +fn test_query_with_anonymous_error_node() { + let language = get_test_fixture_language("anonymous_error"); + let mut parser = Parser::new(); + parser.set_language(&language).unwrap(); + + let source = "ERROR"; + + let tree = parser.parse(source, None).unwrap(); + let query = Query::new( + &language, + r#" + "ERROR" @error + (document "ERROR" @error) + "#, + ) + .unwrap(); + let mut cursor = QueryCursor::new(); + let matches = cursor.matches(&query, tree.root_node(), source.as_bytes()); + let matches = collect_matches(matches, &query, source); + + assert_eq!( + matches, + vec![(1, vec![("error", "ERROR")]), (0, vec![("error", "ERROR")])] + ); +} + +#[test] +fn test_query_allows_error_nodes_with_children() { + allocations::record(|| { + let language = get_language("cpp"); + + let code = "SomeStruct foo{.bar{}};"; + + let mut parser = Parser::new(); + parser.set_language(&language).unwrap(); + + let tree = parser.parse(code, None).unwrap(); + let root = tree.root_node(); + + let query = Query::new(&language, "(initializer_list (ERROR) @error)").unwrap(); + let mut cursor = QueryCursor::new(); + + let matches = cursor.matches(&query, root, code.as_bytes()); + let matches = collect_matches(matches, &query, code); + assert_eq!(matches, &[(0, vec![("error", ".bar")])]); + }); +} + +#[test] +fn test_query_assertion_on_unreachable_node_with_child() { + // The `await_binding` rule is unreachable because it has a lower precedence than + // `identifier`, so we'll always reduce to an expression of type `identifier` + // instead whenever we see the token `await` followed by an identifier. + // + // A query that tries to capture the `await` token in the `await_binding` rule + // should not cause an assertion failure during query analysis. + let grammar = r#" +export default grammar({ + name: "query_assertion_crash", + + rules: { + source_file: $ => repeat($.expression), + + expression: $ => choice( + $.await_binding, + $.await_expr, + $.equal_expr, + prec(3, $.identifier), + ), + + await_binding: $ => prec(1, seq('await', $.identifier, '=', $.expression)), + + await_expr: $ => prec(1, seq('await', $.expression)), + + equal_expr: $ => prec.right(2, seq($.expression, '=', $.expression)), + + identifier: _ => /[a-z]+/, + } +}); + "#; + + let file = tempfile::NamedTempFile::with_suffix(".js").unwrap(); + std::fs::write(file.path(), grammar).unwrap(); + + let grammar_json = load_grammar_file(file.path(), None).unwrap(); + + let (parser_name, parser_code) = generate_parser(&grammar_json).unwrap(); + + let language = get_test_language(&parser_name, &parser_code, None); + + let query_result = Query::new(&language, r#"(await_binding "await")"#); + + assert!(query_result.is_err()); + assert_eq!( + query_result.unwrap_err(), + QueryError { + kind: QueryErrorKind::Structure, + row: 0, + offset: 0, + column: 0, + message: ["(await_binding \"await\")", "^"].join("\n"), + } + ); +} + +#[test] +fn test_query_supertype_with_anonymous_node() { + let grammar = r#" +export default grammar({ + name: "supertype_anonymous_test", + + extras: $ => [/\s/, $.comment], + + supertypes: $ => [$.expression], + + word: $ => $.identifier, + + rules: { + source_file: $ => repeat($.expression), + + expression: $ => choice( + $.function_call, + '()' // an empty tuple, which should be queryable with the supertype syntax + ), + + function_call: $ => seq($.identifier, '()'), + + identifier: _ => /[a-zA-Z_][a-zA-Z0-9_]*/, + + comment: _ => token(seq('//', /.*/)), + } +}); + "#; + + let file = tempfile::NamedTempFile::with_suffix(".js").unwrap(); + std::fs::write(file.path(), grammar).unwrap(); + + let grammar_json = load_grammar_file(file.path(), None).unwrap(); + + let (parser_name, parser_code) = generate_parser(&grammar_json).unwrap(); + + let language = get_test_language(&parser_name, &parser_code, None); + + let query_result = Query::new(&language, r#"(expression/"()") @tuple"#); + + assert!(query_result.is_ok()); + + let query = query_result.unwrap(); + + let source = "foo()\n()"; + + assert_query_matches(&language, &query, source, &[(0, vec![("tuple", "()")])]); +} diff --git a/cli/src/tests/tags_test.rs b/crates/cli/src/tests/tags_test.rs similarity index 93% rename from cli/src/tests/tags_test.rs rename to crates/cli/src/tests/tags_test.rs index cb07fb75..0c9f7111 100644 --- a/cli/src/tests/tags_test.rs +++ b/crates/cli/src/tests/tags_test.rs @@ -1,6 +1,7 @@ use std::{ ffi::{CStr, CString}, fs, ptr, slice, str, + sync::atomic::{AtomicUsize, Ordering}, }; use tree_sitter::Point; @@ -262,34 +263,34 @@ fn test_tags_ruby() { #[test] fn test_tags_cancellation() { - use std::sync::atomic::{AtomicUsize, Ordering}; - allocations::record(|| { // Large javascript document - let source = (0..500) - .map(|_| "/* hi */ class A { /* ok */ b() {} }\n") - .collect::(); - + let source = "/* hi */ class A { /* ok */ b() {} }\n".repeat(500); let cancellation_flag = AtomicUsize::new(0); let language = get_language("javascript"); let tags_config = TagsConfiguration::new(language, JS_TAG_QUERY, "").unwrap(); - let mut tag_context = TagsContext::new(); let tags = tag_context .generate_tags(&tags_config, source.as_bytes(), Some(&cancellation_flag)) .unwrap(); - for (i, tag) in tags.0.enumerate() { + let found_cancellation_error = tags.0.enumerate().any(|(i, tag)| { if i == 150 { cancellation_flag.store(1, Ordering::SeqCst); } - if let Err(e) = tag { - assert_eq!(e, Error::Cancelled); - return; + match tag { + Ok(_) => false, + Err(Error::Cancelled) => true, + Err(e) => { + unreachable!("Unexpected error type while iterating tags: {e}") + } } - } + }); - panic!("Expected to halt tagging with an error"); + assert!( + found_cancellation_error, + "Expected to halt tagging with a cancellation error" + ); }); } @@ -401,8 +402,11 @@ fn test_tags_via_c_api() { let syntax_types = unsafe { let mut len = 0; - let ptr = - c::ts_tagger_syntax_kinds_for_scope_name(tagger, c_scope_name.as_ptr(), &mut len); + let ptr = c::ts_tagger_syntax_kinds_for_scope_name( + tagger, + c_scope_name.as_ptr(), + &raw mut len, + ); slice::from_raw_parts(ptr, len as usize) .iter() .map(|i| CStr::from_ptr(*i).to_str().unwrap()) diff --git a/cli/src/tests/test_highlight_test.rs b/crates/cli/src/tests/test_highlight_test.rs similarity index 83% rename from cli/src/tests/test_highlight_test.rs rename to crates/cli/src/tests/test_highlight_test.rs index 054e33f8..e8567d6d 100644 --- a/cli/src/tests/test_highlight_test.rs +++ b/crates/cli/src/tests/test_highlight_test.rs @@ -23,7 +23,7 @@ fn test_highlight_test_with_basic_test() { "// hi", "var abc = function(d) {", " // ^ function", - " // ^ keyword", + " // ^^^ keyword", " return d + e;", " // ^ variable", " // ^ !variable", @@ -39,12 +39,12 @@ fn test_highlight_test_with_basic_test() { assert_eq!( assertions, &[ - Assertion::new(1, 5, false, String::from("function")), - Assertion::new(1, 11, false, String::from("keyword")), - Assertion::new(4, 9, false, String::from("variable")), - Assertion::new(4, 11, true, String::from("variable")), - Assertion::new(8, 5, false, String::from("function")), - Assertion::new(8, 11, false, String::from("keyword")), + Assertion::new(1, 5, 1, false, String::from("function")), + Assertion::new(1, 11, 3, false, String::from("keyword")), + Assertion::new(4, 9, 1, false, String::from("variable")), + Assertion::new(4, 11, 1, true, String::from("variable")), + Assertion::new(8, 5, 1, false, String::from("function")), + Assertion::new(8, 11, 1, false, String::from("keyword")), ] ); diff --git a/cli/src/tests/test_tags_test.rs b/crates/cli/src/tests/test_tags_test.rs similarity index 83% rename from cli/src/tests/test_tags_test.rs rename to crates/cli/src/tests/test_tags_test.rs index 5f7b88fc..c9019a71 100644 --- a/cli/src/tests/test_tags_test.rs +++ b/crates/cli/src/tests/test_tags_test.rs @@ -30,10 +30,10 @@ fn test_tags_test_with_basic_test() { assert_eq!( assertions, &[ - Assertion::new(1, 4, false, String::from("definition.function")), - Assertion::new(3, 9, false, String::from("reference.call")), - Assertion::new(5, 11, false, String::from("reference.call")), - Assertion::new(5, 13, true, String::from("variable.parameter")), + Assertion::new(1, 4, 1, false, String::from("definition.function")), + Assertion::new(3, 9, 1, false, String::from("reference.call")), + Assertion::new(5, 11, 1, false, String::from("reference.call")), + Assertion::new(5, 13, 1, true, String::from("variable.parameter")), ] ); diff --git a/cli/src/tests/text_provider_test.rs b/crates/cli/src/tests/text_provider_test.rs similarity index 92% rename from cli/src/tests/text_provider_test.rs rename to crates/cli/src/tests/text_provider_test.rs index ffedc36b..6c4e8939 100644 --- a/cli/src/tests/text_provider_test.rs +++ b/crates/cli/src/tests/text_provider_test.rs @@ -21,7 +21,6 @@ where let mut parser = Parser::new(); parser.set_language(&language).unwrap(); let tree = parser.parse_with_options(callback, None, None).unwrap(); - // eprintln!("{}", tree.clone().root_node().to_sexp()); assert_eq!("comment", tree.root_node().child(0).unwrap().kind()); (tree, language) } @@ -107,6 +106,19 @@ fn test_text_provider_for_arc_of_bytes_slice() { check_parsing(text.clone(), text.as_ref()); } +#[test] +fn test_text_provider_for_vec_utf16_text() { + let source_text = "你好".encode_utf16().collect::>(); + + let language = get_language("c"); + let mut parser = Parser::new(); + parser.set_language(&language).unwrap(); + let tree = parser.parse_utf16_le(&source_text, None).unwrap(); + + let tree_text = tree.root_node().utf16_text(&source_text); + assert_eq!(source_text, tree_text); +} + #[test] fn test_text_provider_callback_with_str_slice() { let text: &str = "// comment"; diff --git a/cli/src/tests/tree_test.rs b/crates/cli/src/tests/tree_test.rs similarity index 95% rename from cli/src/tests/tree_test.rs rename to crates/cli/src/tests/tree_test.rs index 083955b1..a4941bb6 100644 --- a/cli/src/tests/tree_test.rs +++ b/crates/cli/src/tests/tree_test.rs @@ -3,7 +3,11 @@ use std::str; use tree_sitter::{InputEdit, Parser, Point, Range, Tree}; use super::helpers::fixtures::get_language; -use crate::{fuzz::edits::Edit, parse::perform_edit, tests::invert_edit}; +use crate::{ + fuzz::edits::Edit, + parse::perform_edit, + tests::{helpers::fixtures::get_test_fixture_language, invert_edit}, +}; #[test] fn test_tree_edit() { @@ -377,6 +381,40 @@ fn test_tree_cursor() { assert_eq!(copy.node().kind(), "struct_item"); } +#[test] +fn test_tree_cursor_previous_sibling_with_aliases() { + let mut parser = Parser::new(); + parser + .set_language(&get_test_fixture_language("aliases_in_root")) + .unwrap(); + + let text = "# comment\n# \nfoo foo"; + let tree = parser.parse(text, None).unwrap(); + let mut cursor = tree.walk(); + assert_eq!(cursor.node().kind(), "document"); + + cursor.goto_first_child(); + assert_eq!(cursor.node().kind(), "comment"); + + assert!(cursor.goto_next_sibling()); + assert_eq!(cursor.node().kind(), "comment"); + + assert!(cursor.goto_next_sibling()); + assert_eq!(cursor.node().kind(), "bar"); + + assert!(cursor.goto_previous_sibling()); + assert_eq!(cursor.node().kind(), "comment"); + + assert!(cursor.goto_previous_sibling()); + assert_eq!(cursor.node().kind(), "comment"); + + assert!(cursor.goto_next_sibling()); + assert_eq!(cursor.node().kind(), "comment"); + + assert!(cursor.goto_next_sibling()); + assert_eq!(cursor.node().kind(), "bar"); +} + #[test] fn test_tree_cursor_previous_sibling() { let mut parser = Parser::new(); diff --git a/cli/src/tests/wasm_language_test.rs b/crates/cli/src/tests/wasm_language_test.rs similarity index 87% rename from cli/src/tests/wasm_language_test.rs rename to crates/cli/src/tests/wasm_language_test.rs index 34584dae..d62095c0 100644 --- a/cli/src/tests/wasm_language_test.rs +++ b/crates/cli/src/tests/wasm_language_test.rs @@ -1,17 +1,13 @@ use std::fs; -use lazy_static::lazy_static; use streaming_iterator::StreamingIterator; -use tree_sitter::{ - wasmtime::Engine, Parser, Query, QueryCursor, WasmError, WasmErrorKind, WasmStore, +use tree_sitter::{Parser, Query, QueryCursor, WasmError, WasmErrorKind, WasmStore}; + +use crate::tests::helpers::{ + allocations, + fixtures::{get_test_fixture_language_wasm, ENGINE, WASM_DIR}, }; -use crate::tests::helpers::{allocations, fixtures::WASM_DIR}; - -lazy_static! { - static ref ENGINE: Engine = Engine::default(); -} - #[test] fn test_wasm_stdlib_symbols() { let symbols = tree_sitter::wasm_stdlib_symbols().collect::>(); @@ -95,6 +91,33 @@ fn test_load_wasm_javascript_language() { }); } +#[test] +fn test_load_wasm_python_language() { + allocations::record(|| { + let mut store = WasmStore::new(&ENGINE).unwrap(); + let mut parser = Parser::new(); + let wasm = fs::read(WASM_DIR.join("tree-sitter-python.wasm")).unwrap(); + let language = store.load_language("python", &wasm).unwrap(); + parser.set_wasm_store(store).unwrap(); + parser.set_language(&language).unwrap(); + let tree = parser.parse("a = b\nc = d", None).unwrap(); + assert_eq!(tree.root_node().to_sexp(), "(module (expression_statement (assignment left: (identifier) right: (identifier))) (expression_statement (assignment left: (identifier) right: (identifier))))"); + }); +} + +#[test] +fn test_load_fixture_language_wasm() { + allocations::record(|| { + let store = WasmStore::new(&ENGINE).unwrap(); + let mut parser = Parser::new(); + let language = get_test_fixture_language_wasm("epsilon_external_tokens"); + parser.set_wasm_store(store).unwrap(); + parser.set_language(&language).unwrap(); + let tree = parser.parse("hello", None).unwrap(); + assert_eq!(tree.root_node().to_sexp(), "(document (zero_width))"); + }); +} + #[test] fn test_load_multiple_wasm_languages() { allocations::record(|| { @@ -119,7 +142,7 @@ fn test_load_multiple_wasm_languages() { let mut query_cursor = QueryCursor::new(); // First, parse with the store that originally loaded the languages. - // Then parse with a new parser and wasm store, so that the languages + // Then parse with a new parser and Wasm store, so that the languages // are added one-by-one, in between parses. for mut parser in [parser, parser2] { for _ in 0..2 { @@ -229,7 +252,7 @@ fn test_load_wasm_errors() { store.load_language("rust", bad_wasm).unwrap_err(), WasmError { kind: WasmErrorKind::Parse, - message: "failed to parse dylink section of wasm module".into(), + message: "failed to parse dylink section of Wasm module".into(), } ); diff --git a/cli/src/lib.rs b/crates/cli/src/tree_sitter_cli.rs similarity index 77% rename from cli/src/lib.rs rename to crates/cli/src/tree_sitter_cli.rs index 657e9d9c..3960d961 100644 --- a/cli/src/lib.rs +++ b/crates/cli/src/tree_sitter_cli.rs @@ -1,8 +1,9 @@ -#![doc = include_str!("../README.md")] +#![cfg_attr(not(any(test, doctest)), doc = include_str!("../README.md"))] pub mod fuzz; pub mod highlight; pub mod init; +pub mod input; pub mod logger; pub mod parse; pub mod playground; @@ -19,6 +20,5 @@ pub mod wasm; #[cfg(test)] mod tests; -// To run compile fail tests #[cfg(doctest)] mod tests; diff --git a/cli/src/util.rs b/crates/cli/src/util.rs similarity index 99% rename from cli/src/util.rs rename to crates/cli/src/util.rs index fd4f4699..72968db8 100644 --- a/cli/src/util.rs +++ b/crates/cli/src/util.rs @@ -9,6 +9,7 @@ use std::{ use anyhow::{anyhow, Context, Result}; use indoc::indoc; +use log::error; use tree_sitter::{Parser, Tree}; use tree_sitter_config::Config; use tree_sitter_loader::Config as LoaderConfig; @@ -120,7 +121,7 @@ impl Drop for LogSession { webbrowser::open(&self.path.to_string_lossy()).unwrap(); } } else { - eprintln!( + error!( "Dot failed: {} {}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) diff --git a/crates/cli/src/version.rs b/crates/cli/src/version.rs new file mode 100644 index 00000000..757306c7 --- /dev/null +++ b/crates/cli/src/version.rs @@ -0,0 +1,396 @@ +use std::{fs, path::PathBuf, process::Command}; + +use clap::ValueEnum; +use log::{info, warn}; +use regex::Regex; +use semver::Version as SemverVersion; +use std::cmp::Ordering; +use tree_sitter_loader::TreeSitterJSON; + +#[derive(Clone, Copy, Default, ValueEnum)] +pub enum BumpLevel { + #[default] + Patch, + Minor, + Major, +} + +pub struct Version { + pub version: Option, + pub current_dir: PathBuf, + pub bump: Option, +} + +#[derive(thiserror::Error, Debug)] +pub enum VersionError { + #[error(transparent)] + Json(#[from] serde_json::Error), + #[error(transparent)] + Io(#[from] std::io::Error), + #[error("Failed to update one or more files:\n\n{0}")] + Update(UpdateErrors), +} + +#[derive(thiserror::Error, Debug)] +pub struct UpdateErrors(Vec); + +impl std::fmt::Display for UpdateErrors { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + for error in &self.0 { + writeln!(f, "{error}\n")?; + } + Ok(()) + } +} + +#[derive(thiserror::Error, Debug)] +pub enum UpdateError { + #[error("Failed to update {1}:\n{0}")] + Io(std::io::Error, PathBuf), + #[error("Failed to run `{0}`:\n{1}")] + Command(&'static str, String), +} + +impl Version { + #[must_use] + pub const fn new( + version: Option, + current_dir: PathBuf, + bump: Option, + ) -> Self { + Self { + version, + current_dir, + bump, + } + } + + pub fn run(mut self) -> Result<(), VersionError> { + let tree_sitter_json = self.current_dir.join("tree-sitter.json"); + + let tree_sitter_json = + serde_json::from_str::(&fs::read_to_string(tree_sitter_json)?)?; + + let current_version = tree_sitter_json.metadata.version; + self.version = match (self.version.is_some(), self.bump) { + (false, None) => { + info!("Current version: {current_version}"); + return Ok(()); + } + (true, None) => self.version, + (false, Some(bump)) => { + let mut v = current_version.clone(); + match bump { + BumpLevel::Patch => v.patch += 1, + BumpLevel::Minor => { + v.minor += 1; + v.patch = 0; + } + BumpLevel::Major => { + v.major += 1; + v.minor = 0; + v.patch = 0; + } + } + Some(v) + } + (true, Some(_)) => unreachable!(), + }; + + let new_version = self.version.as_ref().unwrap(); + match new_version.cmp(¤t_version) { + Ordering::Less => { + warn!("New version is lower than current!"); + warn!("Reverting version {current_version} to {new_version}"); + } + Ordering::Greater => { + info!("Bumping version {current_version} to {new_version}"); + } + Ordering::Equal => { + info!("Keeping version {current_version}"); + } + } + + let is_multigrammar = tree_sitter_json.grammars.len() > 1; + + let mut errors = Vec::new(); + + // Helper to push errors into the errors vector, returns true if an error was pushed + let mut push_err = |result: Result<(), UpdateError>| -> bool { + if let Err(e) = result { + errors.push(e); + return true; + } + false + }; + + push_err(self.update_treesitter_json()); + + // Only update Cargo.lock if Cargo.toml was updated + push_err(self.update_cargo_toml()).then(|| push_err(self.update_cargo_lock())); + + // Only update package-lock.json if package.json was updated + push_err(self.update_package_json()).then(|| push_err(self.update_package_lock_json())); + + push_err(self.update_makefile(is_multigrammar)); + push_err(self.update_cmakelists_txt()); + push_err(self.update_pyproject_toml()); + push_err(self.update_zig_zon()); + + if errors.is_empty() { + Ok(()) + } else { + Err(VersionError::Update(UpdateErrors(errors))) + } + } + + fn update_file_with(&self, path: &PathBuf, update_fn: F) -> Result<(), UpdateError> + where + F: Fn(&str) -> String, + { + let content = fs::read_to_string(path).map_err(|e| UpdateError::Io(e, path.clone()))?; + let updated_content = update_fn(&content); + fs::write(path, updated_content).map_err(|e| UpdateError::Io(e, path.clone())) + } + + fn update_treesitter_json(&self) -> Result<(), UpdateError> { + let json_path = self.current_dir.join("tree-sitter.json"); + self.update_file_with(&json_path, |content| { + content + .lines() + .map(|line| { + if line.contains("\"version\":") { + let prefix_index = + line.find("\"version\":").unwrap() + "\"version\":".len(); + let start_quote = + line[prefix_index..].find('"').unwrap() + prefix_index + 1; + let end_quote = + line[start_quote + 1..].find('"').unwrap() + start_quote + 1; + + format!( + "{}{}{}", + &line[..start_quote], + self.version.as_ref().unwrap(), + &line[end_quote..] + ) + } else { + line.to_string() + } + }) + .collect::>() + .join("\n") + + "\n" + }) + } + + fn update_cargo_toml(&self) -> Result<(), UpdateError> { + let cargo_toml_path = self.current_dir.join("Cargo.toml"); + if !cargo_toml_path.exists() { + return Ok(()); + } + + self.update_file_with(&cargo_toml_path, |content| { + content + .lines() + .map(|line| { + if line.starts_with("version =") { + format!("version = \"{}\"", self.version.as_ref().unwrap()) + } else { + line.to_string() + } + }) + .collect::>() + .join("\n") + + "\n" + })?; + + Ok(()) + } + + fn update_cargo_lock(&self) -> Result<(), UpdateError> { + if self.current_dir.join("Cargo.lock").exists() { + let Ok(cmd) = Command::new("cargo") + .arg("generate-lockfile") + .arg("--offline") + .current_dir(&self.current_dir) + .output() + else { + return Ok(()); // cargo is not `executable`, ignore + }; + + if !cmd.status.success() { + let stderr = String::from_utf8_lossy(&cmd.stderr); + return Err(UpdateError::Command( + "cargo generate-lockfile", + stderr.to_string(), + )); + } + } + + Ok(()) + } + + fn update_package_json(&self) -> Result<(), UpdateError> { + let package_json_path = self.current_dir.join("package.json"); + if !package_json_path.exists() { + return Ok(()); + } + + self.update_file_with(&package_json_path, |content| { + content + .lines() + .map(|line| { + if line.contains("\"version\":") { + let prefix_index = + line.find("\"version\":").unwrap() + "\"version\":".len(); + let start_quote = + line[prefix_index..].find('"').unwrap() + prefix_index + 1; + let end_quote = + line[start_quote + 1..].find('"').unwrap() + start_quote + 1; + + format!( + "{}{}{}", + &line[..start_quote], + self.version.as_ref().unwrap(), + &line[end_quote..] + ) + } else { + line.to_string() + } + }) + .collect::>() + .join("\n") + + "\n" + })?; + + Ok(()) + } + + fn update_package_lock_json(&self) -> Result<(), UpdateError> { + if self.current_dir.join("package-lock.json").exists() { + let Ok(cmd) = Command::new("npm") + .arg("install") + .arg("--package-lock-only") + .current_dir(&self.current_dir) + .output() + else { + return Ok(()); // npm is not `executable`, ignore + }; + + if !cmd.status.success() { + let stderr = String::from_utf8_lossy(&cmd.stderr); + return Err(UpdateError::Command("npm install", stderr.to_string())); + } + } + + Ok(()) + } + + fn update_makefile(&self, is_multigrammar: bool) -> Result<(), UpdateError> { + let makefile_path = if is_multigrammar { + self.current_dir.join("common").join("common.mak") + } else { + self.current_dir.join("Makefile") + }; + + self.update_file_with(&makefile_path, |content| { + content + .lines() + .map(|line| { + if line.starts_with("VERSION") { + format!("VERSION := {}", self.version.as_ref().unwrap()) + } else { + line.to_string() + } + }) + .collect::>() + .join("\n") + + "\n" + })?; + + Ok(()) + } + + fn update_cmakelists_txt(&self) -> Result<(), UpdateError> { + let cmake_lists_path = self.current_dir.join("CMakeLists.txt"); + if !cmake_lists_path.exists() { + return Ok(()); + } + + self.update_file_with(&cmake_lists_path, |content| { + let re = Regex::new(r#"(\s*VERSION\s+)"[0-9]+\.[0-9]+\.[0-9]+""#) + .expect("Failed to compile regex"); + re.replace( + content, + format!(r#"$1"{}""#, self.version.as_ref().unwrap()), + ) + .to_string() + })?; + + Ok(()) + } + + fn update_pyproject_toml(&self) -> Result<(), UpdateError> { + let pyproject_toml_path = self.current_dir.join("pyproject.toml"); + if !pyproject_toml_path.exists() { + return Ok(()); + } + + self.update_file_with(&pyproject_toml_path, |content| { + content + .lines() + .map(|line| { + if line.starts_with("version =") { + format!("version = \"{}\"", self.version.as_ref().unwrap()) + } else { + line.to_string() + } + }) + .collect::>() + .join("\n") + + "\n" + })?; + + Ok(()) + } + + fn update_zig_zon(&self) -> Result<(), UpdateError> { + let zig_zon_path = self.current_dir.join("build.zig.zon"); + if !zig_zon_path.exists() { + return Ok(()); + } + + self.update_file_with(&zig_zon_path, |content| { + let zig_version_prefix = ".version ="; + content + .lines() + .map(|line| { + if line + .trim_start_matches(|c: char| c.is_ascii_whitespace()) + .starts_with(zig_version_prefix) + { + let prefix_index = + line.find(zig_version_prefix).unwrap() + zig_version_prefix.len(); + let start_quote = + line[prefix_index..].find('"').unwrap() + prefix_index + 1; + let end_quote = + line[start_quote + 1..].find('"').unwrap() + start_quote + 1; + + format!( + "{}{}{}", + &line[..start_quote], + self.version.as_ref().unwrap(), + &line[end_quote..] + ) + } else { + line.to_string() + } + }) + .collect::>() + .join("\n") + + "\n" + })?; + + Ok(()) + } +} diff --git a/cli/src/wasm.rs b/crates/cli/src/wasm.rs similarity index 81% rename from cli/src/wasm.rs rename to crates/cli/src/wasm.rs index eca6ac24..09fb459b 100644 --- a/cli/src/wasm.rs +++ b/crates/cli/src/wasm.rs @@ -5,13 +5,13 @@ use std::{ use anyhow::{anyhow, Context, Result}; use tree_sitter::wasm_stdlib_symbols; -use tree_sitter_generate::parse_grammar::GrammarJSON; +use tree_sitter_generate::{load_grammar_file, parse_grammar::GrammarJSON}; use tree_sitter_loader::Loader; use wasmparser::Parser; pub fn load_language_wasm_file(language_dir: &Path) -> Result<(String, Vec)> { let grammar_name = get_grammar_name(language_dir) - .with_context(|| "Failed to get wasm filename") + .with_context(|| "Failed to get Wasm filename") .unwrap(); let wasm_filename = format!("tree-sitter-{grammar_name}.wasm"); let contents = fs::read(language_dir.join(&wasm_filename)).with_context(|| { @@ -23,39 +23,44 @@ pub fn load_language_wasm_file(language_dir: &Path) -> Result<(String, Vec)> pub fn get_grammar_name(language_dir: &Path) -> Result { let src_dir = language_dir.join("src"); let grammar_json_path = src_dir.join("grammar.json"); - let grammar_json = fs::read_to_string(&grammar_json_path) - .with_context(|| format!("Failed to read grammar file {grammar_json_path:?}"))?; - let grammar: GrammarJSON = serde_json::from_str(&grammar_json) - .with_context(|| format!("Failed to parse grammar file {grammar_json_path:?}"))?; + let grammar_json = fs::read_to_string(&grammar_json_path).with_context(|| { + format!( + "Failed to read grammar file {}", + grammar_json_path.display() + ) + })?; + let grammar: GrammarJSON = serde_json::from_str(&grammar_json).with_context(|| { + format!( + "Failed to parse grammar file {}", + grammar_json_path.display() + ) + })?; Ok(grammar.name) } pub fn compile_language_to_wasm( loader: &Loader, - root_dir: Option<&Path>, language_dir: &Path, output_dir: &Path, output_file: Option, - force_docker: bool, ) -> Result<()> { - let grammar_name = get_grammar_name(language_dir)?; + let grammar_name = get_grammar_name(language_dir) + .or_else(|_| load_grammar_file(&language_dir.join("grammar.js"), None))?; let output_filename = output_file.unwrap_or_else(|| output_dir.join(format!("tree-sitter-{grammar_name}.wasm"))); let src_path = language_dir.join("src"); let scanner_path = loader.get_scanner_path(&src_path); loader.compile_parser_to_wasm( &grammar_name, - root_dir, &src_path, scanner_path .as_ref() .and_then(|p| Some(Path::new(p.file_name()?))), &output_filename, - force_docker, )?; // Exit with an error if the external scanner uses symbols from the - // C or C++ standard libraries that aren't available to wasm parsers. + // C or C++ standard libraries that aren't available to Wasm parsers. let stdlib_symbols = wasm_stdlib_symbols().collect::>(); let dylink_symbols = [ "__indirect_function_table", @@ -94,7 +99,7 @@ pub fn compile_language_to_wasm( if !missing_symbols.is_empty() { Err(anyhow!( concat!( - "This external scanner uses a symbol that isn't available to wasm parsers.\n", + "This external scanner uses a symbol that isn't available to Wasm parsers.\n", "\n", "Missing symbols:\n", " {}\n", diff --git a/cli/config/Cargo.toml b/crates/config/Cargo.toml similarity index 73% rename from cli/config/Cargo.toml rename to crates/config/Cargo.toml index 5ad7b88f..641b434b 100644 --- a/cli/config/Cargo.toml +++ b/crates/config/Cargo.toml @@ -8,15 +8,20 @@ rust-version.workspace = true readme = "README.md" homepage.workspace = true repository.workspace = true +documentation = "https://docs.rs/tree-sitter-config" license.workspace = true keywords.workspace = true categories.workspace = true +[lib] +path = "src/tree_sitter_config.rs" + [lints] workspace = true [dependencies] -anyhow.workspace = true -dirs.workspace = true +etcetera.workspace = true +log.workspace = true serde.workspace = true serde_json.workspace = true +thiserror.workspace = true diff --git a/crates/config/LICENSE b/crates/config/LICENSE new file mode 100644 index 00000000..971b81f9 --- /dev/null +++ b/crates/config/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2018 Max Brunsfeld + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cli/config/README.md b/crates/config/README.md similarity index 100% rename from cli/config/README.md rename to crates/config/README.md diff --git a/cli/config/src/lib.rs b/crates/config/src/tree_sitter_config.rs similarity index 56% rename from cli/config/src/lib.rs rename to crates/config/src/tree_sitter_config.rs index 6d240c52..17b1d384 100644 --- a/cli/config/src/lib.rs +++ b/crates/config/src/tree_sitter_config.rs @@ -1,10 +1,54 @@ -#![doc = include_str!("../README.md")] +#![cfg_attr(not(any(test, doctest)), doc = include_str!("../README.md"))] -use std::{env, fs, path::PathBuf}; +use std::{ + env, fs, + path::{Path, PathBuf}, +}; -use anyhow::{anyhow, Context, Result}; +use etcetera::BaseStrategy as _; +use log::warn; use serde::{Deserialize, Serialize}; use serde_json::Value; +use thiserror::Error; + +pub type ConfigResult = Result; + +#[derive(Debug, Error)] +pub enum ConfigError { + #[error("Bad JSON config {0} -- {1}")] + ConfigRead(String, serde_json::Error), + #[error(transparent)] + HomeDir(#[from] etcetera::HomeDirError), + #[error(transparent)] + IO(IoError), + #[error(transparent)] + Serialization(#[from] serde_json::Error), +} + +#[derive(Debug, Error)] +pub struct IoError { + pub error: std::io::Error, + pub path: Option, +} + +impl IoError { + fn new(error: std::io::Error, path: Option<&Path>) -> Self { + Self { + error, + path: path.map(|p| p.to_string_lossy().to_string()), + } + } +} + +impl std::fmt::Display for IoError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.error)?; + if let Some(ref path) = self.path { + write!(f, " ({path})")?; + } + Ok(()) + } +} /// Holds the contents of tree-sitter's configuration file. /// @@ -21,7 +65,7 @@ pub struct Config { } impl Config { - pub fn find_config_file() -> Result> { + pub fn find_config_file() -> ConfigResult> { if let Ok(path) = env::var("TREE_SITTER_DIR") { let mut path = PathBuf::from(path); path.push("config.json"); @@ -38,8 +82,28 @@ impl Config { return Ok(Some(xdg_path)); } - let legacy_path = dirs::home_dir() - .ok_or_else(|| anyhow!("Cannot determine home directory"))? + if cfg!(target_os = "macos") { + let legacy_apple_path = etcetera::base_strategy::Apple::new()? + .data_dir() // `$HOME/Library/Application Support/` + .join("tree-sitter") + .join("config.json"); + if legacy_apple_path.is_file() { + let xdg_dir = xdg_path.parent().unwrap(); + fs::create_dir_all(xdg_dir) + .map_err(|e| ConfigError::IO(IoError::new(e, Some(xdg_dir))))?; + fs::rename(&legacy_apple_path, &xdg_path).map_err(|e| { + ConfigError::IO(IoError::new(e, Some(legacy_apple_path.as_path()))) + })?; + warn!( + "Your config.json file has been automatically migrated from \"{}\" to \"{}\"", + legacy_apple_path.display(), + xdg_path.display() + ); + return Ok(Some(xdg_path)); + } + } + + let legacy_path = etcetera::home_dir()? .join(".tree-sitter") .join("config.json"); if legacy_path.is_file() { @@ -49,9 +113,9 @@ impl Config { Ok(None) } - fn xdg_config_file() -> Result { - let xdg_path = dirs::config_dir() - .ok_or_else(|| anyhow!("Cannot determine config directory"))? + fn xdg_config_file() -> ConfigResult { + let xdg_path = etcetera::choose_base_strategy()? + .config_dir() .join("tree-sitter") .join("config.json"); Ok(xdg_path) @@ -63,10 +127,10 @@ impl Config { /// - Location specified by the path parameter if provided /// - `$TREE_SITTER_DIR/config.json`, if the `TREE_SITTER_DIR` environment variable is set /// - `tree-sitter/config.json` in your default user configuration directory, as determined by - /// [`dirs::config_dir`](https://docs.rs/dirs/*/dirs/fn.config_dir.html) + /// [`etcetera::choose_base_strategy`](https://docs.rs/etcetera/*/etcetera/#basestrategy) /// - `$HOME/.tree-sitter/config.json` as a fallback from where tree-sitter _used_ to store /// its configuration - pub fn load(path: Option) -> Result { + pub fn load(path: Option) -> ConfigResult { let location = if let Some(path) = path { path } else if let Some(path) = Self::find_config_file()? { @@ -76,9 +140,9 @@ impl Config { }; let content = fs::read_to_string(&location) - .with_context(|| format!("Failed to read {}", &location.to_string_lossy()))?; + .map_err(|e| ConfigError::IO(IoError::new(e, Some(location.as_path()))))?; let config = serde_json::from_str(&content) - .with_context(|| format!("Bad JSON config {}", &location.to_string_lossy()))?; + .map_err(|e| ConfigError::ConfigRead(location.to_string_lossy().to_string(), e))?; Ok(Self { location, config }) } @@ -88,7 +152,7 @@ impl Config { /// disk. /// /// (Note that this is typically only done by the `tree-sitter init-config` command.) - pub fn initial() -> Result { + pub fn initial() -> ConfigResult { let location = if let Ok(path) = env::var("TREE_SITTER_DIR") { let mut path = PathBuf::from(path); path.push("config.json"); @@ -101,17 +165,20 @@ impl Config { } /// Saves this configuration to the file that it was originally loaded from. - pub fn save(&self) -> Result<()> { + pub fn save(&self) -> ConfigResult<()> { let json = serde_json::to_string_pretty(&self.config)?; - fs::create_dir_all(self.location.parent().unwrap())?; - fs::write(&self.location, json)?; + let config_dir = self.location.parent().unwrap(); + fs::create_dir_all(config_dir) + .map_err(|e| ConfigError::IO(IoError::new(e, Some(config_dir))))?; + fs::write(&self.location, json) + .map_err(|e| ConfigError::IO(IoError::new(e, Some(self.location.as_path()))))?; Ok(()) } /// Parses a component-specific configuration from the configuration file. The type `C` must /// be [deserializable](https://docs.rs/serde/*/serde/trait.Deserialize.html) from a JSON /// object, and must only include the fields relevant to that component. - pub fn get(&self) -> Result + pub fn get(&self) -> ConfigResult where C: for<'de> Deserialize<'de>, { @@ -122,7 +189,7 @@ impl Config { /// Adds a component-specific configuration to the configuration file. The type `C` must be /// [serializable](https://docs.rs/serde/*/serde/trait.Serialize.html) into a JSON object, and /// must only include the fields relevant to that component. - pub fn add(&mut self, config: C) -> Result<()> + pub fn add(&mut self, config: C) -> ConfigResult<()> where C: Serialize, { diff --git a/cli/generate/Cargo.toml b/crates/generate/Cargo.toml similarity index 55% rename from cli/generate/Cargo.toml rename to crates/generate/Cargo.toml index 8f374ae1..e55be890 100644 --- a/cli/generate/Cargo.toml +++ b/crates/generate/Cargo.toml @@ -8,27 +8,44 @@ rust-version.workspace = true readme = "README.md" homepage.workspace = true repository.workspace = true +documentation = "https://docs.rs/tree-sitter-generate" license.workspace = true keywords.workspace = true categories.workspace = true +[lib] +path = "src/generate.rs" + [lints] workspace = true +[features] +default = ["qjs-rt"] +load = ["dep:semver"] +qjs-rt = ["load", "rquickjs", "pathdiff"] + [dependencies] -anyhow.workspace = true -heck.workspace = true +bitflags = "2.9.4" +dunce = "1.0.5" indexmap.workspace = true indoc.workspace = true -lazy_static.workspace = true log.workspace = true +pathdiff = { version = "0.2.3", optional = true } regex.workspace = true regex-syntax.workspace = true +rquickjs = { version = "0.11.0", optional = true, features = [ + "bindgen", + "loader", + "macro", + "phf", +] } rustc-hash.workspace = true -semver.workspace = true +semver = { workspace = true, optional = true } serde.workspace = true serde_json.workspace = true smallbitvec.workspace = true -url.workspace = true +thiserror.workspace = true +topological-sort.workspace = true -tree-sitter.workspace = true +[dev-dependencies] +tempfile.workspace = true diff --git a/crates/generate/LICENSE b/crates/generate/LICENSE new file mode 100644 index 00000000..971b81f9 --- /dev/null +++ b/crates/generate/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2018 Max Brunsfeld + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cli/generate/README.md b/crates/generate/README.md similarity index 100% rename from cli/generate/README.md rename to crates/generate/README.md diff --git a/cli/generate/src/build_tables/mod.rs b/crates/generate/src/build_tables.rs similarity index 82% rename from cli/generate/src/build_tables/mod.rs rename to crates/generate/src/build_tables.rs index 3d7ee4d7..8c6ef2a4 100644 --- a/cli/generate/src/build_tables/mod.rs +++ b/crates/generate/src/build_tables.rs @@ -8,14 +8,16 @@ mod token_conflicts; use std::collections::{BTreeSet, HashMap}; -use anyhow::Result; pub use build_lex_table::LARGE_CHARACTER_RANGE_COUNT; -use log::info; +use build_parse_table::BuildTableResult; +pub use build_parse_table::ParseTableBuilderError; +use log::{debug, info}; use self::{ build_lex_table::build_lex_table, build_parse_table::{build_parse_table, ParseStateInfo}, coincident_tokens::CoincidentTokenIndex, + item_set_builder::ParseItemSetBuilder, minimize_parse_table::minimize_parse_table, token_conflicts::TokenConflictMap, }; @@ -25,13 +27,13 @@ use crate::{ node_types::VariableInfo, rules::{AliasMap, Symbol, SymbolType, TokenSet}, tables::{LexTable, ParseAction, ParseTable, ParseTableEntry}, + OptLevel, }; pub struct Tables { pub parse_table: ParseTable, pub main_lex_table: LexTable, pub keyword_lex_table: LexTable, - pub word_token: Option, pub large_character_sets: Vec<(Option, CharacterSet)>, } @@ -42,9 +44,17 @@ pub fn build_tables( variable_info: &[VariableInfo], inlines: &InlinedProductionMap, report_symbol_name: Option<&str>, -) -> Result { - let (mut parse_table, following_tokens, parse_state_info) = - build_parse_table(syntax_grammar, lexical_grammar, inlines, variable_info)?; + optimizations: OptLevel, +) -> BuildTableResult { + let item_set_builder = ParseItemSetBuilder::new(syntax_grammar, lexical_grammar, inlines); + let following_tokens = + get_following_tokens(syntax_grammar, lexical_grammar, inlines, &item_set_builder); + let (mut parse_table, parse_state_info) = build_parse_table( + syntax_grammar, + lexical_grammar, + item_set_builder, + variable_info, + )?; let token_conflict_map = TokenConflictMap::new(lexical_grammar, following_tokens); let coincident_token_index = CoincidentTokenIndex::new(&parse_table, lexical_grammar); let keywords = identify_keywords( @@ -70,6 +80,7 @@ pub fn build_tables( simple_aliases, &token_conflict_map, &keywords, + optimizations, ); let lex_tables = build_lex_table( &mut parse_table, @@ -92,15 +103,59 @@ pub fn build_tables( ); } + if parse_table.states.len() > u16::MAX as usize { + Err(ParseTableBuilderError::StateCount(parse_table.states.len()))?; + } + Ok(Tables { parse_table, main_lex_table: lex_tables.main_lex_table, keyword_lex_table: lex_tables.keyword_lex_table, large_character_sets: lex_tables.large_character_sets, - word_token: syntax_grammar.word_token, }) } +fn get_following_tokens( + syntax_grammar: &SyntaxGrammar, + lexical_grammar: &LexicalGrammar, + inlines: &InlinedProductionMap, + builder: &ParseItemSetBuilder, +) -> Vec { + let mut result = vec![TokenSet::new(); lexical_grammar.variables.len()]; + let productions = syntax_grammar + .variables + .iter() + .flat_map(|v| &v.productions) + .chain(&inlines.productions); + let all_tokens = (0..result.len()) + .map(Symbol::terminal) + .collect::(); + for production in productions { + for i in 1..production.steps.len() { + let left_tokens = builder.last_set(&production.steps[i - 1].symbol); + let right_tokens = builder.first_set(&production.steps[i].symbol); + let right_reserved_tokens = builder.reserved_first_set(&production.steps[i].symbol); + for left_token in left_tokens.iter() { + if left_token.is_terminal() { + result[left_token.index].insert_all_terminals(right_tokens); + if let Some(reserved_tokens) = right_reserved_tokens { + result[left_token.index].insert_all_terminals(reserved_tokens); + } + } + } + } + } + for extra in &syntax_grammar.extra_symbols { + if extra.is_terminal() { + for entry in &mut result { + entry.insert(*extra); + } + result[extra.index] = all_tokens.clone(); + } + } + result +} + fn populate_error_state( parse_table: &mut ParseTable, syntax_grammar: &SyntaxGrammar, @@ -124,7 +179,7 @@ fn populate_error_state( if conflicts_with_other_tokens { None } else { - info!( + debug!( "error recovery - token {} has no conflicts", lexical_grammar.variables[i].name ); @@ -150,14 +205,14 @@ fn populate_error_state( !coincident_token_index.contains(symbol, *t) && token_conflict_map.does_conflict(symbol.index, t.index) }) { - info!( + debug!( "error recovery - exclude token {} because of conflict with {}", lexical_grammar.variables[i].name, lexical_grammar.variables[t.index].name ); continue; } } - info!( + debug!( "error recovery - include token {}", lexical_grammar.variables[i].name ); @@ -290,7 +345,7 @@ fn identify_keywords( && token_conflict_map.does_match_same_string(i, word_token.index) && !token_conflict_map.does_match_different_string(i, word_token.index) { - info!( + debug!( "Keywords - add candidate {}", lexical_grammar.variables[i].name ); @@ -309,7 +364,7 @@ fn identify_keywords( if other_token != *token && token_conflict_map.does_match_same_string(other_token.index, token.index) { - info!( + debug!( "Keywords - exclude {} because it matches the same string as {}", lexical_grammar.variables[token.index].name, lexical_grammar.variables[other_token.index].name @@ -351,7 +406,7 @@ fn identify_keywords( word_token.index, other_index, ) { - info!( + debug!( "Keywords - exclude {} because of conflict with {}", lexical_grammar.variables[token.index].name, lexical_grammar.variables[other_index].name @@ -360,7 +415,7 @@ fn identify_keywords( } } - info!( + debug!( "Keywords - include {}", lexical_grammar.variables[token.index].name, ); @@ -414,9 +469,9 @@ fn report_state_info<'a>( for (i, state) in parse_table.states.iter().enumerate() { all_state_indices.insert(i); let item_set = &parse_state_info[state.id]; - for (item, _) in &item_set.1.entries { - if !item.is_augmented() { - symbols_with_state_indices[item.variable_index as usize] + for entry in &item_set.1.entries { + if !entry.item.is_augmented() { + symbols_with_state_indices[entry.item.variable_index as usize] .1 .insert(i); } @@ -432,14 +487,14 @@ fn report_state_info<'a>( .max() .unwrap(); for (symbol, states) in &symbols_with_state_indices { - eprintln!( + info!( "{:width$}\t{}", syntax_grammar.variables[symbol.index].name, states.len(), width = max_symbol_name_length ); } - eprintln!(); + info!(""); let state_indices = if report_symbol_name == "*" { Some(&all_state_indices) @@ -462,22 +517,27 @@ fn report_state_info<'a>( for state_index in state_indices { let id = parse_table.states[state_index].id; let (preceding_symbols, item_set) = &parse_state_info[id]; - eprintln!("state index: {state_index}"); - eprintln!("state id: {id}"); - eprint!("symbol sequence:"); - for symbol in preceding_symbols { - let name = if symbol.is_terminal() { - &lexical_grammar.variables[symbol.index].name - } else if symbol.is_external() { - &syntax_grammar.external_tokens[symbol.index].name - } else { - &syntax_grammar.variables[symbol.index].name - }; - eprint!(" {name}"); - } - eprintln!( + info!("state index: {state_index}"); + info!("state id: {id}"); + info!( + "symbol sequence: {}", + preceding_symbols + .iter() + .map(|symbol| { + if symbol.is_terminal() { + lexical_grammar.variables[symbol.index].name.clone() + } else if symbol.is_external() { + syntax_grammar.external_tokens[symbol.index].name.clone() + } else { + syntax_grammar.variables[symbol.index].name.clone() + } + }) + .collect::>() + .join(" ") + ); + info!( "\nitems:\n{}", - self::item::ParseItemSetDisplay(item_set, syntax_grammar, lexical_grammar,), + item::ParseItemSetDisplay(item_set, syntax_grammar, lexical_grammar), ); } } diff --git a/cli/generate/src/build_tables/build_lex_table.rs b/crates/generate/src/build_tables/build_lex_table.rs similarity index 97% rename from cli/generate/src/build_tables/build_lex_table.rs rename to crates/generate/src/build_tables/build_lex_table.rs index c96e7013..9d0d4fb7 100644 --- a/cli/generate/src/build_tables/build_lex_table.rs +++ b/crates/generate/src/build_tables/build_lex_table.rs @@ -3,7 +3,7 @@ use std::{ mem, }; -use log::info; +use log::debug; use super::{coincident_tokens::CoincidentTokenIndex, token_conflicts::TokenConflictMap}; use crate::{ @@ -43,15 +43,17 @@ pub fn build_lex_table( let tokens = state .terminal_entries .keys() + .copied() + .chain(state.reserved_words.iter()) .filter_map(|token| { if token.is_terminal() { - if keywords.contains(token) { + if keywords.contains(&token) { syntax_grammar.word_token } else { - Some(*token) + Some(token) } } else if token.is_eof() { - Some(*token) + Some(token) } else { None } @@ -174,9 +176,8 @@ impl<'a> LexTableBuilder<'a> { let (state_id, is_new) = self.add_state(nfa_states, eof_valid); if is_new { - info!( - "entry point state: {}, tokens: {:?}", - state_id, + debug!( + "entry point state: {state_id}, tokens: {:?}", tokens .iter() .map(|t| &self.lexical_grammar.variables[t.index].name) @@ -357,9 +358,7 @@ fn minimize_lex_table(table: &mut LexTable, parse_table: &mut ParseTable) { &mut group_ids_by_state_id, 1, lex_states_differ, - ) { - continue; - } + ) {} let mut new_states = Vec::with_capacity(state_ids_by_group_id.len()); for state_ids in &state_ids_by_group_id { diff --git a/cli/generate/src/build_tables/build_parse_table.rs b/crates/generate/src/build_tables/build_parse_table.rs similarity index 68% rename from cli/generate/src/build_tables/build_parse_table.rs rename to crates/generate/src/build_tables/build_parse_table.rs index 353f6667..66f29609 100644 --- a/cli/generate/src/build_tables/build_parse_table.rs +++ b/crates/generate/src/build_tables/build_parse_table.rs @@ -1,22 +1,21 @@ use std::{ cmp::Ordering, - collections::{BTreeMap, HashMap, HashSet, VecDeque}, - fmt::Write, + collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque}, hash::BuildHasherDefault, }; -use anyhow::{anyhow, Result}; use indexmap::{map::Entry, IndexMap}; +use log::warn; use rustc_hash::FxHasher; +use serde::Serialize; +use thiserror::Error; use super::{ - item::{ParseItem, ParseItemSet, ParseItemSetCore}, + item::{ParseItem, ParseItemSet, ParseItemSetCore, ParseItemSetEntry}, item_set_builder::ParseItemSetBuilder, }; use crate::{ - grammars::{ - InlinedProductionMap, LexicalGrammar, PrecedenceEntry, SyntaxGrammar, VariableType, - }, + grammars::{LexicalGrammar, PrecedenceEntry, ReservedWordSetId, SyntaxGrammar, VariableType}, node_types::VariableInfo, rules::{Associativity, Precedence, Symbol, SymbolType, TokenSet}, tables::{ @@ -66,8 +65,208 @@ struct ParseTableBuilder<'a> { parse_table: ParseTable, } +pub type BuildTableResult = Result; + +#[derive(Debug, Error, Serialize)] +pub enum ParseTableBuilderError { + #[error("Unresolved conflict for symbol sequence:\n\n{0}")] + Conflict(#[from] ConflictError), + #[error("Extra rules must have unambiguous endings. Conflicting rules: {0}")] + AmbiguousExtra(#[from] AmbiguousExtraError), + #[error( + "The non-terminal rule `{0}` is used in a non-terminal `extra` rule, which is not allowed." + )] + ImproperNonTerminalExtra(String), + #[error("State count `{0}` exceeds the max value {max}.", max=u16::MAX)] + StateCount(usize), +} + +#[derive(Default, Debug, Serialize, Error)] +pub struct ConflictError { + pub symbol_sequence: Vec, + pub conflicting_lookahead: String, + pub possible_interpretations: Vec, + pub possible_resolutions: Vec, +} + +#[derive(Default, Debug, Serialize, Error)] +pub struct Interpretation { + pub preceding_symbols: Vec, + pub variable_name: String, + pub production_step_symbols: Vec, + pub step_index: u32, + pub done: bool, + pub conflicting_lookahead: String, + pub precedence: Option, + pub associativity: Option, +} + +#[derive(Debug, Serialize)] +pub enum Resolution { + Precedence { symbols: Vec }, + Associativity { symbols: Vec }, + AddConflict { symbols: Vec }, +} + +#[derive(Debug, Serialize, Error)] +pub struct AmbiguousExtraError { + pub parent_symbols: Vec, +} + +impl std::fmt::Display for ConflictError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + for symbol in &self.symbol_sequence { + write!(f, " {symbol}")?; + } + writeln!(f, " • {} …\n", self.conflicting_lookahead)?; + + writeln!(f, "Possible interpretations:\n")?; + let mut interpretations = self + .possible_interpretations + .iter() + .map(|i| { + let line = i.to_string(); + let prec_line = if let (Some(precedence), Some(associativity)) = + (&i.precedence, &i.associativity) + { + Some(format!( + "(precedence: {precedence}, associativity: {associativity})", + )) + } else { + i.precedence + .as_ref() + .map(|precedence| format!("(precedence: {precedence})")) + }; + + (line, prec_line) + }) + .collect::>(); + let max_interpretation_length = interpretations + .iter() + .map(|i| i.0.chars().count()) + .max() + .unwrap(); + interpretations.sort_unstable(); + for (i, (line, prec_suffix)) in interpretations.into_iter().enumerate() { + write!(f, " {}:", i + 1).unwrap(); + write!(f, "{line}")?; + if let Some(prec_suffix) = prec_suffix { + write!( + f, + "{:1$}", + "", + max_interpretation_length.saturating_sub(line.chars().count()) + 2 + )?; + write!(f, "{prec_suffix}")?; + } + writeln!(f)?; + } + + writeln!(f, "\nPossible resolutions:\n")?; + for (i, resolution) in self.possible_resolutions.iter().enumerate() { + writeln!(f, " {}: {resolution}", i + 1)?; + } + Ok(()) + } +} + +impl std::fmt::Display for Interpretation { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + for symbol in &self.preceding_symbols { + write!(f, " {symbol}")?; + } + write!(f, " ({}", self.variable_name)?; + for (i, symbol) in self.production_step_symbols.iter().enumerate() { + if i == self.step_index as usize { + write!(f, " •")?; + } + write!(f, " {symbol}")?; + } + write!(f, ")")?; + if self.done { + write!(f, " • {} …", self.conflicting_lookahead)?; + } + Ok(()) + } +} + +impl std::fmt::Display for Resolution { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + Self::Precedence { symbols } => { + write!(f, "Specify a higher precedence in ")?; + for (i, symbol) in symbols.iter().enumerate() { + if i > 0 { + write!(f, " and ")?; + } + write!(f, "`{symbol}`")?; + } + write!(f, " than in the other rules.")?; + } + Self::Associativity { symbols } => { + write!(f, "Specify a left or right associativity in ")?; + for (i, symbol) in symbols.iter().enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "`{symbol}`")?; + } + } + Self::AddConflict { symbols } => { + write!(f, "Add a conflict for these rules: ")?; + for (i, symbol) in symbols.iter().enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "`{symbol}`")?; + } + } + } + Ok(()) + } +} + +impl std::fmt::Display for AmbiguousExtraError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + for (i, symbol) in self.parent_symbols.iter().enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "{symbol}")?; + } + Ok(()) + } +} + impl<'a> ParseTableBuilder<'a> { - fn build(mut self) -> Result<(ParseTable, Vec>)> { + fn new( + syntax_grammar: &'a SyntaxGrammar, + lexical_grammar: &'a LexicalGrammar, + item_set_builder: ParseItemSetBuilder<'a>, + variable_info: &'a [VariableInfo], + ) -> Self { + Self { + syntax_grammar, + lexical_grammar, + item_set_builder, + variable_info, + non_terminal_extra_states: Vec::new(), + state_ids_by_item_set: IndexMap::default(), + core_ids_by_core: HashMap::new(), + parse_state_info_by_id: Vec::new(), + parse_state_queue: VecDeque::new(), + actual_conflicts: syntax_grammar.expected_conflicts.iter().cloned().collect(), + parse_table: ParseTable { + states: Vec::new(), + symbols: Vec::new(), + external_lex_states: Vec::new(), + production_infos: Vec::new(), + max_aliased_production_length: 1, + }, + } + } + + fn build(mut self) -> BuildTableResult<(ParseTable, Vec>)> { // Ensure that the empty alias sequence has index 0. self.parse_table .production_infos @@ -80,10 +279,13 @@ impl<'a> ParseTableBuilder<'a> { self.add_parse_state( &Vec::new(), &Vec::new(), - ParseItemSet::with(std::iter::once(( - ParseItem::start(), - std::iter::once(Symbol::end()).collect(), - ))), + ParseItemSet { + entries: vec![ParseItemSetEntry { + item: ParseItem::start(), + lookaheads: std::iter::once(Symbol::end()).collect(), + following_reserved_word_set: ReservedWordSetId::default(), + }], + }, ); // Compute the possible item sets for non-terminal extras. @@ -99,23 +301,35 @@ impl<'a> ParseTableBuilder<'a> { non_terminal_extra_item_sets_by_first_terminal .entry(production.first_symbol().unwrap()) .or_insert_with(ParseItemSet::default) - .insert( - ParseItem { - variable_index: extra_non_terminal.index as u32, - production, - step_index: 1, - has_preceding_inherited_fields: false, - }, - &std::iter::once(Symbol::end_of_nonterminal_extra()).collect(), - ); + .insert(ParseItem { + variable_index: extra_non_terminal.index as u32, + production, + step_index: 1, + has_preceding_inherited_fields: false, + }) + .lookaheads + .insert(Symbol::end_of_nonterminal_extra()); } } + let non_terminal_sets_len = non_terminal_extra_item_sets_by_first_terminal.len(); + self.non_terminal_extra_states + .reserve(non_terminal_sets_len); + self.parse_state_info_by_id.reserve(non_terminal_sets_len); + self.parse_table.states.reserve(non_terminal_sets_len); + self.parse_state_queue.reserve(non_terminal_sets_len); // Add a state for each starting terminal of a non-terminal extra rule. for (terminal, item_set) in non_terminal_extra_item_sets_by_first_terminal { - self.non_terminal_extra_states - .push((terminal, self.parse_table.states.len())); - self.add_parse_state(&Vec::new(), &Vec::new(), item_set); + if terminal.is_non_terminal() { + Err(ParseTableBuilderError::ImproperNonTerminalExtra( + self.symbol_name(&terminal), + ))?; + } + + // Add the parse state, and *then* push the terminal and the state id into the + // list of nonterminal extra states + let state_id = self.add_parse_state(&Vec::new(), &Vec::new(), item_set); + self.non_terminal_extra_states.push((terminal, state_id)); } while let Some(entry) = self.parse_state_queue.pop_front() { @@ -132,17 +346,21 @@ impl<'a> ParseTableBuilder<'a> { } if !self.actual_conflicts.is_empty() { - println!("Warning: unnecessary conflicts"); - for conflict in &self.actual_conflicts { - println!( - " {}", - conflict - .iter() - .map(|symbol| format!("`{}`", self.symbol_name(symbol))) - .collect::>() - .join(", ") - ); - } + warn!( + "unnecessary conflicts:\n {}", + &self + .actual_conflicts + .iter() + .map(|conflict| { + conflict + .iter() + .map(|symbol| format!("`{}`", self.symbol_name(symbol))) + .collect::>() + .join(", ") + }) + .collect::>() + .join("\n ") + ); } Ok((self.parse_table, self.parse_state_info_by_id)) @@ -176,6 +394,7 @@ impl<'a> ParseTableBuilder<'a> { external_lex_state_id: 0, terminal_entries: IndexMap::default(), nonterminal_entries: IndexMap::default(), + reserved_words: TokenSet::default(), core_id, }); self.parse_state_queue.push_back(ParseStateQueueEntry { @@ -194,7 +413,7 @@ impl<'a> ParseTableBuilder<'a> { mut preceding_auxiliary_symbols: AuxiliarySymbolSequence, state_id: ParseStateId, item_set: &ParseItemSet<'a>, - ) -> Result<()> { + ) -> BuildTableResult<()> { let mut terminal_successors = BTreeMap::new(); let mut non_terminal_successors = BTreeMap::new(); let mut lookaheads_with_conflicts = TokenSet::new(); @@ -202,13 +421,18 @@ impl<'a> ParseTableBuilder<'a> { // Each item in the item set contributes to either or a Shift action or a Reduce // action in this state. - for (item, lookaheads) in &item_set.entries { + for ParseItemSetEntry { + item, + lookaheads, + following_reserved_word_set: reserved_lookaheads, + } in &item_set.entries + { // If the item is unfinished, then this state has a transition for the item's // next symbol. Advance the item to its next step and insert the resulting // item into the successor item set. if let Some(next_symbol) = item.symbol() { let mut successor = item.successor(); - if next_symbol.is_non_terminal() { + let successor_set = if next_symbol.is_non_terminal() { let variable = &self.syntax_grammar.variables[next_symbol.index]; // Keep track of where auxiliary non-terminals (repeat symbols) are @@ -237,13 +461,16 @@ impl<'a> ParseTableBuilder<'a> { non_terminal_successors .entry(next_symbol) .or_insert_with(ParseItemSet::default) - .insert(successor, lookaheads); } else { terminal_successors .entry(next_symbol) .or_insert_with(ParseItemSet::default) - .insert(successor, lookaheads); - } + }; + let successor_entry = successor_set.insert(successor); + successor_entry.lookaheads.insert_all(lookaheads); + successor_entry.following_reserved_word_set = successor_entry + .following_reserved_word_set + .max(*reserved_lookaheads); } // If the item is finished, then add a Reduce action to this state based // on this item. @@ -370,7 +597,7 @@ impl<'a> ParseTableBuilder<'a> { )?; } - // Finally, add actions for the grammar's `extra` symbols. + // Add actions for the grammar's `extra` symbols. let state = &mut self.parse_table.states[state_id]; let is_end_of_non_terminal_extra = state.is_end_of_non_terminal_extra(); @@ -382,7 +609,7 @@ impl<'a> ParseTableBuilder<'a> { let parent_symbols = item_set .entries .iter() - .filter_map(|(item, _)| { + .filter_map(|ParseItemSetEntry { item, .. }| { if !item.is_augmented() && item.step_index > 0 { Some(item.variable_index) } else { @@ -390,15 +617,18 @@ impl<'a> ParseTableBuilder<'a> { } }) .collect::>(); - let mut message = - "Extra rules must have unambiguous endings. Conflicting rules: ".to_string(); - for (i, variable_index) in parent_symbols.iter().enumerate() { - if i > 0 { - message += ", "; - } - message += &self.syntax_grammar.variables[*variable_index as usize].name; - } - return Err(anyhow!(message)); + let parent_symbol_names = parent_symbols + .iter() + .map(|&variable_index| { + self.syntax_grammar.variables[variable_index as usize] + .name + .clone() + }) + .collect::>(); + + Err(AmbiguousExtraError { + parent_symbols: parent_symbol_names, + })?; } } // Add actions for the start tokens of each non-terminal extra rule. @@ -436,6 +666,30 @@ impl<'a> ParseTableBuilder<'a> { } } + if let Some(keyword_capture_token) = self.syntax_grammar.word_token { + let reserved_word_set_id = item_set + .entries + .iter() + .filter_map(|entry| { + if let Some(next_step) = entry.item.step() { + if next_step.symbol == keyword_capture_token { + Some(next_step.reserved_word_set_id) + } else { + None + } + } else if entry.lookaheads.contains(&keyword_capture_token) { + Some(entry.following_reserved_word_set) + } else { + None + } + }) + .max(); + if let Some(reserved_word_set_id) = reserved_word_set_id { + state.reserved_words = + self.syntax_grammar.reserved_word_sets[reserved_word_set_id.0].clone(); + } + } + Ok(()) } @@ -447,7 +701,7 @@ impl<'a> ParseTableBuilder<'a> { preceding_auxiliary_symbols: &[AuxiliarySymbolInfo], conflicting_lookahead: Symbol, reduction_info: &ReductionInfo, - ) -> Result<()> { + ) -> BuildTableResult<()> { let entry = self.parse_table.states[state_id] .terminal_entries .get_mut(&conflicting_lookahead) @@ -461,8 +715,11 @@ impl<'a> ParseTableBuilder<'a> { // precedence, and there can still be SHIFT/REDUCE conflicts. let mut considered_associativity = false; let mut shift_precedence = Vec::<(&Precedence, Symbol)>::new(); - let mut conflicting_items = HashSet::new(); - for (item, lookaheads) in &item_set.entries { + let mut conflicting_items = BTreeSet::new(); + for ParseItemSetEntry { + item, lookaheads, .. + } in &item_set.entries + { if let Some(step) = item.step() { if item.step_index > 0 && self @@ -599,93 +856,55 @@ impl<'a> ParseTableBuilder<'a> { return Ok(()); } - let mut msg = "Unresolved conflict for symbol sequence:\n\n".to_string(); + let mut conflict_error = ConflictError::default(); for symbol in preceding_symbols { - write!(&mut msg, " {}", self.symbol_name(symbol)).unwrap(); + conflict_error + .symbol_sequence + .push(self.symbol_name(symbol)); } + conflict_error.conflicting_lookahead = self.symbol_name(&conflicting_lookahead); - writeln!( - &mut msg, - " • {} …\n", - self.symbol_name(&conflicting_lookahead) - ) - .unwrap(); - writeln!(&mut msg, "Possible interpretations:\n").unwrap(); - - let mut interpretations = conflicting_items + let interpretations = conflicting_items .iter() .map(|item| { - let mut line = String::new(); - for preceding_symbol in preceding_symbols + let preceding_symbols = preceding_symbols .iter() .take(preceding_symbols.len() - item.step_index as usize) - { - write!(&mut line, " {}", self.symbol_name(preceding_symbol)).unwrap(); - } + .map(|symbol| self.symbol_name(symbol)) + .collect::>(); - write!( - &mut line, - " ({}", - &self.syntax_grammar.variables[item.variable_index as usize].name - ) - .unwrap(); + let variable_name = self.syntax_grammar.variables[item.variable_index as usize] + .name + .clone(); - for (j, step) in item.production.steps.iter().enumerate() { - if j as u32 == item.step_index { - write!(&mut line, " •").unwrap(); - } - write!(&mut line, " {}", self.symbol_name(&step.symbol)).unwrap(); - } + let production_step_symbols = item + .production + .steps + .iter() + .map(|step| self.symbol_name(&step.symbol)) + .collect::>(); - write!(&mut line, ")").unwrap(); - - if item.is_done() { - write!( - &mut line, - " • {} …", - self.symbol_name(&conflicting_lookahead) - ) - .unwrap(); - } - - let precedence = item.precedence(); - let associativity = item.associativity(); - - let prec_line = if let Some(associativity) = associativity { - Some(format!( - "(precedence: {precedence}, associativity: {associativity:?})", - )) - } else if !precedence.is_none() { - Some(format!("(precedence: {precedence})")) - } else { - None + let precedence = match item.precedence() { + Precedence::None => None, + _ => Some(item.precedence().to_string()), }; - (line, prec_line) + let associativity = item.associativity().map(|assoc| format!("{assoc:?}")); + + Interpretation { + preceding_symbols, + variable_name, + production_step_symbols, + step_index: item.step_index, + done: item.is_done(), + conflicting_lookahead: self.symbol_name(&conflicting_lookahead), + precedence, + associativity, + } }) .collect::>(); + conflict_error.possible_interpretations = interpretations; - let max_interpretation_length = interpretations - .iter() - .map(|i| i.0.chars().count()) - .max() - .unwrap(); - interpretations.sort_unstable(); - for (i, (line, prec_suffix)) in interpretations.into_iter().enumerate() { - write!(&mut msg, " {}:", i + 1).unwrap(); - msg += &line; - if let Some(prec_suffix) = prec_suffix { - for _ in line.chars().count()..max_interpretation_length { - msg.push(' '); - } - msg += " "; - msg += &prec_suffix; - } - msg.push('\n'); - } - - let mut resolution_count = 0; - writeln!(&mut msg, "\nPossible resolutions:\n").unwrap(); let mut shift_items = Vec::new(); let mut reduce_items = Vec::new(); for item in conflicting_items { @@ -698,76 +917,57 @@ impl<'a> ParseTableBuilder<'a> { shift_items.sort_unstable(); reduce_items.sort_unstable(); - let list_rule_names = |mut msg: &mut String, items: &[&ParseItem]| { + let get_rule_names = |items: &[&ParseItem]| -> Vec { let mut last_rule_id = None; + let mut result = Vec::with_capacity(items.len()); for item in items { if last_rule_id == Some(item.variable_index) { continue; } - - if last_rule_id.is_some() { - write!(&mut msg, " and").unwrap(); - } - last_rule_id = Some(item.variable_index); - write!( - msg, - " `{}`", - self.symbol_name(&Symbol::non_terminal(item.variable_index as usize)) - ) - .unwrap(); + result.push(self.symbol_name(&Symbol::non_terminal(item.variable_index as usize))); } + + result }; if actual_conflict.len() > 1 { if !shift_items.is_empty() { - resolution_count += 1; - write!( - &mut msg, - " {resolution_count}: Specify a higher precedence in", - ) - .unwrap(); - list_rule_names(&mut msg, &shift_items); - writeln!(&mut msg, " than in the other rules.").unwrap(); + let names = get_rule_names(&shift_items); + conflict_error + .possible_resolutions + .push(Resolution::Precedence { symbols: names }); } for item in &reduce_items { - resolution_count += 1; - writeln!( - &mut msg, - " {resolution_count}: Specify a higher precedence in `{}` than in the other rules.", - self.symbol_name(&Symbol::non_terminal(item.variable_index as usize)) - ) - .unwrap(); + let name = self.symbol_name(&Symbol::non_terminal(item.variable_index as usize)); + conflict_error + .possible_resolutions + .push(Resolution::Precedence { + symbols: vec![name], + }); } } if considered_associativity { - resolution_count += 1; - write!( - &mut msg, - " {resolution_count}: Specify a left or right associativity in", - ) - .unwrap(); - list_rule_names(&mut msg, &reduce_items); - writeln!(&mut msg).unwrap(); + let names = get_rule_names(&reduce_items); + conflict_error + .possible_resolutions + .push(Resolution::Associativity { symbols: names }); } - resolution_count += 1; - write!( - &mut msg, - " {resolution_count}: Add a conflict for these rules: ", - ) - .unwrap(); - for (i, symbol) in actual_conflict.iter().enumerate() { - if i > 0 { - write!(&mut msg, ", ").unwrap(); - } - write!(&mut msg, "`{}`", self.symbol_name(symbol)).unwrap(); - } - writeln!(&mut msg).unwrap(); + conflict_error + .possible_resolutions + .push(Resolution::AddConflict { + symbols: actual_conflict + .iter() + .map(|s| self.symbol_name(s)) + .collect(), + }); - Err(anyhow!(msg)) + self.actual_conflicts.insert(actual_conflict); + + Err(conflict_error)? } fn compare_precedence( @@ -836,7 +1036,7 @@ impl<'a> ParseTableBuilder<'a> { let parent_symbols = item_set .entries .iter() - .filter_map(|(item, _)| { + .filter_map(|ParseItemSetEntry { item, .. }| { let variable_index = item.variable_index as usize; if item.symbol() == Some(symbol) && !self.syntax_grammar.variables[variable_index].is_auxiliary() @@ -924,84 +1124,24 @@ impl<'a> ParseTableBuilder<'a> { if variable.kind == VariableType::Named { variable.name.clone() } else { - format!("'{}'", &variable.name) + format!("'{}'", variable.name) } } } } } -fn populate_following_tokens( - result: &mut [TokenSet], - grammar: &SyntaxGrammar, - inlines: &InlinedProductionMap, - builder: &ParseItemSetBuilder, -) { - let productions = grammar - .variables - .iter() - .flat_map(|v| &v.productions) - .chain(&inlines.productions); - let all_tokens = (0..result.len()) - .map(Symbol::terminal) - .collect::(); - for production in productions { - for i in 1..production.steps.len() { - let left_tokens = builder.last_set(&production.steps[i - 1].symbol); - let right_tokens = builder.first_set(&production.steps[i].symbol); - for left_token in left_tokens.iter() { - if left_token.is_terminal() { - result[left_token.index].insert_all_terminals(right_tokens); - } - } - } - } - for extra in &grammar.extra_symbols { - if extra.is_terminal() { - for entry in result.iter_mut() { - entry.insert(*extra); - } - result[extra.index].clone_from(&all_tokens); - } - } -} - pub fn build_parse_table<'a>( syntax_grammar: &'a SyntaxGrammar, lexical_grammar: &'a LexicalGrammar, - inlines: &'a InlinedProductionMap, + item_set_builder: ParseItemSetBuilder<'a>, variable_info: &'a [VariableInfo], -) -> Result<(ParseTable, Vec, Vec>)> { - let actual_conflicts = syntax_grammar.expected_conflicts.iter().cloned().collect(); - let item_set_builder = ParseItemSetBuilder::new(syntax_grammar, lexical_grammar, inlines); - let mut following_tokens = vec![TokenSet::new(); lexical_grammar.variables.len()]; - populate_following_tokens( - &mut following_tokens, - syntax_grammar, - inlines, - &item_set_builder, - ); - - let (table, item_sets) = ParseTableBuilder { +) -> BuildTableResult<(ParseTable, Vec>)> { + ParseTableBuilder::new( syntax_grammar, lexical_grammar, item_set_builder, variable_info, - non_terminal_extra_states: Vec::new(), - actual_conflicts, - state_ids_by_item_set: IndexMap::default(), - core_ids_by_core: HashMap::new(), - parse_state_info_by_id: Vec::new(), - parse_state_queue: VecDeque::new(), - parse_table: ParseTable { - states: Vec::new(), - symbols: Vec::new(), - external_lex_states: Vec::new(), - production_infos: Vec::new(), - max_aliased_production_length: 1, - }, - } - .build()?; - - Ok((table, following_tokens, item_sets)) + ) + .build() } diff --git a/cli/generate/src/build_tables/coincident_tokens.rs b/crates/generate/src/build_tables/coincident_tokens.rs similarity index 100% rename from cli/generate/src/build_tables/coincident_tokens.rs rename to crates/generate/src/build_tables/coincident_tokens.rs diff --git a/cli/generate/src/build_tables/item.rs b/crates/generate/src/build_tables/item.rs similarity index 74% rename from cli/generate/src/build_tables/item.rs rename to crates/generate/src/build_tables/item.rs index e20b1a8f..cd70ce74 100644 --- a/cli/generate/src/build_tables/item.rs +++ b/crates/generate/src/build_tables/item.rs @@ -2,30 +2,31 @@ use std::{ cmp::Ordering, fmt, hash::{Hash, Hasher}, + sync::LazyLock, }; -use lazy_static::lazy_static; - use crate::{ - grammars::{LexicalGrammar, Production, ProductionStep, SyntaxGrammar}, + grammars::{ + LexicalGrammar, Production, ProductionStep, ReservedWordSetId, SyntaxGrammar, + NO_RESERVED_WORDS, + }, rules::{Associativity, Precedence, Symbol, SymbolType, TokenSet}, }; -lazy_static! { - static ref START_PRODUCTION: Production = Production { - dynamic_precedence: 0, - steps: vec![ProductionStep { - symbol: Symbol { - index: 0, - kind: SymbolType::NonTerminal, - }, - precedence: Precedence::None, - associativity: None, - alias: None, - field_name: None, - }], - }; -} +static START_PRODUCTION: LazyLock = LazyLock::new(|| Production { + dynamic_precedence: 0, + steps: vec![ProductionStep { + symbol: Symbol { + index: 0, + kind: SymbolType::NonTerminal, + }, + precedence: Precedence::None, + associativity: None, + alias: None, + field_name: None, + reserved_word_set_id: NO_RESERVED_WORDS, + }], +}); /// A [`ParseItem`] represents an in-progress match of a single production in a grammar. #[derive(Clone, Copy, Debug)] @@ -58,7 +59,14 @@ pub struct ParseItem<'a> { /// to a state in the final parse table. #[derive(Clone, Debug, PartialEq, Eq, Default)] pub struct ParseItemSet<'a> { - pub entries: Vec<(ParseItem<'a>, TokenSet)>, + pub entries: Vec>, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ParseItemSetEntry<'a> { + pub item: ParseItem<'a>, + pub lookaheads: TokenSet, + pub following_reserved_word_set: ReservedWordSetId, } /// A [`ParseItemSetCore`] is like a [`ParseItemSet`], but without the lookahead @@ -152,30 +160,26 @@ impl<'a> ParseItem<'a> { } impl<'a> ParseItemSet<'a> { - pub fn with(elements: impl IntoIterator, TokenSet)>) -> Self { - let mut result = Self::default(); - for (item, lookaheads) in elements { - result.insert(item, &lookaheads); - } - result - } - - pub fn insert(&mut self, item: ParseItem<'a>, lookaheads: &TokenSet) -> &mut TokenSet { - match self.entries.binary_search_by(|(i, _)| i.cmp(&item)) { + pub fn insert(&mut self, item: ParseItem<'a>) -> &mut ParseItemSetEntry<'a> { + match self.entries.binary_search_by(|e| e.item.cmp(&item)) { Err(i) => { - self.entries.insert(i, (item, lookaheads.clone())); - &mut self.entries[i].1 - } - Ok(i) => { - self.entries[i].1.insert_all(lookaheads); - &mut self.entries[i].1 + self.entries.insert( + i, + ParseItemSetEntry { + item, + lookaheads: TokenSet::new(), + following_reserved_word_set: ReservedWordSetId::default(), + }, + ); + &mut self.entries[i] } + Ok(i) => &mut self.entries[i], } } pub fn core(&self) -> ParseItemSetCore<'a> { ParseItemSetCore { - entries: self.entries.iter().map(|e| e.0).collect(), + entries: self.entries.iter().map(|e| e.item).collect(), } } } @@ -188,35 +192,42 @@ impl fmt::Display for ParseItemDisplay<'_> { write!( f, "{} →", - &self.1.variables[self.0.variable_index as usize].name + self.1.variables[self.0.variable_index as usize].name )?; } for (i, step) in self.0.production.steps.iter().enumerate() { if i == self.0.step_index as usize { write!(f, " •")?; - if let Some(associativity) = step.associativity { - if step.precedence.is_none() { - write!(f, " ({associativity:?})")?; - } else { - write!(f, " ({} {associativity:?})", step.precedence)?; + if !step.precedence.is_none() + || step.associativity.is_some() + || step.reserved_word_set_id != ReservedWordSetId::default() + { + write!(f, " (")?; + if !step.precedence.is_none() { + write!(f, " {}", step.precedence)?; } - } else if !step.precedence.is_none() { - write!(f, " ({})", step.precedence)?; + if let Some(associativity) = step.associativity { + write!(f, " {associativity:?}")?; + } + if step.reserved_word_set_id != ReservedWordSetId::default() { + write!(f, "reserved: {}", step.reserved_word_set_id)?; + } + write!(f, " )")?; } } write!(f, " ")?; if step.symbol.is_terminal() { if let Some(variable) = self.2.variables.get(step.symbol.index) { - write!(f, "{}", &variable.name)?; + write!(f, "{}", variable.name)?; } else { write!(f, "terminal-{}", step.symbol.index)?; } } else if step.symbol.is_external() { - write!(f, "{}", &self.1.external_tokens[step.symbol.index].name)?; + write!(f, "{}", self.1.external_tokens[step.symbol.index].name)?; } else { - write!(f, "{}", &self.1.variables[step.symbol.index].name)?; + write!(f, "{}", self.1.variables[step.symbol.index].name)?; } if let Some(alias) = &step.alias { @@ -243,6 +254,32 @@ impl fmt::Display for ParseItemDisplay<'_> { } } +const fn escape_invisible(c: char) -> Option<&'static str> { + Some(match c { + '\n' => "\\n", + '\r' => "\\r", + '\t' => "\\t", + '\0' => "\\0", + '\\' => "\\\\", + '\x0b' => "\\v", + '\x0c' => "\\f", + _ => return None, + }) +} + +fn display_variable_name(source: &str) -> String { + source + .chars() + .fold(String::with_capacity(source.len()), |mut acc, c| { + if let Some(esc) = escape_invisible(c) { + acc.push_str(esc); + } else { + acc.push(c); + } + acc + }) +} + impl fmt::Display for TokenSetDisplay<'_> { fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { write!(f, "[")?; @@ -253,14 +290,14 @@ impl fmt::Display for TokenSetDisplay<'_> { if symbol.is_terminal() { if let Some(variable) = self.2.variables.get(symbol.index) { - write!(f, "{}", &variable.name)?; + write!(f, "{}", display_variable_name(&variable.name))?; } else { write!(f, "terminal-{}", symbol.index)?; } } else if symbol.is_external() { - write!(f, "{}", &self.1.external_tokens[symbol.index].name)?; + write!(f, "{}", self.1.external_tokens[symbol.index].name)?; } else { - write!(f, "{}", &self.1.variables[symbol.index].name)?; + write!(f, "{}", self.1.variables[symbol.index].name)?; } } write!(f, "]")?; @@ -270,13 +307,21 @@ impl fmt::Display for TokenSetDisplay<'_> { impl fmt::Display for ParseItemSetDisplay<'_> { fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { - for (item, lookaheads) in &self.0.entries { - writeln!( + for entry in &self.0.entries { + write!( f, "{}\t{}", - ParseItemDisplay(item, self.1, self.2), - TokenSetDisplay(lookaheads, self.1, self.2) + ParseItemDisplay(&entry.item, self.1, self.2), + TokenSetDisplay(&entry.lookaheads, self.1, self.2), )?; + if entry.following_reserved_word_set != ReservedWordSetId::default() { + write!( + f, + "\treserved word set: {}", + entry.following_reserved_word_set + )?; + } + writeln!(f)?; } Ok(()) } @@ -296,7 +341,7 @@ impl Hash for ParseItem<'_> { // this item, unless any of the following are true: // * the children have fields // * the children have aliases - // * the children are hidden and + // * the children are hidden and represent rules that have fields. // See the docs for `has_preceding_inherited_fields`. for step in &self.production.steps[0..self.step_index as usize] { step.alias.hash(hasher); @@ -399,9 +444,10 @@ impl Eq for ParseItem<'_> {} impl Hash for ParseItemSet<'_> { fn hash(&self, hasher: &mut H) { hasher.write_usize(self.entries.len()); - for (item, lookaheads) in &self.entries { - item.hash(hasher); - lookaheads.hash(hasher); + for entry in &self.entries { + entry.item.hash(hasher); + entry.lookaheads.hash(hasher); + entry.following_reserved_word_set.hash(hasher); } } } diff --git a/cli/generate/src/build_tables/item_set_builder.rs b/crates/generate/src/build_tables/item_set_builder.rs similarity index 54% rename from cli/generate/src/build_tables/item_set_builder.rs rename to crates/generate/src/build_tables/item_set_builder.rs index aa40dd85..44e05702 100644 --- a/cli/generate/src/build_tables/item_set_builder.rs +++ b/crates/generate/src/build_tables/item_set_builder.rs @@ -3,9 +3,9 @@ use std::{ fmt, }; -use super::item::{ParseItem, ParseItemDisplay, ParseItemSet, TokenSetDisplay}; +use super::item::{ParseItem, ParseItemDisplay, ParseItemSet, ParseItemSetEntry, TokenSetDisplay}; use crate::{ - grammars::{InlinedProductionMap, LexicalGrammar, SyntaxGrammar}, + grammars::{InlinedProductionMap, LexicalGrammar, ReservedWordSetId, SyntaxGrammar}, rules::{Symbol, SymbolType, TokenSet}, }; @@ -15,9 +15,10 @@ struct TransitiveClosureAddition<'a> { info: FollowSetInfo, } -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, Default, PartialEq, Eq)] struct FollowSetInfo { lookaheads: TokenSet, + reserved_lookaheads: ReservedWordSetId, propagates_lookaheads: bool, } @@ -25,6 +26,7 @@ pub struct ParseItemSetBuilder<'a> { syntax_grammar: &'a SyntaxGrammar, lexical_grammar: &'a LexicalGrammar, first_sets: HashMap, + reserved_first_sets: HashMap, last_sets: HashMap, inlines: &'a InlinedProductionMap, transitive_closure_additions: Vec>>, @@ -46,6 +48,7 @@ impl<'a> ParseItemSetBuilder<'a> { syntax_grammar, lexical_grammar, first_sets: HashMap::new(), + reserved_first_sets: HashMap::new(), last_sets: HashMap::new(), inlines, transitive_closure_additions: vec![Vec::new(); syntax_grammar.variables.len()], @@ -54,8 +57,7 @@ impl<'a> ParseItemSetBuilder<'a> { // For each grammar symbol, populate the FIRST and LAST sets: the set of // terminals that appear at the beginning and end that symbol's productions, // respectively. - // - // For a terminal symbol, the FIRST and LAST set just consists of the + // For a terminal symbol, the FIRST and LAST sets just consist of the // terminal itself. for i in 0..lexical_grammar.variables.len() { let symbol = Symbol::terminal(i); @@ -63,6 +65,9 @@ impl<'a> ParseItemSetBuilder<'a> { set.insert(symbol); result.first_sets.insert(symbol, set.clone()); result.last_sets.insert(symbol, set); + result + .reserved_first_sets + .insert(symbol, ReservedWordSetId::default()); } for i in 0..syntax_grammar.external_tokens.len() { @@ -71,12 +76,15 @@ impl<'a> ParseItemSetBuilder<'a> { set.insert(symbol); result.first_sets.insert(symbol, set.clone()); result.last_sets.insert(symbol, set); + result + .reserved_first_sets + .insert(symbol, ReservedWordSetId::default()); } - // The FIRST set of a non-terminal `i` is the union of the following sets: - // * the set of all terminals that appear at the beginnings of i's productions - // * the FIRST sets of all the non-terminals that appear at the beginnings of i's - // productions + // The FIRST set of a non-terminal `i` is the union of the FIRST sets + // of all the symbols that appear at the beginnings of i's productions. Some + // of these symbols may themselves be non-terminals, so this is a recursive + // definition. // // Rather than computing these sets using recursion, we use an explicit stack // called `symbols_to_process`. @@ -84,37 +92,36 @@ impl<'a> ParseItemSetBuilder<'a> { let mut processed_non_terminals = HashSet::new(); for i in 0..syntax_grammar.variables.len() { let symbol = Symbol::non_terminal(i); + let first_set = result.first_sets.entry(symbol).or_default(); + let reserved_first_set = result.reserved_first_sets.entry(symbol).or_default(); - let first_set = result - .first_sets - .entry(symbol) - .or_insert_with(TokenSet::new); processed_non_terminals.clear(); symbols_to_process.clear(); symbols_to_process.push(symbol); - while let Some(current_symbol) = symbols_to_process.pop() { - if current_symbol.is_terminal() || current_symbol.is_external() { - first_set.insert(current_symbol); - } else if processed_non_terminals.insert(current_symbol) { - for production in &syntax_grammar.variables[current_symbol.index].productions { - if let Some(step) = production.steps.first() { + while let Some(sym) = symbols_to_process.pop() { + for production in &syntax_grammar.variables[sym.index].productions { + if let Some(step) = production.steps.first() { + if step.symbol.is_terminal() || step.symbol.is_external() { + first_set.insert(step.symbol); + } else if processed_non_terminals.insert(step.symbol) { symbols_to_process.push(step.symbol); } + *reserved_first_set = (*reserved_first_set).max(step.reserved_word_set_id); } } } // The LAST set is defined in a similar way to the FIRST set. - let last_set = result.last_sets.entry(symbol).or_insert_with(TokenSet::new); + let last_set = result.last_sets.entry(symbol).or_default(); processed_non_terminals.clear(); symbols_to_process.clear(); symbols_to_process.push(symbol); - while let Some(current_symbol) = symbols_to_process.pop() { - if current_symbol.is_terminal() || current_symbol.is_external() { - last_set.insert(current_symbol); - } else if processed_non_terminals.insert(current_symbol) { - for production in &syntax_grammar.variables[current_symbol.index].productions { - if let Some(step) = production.steps.last() { + while let Some(sym) = symbols_to_process.pop() { + for production in &syntax_grammar.variables[sym.index].productions { + if let Some(step) = production.steps.last() { + if step.symbol.is_terminal() || step.symbol.is_external() { + last_set.insert(step.symbol); + } else if processed_non_terminals.insert(step.symbol) { symbols_to_process.push(step.symbol); } } @@ -124,67 +131,75 @@ impl<'a> ParseItemSetBuilder<'a> { // To compute an item set's transitive closure, we find each item in the set // whose next symbol is a non-terminal, and we add new items to the set for - // each of that symbols' productions. These productions might themselves begin + // each of that symbol's productions. These productions might themselves begin // with non-terminals, so the process continues recursively. In this process, // the total set of entries that get added depends only on two things: - // * the set of non-terminal symbols that occur at each item's current position - // * the set of terminals that occurs after each of these non-terminal symbols + // + // * the non-terminal symbol that occurs next in each item + // + // * the set of terminals that can follow that non-terminal symbol in the item // // So we can avoid a lot of duplicated recursive work by precomputing, for each // non-terminal symbol `i`, a final list of *additions* that must be made to an - // item set when `i` occurs as the next symbol in one if its core items. The - // structure of an *addition* is as follows: - // * `item` - the new item that must be added as part of the expansion of `i` - // * `lookaheads` - lookahead tokens that can always come after that item in the expansion - // of `i` - // * `propagates_lookaheads` - a boolean indicating whether or not `item` can occur at the - // *end* of the expansion of `i`, so that i's own current lookahead tokens can occur - // after `item`. + // item set when symbol `i` occurs as the next symbol in one if its core items. + // The structure of a precomputed *addition* is as follows: // - // Again, rather than computing these additions recursively, we use an explicit - // stack called `entries_to_process`. + // * `item` - the new item that must be added as part of the expansion of the symbol `i`. + // + // * `lookaheads` - the set of possible lookahead tokens that can always come after `item` + // in an expansion of symbol `i`. + // + // * `reserved_lookaheads` - the set of reserved lookahead lookahead tokens that can + // always come after `item` in the expansion of symbol `i`. + // + // * `propagates_lookaheads` - a boolean indicating whether or not `item` can occur at the + // *end* of the expansion of symbol `i`, so that i's own current lookahead tokens can + // occur after `item`. + // + // Rather than computing these additions recursively, we use an explicit stack. + let empty_lookaheads = TokenSet::new(); + let mut stack = Vec::new(); + let mut follow_set_info_by_non_terminal = HashMap::::new(); for i in 0..syntax_grammar.variables.len() { - let empty_lookaheads = TokenSet::new(); - let mut entries_to_process = vec![(i, &empty_lookaheads, true)]; - // First, build up a map whose keys are all of the non-terminals that can // appear at the beginning of non-terminal `i`, and whose values store - // information about the tokens that can follow each non-terminal. - let mut follow_set_info_by_non_terminal = HashMap::new(); - while let Some(entry) = entries_to_process.pop() { - let (variable_index, lookaheads, propagates_lookaheads) = entry; - let existing_info = follow_set_info_by_non_terminal - .entry(variable_index) - .or_insert_with(|| FollowSetInfo { - lookaheads: TokenSet::new(), - propagates_lookaheads: false, - }); - - let did_add_follow_set_info; - if propagates_lookaheads { - did_add_follow_set_info = !existing_info.propagates_lookaheads; - existing_info.propagates_lookaheads = true; - } else { - did_add_follow_set_info = existing_info.lookaheads.insert_all(lookaheads); + // information about the tokens that can follow those non-terminals. + stack.clear(); + stack.push((i, &empty_lookaheads, ReservedWordSetId::default(), true)); + follow_set_info_by_non_terminal.clear(); + while let Some((sym_ix, lookaheads, reserved_word_set_id, propagates_lookaheads)) = + stack.pop() + { + let mut did_add = false; + let info = follow_set_info_by_non_terminal.entry(sym_ix).or_default(); + did_add |= info.lookaheads.insert_all(lookaheads); + if reserved_word_set_id > info.reserved_lookaheads { + info.reserved_lookaheads = reserved_word_set_id; + did_add = true; + } + did_add |= propagates_lookaheads && !info.propagates_lookaheads; + info.propagates_lookaheads |= propagates_lookaheads; + if !did_add { + continue; } - if did_add_follow_set_info { - for production in &syntax_grammar.variables[variable_index].productions { - if let Some(symbol) = production.first_symbol() { - if symbol.is_non_terminal() { - if production.steps.len() == 1 { - entries_to_process.push(( - symbol.index, - lookaheads, - propagates_lookaheads, - )); - } else { - entries_to_process.push(( - symbol.index, - &result.first_sets[&production.steps[1].symbol], - false, - )); - } + for production in &syntax_grammar.variables[sym_ix].productions { + if let Some(symbol) = production.first_symbol() { + if symbol.is_non_terminal() { + if let Some(next_step) = production.steps.get(1) { + stack.push(( + symbol.index, + &result.first_sets[&next_step.symbol], + result.reserved_first_sets[&next_step.symbol], + false, + )); + } else { + stack.push(( + symbol.index, + lookaheads, + reserved_word_set_id, + propagates_lookaheads, + )); } } } @@ -194,7 +209,7 @@ impl<'a> ParseItemSetBuilder<'a> { // Store all of those non-terminals' productions, along with their associated // lookahead info, as *additions* associated with non-terminal `i`. let additions_for_non_terminal = &mut result.transitive_closure_additions[i]; - for (variable_index, follow_set_info) in follow_set_info_by_non_terminal { + for (&variable_index, follow_set_info) in &follow_set_info_by_non_terminal { let variable = &syntax_grammar.variables[variable_index]; let non_terminal = Symbol::non_terminal(variable_index); let variable_index = variable_index as u32; @@ -239,20 +254,23 @@ impl<'a> ParseItemSetBuilder<'a> { pub fn transitive_closure(&self, item_set: &ParseItemSet<'a>) -> ParseItemSet<'a> { let mut result = ParseItemSet::default(); - for (item, lookaheads) in &item_set.entries { + for entry in &item_set.entries { if let Some(productions) = self .inlines - .inlined_productions(item.production, item.step_index) + .inlined_productions(entry.item.production, entry.item.step_index) { for production in productions { self.add_item( &mut result, - item.substitute_production(production), - lookaheads, + &ParseItemSetEntry { + item: entry.item.substitute_production(production), + lookaheads: entry.lookaheads.clone(), + following_reserved_word_set: entry.following_reserved_word_set, + }, ); } } else { - self.add_item(&mut result, *item, lookaheads); + self.add_item(&mut result, entry); } } result @@ -262,30 +280,64 @@ impl<'a> ParseItemSetBuilder<'a> { &self.first_sets[symbol] } + pub fn reserved_first_set(&self, symbol: &Symbol) -> Option<&TokenSet> { + let id = *self.reserved_first_sets.get(symbol)?; + Some(&self.syntax_grammar.reserved_word_sets[id.0]) + } + pub fn last_set(&self, symbol: &Symbol) -> &TokenSet { &self.last_sets[symbol] } - fn add_item(&self, set: &mut ParseItemSet<'a>, item: ParseItem<'a>, lookaheads: &TokenSet) { - if let Some(step) = item.step() { + fn add_item(&self, set: &mut ParseItemSet<'a>, entry: &ParseItemSetEntry<'a>) { + if let Some(step) = entry.item.step() { if step.symbol.is_non_terminal() { - let next_step = item.successor().step(); + let next_step = entry.item.successor().step(); // Determine which tokens can follow this non-terminal. - let following_tokens = next_step.map_or(lookaheads, |next_step| { - self.first_sets.get(&next_step.symbol).unwrap() - }); + let (following_tokens, following_reserved_tokens) = + if let Some(next_step) = next_step { + ( + self.first_sets.get(&next_step.symbol).unwrap(), + *self.reserved_first_sets.get(&next_step.symbol).unwrap(), + ) + } else { + (&entry.lookaheads, entry.following_reserved_word_set) + }; // Use the pre-computed *additions* to expand the non-terminal. for addition in &self.transitive_closure_additions[step.symbol.index] { - let lookaheads = set.insert(addition.item, &addition.info.lookaheads); + let entry = set.insert(addition.item); + entry.lookaheads.insert_all(&addition.info.lookaheads); + + if let Some(word_token) = self.syntax_grammar.word_token { + if addition.info.lookaheads.contains(&word_token) { + entry.following_reserved_word_set = entry + .following_reserved_word_set + .max(addition.info.reserved_lookaheads); + } + } + if addition.info.propagates_lookaheads { - lookaheads.insert_all(following_tokens); + entry.lookaheads.insert_all(following_tokens); + + if let Some(word_token) = self.syntax_grammar.word_token { + if following_tokens.contains(&word_token) { + entry.following_reserved_word_set = entry + .following_reserved_word_set + .max(following_reserved_tokens); + } + } } } } } - set.insert(item, lookaheads); + + let e = set.insert(entry.item); + e.lookaheads.insert_all(&entry.lookaheads); + e.following_reserved_word_set = e + .following_reserved_word_set + .max(entry.following_reserved_word_set); } } diff --git a/cli/generate/src/build_tables/minimize_parse_table.rs b/crates/generate/src/build_tables/minimize_parse_table.rs similarity index 87% rename from cli/generate/src/build_tables/minimize_parse_table.rs rename to crates/generate/src/build_tables/minimize_parse_table.rs index 70dd145e..6c26f1c4 100644 --- a/cli/generate/src/build_tables/minimize_parse_table.rs +++ b/crates/generate/src/build_tables/minimize_parse_table.rs @@ -3,7 +3,7 @@ use std::{ mem, }; -use log::info; +use log::debug; use super::token_conflicts::TokenConflictMap; use crate::{ @@ -11,6 +11,7 @@ use crate::{ grammars::{LexicalGrammar, SyntaxGrammar, VariableType}, rules::{AliasMap, Symbol, TokenSet}, tables::{GotoAction, ParseAction, ParseState, ParseStateId, ParseTable, ParseTableEntry}, + OptLevel, }; pub fn minimize_parse_table( @@ -20,6 +21,7 @@ pub fn minimize_parse_table( simple_aliases: &AliasMap, token_conflict_map: &TokenConflictMap, keywords: &TokenSet, + optimizations: OptLevel, ) { let mut minimizer = Minimizer { parse_table, @@ -29,7 +31,9 @@ pub fn minimize_parse_table( keywords, simple_aliases, }; - minimizer.merge_compatible_states(); + if optimizations.contains(OptLevel::MergeStates) { + minimizer.merge_compatible_states(); + } minimizer.remove_unit_reductions(); minimizer.remove_unused_states(); minimizer.reorder_states_by_descending_size(); @@ -151,9 +155,7 @@ impl Minimizer<'_> { &mut group_ids_by_state_id, 0, |left, right, groups| self.state_successors_differ(left, right, groups), - ) { - continue; - } + ) {} let error_group_index = state_ids_by_group_id .iter() @@ -170,17 +172,12 @@ impl Minimizer<'_> { let mut new_states = Vec::with_capacity(state_ids_by_group_id.len()); for state_ids in &state_ids_by_group_id { // Initialize the new state based on the first old state in the group. - let mut parse_state = ParseState::default(); - mem::swap(&mut parse_state, &mut self.parse_table.states[state_ids[0]]); + let mut parse_state = mem::take(&mut self.parse_table.states[state_ids[0]]); // Extend the new state with all of the actions from the other old states // in the group. for state_id in &state_ids[1..] { - let mut other_parse_state = ParseState::default(); - mem::swap( - &mut other_parse_state, - &mut self.parse_table.states[*state_id], - ); + let other_parse_state = mem::take(&mut self.parse_table.states[*state_id]); parse_state .terminal_entries @@ -188,6 +185,12 @@ impl Minimizer<'_> { parse_state .nonterminal_entries .extend(other_parse_state.nonterminal_entries); + parse_state + .reserved_words + .insert_all(&other_parse_state.reserved_words); + for symbol in parse_state.terminal_entries.keys() { + parse_state.reserved_words.remove(symbol); + } } // Update the new state's outgoing references using the new grouping. @@ -216,24 +219,14 @@ impl Minimizer<'_> { ) { return true; } - } else if self.token_conflicts( - left_state.id, - right_state.id, - right_state.terminal_entries.keys(), - *token, - ) { + } else if self.token_conflicts(left_state.id, right_state.id, right_state, *token) { return true; } } for token in right_state.terminal_entries.keys() { if !left_state.terminal_entries.contains_key(token) - && self.token_conflicts( - left_state.id, - right_state.id, - left_state.terminal_entries.keys(), - *token, - ) + && self.token_conflicts(left_state.id, right_state.id, left_state, *token) { return true; } @@ -255,7 +248,7 @@ impl Minimizer<'_> { let group1 = group_ids_by_state_id[*s1]; let group2 = group_ids_by_state_id[*s2]; if group1 != group2 { - info!( + debug!( "split states {} {} - successors for {} are split: {s1} {s2}", state1.id, state2.id, @@ -271,12 +264,12 @@ impl Minimizer<'_> { for (symbol, s1) in &state1.nonterminal_entries { if let Some(s2) = state2.nonterminal_entries.get(symbol) { match (s1, s2) { - (GotoAction::ShiftExtra, GotoAction::ShiftExtra) => continue, + (GotoAction::ShiftExtra, GotoAction::ShiftExtra) => {} (GotoAction::Goto(s1), GotoAction::Goto(s2)) => { let group1 = group_ids_by_state_id[*s1]; let group2 = group_ids_by_state_id[*s2]; if group1 != group2 { - info!( + debug!( "split states {} {} - successors for {} are split: {s1} {s2}", state1.id, state2.id, @@ -306,16 +299,14 @@ impl Minimizer<'_> { let actions1 = &entry1.actions; let actions2 = &entry2.actions; if actions1.len() != actions2.len() { - info!( + debug!( "split states {state_id1} {state_id2} - differing action counts for token {}", self.symbol_name(token) ); return true; } - for (i, action1) in actions1.iter().enumerate() { - let action2 = &actions2[i]; - + for (action1, action2) in actions1.iter().zip(actions2.iter()) { // Two shift actions are equivalent if their destinations are in the same group. if let ( ParseAction::Shift { @@ -333,13 +324,13 @@ impl Minimizer<'_> { if group1 == group2 && is_repetition1 == is_repetition2 { continue; } - info!( + debug!( "split states {state_id1} {state_id2} - successors for {} are split: {s1} {s2}", self.symbol_name(token), ); return true; } else if action1 != action2 { - info!( + debug!( "split states {state_id1} {state_id2} - unequal actions for {}", self.symbol_name(token), ); @@ -350,28 +341,32 @@ impl Minimizer<'_> { false } - fn token_conflicts<'b>( + fn token_conflicts( &self, left_id: ParseStateId, right_id: ParseStateId, - existing_tokens: impl Iterator, + right_state: &ParseState, new_token: Symbol, ) -> bool { if new_token == Symbol::end_of_nonterminal_extra() { - info!("split states {left_id} {right_id} - end of non-terminal extra",); + debug!("split states {left_id} {right_id} - end of non-terminal extra",); return true; } // Do not add external tokens; they could conflict lexically with any of the state's // existing lookahead tokens. if new_token.is_external() { - info!( + debug!( "split states {left_id} {right_id} - external token {}", self.symbol_name(&new_token), ); return true; } + if right_state.reserved_words.contains(&new_token) { + return false; + } + // Do not add tokens which are both internal and external. Their validity could // influence the behavior of the external scanner. if self @@ -380,7 +375,7 @@ impl Minimizer<'_> { .iter() .any(|external| external.corresponding_internal_token == Some(new_token)) { - info!( + debug!( "split states {left_id} {right_id} - internal/external token {}", self.symbol_name(&new_token), ); @@ -388,23 +383,30 @@ impl Minimizer<'_> { } // Do not add a token if it conflicts with an existing token. - for token in existing_tokens { - if token.is_terminal() - && !(self.syntax_grammar.word_token == Some(*token) - && self.keywords.contains(&new_token)) - && !(self.syntax_grammar.word_token == Some(new_token) - && self.keywords.contains(token)) - && (self + for token in right_state.terminal_entries.keys().copied() { + if !token.is_terminal() { + continue; + } + if self.syntax_grammar.word_token == Some(token) && self.keywords.contains(&new_token) { + continue; + } + if self.syntax_grammar.word_token == Some(new_token) && self.keywords.contains(&token) { + continue; + } + + if self + .token_conflict_map + .does_conflict(new_token.index, token.index) + || self .token_conflict_map - .does_conflict(new_token.index, token.index) - || self - .token_conflict_map - .does_match_same_string(new_token.index, token.index)) + .does_match_same_string(new_token.index, token.index) { - info!( - "split states {left_id} {right_id} - token {} conflicts with {}", + debug!( + "split states {} {} - token {} conflicts with {}", + left_id, + right_id, self.symbol_name(&new_token), - self.symbol_name(token), + self.symbol_name(&token), ); return true; } diff --git a/cli/generate/src/build_tables/token_conflicts.rs b/crates/generate/src/build_tables/token_conflicts.rs similarity index 99% rename from cli/generate/src/build_tables/token_conflicts.rs rename to crates/generate/src/build_tables/token_conflicts.rs index bacac1b4..d72effd4 100644 --- a/cli/generate/src/build_tables/token_conflicts.rs +++ b/crates/generate/src/build_tables/token_conflicts.rs @@ -28,7 +28,7 @@ pub struct TokenConflictMap<'a> { impl<'a> TokenConflictMap<'a> { /// Create a token conflict map based on a lexical grammar, which describes the structure - /// each token, and a `following_token` map, which indicates which tokens may be appear + /// of each token, and a `following_token` map, which indicates which tokens may be appear /// immediately after each other token. /// /// This analyzes the possible kinds of overlap between each pair of tokens and stores diff --git a/cli/generate/src/dedup.rs b/crates/generate/src/dedup.rs similarity index 92% rename from cli/generate/src/dedup.rs rename to crates/generate/src/dedup.rs index fffe2675..c49c73d0 100644 --- a/cli/generate/src/dedup.rs +++ b/crates/generate/src/dedup.rs @@ -3,7 +3,7 @@ pub fn split_state_id_groups( state_ids_by_group_id: &mut Vec>, group_ids_by_state_id: &mut [usize], start_group_id: usize, - mut f: impl FnMut(&S, &S, &[usize]) -> bool, + mut should_split: impl FnMut(&S, &S, &[usize]) -> bool, ) -> bool { let mut result = false; @@ -33,7 +33,7 @@ pub fn split_state_id_groups( } let right_state = &states[right_state_id]; - if f(left_state, right_state, group_ids_by_state_id) { + if should_split(left_state, right_state, group_ids_by_state_id) { split_state_ids.push(right_state_id); } diff --git a/cli/generate/src/dsl.js b/crates/generate/src/dsl.js similarity index 85% rename from cli/generate/src/dsl.js rename to crates/generate/src/dsl.js index 0c4d179d..f522fd7f 100644 --- a/cli/generate/src/dsl.js +++ b/crates/generate/src/dsl.js @@ -16,6 +16,7 @@ function alias(rule, value) { result.value = value.symbol.name; return result; case Object: + case GrammarSymbol: if (typeof value.type === 'string' && value.type === 'SYMBOL') { result.named = true; result.value = value.name; @@ -69,7 +70,7 @@ function prec(number, rule) { }; } -prec.left = function(number, rule) { +prec.left = function (number, rule) { if (rule == null) { rule = number; number = 0; @@ -91,7 +92,7 @@ prec.left = function(number, rule) { }; } -prec.right = function(number, rule) { +prec.right = function (number, rule) { if (rule == null) { rule = number; number = 0; @@ -113,7 +114,7 @@ prec.right = function(number, rule) { }; } -prec.dynamic = function(number, rule) { +prec.dynamic = function (number, rule) { checkPrecedence(number); checkArguments( arguments, @@ -153,11 +154,26 @@ function seq(...elements) { }; } -function sym(name) { +class GrammarSymbol { + constructor(name) { + this.type = "SYMBOL"; + this.name = name; + } +} + +function reserved(wordset, rule) { + if (typeof wordset !== 'string') { + throw new Error('Invalid reserved word set name: ' + wordset) + } return { - type: "SYMBOL", - name - }; + type: "RESERVED", + content: normalize(rule), + context_name: wordset, + } +} + +function sym(name) { + return new GrammarSymbol(name); } function token(value) { @@ -168,7 +184,7 @@ function token(value) { }; } -token.immediate = function(value) { +token.immediate = function (value) { checkArguments(arguments, arguments.length, token.immediate, 'token.immediate', '', 'literal'); return { type: "IMMEDIATE_TOKEN", @@ -195,6 +211,11 @@ function normalize(value) { type: 'PATTERN', value: value.source }; + case RustRegex: + return { + type: 'PATTERN', + value: value.value + }; case ReferenceError: throw value default: @@ -236,6 +257,7 @@ function grammar(baseGrammar, options) { inline: [], supertypes: [], precedences: [], + reserved: {}, }; } else { baseGrammar = baseGrammar.grammar; @@ -309,6 +331,28 @@ function grammar(baseGrammar, options) { } } + let reserved = baseGrammar.reserved; + if (options.reserved) { + if (typeof options.reserved !== "object") { + throw new Error("Grammar's 'reserved' property must be an object."); + } + + for (const reservedWordSetName of Object.keys(options.reserved)) { + const reservedWordSetFn = options.reserved[reservedWordSetName] + if (typeof reservedWordSetFn !== "function") { + throw new Error(`Grammar reserved word sets must all be functions. '${reservedWordSetName}' is not.`); + } + + const reservedTokens = reservedWordSetFn.call(ruleBuilder, ruleBuilder, baseGrammar.reserved[reservedWordSetName]); + + if (!Array.isArray(reservedTokens)) { + throw new Error(`Grammar's reserved word set functions must all return arrays of rules. '${reservedWordSetName}' does not.`); + } + + reserved[reservedWordSetName] = reservedTokens.map(normalize); + } + } + let extras = baseGrammar.extras.slice(); if (options.extras) { if (typeof options.extras !== "function") { @@ -439,10 +483,17 @@ function grammar(baseGrammar, options) { externals, inline, supertypes, + reserved, }, }; } +class RustRegex { + constructor(value) { + this.value = value; + } +} + function checkArguments(args, ruleCount, caller, callerName, suffix = '', argType = 'rule') { // Allow for .map() usage where additional arguments are index and the entire array. const isMapCall = ruleCount === 3 && typeof args[1] === 'number' && Array.isArray(args[2]); @@ -466,6 +517,7 @@ function checkPrecedence(value) { } function getEnv(name) { + if (globalThis.native) return globalThis.__ts_grammar_path; if (globalThis.process) return process.env[name]; // Node/Bun if (globalThis.Deno) return Deno.env.get(name); // Deno throw Error("Unsupported JS runtime"); @@ -478,20 +530,31 @@ globalThis.optional = optional; globalThis.prec = prec; globalThis.repeat = repeat; globalThis.repeat1 = repeat1; +globalThis.reserved = reserved; globalThis.seq = seq; globalThis.sym = sym; globalThis.token = token; globalThis.grammar = grammar; globalThis.field = field; +globalThis.RustRegex = RustRegex; + +const grammarPath = getEnv("TREE_SITTER_GRAMMAR_PATH"); +let result = await import(grammarPath); +let grammarObj = result.default?.grammar ?? result.grammar; + +if (globalThis.native && !grammarObj) { + grammarObj = module.exports.grammar; +} -const result = await import(getEnv("TREE_SITTER_GRAMMAR_PATH")); const object = { "$schema": "https://tree-sitter.github.io/tree-sitter/assets/schemas/grammar.schema.json", - ...(result.default?.grammar ?? result.grammar) + ...grammarObj, }; const output = JSON.stringify(object); -if (globalThis.process) { // Node/Bun +if (globalThis.native) { + globalThis.output = output; +} else if (globalThis.process) { // Node/Bun process.stdout.write(output); } else if (globalThis.Deno) { // Deno Deno.stdout.writeSync(new TextEncoder().encode(output)); diff --git a/crates/generate/src/generate.rs b/crates/generate/src/generate.rs new file mode 100644 index 00000000..6a005637 --- /dev/null +++ b/crates/generate/src/generate.rs @@ -0,0 +1,621 @@ +use std::{collections::BTreeMap, sync::LazyLock}; +#[cfg(feature = "load")] +use std::{ + env, fs, + io::Write, + path::{Path, PathBuf}, + process::{Command, Stdio}, +}; + +use bitflags::bitflags; +use log::warn; +use node_types::VariableInfo; +use regex::{Regex, RegexBuilder}; +use rules::{Alias, Symbol}; +#[cfg(feature = "load")] +use semver::Version; +#[cfg(feature = "load")] +use serde::Deserialize; +use serde::Serialize; +use thiserror::Error; + +mod build_tables; +mod dedup; +mod grammars; +mod nfa; +mod node_types; +pub mod parse_grammar; +mod prepare_grammar; +#[cfg(feature = "qjs-rt")] +mod quickjs; +mod render; +mod rules; +mod tables; + +use build_tables::build_tables; +pub use build_tables::ParseTableBuilderError; +use grammars::{InlinedProductionMap, InputGrammar, LexicalGrammar, SyntaxGrammar}; +pub use node_types::{SuperTypeCycleError, VariableInfoError}; +use parse_grammar::parse_grammar; +pub use parse_grammar::ParseGrammarError; +use prepare_grammar::prepare_grammar; +pub use prepare_grammar::PrepareGrammarError; +use render::render_c_code; +pub use render::{ABI_VERSION_MAX, ABI_VERSION_MIN}; + +static JSON_COMMENT_REGEX: LazyLock = LazyLock::new(|| { + RegexBuilder::new("^\\s*//.*") + .multi_line(true) + .build() + .unwrap() +}); + +struct JSONOutput { + #[cfg(feature = "load")] + node_types_json: String, + syntax_grammar: SyntaxGrammar, + lexical_grammar: LexicalGrammar, + inlines: InlinedProductionMap, + simple_aliases: BTreeMap, + variable_info: Vec, +} + +struct GeneratedParser { + c_code: String, + #[cfg(feature = "load")] + node_types_json: String, +} + +// NOTE: This constant must be kept in sync with the definition of +// `TREE_SITTER_LANGUAGE_VERSION` in `lib/include/tree_sitter/api.h`. +const LANGUAGE_VERSION: usize = 15; + +pub const ALLOC_HEADER: &str = include_str!("templates/alloc.h"); +pub const ARRAY_HEADER: &str = include_str!("templates/array.h"); +pub const PARSER_HEADER: &str = include_str!("parser.h.inc"); + +pub type GenerateResult = Result; + +#[derive(Debug, Error, Serialize)] +pub enum GenerateError { + #[error("Error with specified path -- {0}")] + GrammarPath(String), + #[error(transparent)] + IO(IoError), + #[cfg(feature = "load")] + #[error(transparent)] + LoadGrammarFile(#[from] LoadGrammarError), + #[error(transparent)] + ParseGrammar(#[from] ParseGrammarError), + #[error(transparent)] + Prepare(#[from] PrepareGrammarError), + #[error(transparent)] + VariableInfo(#[from] VariableInfoError), + #[error(transparent)] + BuildTables(#[from] ParseTableBuilderError), + #[cfg(feature = "load")] + #[error(transparent)] + ParseVersion(#[from] ParseVersionError), + #[error(transparent)] + SuperTypeCycle(#[from] SuperTypeCycleError), +} + +#[derive(Debug, Error, Serialize)] +pub struct IoError { + pub error: String, + pub path: Option, +} + +impl IoError { + fn new(error: &std::io::Error, path: Option<&Path>) -> Self { + Self { + error: error.to_string(), + path: path.map(|p| p.to_string_lossy().to_string()), + } + } +} + +impl std::fmt::Display for IoError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.error)?; + if let Some(ref path) = self.path { + write!(f, " ({path})")?; + } + Ok(()) + } +} + +#[cfg(feature = "load")] +pub type LoadGrammarFileResult = Result; + +#[cfg(feature = "load")] +#[derive(Debug, Error, Serialize)] +pub enum LoadGrammarError { + #[error("Path to a grammar file with `.js` or `.json` extension is required")] + InvalidPath, + #[error("Failed to load grammar.js -- {0}")] + LoadJSGrammarFile(#[from] JSError), + #[error("Failed to load grammar.json -- {0}")] + IO(IoError), + #[error("Unknown grammar file extension: {0:?}")] + FileExtension(PathBuf), +} + +#[cfg(feature = "load")] +#[derive(Debug, Error, Serialize)] +pub enum ParseVersionError { + #[error("{0}")] + Version(String), + #[error("{0}")] + JSON(String), + #[error(transparent)] + IO(IoError), +} + +#[cfg(feature = "load")] +pub type JSResult = Result; + +#[cfg(feature = "load")] +#[derive(Debug, Error, Serialize)] +pub enum JSError { + #[error("Failed to run `{runtime}` -- {error}")] + JSRuntimeSpawn { runtime: String, error: String }, + #[error("Got invalid UTF8 from `{runtime}` -- {error}")] + JSRuntimeUtf8 { runtime: String, error: String }, + #[error("`{runtime}` process exited with status {code}")] + JSRuntimeExit { runtime: String, code: i32 }, + #[error("Failed to open stdin for `{runtime}`")] + JSRuntimeStdin { runtime: String }, + #[error("Failed to write {item} to `{runtime}`'s stdin -- {error}")] + JSRuntimeWrite { + runtime: String, + item: String, + error: String, + }, + #[error("Failed to read output from `{runtime}` -- {error}")] + JSRuntimeRead { runtime: String, error: String }, + #[error(transparent)] + IO(IoError), + #[cfg(feature = "qjs-rt")] + #[error("Failed to get relative path")] + RelativePath, + #[error("Could not parse this package's version as semver -- {0}")] + Semver(String), + #[error("Failed to serialze grammar JSON -- {0}")] + Serialzation(String), + #[cfg(feature = "qjs-rt")] + #[error("QuickJS error: {0}")] + QuickJS(String), +} + +#[cfg(feature = "load")] +impl From for JSError { + fn from(value: serde_json::Error) -> Self { + Self::Serialzation(value.to_string()) + } +} + +#[cfg(feature = "load")] +impl From for JSError { + fn from(value: semver::Error) -> Self { + Self::Semver(value.to_string()) + } +} + +#[cfg(feature = "qjs-rt")] +impl From for JSError { + fn from(value: rquickjs::Error) -> Self { + Self::QuickJS(value.to_string()) + } +} + +bitflags! { + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + pub struct OptLevel: u32 { + const MergeStates = 1 << 0; + } +} + +impl Default for OptLevel { + fn default() -> Self { + Self::MergeStates + } +} + +#[cfg(feature = "load")] +#[allow(clippy::too_many_arguments)] +pub fn generate_parser_in_directory( + repo_path: T, + out_path: Option, + grammar_path: Option, + mut abi_version: usize, + report_symbol_name: Option<&str>, + js_runtime: Option<&str>, + generate_parser: bool, + optimizations: OptLevel, +) -> GenerateResult<()> +where + T: Into, + U: Into, + V: Into, +{ + let mut repo_path: PathBuf = repo_path.into(); + + // Populate a new empty grammar directory. + let grammar_path = if let Some(path) = grammar_path { + let path_buf: PathBuf = path.into(); + if !path_buf + .try_exists() + .map_err(|e| GenerateError::GrammarPath(e.to_string()))? + { + fs::create_dir_all(&path_buf) + .map_err(|e| GenerateError::IO(IoError::new(&e, Some(path_buf.as_path()))))?; + repo_path = path_buf; + repo_path.join("grammar.js") + } else { + path_buf + } + } else { + repo_path.join("grammar.js") + }; + + // Read the grammar file. + let grammar_json = load_grammar_file(&grammar_path, js_runtime)?; + + let src_path = out_path.map_or_else(|| repo_path.join("src"), |p| p.into()); + let header_path = src_path.join("tree_sitter"); + + // Ensure that the output directory exists + fs::create_dir_all(&src_path) + .map_err(|e| GenerateError::IO(IoError::new(&e, Some(src_path.as_path()))))?; + + if grammar_path.file_name().unwrap() != "grammar.json" { + fs::write(src_path.join("grammar.json"), &grammar_json) + .map_err(|e| GenerateError::IO(IoError::new(&e, Some(src_path.as_path()))))?; + } + + // If our job is only to generate `grammar.json` and not `parser.c`, stop here. + let input_grammar = parse_grammar(&grammar_json)?; + + if !generate_parser { + let node_types_json = generate_node_types_from_grammar(&input_grammar)?.node_types_json; + write_file(&src_path.join("node-types.json"), node_types_json)?; + return Ok(()); + } + + let semantic_version = read_grammar_version(&repo_path)?; + + if semantic_version.is_none() && abi_version > ABI_VERSION_MIN { + warn!( + concat!( + "No `tree-sitter.json` file found in your grammar, ", + "this file is required to generate with ABI {}. ", + "Using ABI version {} instead.\n", + "This file can be set up with `tree-sitter init`. ", + "For more information, see https://tree-sitter.github.io/tree-sitter/cli/init." + ), + abi_version, ABI_VERSION_MIN + ); + abi_version = ABI_VERSION_MIN; + } + + // Generate the parser and related files. + let GeneratedParser { + c_code, + node_types_json, + } = generate_parser_for_grammar_with_opts( + &input_grammar, + abi_version, + semantic_version.map(|v| (v.major as u8, v.minor as u8, v.patch as u8)), + report_symbol_name, + optimizations, + )?; + + write_file(&src_path.join("parser.c"), c_code)?; + write_file(&src_path.join("node-types.json"), node_types_json)?; + fs::create_dir_all(&header_path) + .map_err(|e| GenerateError::IO(IoError::new(&e, Some(header_path.as_path()))))?; + write_file(&header_path.join("alloc.h"), ALLOC_HEADER)?; + write_file(&header_path.join("array.h"), ARRAY_HEADER)?; + write_file(&header_path.join("parser.h"), PARSER_HEADER)?; + + Ok(()) +} + +pub fn generate_parser_for_grammar( + grammar_json: &str, + semantic_version: Option<(u8, u8, u8)>, +) -> GenerateResult<(String, String)> { + let grammar_json = JSON_COMMENT_REGEX.replace_all(grammar_json, "\n"); + let input_grammar = parse_grammar(&grammar_json)?; + let parser = generate_parser_for_grammar_with_opts( + &input_grammar, + LANGUAGE_VERSION, + semantic_version, + None, + OptLevel::empty(), + )?; + Ok((input_grammar.name, parser.c_code)) +} + +fn generate_node_types_from_grammar(input_grammar: &InputGrammar) -> GenerateResult { + let (syntax_grammar, lexical_grammar, inlines, simple_aliases) = + prepare_grammar(input_grammar)?; + let variable_info = + node_types::get_variable_info(&syntax_grammar, &lexical_grammar, &simple_aliases)?; + + #[cfg(feature = "load")] + let node_types_json = node_types::generate_node_types_json( + &syntax_grammar, + &lexical_grammar, + &simple_aliases, + &variable_info, + )?; + Ok(JSONOutput { + #[cfg(feature = "load")] + node_types_json: serde_json::to_string_pretty(&node_types_json).unwrap(), + syntax_grammar, + lexical_grammar, + inlines, + simple_aliases, + variable_info, + }) +} + +fn generate_parser_for_grammar_with_opts( + input_grammar: &InputGrammar, + abi_version: usize, + semantic_version: Option<(u8, u8, u8)>, + report_symbol_name: Option<&str>, + optimizations: OptLevel, +) -> GenerateResult { + let JSONOutput { + syntax_grammar, + lexical_grammar, + inlines, + simple_aliases, + variable_info, + #[cfg(feature = "load")] + node_types_json, + } = generate_node_types_from_grammar(input_grammar)?; + let supertype_symbol_map = + node_types::get_supertype_symbol_map(&syntax_grammar, &simple_aliases, &variable_info); + let tables = build_tables( + &syntax_grammar, + &lexical_grammar, + &simple_aliases, + &variable_info, + &inlines, + report_symbol_name, + optimizations, + )?; + let c_code = render_c_code( + &input_grammar.name, + tables, + syntax_grammar, + lexical_grammar, + simple_aliases, + abi_version, + semantic_version, + supertype_symbol_map, + ); + Ok(GeneratedParser { + c_code, + #[cfg(feature = "load")] + node_types_json, + }) +} + +/// This will read the `tree-sitter.json` config file and attempt to extract the version. +/// +/// If the file is not found in the current directory or any of its parent directories, this will +/// return `None` to maintain backwards compatibility. If the file is found but the version cannot +/// be parsed as semver, this will return an error. +#[cfg(feature = "load")] +fn read_grammar_version(repo_path: &Path) -> Result, ParseVersionError> { + #[derive(Deserialize)] + struct TreeSitterJson { + metadata: Metadata, + } + + #[derive(Deserialize)] + struct Metadata { + version: String, + } + + let filename = "tree-sitter.json"; + let mut path = repo_path.join(filename); + + loop { + let json = path + .exists() + .then(|| { + let contents = fs::read_to_string(path.as_path()) + .map_err(|e| ParseVersionError::IO(IoError::new(&e, Some(path.as_path()))))?; + serde_json::from_str::(&contents).map_err(|e| { + ParseVersionError::JSON(format!("Failed to parse `{}` -- {e}", path.display())) + }) + }) + .transpose()?; + if let Some(json) = json { + return Version::parse(&json.metadata.version) + .map_err(|e| { + ParseVersionError::Version(format!( + "Failed to parse `{}` version as semver -- {e}", + path.display() + )) + }) + .map(Some); + } + path.pop(); // filename + if !path.pop() { + return Ok(None); + } + path.push(filename); + } +} + +#[cfg(feature = "load")] +pub fn load_grammar_file( + grammar_path: &Path, + js_runtime: Option<&str>, +) -> LoadGrammarFileResult { + if grammar_path.is_dir() { + Err(LoadGrammarError::InvalidPath)?; + } + match grammar_path.extension().and_then(|e| e.to_str()) { + Some("js") => Ok(load_js_grammar_file(grammar_path, js_runtime)?), + Some("json") => Ok(fs::read_to_string(grammar_path) + .map_err(|e| LoadGrammarError::IO(IoError::new(&e, Some(grammar_path))))?), + _ => Err(LoadGrammarError::FileExtension(grammar_path.to_owned()))?, + } +} + +#[cfg(feature = "load")] +fn load_js_grammar_file(grammar_path: &Path, js_runtime: Option<&str>) -> JSResult { + let grammar_path = dunce::canonicalize(grammar_path) + .map_err(|e| JSError::IO(IoError::new(&e, Some(grammar_path))))?; + + #[cfg(feature = "qjs-rt")] + if js_runtime == Some("native") { + return quickjs::execute_native_runtime(&grammar_path); + } + + // The "file:///" prefix is incompatible with the quickjs runtime, but is required + // for node and bun + #[cfg(windows)] + let grammar_path = PathBuf::from(format!("file:///{}", grammar_path.display())); + + let js_runtime = js_runtime.unwrap_or("node"); + + let mut js_command = Command::new(js_runtime); + match js_runtime { + "node" => { + js_command.args(["--input-type=module", "-"]); + } + "bun" => { + js_command.arg("-"); + } + "deno" => { + js_command.args(["run", "--allow-all", "-"]); + } + _ => {} + } + + let mut js_process = js_command + .env("TREE_SITTER_GRAMMAR_PATH", grammar_path) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .map_err(|e| JSError::JSRuntimeSpawn { + runtime: js_runtime.to_string(), + error: e.to_string(), + })?; + + let mut js_stdin = js_process + .stdin + .take() + .ok_or_else(|| JSError::JSRuntimeStdin { + runtime: js_runtime.to_string(), + })?; + + let cli_version = Version::parse(env!("CARGO_PKG_VERSION"))?; + write!( + js_stdin, + "globalThis.TREE_SITTER_CLI_VERSION_MAJOR = {}; + globalThis.TREE_SITTER_CLI_VERSION_MINOR = {}; + globalThis.TREE_SITTER_CLI_VERSION_PATCH = {};", + cli_version.major, cli_version.minor, cli_version.patch, + ) + .map_err(|e| JSError::JSRuntimeWrite { + runtime: js_runtime.to_string(), + item: "tree-sitter version".to_string(), + error: e.to_string(), + })?; + js_stdin + .write(include_bytes!("./dsl.js")) + .map_err(|e| JSError::JSRuntimeWrite { + runtime: js_runtime.to_string(), + item: "grammar dsl".to_string(), + error: e.to_string(), + })?; + drop(js_stdin); + + let output = js_process + .wait_with_output() + .map_err(|e| JSError::JSRuntimeRead { + runtime: js_runtime.to_string(), + error: e.to_string(), + })?; + match output.status.code() { + Some(0) => { + let stdout = String::from_utf8(output.stdout).map_err(|e| JSError::JSRuntimeUtf8 { + runtime: js_runtime.to_string(), + error: e.to_string(), + })?; + + let mut grammar_json = &stdout[..]; + + if let Some(pos) = stdout.rfind('\n') { + // If there's a newline, split the last line from the rest of the output + let node_output = &stdout[..pos]; + grammar_json = &stdout[pos + 1..]; + + let mut stdout = std::io::stdout().lock(); + stdout + .write_all(node_output.as_bytes()) + .map_err(|e| JSError::IO(IoError::new(&e, None)))?; + stdout + .write_all(b"\n") + .map_err(|e| JSError::IO(IoError::new(&e, None)))?; + stdout + .flush() + .map_err(|e| JSError::IO(IoError::new(&e, None)))?; + } + + Ok(serde_json::to_string_pretty(&serde_json::from_str::< + serde_json::Value, + >(grammar_json)?)?) + } + Some(code) => Err(JSError::JSRuntimeExit { + runtime: js_runtime.to_string(), + code, + }), + None => Err(JSError::JSRuntimeExit { + runtime: js_runtime.to_string(), + code: -1, + }), + } +} + +#[cfg(feature = "load")] +pub fn write_file(path: &Path, body: impl AsRef<[u8]>) -> GenerateResult<()> { + fs::write(path, body).map_err(|e| GenerateError::IO(IoError::new(&e, Some(path)))) +} + +#[cfg(test)] +mod tests { + use super::{LANGUAGE_VERSION, PARSER_HEADER}; + #[test] + fn test_language_versions_are_in_sync() { + let api_h = include_str!("../../../lib/include/tree_sitter/api.h"); + let api_language_version = api_h + .lines() + .find_map(|line| { + line.trim() + .strip_prefix("#define TREE_SITTER_LANGUAGE_VERSION ") + .and_then(|v| v.parse::().ok()) + }) + .expect("Failed to find TREE_SITTER_LANGUAGE_VERSION definition in api.h"); + assert_eq!(LANGUAGE_VERSION, api_language_version); + } + + #[test] + fn test_parser_header_in_sync() { + let parser_h = include_str!("../../../lib/src/parser.h"); + assert!( + parser_h == PARSER_HEADER, + "parser.h.inc is out of sync with lib/src/parser.h. Run: cp lib/src/parser.h crates/generate/src/parser.h.inc" + ); + } +} diff --git a/cli/generate/src/grammars.rs b/crates/generate/src/grammars.rs similarity index 79% rename from cli/generate/src/grammars.rs rename to crates/generate/src/grammars.rs index fab07afb..c6e0acdd 100644 --- a/cli/generate/src/grammars.rs +++ b/crates/generate/src/grammars.rs @@ -2,7 +2,7 @@ use std::{collections::HashMap, fmt}; use super::{ nfa::Nfa, - rules::{Alias, Associativity, Precedence, Rule, Symbol}, + rules::{Alias, Associativity, Precedence, Rule, Symbol, TokenSet}, }; #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] @@ -39,6 +39,13 @@ pub struct InputGrammar { pub variables_to_inline: Vec, pub supertype_symbols: Vec, pub word_token: Option, + pub reserved_words: Vec>, +} + +#[derive(Debug, Default, PartialEq, Eq)] +pub struct ReservedWordContext { + pub name: String, + pub reserved_words: Vec, } // Extracted lexical grammar @@ -66,8 +73,20 @@ pub struct ProductionStep { pub associativity: Option, pub alias: Option, pub field_name: Option, + pub reserved_word_set_id: ReservedWordSetId, } +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct ReservedWordSetId(pub usize); + +impl fmt::Display for ReservedWordSetId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +pub const NO_RESERVED_WORDS: ReservedWordSetId = ReservedWordSetId(usize::MAX); + #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct Production { pub steps: Vec, @@ -104,51 +123,44 @@ pub struct SyntaxGrammar { pub variables_to_inline: Vec, pub word_token: Option, pub precedence_orderings: Vec>, + pub reserved_word_sets: Vec, } #[cfg(test)] impl ProductionStep { #[must_use] - pub const fn new(symbol: Symbol) -> Self { + pub fn new(symbol: Symbol) -> Self { Self { symbol, precedence: Precedence::None, associativity: None, alias: None, field_name: None, + reserved_word_set_id: ReservedWordSetId::default(), } } - pub fn with_prec(self, precedence: Precedence, associativity: Option) -> Self { - Self { - symbol: self.symbol, - precedence, - associativity, - alias: self.alias, - field_name: self.field_name, - } + pub fn with_prec( + mut self, + precedence: Precedence, + associativity: Option, + ) -> Self { + self.precedence = precedence; + self.associativity = associativity; + self } - pub fn with_alias(self, value: &str, is_named: bool) -> Self { - Self { - symbol: self.symbol, - precedence: self.precedence, - associativity: self.associativity, - alias: Some(Alias { - value: value.to_string(), - is_named, - }), - field_name: self.field_name, - } + pub fn with_alias(mut self, value: &str, is_named: bool) -> Self { + self.alias = Some(Alias { + value: value.to_string(), + is_named, + }); + self } - pub fn with_field_name(self, name: &str) -> Self { - Self { - symbol: self.symbol, - precedence: self.precedence, - associativity: self.associativity, - alias: self.alias, - field_name: Some(name.to_string()), - } + + pub fn with_field_name(mut self, name: &str) -> Self { + self.field_name = Some(name.to_string()); + self } } @@ -241,7 +253,7 @@ impl InlinedProductionMap { step_index: u32, ) -> Option + 'a> { self.production_map - .get(&(production as *const Production, step_index)) + .get(&(std::ptr::from_ref::(production), step_index)) .map(|production_indices| { production_indices .iter() diff --git a/cli/generate/src/nfa.rs b/crates/generate/src/nfa.rs similarity index 98% rename from cli/generate/src/nfa.rs rename to crates/generate/src/nfa.rs index 3d14b513..eecbc40b 100644 --- a/cli/generate/src/nfa.rs +++ b/crates/generate/src/nfa.rs @@ -434,6 +434,7 @@ impl Nfa { } pub fn last_state_id(&self) -> u32 { + assert!(!self.states.is_empty()); self.states.len() as u32 - 1 } } @@ -949,20 +950,19 @@ mod tests { assert_eq!( left.remove_intersection(&mut right), row.intersection, - "row {}a: {:?} && {:?}", - i, + "row {i}a: {:?} && {:?}", row.left, row.right ); assert_eq!( left, row.left_only, - "row {}a: {:?} - {:?}", - i, row.left, row.right + "row {i}a: {:?} - {:?}", + row.left, row.right ); assert_eq!( right, row.right_only, - "row {}a: {:?} - {:?}", - i, row.right, row.left + "row {i}a: {:?} - {:?}", + row.right, row.left ); let mut left = row.left.clone(); @@ -970,27 +970,25 @@ mod tests { assert_eq!( right.remove_intersection(&mut left), row.intersection, - "row {}b: {:?} && {:?}", - i, + "row {i}b: {:?} && {:?}", row.left, row.right ); assert_eq!( left, row.left_only, - "row {}b: {:?} - {:?}", - i, row.left, row.right + "row {i}b: {:?} - {:?}", + row.left, row.right ); assert_eq!( right, row.right_only, - "row {}b: {:?} - {:?}", - i, row.right, row.left + "row {i}b: {:?} - {:?}", + row.right, row.left ); assert_eq!( row.left.clone().difference(row.right.clone()), row.left_only, - "row {}b: {:?} -- {:?}", - i, + "row {i}b: {:?} -- {:?}", row.left, row.right ); diff --git a/cli/generate/src/node_types.rs b/crates/generate/src/node_types.rs similarity index 86% rename from cli/generate/src/node_types.rs rename to crates/generate/src/node_types.rs index debd8ae1..2dde0c49 100644 --- a/cli/generate/src/node_types.rs +++ b/crates/generate/src/node_types.rs @@ -1,10 +1,7 @@ -use std::{ - cmp::Ordering, - collections::{BTreeMap, HashMap, HashSet}, -}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; -use anyhow::{anyhow, Result}; use serde::Serialize; +use thiserror::Error; use super::{ grammars::{LexicalGrammar, SyntaxGrammar, VariableType}, @@ -32,12 +29,15 @@ pub struct VariableInfo { } #[derive(Debug, Serialize, PartialEq, Eq, Default, PartialOrd, Ord)] +#[cfg(feature = "load")] pub struct NodeInfoJSON { #[serde(rename = "type")] kind: String, named: bool, #[serde(skip_serializing_if = "std::ops::Not::not")] root: bool, + #[serde(skip_serializing_if = "std::ops::Not::not")] + extra: bool, #[serde(skip_serializing_if = "Option::is_none")] fields: Option>, #[serde(skip_serializing_if = "Option::is_none")] @@ -47,6 +47,7 @@ pub struct NodeInfoJSON { } #[derive(Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg(feature = "load")] pub struct NodeTypeJSON { #[serde(rename = "type")] kind: String, @@ -54,6 +55,7 @@ pub struct NodeTypeJSON { } #[derive(Debug, Serialize, PartialEq, Eq, PartialOrd, Ord)] +#[cfg(feature = "load")] pub struct FieldInfoJSON { multiple: bool, required: bool, @@ -67,6 +69,7 @@ pub struct ChildQuantity { multiple: bool, } +#[cfg(feature = "load")] impl Default for FieldInfoJSON { fn default() -> Self { Self { @@ -102,7 +105,7 @@ impl ChildQuantity { } } - fn append(&mut self, other: Self) { + const fn append(&mut self, other: Self) { if other.exists { if self.exists || other.multiple { self.multiple = true; @@ -114,7 +117,7 @@ impl ChildQuantity { } } - fn union(&mut self, other: Self) -> bool { + const fn union(&mut self, other: Self) -> bool { let mut result = false; if !self.exists && other.exists { result = true; @@ -132,6 +135,14 @@ impl ChildQuantity { } } +pub type VariableInfoResult = Result; + +#[derive(Debug, Error, Serialize)] +pub enum VariableInfoError { + #[error("Grammar error: Supertype symbols must always have a single visible child, but `{0}` can have multiple")] + InvalidSupertype(String), +} + /// Compute a summary of the public-facing structure of each variable in the /// grammar. Each variable in the grammar corresponds to a distinct public-facing /// node type. @@ -157,7 +168,7 @@ pub fn get_variable_info( syntax_grammar: &SyntaxGrammar, lexical_grammar: &LexicalGrammar, default_aliases: &AliasMap, -) -> Result> { +) -> VariableInfoResult> { let child_type_is_visible = |t: &ChildType| { variable_type_for_child_type(t, syntax_grammar, lexical_grammar) >= VariableType::Anonymous }; @@ -338,13 +349,7 @@ pub fn get_variable_info( for supertype_symbol in &syntax_grammar.supertype_symbols { if result[supertype_symbol.index].has_multi_step_production { let variable = &syntax_grammar.variables[supertype_symbol.index]; - return Err(anyhow!( - concat!( - "Grammar error: Supertype symbols must always ", - "have a single visible child, but `{}` can have multiple" - ), - variable.name - )); + Err(VariableInfoError::InvalidSupertype(variable.name.clone()))?; } } @@ -369,12 +374,105 @@ pub fn get_variable_info( Ok(result) } +fn get_aliases_by_symbol( + syntax_grammar: &SyntaxGrammar, + default_aliases: &AliasMap, +) -> HashMap>> { + let mut aliases_by_symbol = HashMap::new(); + for (symbol, alias) in default_aliases { + aliases_by_symbol.insert(*symbol, { + let mut aliases = BTreeSet::new(); + aliases.insert(Some(alias.clone())); + aliases + }); + } + for extra_symbol in &syntax_grammar.extra_symbols { + if !default_aliases.contains_key(extra_symbol) { + aliases_by_symbol + .entry(*extra_symbol) + .or_insert_with(BTreeSet::new) + .insert(None); + } + } + for variable in &syntax_grammar.variables { + for production in &variable.productions { + for step in &production.steps { + aliases_by_symbol + .entry(step.symbol) + .or_insert_with(BTreeSet::new) + .insert( + step.alias + .as_ref() + .or_else(|| default_aliases.get(&step.symbol)) + .cloned(), + ); + } + } + } + aliases_by_symbol.insert( + Symbol::non_terminal(0), + std::iter::once(&None).cloned().collect(), + ); + aliases_by_symbol +} + +pub fn get_supertype_symbol_map( + syntax_grammar: &SyntaxGrammar, + default_aliases: &AliasMap, + variable_info: &[VariableInfo], +) -> BTreeMap> { + let aliases_by_symbol = get_aliases_by_symbol(syntax_grammar, default_aliases); + let mut supertype_symbol_map = BTreeMap::new(); + + let mut symbols_by_alias = HashMap::new(); + for (symbol, aliases) in &aliases_by_symbol { + for alias in aliases.iter().flatten() { + symbols_by_alias + .entry(alias) + .or_insert_with(Vec::new) + .push(*symbol); + } + } + + for (i, info) in variable_info.iter().enumerate() { + let symbol = Symbol::non_terminal(i); + if syntax_grammar.supertype_symbols.contains(&symbol) { + let subtypes = info.children.types.clone(); + supertype_symbol_map.insert(symbol, subtypes); + } + } + supertype_symbol_map +} + +#[cfg(feature = "load")] +pub type SuperTypeCycleResult = Result; + +#[derive(Debug, Error, Serialize)] +pub struct SuperTypeCycleError { + items: Vec, +} + +impl std::fmt::Display for SuperTypeCycleError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Dependency cycle detected in node types:")?; + for (i, item) in self.items.iter().enumerate() { + write!(f, " {item}")?; + if i < self.items.len() - 1 { + write!(f, ",")?; + } + } + + Ok(()) + } +} + +#[cfg(feature = "load")] pub fn generate_node_types_json( syntax_grammar: &SyntaxGrammar, lexical_grammar: &LexicalGrammar, default_aliases: &AliasMap, variable_info: &[VariableInfo], -) -> Vec { +) -> SuperTypeCycleResult> { let mut node_types_json = BTreeMap::new(); let child_type_to_node_type = |child_type: &ChildType| match child_type { @@ -430,41 +528,32 @@ pub fn generate_node_types_json( } }; - let mut aliases_by_symbol = HashMap::new(); - for (symbol, alias) in default_aliases { - aliases_by_symbol.insert(*symbol, { - let mut aliases = HashSet::new(); - aliases.insert(Some(alias.clone())); - aliases - }); - } - for extra_symbol in &syntax_grammar.extra_symbols { - if !default_aliases.contains_key(extra_symbol) { + let aliases_by_symbol = get_aliases_by_symbol(syntax_grammar, default_aliases); + + let empty = BTreeSet::new(); + let extra_names = syntax_grammar + .extra_symbols + .iter() + .flat_map(|symbol| { aliases_by_symbol - .entry(*extra_symbol) - .or_insert_with(HashSet::new) - .insert(None); - } - } - for variable in &syntax_grammar.variables { - for production in &variable.productions { - for step in &production.steps { - aliases_by_symbol - .entry(step.symbol) - .or_insert_with(HashSet::new) - .insert( - step.alias - .as_ref() - .or_else(|| default_aliases.get(&step.symbol)) - .cloned(), - ); - } - } - } - aliases_by_symbol.insert( - Symbol::non_terminal(0), - std::iter::once(&None).cloned().collect(), - ); + .get(symbol) + .unwrap_or(&empty) + .iter() + .map(|alias| { + alias.as_ref().map_or( + match symbol.kind { + SymbolType::NonTerminal => &syntax_grammar.variables[symbol.index].name, + SymbolType::Terminal => &lexical_grammar.variables[symbol.index].name, + SymbolType::External => { + &syntax_grammar.external_tokens[symbol.index].name + } + _ => unreachable!(), + }, + |alias| &alias.value, + ) + }) + }) + .collect::>(); let mut subtype_map = Vec::new(); for (i, info) in variable_info.iter().enumerate() { @@ -478,6 +567,7 @@ pub fn generate_node_types_json( kind: variable.name.clone(), named: true, root: false, + extra: extra_names.contains(&variable.name), fields: None, children: None, subtypes: None, @@ -499,7 +589,7 @@ pub fn generate_node_types_json( } else if !syntax_grammar.variables_to_inline.contains(&symbol) { // If a rule is aliased under multiple names, then its information // contributes to multiple entries in the final JSON. - for alias in aliases_by_symbol.get(&symbol).unwrap_or(&HashSet::new()) { + for alias in aliases_by_symbol.get(&symbol).unwrap_or(&BTreeSet::new()) { let kind; let is_named; if let Some(alias) = alias { @@ -521,6 +611,7 @@ pub fn generate_node_types_json( kind: kind.clone(), named: is_named, root: i == 0, + extra: extra_names.contains(&kind), fields: Some(BTreeMap::new()), children: None, subtypes: None, @@ -559,15 +650,33 @@ pub fn generate_node_types_json( } } - // Sort the subtype map so that subtypes are listed before their supertypes. - subtype_map.sort_by(|a, b| { - if b.1.contains(&a.0) { - Ordering::Less - } else if a.1.contains(&b.0) { - Ordering::Greater - } else { - Ordering::Equal + // Sort the subtype map topologically so that subtypes are listed before their supertypes. + let mut sorted_kinds = Vec::with_capacity(subtype_map.len()); + let mut top_sort = topological_sort::TopologicalSort::::new(); + for (supertype, subtypes) in &subtype_map { + for subtype in subtypes { + top_sort.add_dependency(subtype.kind.clone(), supertype.kind.clone()); } + } + loop { + let mut next_kinds = top_sort.pop_all(); + match (next_kinds.is_empty(), top_sort.is_empty()) { + (true, true) => break, + (true, false) => { + let mut items = top_sort.collect::>(); + items.sort(); + return Err(SuperTypeCycleError { items }); + } + (false, _) => { + next_kinds.sort(); + sorted_kinds.extend(next_kinds); + } + } + } + subtype_map.sort_by(|a, b| { + let a_idx = sorted_kinds.iter().position(|n| n.eq(&a.0.kind)).unwrap(); + let b_idx = sorted_kinds.iter().position(|n| n.eq(&b.0.kind)).unwrap(); + a_idx.cmp(&b_idx) }); for node_type_json in node_types_json.values_mut() { @@ -591,7 +700,6 @@ pub fn generate_node_types_json( let mut anonymous_node_types = Vec::new(); - let empty = HashSet::new(); let regular_tokens = lexical_grammar .variables .iter() @@ -636,6 +744,7 @@ pub fn generate_node_types_json( kind: name.clone(), named: true, root: false, + extra: extra_names.contains(&name), fields: None, children: None, subtypes: None, @@ -653,6 +762,7 @@ pub fn generate_node_types_json( kind: name.clone(), named: false, root: false, + extra: extra_names.contains(&name), fields: None, children: None, subtypes: None, @@ -673,11 +783,15 @@ pub fn generate_node_types_json( a_is_leaf.cmp(&b_is_leaf) }) .then_with(|| a.kind.cmp(&b.kind)) + .then_with(|| a.named.cmp(&b.named)) + .then_with(|| a.root.cmp(&b.root)) + .then_with(|| a.extra.cmp(&b.extra)) }); result.dedup(); - result + Ok(result) } +#[cfg(feature = "load")] fn process_supertypes(info: &mut FieldInfoJSON, subtype_map: &[(NodeTypeJSON, Vec)]) { for (supertype, subtypes) in subtype_map { if info.types.contains(supertype) { @@ -714,17 +828,17 @@ fn extend_sorted<'a, T>(vec: &mut Vec, values: impl IntoIterator>(), @@ -1484,6 +1740,7 @@ mod tests { kind: "b".to_string(), named: true, root: false, + extra: false, subtypes: None, children: Some(FieldInfoJSON { multiple: true, @@ -1798,7 +2055,7 @@ mod tests { ); } - fn get_node_types(grammar: &InputGrammar) -> Vec { + fn get_node_types(grammar: &InputGrammar) -> SuperTypeCycleResult> { let (syntax_grammar, lexical_grammar, _, default_aliases) = prepare_grammar(grammar).unwrap(); let variable_info = diff --git a/crates/generate/src/parse_grammar.rs b/crates/generate/src/parse_grammar.rs new file mode 100644 index 00000000..de8f0a97 --- /dev/null +++ b/crates/generate/src/parse_grammar.rs @@ -0,0 +1,460 @@ +use std::collections::HashSet; + +use log::warn; +use regex::Regex; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use thiserror::Error; + +use crate::{ + grammars::{InputGrammar, PrecedenceEntry, ReservedWordContext, Variable, VariableType}, + rules::{Precedence, Rule}, +}; + +#[derive(Deserialize)] +#[serde(tag = "type")] +#[allow(non_camel_case_types)] +#[allow(clippy::upper_case_acronyms)] +enum RuleJSON { + ALIAS { + content: Box, + named: bool, + value: String, + }, + BLANK, + STRING { + value: String, + }, + PATTERN { + value: String, + flags: Option, + }, + SYMBOL { + name: String, + }, + CHOICE { + members: Vec, + }, + FIELD { + name: String, + content: Box, + }, + SEQ { + members: Vec, + }, + REPEAT { + content: Box, + }, + REPEAT1 { + content: Box, + }, + PREC_DYNAMIC { + value: i32, + content: Box, + }, + PREC_LEFT { + value: PrecedenceValueJSON, + content: Box, + }, + PREC_RIGHT { + value: PrecedenceValueJSON, + content: Box, + }, + PREC { + value: PrecedenceValueJSON, + content: Box, + }, + TOKEN { + content: Box, + }, + IMMEDIATE_TOKEN { + content: Box, + }, + RESERVED { + context_name: String, + content: Box, + }, +} + +#[derive(Deserialize)] +#[serde(untagged)] +enum PrecedenceValueJSON { + Integer(i32), + Name(String), +} + +#[derive(Deserialize)] +pub struct GrammarJSON { + pub name: String, + rules: Map, + #[serde(default)] + precedences: Vec>, + #[serde(default)] + conflicts: Vec>, + #[serde(default)] + externals: Vec, + #[serde(default)] + extras: Vec, + #[serde(default)] + inline: Vec, + #[serde(default)] + supertypes: Vec, + #[serde(default)] + word: Option, + #[serde(default)] + reserved: Map, +} + +pub type ParseGrammarResult = Result; + +#[derive(Debug, Error, Serialize)] +pub enum ParseGrammarError { + #[error("{0}")] + Serialization(String), + #[error("Rules in the `extras` array must not contain empty strings")] + InvalidExtra, + #[error("Invalid rule in precedences array. Only strings and symbols are allowed")] + Unexpected, + #[error("Reserved word sets must be arrays")] + InvalidReservedWordSet, + #[error("Grammar Error: Unexpected rule `{0}` in `token()` call")] + UnexpectedRule(String), +} + +impl From for ParseGrammarError { + fn from(value: serde_json::Error) -> Self { + Self::Serialization(value.to_string()) + } +} + +/// Check if a rule is referenced by another rule. +/// +/// This function is used to determine if a variable is used in a given rule, +/// and `is_other` indicates if the rule is an external, and if it is, +/// to not assume that a named symbol that is equal to itself means it's being referenced. +/// +/// For example, if we have an external rule **and** a normal rule both called `foo`, +/// `foo` should not be thought of as directly used unless it's used within another rule. +fn rule_is_referenced(rule: &Rule, target: &str, is_external: bool) -> bool { + match rule { + Rule::NamedSymbol(name) => name == target && !is_external, + Rule::Choice(rules) | Rule::Seq(rules) => { + rules.iter().any(|r| rule_is_referenced(r, target, false)) + } + Rule::Metadata { rule, .. } | Rule::Reserved { rule, .. } => { + rule_is_referenced(rule, target, is_external) + } + Rule::Repeat(inner) => rule_is_referenced(inner, target, false), + Rule::Blank | Rule::String(_) | Rule::Pattern(_, _) | Rule::Symbol(_) => false, + } +} + +fn variable_is_used( + grammar_rules: &[(String, Rule)], + extras: &[Rule], + externals: &[Rule], + target_name: &str, + in_progress: &mut HashSet, +) -> bool { + let root = &grammar_rules.first().unwrap().0; + if target_name == root { + return true; + } + + if extras + .iter() + .any(|rule| rule_is_referenced(rule, target_name, false)) + { + return true; + } + + if externals + .iter() + .any(|rule| rule_is_referenced(rule, target_name, true)) + { + return true; + } + + in_progress.insert(target_name.to_string()); + let result = grammar_rules + .iter() + .filter(|(key, _)| *key != target_name) + .any(|(name, rule)| { + if !rule_is_referenced(rule, target_name, false) || in_progress.contains(name) { + return false; + } + variable_is_used(grammar_rules, extras, externals, name, in_progress) + }); + in_progress.remove(target_name); + + result +} + +pub(crate) fn parse_grammar(input: &str) -> ParseGrammarResult { + let mut grammar_json = serde_json::from_str::(input)?; + + let mut extra_symbols = + grammar_json + .extras + .into_iter() + .try_fold(Vec::::new(), |mut acc, item| { + let rule = parse_rule(item, false)?; + if let Rule::String(ref value) = rule { + if value.is_empty() { + Err(ParseGrammarError::InvalidExtra)?; + } + } + acc.push(rule); + ParseGrammarResult::Ok(acc) + })?; + + let mut external_tokens = grammar_json + .externals + .into_iter() + .map(|e| parse_rule(e, false)) + .collect::>>()?; + + let mut precedence_orderings = Vec::with_capacity(grammar_json.precedences.len()); + for list in grammar_json.precedences { + let mut ordering = Vec::with_capacity(list.len()); + for entry in list { + ordering.push(match entry { + RuleJSON::STRING { value } => PrecedenceEntry::Name(value), + RuleJSON::SYMBOL { name } => PrecedenceEntry::Symbol(name), + _ => Err(ParseGrammarError::Unexpected)?, + }); + } + precedence_orderings.push(ordering); + } + + let mut variables = Vec::with_capacity(grammar_json.rules.len()); + + let rules = grammar_json + .rules + .into_iter() + .map(|(n, r)| Ok((n, parse_rule(serde_json::from_value(r)?, false)?))) + .collect::>>()?; + + let mut in_progress = HashSet::new(); + + for (name, rule) in &rules { + if grammar_json.word.as_ref().is_none_or(|w| w != name) + && !variable_is_used( + &rules, + &extra_symbols, + &external_tokens, + name, + &mut in_progress, + ) + { + grammar_json.conflicts.retain(|r| !r.contains(name)); + grammar_json.supertypes.retain(|r| r != name); + grammar_json.inline.retain(|r| r != name); + extra_symbols.retain(|r| !rule_is_referenced(r, name, true)); + external_tokens.retain(|r| !rule_is_referenced(r, name, true)); + precedence_orderings.retain(|r| { + !r.iter().any(|e| { + let PrecedenceEntry::Symbol(s) = e else { + return false; + }; + s == name + }) + }); + continue; + } + + if extra_symbols + .iter() + .any(|r| rule_is_referenced(r, name, false)) + { + let inner_rule = if let Rule::Metadata { rule, .. } = rule { + rule + } else { + rule + }; + let matches_empty = match inner_rule { + Rule::String(rule_str) => rule_str.is_empty(), + Rule::Pattern(ref value, _) => Regex::new(value) + .map(|reg| reg.is_match("")) + .unwrap_or(false), + _ => false, + }; + if matches_empty { + warn!( + concat!( + "Named extra rule `{}` matches the empty string. ", + "Inline this to avoid infinite loops while parsing." + ), + name + ); + } + } + variables.push(Variable { + name: name.clone(), + kind: VariableType::Named, + rule: rule.clone(), + }); + } + + let reserved_words = grammar_json + .reserved + .into_iter() + .map(|(name, rule_values)| { + let Value::Array(rule_values) = rule_values else { + Err(ParseGrammarError::InvalidReservedWordSet)? + }; + + let mut reserved_words = Vec::with_capacity(rule_values.len()); + for value in rule_values { + reserved_words.push(parse_rule(serde_json::from_value(value)?, false)?); + } + Ok(ReservedWordContext { + name, + reserved_words, + }) + }) + .collect::>>()?; + + Ok(InputGrammar { + name: grammar_json.name, + word_token: grammar_json.word, + expected_conflicts: grammar_json.conflicts, + supertype_symbols: grammar_json.supertypes, + variables_to_inline: grammar_json.inline, + precedence_orderings, + variables, + extra_symbols, + external_tokens, + reserved_words, + }) +} + +fn parse_rule(json: RuleJSON, is_token: bool) -> ParseGrammarResult { + match json { + RuleJSON::ALIAS { + content, + value, + named, + } => parse_rule(*content, is_token).map(|r| Rule::alias(r, value, named)), + RuleJSON::BLANK => Ok(Rule::Blank), + RuleJSON::STRING { value } => Ok(Rule::String(value)), + RuleJSON::PATTERN { value, flags } => Ok(Rule::Pattern( + value, + flags.map_or(String::new(), |f| { + f.matches(|c| { + if c == 'i' { + true + } else { + // silently ignore unicode flags + if c != 'u' && c != 'v' { + warn!("unsupported flag {c}"); + } + false + } + }) + .collect() + }), + )), + RuleJSON::SYMBOL { name } => { + if is_token { + Err(ParseGrammarError::UnexpectedRule(name))? + } else { + Ok(Rule::NamedSymbol(name)) + } + } + RuleJSON::CHOICE { members } => members + .into_iter() + .map(|m| parse_rule(m, is_token)) + .collect::>>() + .map(Rule::choice), + RuleJSON::FIELD { content, name } => { + parse_rule(*content, is_token).map(|r| Rule::field(name, r)) + } + RuleJSON::SEQ { members } => members + .into_iter() + .map(|m| parse_rule(m, is_token)) + .collect::>>() + .map(Rule::seq), + RuleJSON::REPEAT1 { content } => parse_rule(*content, is_token).map(Rule::repeat), + RuleJSON::REPEAT { content } => { + parse_rule(*content, is_token).map(|m| Rule::choice(vec![Rule::repeat(m), Rule::Blank])) + } + RuleJSON::PREC { value, content } => { + parse_rule(*content, is_token).map(|r| Rule::prec(value.into(), r)) + } + RuleJSON::PREC_LEFT { value, content } => { + parse_rule(*content, is_token).map(|r| Rule::prec_left(value.into(), r)) + } + RuleJSON::PREC_RIGHT { value, content } => { + parse_rule(*content, is_token).map(|r| Rule::prec_right(value.into(), r)) + } + RuleJSON::PREC_DYNAMIC { value, content } => { + parse_rule(*content, is_token).map(|r| Rule::prec_dynamic(value, r)) + } + RuleJSON::RESERVED { + content, + context_name, + } => parse_rule(*content, is_token).map(|r| Rule::Reserved { + rule: Box::new(r), + context_name, + }), + RuleJSON::TOKEN { content } => parse_rule(*content, true).map(Rule::token), + RuleJSON::IMMEDIATE_TOKEN { content } => { + parse_rule(*content, is_token).map(Rule::immediate_token) + } + } +} + +impl From for Precedence { + fn from(val: PrecedenceValueJSON) -> Self { + match val { + PrecedenceValueJSON::Integer(i) => Self::Integer(i), + PrecedenceValueJSON::Name(i) => Self::Name(i), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_grammar() { + let grammar = parse_grammar( + r#"{ + "name": "my_lang", + "rules": { + "file": { + "type": "REPEAT1", + "content": { + "type": "SYMBOL", + "name": "statement" + } + }, + "statement": { + "type": "STRING", + "value": "foo" + } + } + }"#, + ) + .unwrap(); + + assert_eq!(grammar.name, "my_lang"); + assert_eq!( + grammar.variables, + vec![ + Variable { + name: "file".to_string(), + kind: VariableType::Named, + rule: Rule::repeat(Rule::NamedSymbol("statement".to_string())) + }, + Variable { + name: "statement".to_string(), + kind: VariableType::Named, + rule: Rule::String("foo".to_string()) + }, + ] + ); + } +} diff --git a/crates/generate/src/parser.h.inc b/crates/generate/src/parser.h.inc new file mode 100644 index 00000000..858107de --- /dev/null +++ b/crates/generate/src/parser.h.inc @@ -0,0 +1,286 @@ +#ifndef TREE_SITTER_PARSER_H_ +#define TREE_SITTER_PARSER_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +#define ts_builtin_sym_error ((TSSymbol)-1) +#define ts_builtin_sym_end 0 +#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024 + +#ifndef TREE_SITTER_API_H_ +typedef uint16_t TSStateId; +typedef uint16_t TSSymbol; +typedef uint16_t TSFieldId; +typedef struct TSLanguage TSLanguage; +typedef struct TSLanguageMetadata { + uint8_t major_version; + uint8_t minor_version; + uint8_t patch_version; +} TSLanguageMetadata; +#endif + +typedef struct { + TSFieldId field_id; + uint8_t child_index; + bool inherited; +} TSFieldMapEntry; + +// Used to index the field and supertype maps. +typedef struct { + uint16_t index; + uint16_t length; +} TSMapSlice; + +typedef struct { + bool visible; + bool named; + bool supertype; +} TSSymbolMetadata; + +typedef struct TSLexer TSLexer; + +struct TSLexer { + int32_t lookahead; + TSSymbol result_symbol; + void (*advance)(TSLexer *, bool); + void (*mark_end)(TSLexer *); + uint32_t (*get_column)(TSLexer *); + bool (*is_at_included_range_start)(const TSLexer *); + bool (*eof)(const TSLexer *); + void (*log)(const TSLexer *, const char *, ...); +}; + +typedef enum { + TSParseActionTypeShift, + TSParseActionTypeReduce, + TSParseActionTypeAccept, + TSParseActionTypeRecover, +} TSParseActionType; + +typedef union { + struct { + uint8_t type; + TSStateId state; + bool extra; + bool repetition; + } shift; + struct { + uint8_t type; + uint8_t child_count; + TSSymbol symbol; + int16_t dynamic_precedence; + uint16_t production_id; + } reduce; + uint8_t type; +} TSParseAction; + +typedef struct { + uint16_t lex_state; + uint16_t external_lex_state; +} TSLexMode; + +typedef struct { + uint16_t lex_state; + uint16_t external_lex_state; + uint16_t reserved_word_set_id; +} TSLexerMode; + +typedef union { + TSParseAction action; + struct { + uint8_t count; + bool reusable; + } entry; +} TSParseActionEntry; + +typedef struct { + int32_t start; + int32_t end; +} TSCharacterRange; + +struct TSLanguage { + uint32_t abi_version; + uint32_t symbol_count; + uint32_t alias_count; + uint32_t token_count; + uint32_t external_token_count; + uint32_t state_count; + uint32_t large_state_count; + uint32_t production_id_count; + uint32_t field_count; + uint16_t max_alias_sequence_length; + const uint16_t *parse_table; + const uint16_t *small_parse_table; + const uint32_t *small_parse_table_map; + const TSParseActionEntry *parse_actions; + const char * const *symbol_names; + const char * const *field_names; + const TSMapSlice *field_map_slices; + const TSFieldMapEntry *field_map_entries; + const TSSymbolMetadata *symbol_metadata; + const TSSymbol *public_symbol_map; + const uint16_t *alias_map; + const TSSymbol *alias_sequences; + const TSLexerMode *lex_modes; + bool (*lex_fn)(TSLexer *, TSStateId); + bool (*keyword_lex_fn)(TSLexer *, TSStateId); + TSSymbol keyword_capture_token; + struct { + const bool *states; + const TSSymbol *symbol_map; + void *(*create)(void); + void (*destroy)(void *); + bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist); + unsigned (*serialize)(void *, char *); + void (*deserialize)(void *, const char *, unsigned); + } external_scanner; + const TSStateId *primary_state_ids; + const char *name; + const TSSymbol *reserved_words; + uint16_t max_reserved_word_set_size; + uint32_t supertype_count; + const TSSymbol *supertype_symbols; + const TSMapSlice *supertype_map_slices; + const TSSymbol *supertype_map_entries; + TSLanguageMetadata metadata; +}; + +static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, int32_t lookahead) { + uint32_t index = 0; + uint32_t size = len - index; + while (size > 1) { + uint32_t half_size = size / 2; + uint32_t mid_index = index + half_size; + const TSCharacterRange *range = &ranges[mid_index]; + if (lookahead >= range->start && lookahead <= range->end) { + return true; + } else if (lookahead > range->end) { + index = mid_index; + } + size -= half_size; + } + const TSCharacterRange *range = &ranges[index]; + return (lookahead >= range->start && lookahead <= range->end); +} + +/* + * Lexer Macros + */ + +#ifdef _MSC_VER +#define UNUSED __pragma(warning(suppress : 4101)) +#else +#define UNUSED __attribute__((unused)) +#endif + +#define START_LEXER() \ + bool result = false; \ + bool skip = false; \ + UNUSED \ + bool eof = false; \ + int32_t lookahead; \ + goto start; \ + next_state: \ + lexer->advance(lexer, skip); \ + start: \ + skip = false; \ + lookahead = lexer->lookahead; + +#define ADVANCE(state_value) \ + { \ + state = state_value; \ + goto next_state; \ + } + +#define ADVANCE_MAP(...) \ + { \ + static const uint16_t map[] = { __VA_ARGS__ }; \ + for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) { \ + if (map[i] == lookahead) { \ + state = map[i + 1]; \ + goto next_state; \ + } \ + } \ + } + +#define SKIP(state_value) \ + { \ + skip = true; \ + state = state_value; \ + goto next_state; \ + } + +#define ACCEPT_TOKEN(symbol_value) \ + result = true; \ + lexer->result_symbol = symbol_value; \ + lexer->mark_end(lexer); + +#define END_STATE() return result; + +/* + * Parse Table Macros + */ + +#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT) + +#define STATE(id) id + +#define ACTIONS(id) id + +#define SHIFT(state_value) \ + {{ \ + .shift = { \ + .type = TSParseActionTypeShift, \ + .state = (state_value) \ + } \ + }} + +#define SHIFT_REPEAT(state_value) \ + {{ \ + .shift = { \ + .type = TSParseActionTypeShift, \ + .state = (state_value), \ + .repetition = true \ + } \ + }} + +#define SHIFT_EXTRA() \ + {{ \ + .shift = { \ + .type = TSParseActionTypeShift, \ + .extra = true \ + } \ + }} + +#define REDUCE(symbol_name, children, precedence, prod_id) \ + {{ \ + .reduce = { \ + .type = TSParseActionTypeReduce, \ + .symbol = symbol_name, \ + .child_count = children, \ + .dynamic_precedence = precedence, \ + .production_id = prod_id \ + }, \ + }} + +#define RECOVER() \ + {{ \ + .type = TSParseActionTypeRecover \ + }} + +#define ACCEPT_INPUT() \ + {{ \ + .type = TSParseActionTypeAccept \ + }} + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_PARSER_H_ diff --git a/cli/generate/src/prepare_grammar/mod.rs b/crates/generate/src/prepare_grammar.rs similarity index 58% rename from cli/generate/src/prepare_grammar/mod.rs rename to crates/generate/src/prepare_grammar.rs index ea97ac1b..58e0869c 100644 --- a/cli/generate/src/prepare_grammar/mod.rs +++ b/crates/generate/src/prepare_grammar.rs @@ -8,11 +8,18 @@ mod process_inlines; use std::{ cmp::Ordering, - collections::{hash_map, HashMap, HashSet}, + collections::{hash_map, BTreeSet, HashMap, HashSet}, mem, }; -use anyhow::{anyhow, Result}; +pub use expand_tokens::ExpandTokensError; +pub use extract_tokens::ExtractTokensError; +pub use flatten_grammar::FlattenGrammarError; +use indexmap::IndexMap; +pub use intern_symbols::InternSymbolsError; +pub use process_inlines::ProcessInlinesError; +use serde::Serialize; +use thiserror::Error; pub use self::expand_tokens::expand_tokens; use self::{ @@ -27,6 +34,7 @@ use super::{ }, rules::{AliasMap, Precedence, Rule, Symbol}, }; +use crate::grammars::ReservedWordContext; pub struct IntermediateGrammar { variables: Vec, @@ -37,6 +45,7 @@ pub struct IntermediateGrammar { variables_to_inline: Vec, supertype_symbols: Vec, word_token: Option, + reserved_word_sets: Vec>, } pub type InternedGrammar = IntermediateGrammar; @@ -60,21 +69,96 @@ impl Default for IntermediateGrammar { variables_to_inline: Vec::default(), supertype_symbols: Vec::default(), word_token: Option::default(), + reserved_word_sets: Vec::default(), } } } +pub type PrepareGrammarResult = Result; + +#[derive(Debug, Error, Serialize)] +#[error(transparent)] +pub enum PrepareGrammarError { + ValidatePrecedences(#[from] ValidatePrecedenceError), + ValidateIndirectRecursion(#[from] IndirectRecursionError), + InternSymbols(#[from] InternSymbolsError), + ExtractTokens(#[from] ExtractTokensError), + FlattenGrammar(#[from] FlattenGrammarError), + ExpandTokens(#[from] ExpandTokensError), + ProcessInlines(#[from] ProcessInlinesError), +} + +pub type ValidatePrecedenceResult = Result; + +#[derive(Debug, Error, Serialize)] +#[error(transparent)] +pub enum ValidatePrecedenceError { + Undeclared(#[from] UndeclaredPrecedenceError), + Ordering(#[from] ConflictingPrecedenceOrderingError), +} + +#[derive(Debug, Error, Serialize)] +pub struct IndirectRecursionError(pub Vec); + +impl std::fmt::Display for IndirectRecursionError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Grammar contains an indirectly recursive rule: ")?; + for (i, symbol) in self.0.iter().enumerate() { + if i > 0 { + write!(f, " -> ")?; + } + write!(f, "{symbol}")?; + } + Ok(()) + } +} + +#[derive(Debug, Error, Serialize)] +pub struct UndeclaredPrecedenceError { + pub precedence: String, + pub rule: String, +} + +impl std::fmt::Display for UndeclaredPrecedenceError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Undeclared precedence '{}' in rule '{}'", + self.precedence, self.rule + )?; + Ok(()) + } +} + +#[derive(Debug, Error, Serialize)] +pub struct ConflictingPrecedenceOrderingError { + pub precedence_1: String, + pub precedence_2: String, +} + +impl std::fmt::Display for ConflictingPrecedenceOrderingError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Conflicting orderings for precedences {} and {}", + self.precedence_1, self.precedence_2 + )?; + Ok(()) + } +} + /// Transform an input grammar into separate components that are ready /// for parse table construction. pub fn prepare_grammar( input_grammar: &InputGrammar, -) -> Result<( +) -> PrepareGrammarResult<( SyntaxGrammar, LexicalGrammar, InlinedProductionMap, AliasMap, )> { validate_precedences(input_grammar)?; + validate_indirect_recursion(input_grammar)?; let interned_grammar = intern_symbols(input_grammar)?; let (syntax_grammar, lexical_grammar) = extract_tokens(interned_grammar)?; @@ -86,13 +170,94 @@ pub fn prepare_grammar( Ok((syntax_grammar, lexical_grammar, inlines, default_aliases)) } +/// Check for indirect recursion cycles in the grammar that can cause infinite loops while +/// parsing. An indirect recursion cycle occurs when a non-terminal can derive itself through +/// a chain of single-symbol productions (e.g., A -> B, B -> A). +fn validate_indirect_recursion(grammar: &InputGrammar) -> Result<(), IndirectRecursionError> { + let mut epsilon_transitions: IndexMap<&str, BTreeSet> = IndexMap::new(); + + for variable in &grammar.variables { + let productions = get_single_symbol_productions(&variable.rule); + // Filter out rules that *directly* reference themselves, as this doesn't + // cause a parsing loop. + let filtered: BTreeSet = productions + .into_iter() + .filter(|s| s != &variable.name) + .collect(); + epsilon_transitions.insert(variable.name.as_str(), filtered); + } + + for start_symbol in epsilon_transitions.keys() { + let mut visited = BTreeSet::new(); + let mut path = Vec::new(); + if let Some((start_idx, end_idx)) = + get_cycle(start_symbol, &epsilon_transitions, &mut visited, &mut path) + { + let cycle_symbols = path[start_idx..=end_idx] + .iter() + .map(|s| (*s).to_string()) + .collect(); + return Err(IndirectRecursionError(cycle_symbols)); + } + } + + Ok(()) +} + +fn get_single_symbol_productions(rule: &Rule) -> BTreeSet { + match rule { + Rule::NamedSymbol(name) => BTreeSet::from([name.clone()]), + Rule::Choice(choices) => choices + .iter() + .flat_map(get_single_symbol_productions) + .collect(), + Rule::Metadata { rule, .. } => get_single_symbol_productions(rule), + _ => BTreeSet::new(), + } +} + +/// Perform a depth-first search to detect cycles in single state transitions. +fn get_cycle<'a>( + current: &'a str, + transitions: &'a IndexMap<&'a str, BTreeSet>, + visited: &mut BTreeSet<&'a str>, + path: &mut Vec<&'a str>, +) -> Option<(usize, usize)> { + if let Some(first_idx) = path.iter().position(|s| *s == current) { + path.push(current); + return Some((first_idx, path.len() - 1)); + } + + if visited.contains(current) { + return None; + } + + path.push(current); + visited.insert(current); + + if let Some(next_symbols) = transitions.get(current) { + for next in next_symbols { + if let Some(cycle) = get_cycle(next, transitions, visited, path) { + return Some(cycle); + } + } + } + + path.pop(); + None +} + /// Check that all of the named precedences used in the grammar are declared /// within the `precedences` lists, and also that there are no conflicting /// precedence orderings declared in those lists. -fn validate_precedences(grammar: &InputGrammar) -> Result<()> { +fn validate_precedences(grammar: &InputGrammar) -> ValidatePrecedenceResult<()> { // Check that no rule contains a named precedence that is not present in // any of the `precedences` lists. - fn validate(rule_name: &str, rule: &Rule, names: &HashSet<&String>) -> Result<()> { + fn validate( + rule_name: &str, + rule: &Rule, + names: &HashSet<&String>, + ) -> ValidatePrecedenceResult<()> { match rule { Rule::Repeat(rule) => validate(rule_name, rule, names), Rule::Seq(elements) | Rule::Choice(elements) => elements @@ -101,7 +266,10 @@ fn validate_precedences(grammar: &InputGrammar) -> Result<()> { Rule::Metadata { rule, params } => { if let Precedence::Name(n) = ¶ms.precedence { if !names.contains(n) { - return Err(anyhow!("Undeclared precedence '{n}' in rule '{rule_name}'")); + Err(UndeclaredPrecedenceError { + precedence: n.clone(), + rule: rule_name.to_string(), + })?; } } validate(rule_name, rule, names)?; @@ -131,9 +299,10 @@ fn validate_precedences(grammar: &InputGrammar) -> Result<()> { } hash_map::Entry::Occupied(e) => { if e.get() != &ordering { - return Err(anyhow!( - "Conflicting orderings for precedences {entry1} and {entry2}", - )); + Err(ConflictingPrecedenceOrderingError { + precedence_1: entry1.to_string(), + precedence_2: entry2.to_string(), + })?; } } } diff --git a/cli/generate/src/prepare_grammar/expand_repeats.rs b/crates/generate/src/prepare_grammar/expand_repeats.rs similarity index 100% rename from cli/generate/src/prepare_grammar/expand_repeats.rs rename to crates/generate/src/prepare_grammar/expand_repeats.rs diff --git a/cli/generate/src/prepare_grammar/expand_tokens.rs b/crates/generate/src/prepare_grammar/expand_tokens.rs similarity index 89% rename from cli/generate/src/prepare_grammar/expand_tokens.rs rename to crates/generate/src/prepare_grammar/expand_tokens.rs index 84d05981..acfb9ba3 100644 --- a/cli/generate/src/prepare_grammar/expand_tokens.rs +++ b/crates/generate/src/prepare_grammar/expand_tokens.rs @@ -1,9 +1,9 @@ -use anyhow::{anyhow, Context, Result}; -use indoc::indoc; use regex_syntax::{ hir::{Class, Hir, HirKind}, ParserBuilder, }; +use serde::Serialize; +use thiserror::Error; use super::ExtractedLexicalGrammar; use crate::{ @@ -18,6 +18,40 @@ struct NfaBuilder { precedence_stack: Vec, } +pub type ExpandTokensResult = Result; + +#[derive(Debug, Error, Serialize)] +pub enum ExpandTokensError { + #[error( + "The rule `{0}` matches the empty string. +Tree-sitter does not support syntactic rules that match the empty string +unless they are used only as the grammar's start rule. +" + )] + EmptyString(String), + #[error(transparent)] + Processing(ExpandTokensProcessingError), + #[error(transparent)] + ExpandRule(ExpandRuleError), +} + +#[derive(Debug, Error, Serialize)] +pub struct ExpandTokensProcessingError { + rule: String, + error: ExpandRuleError, +} + +impl std::fmt::Display for ExpandTokensProcessingError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!( + f, + "Error processing rule {}: Grammar error: Unexpected rule {:?}", + self.rule, self.error + )?; + Ok(()) + } +} + fn get_implicit_precedence(rule: &Rule) -> i32 { match rule { Rule::String(_) => 2, @@ -41,7 +75,7 @@ const fn get_completion_precedence(rule: &Rule) -> i32 { 0 } -pub fn expand_tokens(mut grammar: ExtractedLexicalGrammar) -> Result { +pub fn expand_tokens(mut grammar: ExtractedLexicalGrammar) -> ExpandTokensResult { let mut builder = NfaBuilder { nfa: Nfa::new(), is_sep: true, @@ -55,17 +89,10 @@ pub fn expand_tokens(mut grammar: ExtractedLexicalGrammar) -> Result Result Result = Result; + +#[derive(Debug, Error, Serialize)] +pub enum ExpandRuleError { + #[error("Grammar error: Unexpected rule {0:?}")] + UnexpectedRule(Rule), + #[error("{0}")] + Parse(String), + #[error(transparent)] + ExpandRegex(ExpandRegexError), +} + +pub type ExpandRegexResult = Result; + +#[derive(Debug, Error, Serialize)] +pub enum ExpandRegexError { + #[error("{0}")] + Utf8(String), + #[error("Regex error: Assertions are not supported")] + Assertion, +} + impl NfaBuilder { - fn expand_rule(&mut self, rule: &Rule, mut next_state_id: u32) -> Result { + fn expand_rule(&mut self, rule: &Rule, mut next_state_id: u32) -> ExpandRuleResult { match rule { Rule::Pattern(s, f) => { // With unicode enabled, `\w`, `\s` and `\d` expand to character sets that are much @@ -124,18 +180,21 @@ impl NfaBuilder { .unicode(true) .utf8(false) .build(); - let hir = parser.parse(&s)?; + let hir = parser + .parse(&s) + .map_err(|e| ExpandRuleError::Parse(e.to_string()))?; self.expand_regex(&hir, next_state_id) + .map_err(ExpandRuleError::ExpandRegex) } Rule::String(s) => { for c in s.chars().rev() { - self.push_advance(CharacterSet::empty().add_char(c), next_state_id); + self.push_advance(CharacterSet::from_char(c), next_state_id); next_state_id = self.nfa.last_state_id(); } Ok(!s.is_empty()) } Rule::Choice(elements) => { - let mut alternative_state_ids = Vec::new(); + let mut alternative_state_ids = Vec::with_capacity(elements.len()); for element in elements { if self.expand_rule(element, next_state_id)? { alternative_state_ids.push(self.nfa.last_state_id()); @@ -189,15 +248,19 @@ impl NfaBuilder { result } Rule::Blank => Ok(false), - _ => Err(anyhow!("Grammar error: Unexpected rule {rule:?}")), + _ => Err(ExpandRuleError::UnexpectedRule(rule.clone()))?, } } - fn expand_regex(&mut self, hir: &Hir, mut next_state_id: u32) -> Result { + fn expand_regex(&mut self, hir: &Hir, mut next_state_id: u32) -> ExpandRegexResult { match hir.kind() { HirKind::Empty => Ok(false), HirKind::Literal(literal) => { - for character in std::str::from_utf8(&literal.0)?.chars().rev() { + for character in std::str::from_utf8(&literal.0) + .map_err(|e| ExpandRegexError::Utf8(e.to_string()))? + .chars() + .rev() + { let char_set = CharacterSet::from_char(character); self.push_advance(char_set, next_state_id); next_state_id = self.nfa.last_state_id(); @@ -234,7 +297,7 @@ impl NfaBuilder { Ok(true) } }, - HirKind::Look(_) => Err(anyhow!("Regex error: Assertions are not supported")), + HirKind::Look(_) => Err(ExpandRegexError::Assertion)?, HirKind::Repetition(repetition) => match (repetition.min, repetition.max) { (0, Some(1)) => self.expand_zero_or_one(&repetition.sub, next_state_id), (1, None) => self.expand_one_or_more(&repetition.sub, next_state_id), @@ -274,7 +337,7 @@ impl NfaBuilder { Ok(result) } HirKind::Alternation(alternations) => { - let mut alternative_state_ids = Vec::new(); + let mut alternative_state_ids = Vec::with_capacity(alternations.len()); for hir in alternations { if self.expand_regex(hir, next_state_id)? { alternative_state_ids.push(self.nfa.last_state_id()); @@ -293,7 +356,7 @@ impl NfaBuilder { } } - fn expand_one_or_more(&mut self, hir: &Hir, next_state_id: u32) -> Result { + fn expand_one_or_more(&mut self, hir: &Hir, next_state_id: u32) -> ExpandRegexResult { self.nfa.states.push(NfaState::Accept { variable_index: 0, precedence: 0, @@ -309,7 +372,7 @@ impl NfaBuilder { } } - fn expand_zero_or_one(&mut self, hir: &Hir, next_state_id: u32) -> Result { + fn expand_zero_or_one(&mut self, hir: &Hir, next_state_id: u32) -> ExpandRegexResult { if self.expand_regex(hir, next_state_id)? { self.push_split(next_state_id); Ok(true) @@ -318,7 +381,7 @@ impl NfaBuilder { } } - fn expand_zero_or_more(&mut self, hir: &Hir, next_state_id: u32) -> Result { + fn expand_zero_or_more(&mut self, hir: &Hir, next_state_id: u32) -> ExpandRegexResult { if self.expand_one_or_more(hir, next_state_id)? { self.push_split(next_state_id); Ok(true) @@ -327,7 +390,12 @@ impl NfaBuilder { } } - fn expand_count(&mut self, hir: &Hir, count: u32, mut next_state_id: u32) -> Result { + fn expand_count( + &mut self, + hir: &Hir, + count: u32, + mut next_state_id: u32, + ) -> ExpandRegexResult { let mut result = false; for _ in 0..count { if self.expand_regex(hir, next_state_id)? { diff --git a/cli/generate/src/prepare_grammar/extract_default_aliases.rs b/crates/generate/src/prepare_grammar/extract_default_aliases.rs similarity index 99% rename from cli/generate/src/prepare_grammar/extract_default_aliases.rs rename to crates/generate/src/prepare_grammar/extract_default_aliases.rs index 68ea1e48..cc977362 100644 --- a/cli/generate/src/prepare_grammar/extract_default_aliases.rs +++ b/crates/generate/src/prepare_grammar/extract_default_aliases.rs @@ -69,9 +69,7 @@ pub(super) fn extract_default_aliases( SymbolType::External => &mut external_status_list[symbol.index], SymbolType::NonTerminal => &mut non_terminal_status_list[symbol.index], SymbolType::Terminal => &mut terminal_status_list[symbol.index], - SymbolType::End | SymbolType::EndOfNonTerminalExtra => { - panic!("Unexpected end token") - } + SymbolType::End | SymbolType::EndOfNonTerminalExtra => panic!("Unexpected end token"), }; status.appears_unaliased = true; } diff --git a/cli/generate/src/prepare_grammar/extract_tokens.rs b/crates/generate/src/prepare_grammar/extract_tokens.rs similarity index 76% rename from cli/generate/src/prepare_grammar/extract_tokens.rs rename to crates/generate/src/prepare_grammar/extract_tokens.rs index 3fd85d6d..a7b4f227 100644 --- a/cli/generate/src/prepare_grammar/extract_tokens.rs +++ b/crates/generate/src/prepare_grammar/extract_tokens.rs @@ -1,16 +1,63 @@ -use std::{collections::HashMap, mem}; +use std::collections::HashMap; -use anyhow::{anyhow, Result}; +use serde::Serialize; +use thiserror::Error; use super::{ExtractedLexicalGrammar, ExtractedSyntaxGrammar, InternedGrammar}; use crate::{ - grammars::{ExternalToken, Variable, VariableType}, + grammars::{ExternalToken, ReservedWordContext, Variable, VariableType}, rules::{MetadataParams, Rule, Symbol, SymbolType}, }; +pub type ExtractTokensResult = Result; + +#[derive(Debug, Error, Serialize)] +pub enum ExtractTokensError { + #[error( + "The rule `{0}` contains an empty string. + +Tree-sitter does not support syntactic rules that contain an empty string +unless they are used only as the grammar's start rule. +" + )] + EmptyString(String), + #[error("Rule '{0}' cannot be used as both an external token and a non-terminal rule")] + ExternalTokenNonTerminal(String), + #[error("Non-symbol rules cannot be used as external tokens")] + NonSymbolExternalToken, + #[error(transparent)] + WordToken(NonTerminalWordTokenError), + #[error("Reserved word '{0}' must be a token")] + NonTokenReservedWord(String), +} + +#[derive(Debug, Error, Serialize)] +pub struct NonTerminalWordTokenError { + pub symbol_name: String, + pub conflicting_symbol_name: Option, +} + +impl std::fmt::Display for NonTerminalWordTokenError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Non-terminal symbol '{}' cannot be used as the word token", + self.symbol_name + )?; + if let Some(conflicting_name) = &self.conflicting_symbol_name { + writeln!( + f, + ", because its rule is duplicated in '{conflicting_name}'", + ) + } else { + writeln!(f) + } + } +} + pub(super) fn extract_tokens( mut grammar: InternedGrammar, -) -> Result<(ExtractedSyntaxGrammar, ExtractedLexicalGrammar)> { +) -> ExtractTokensResult<(ExtractedSyntaxGrammar, ExtractedLexicalGrammar)> { let mut extractor = TokenExtractor { current_variable_name: String::new(), current_variable_token_count: 0, @@ -38,7 +85,7 @@ pub(super) fn extract_tokens( // that pointed to that variable will need to be updated to point to the // variable in the lexical grammar. Symbols that pointed to later variables // will need to have their indices decremented. - let mut variables = Vec::new(); + let mut variables = Vec::with_capacity(grammar.variables.len()); let mut symbol_replacer = SymbolReplacer { replacements: HashMap::new(), }; @@ -105,15 +152,14 @@ pub(super) fn extract_tokens( } } - let mut external_tokens = Vec::new(); + let mut external_tokens = Vec::with_capacity(grammar.external_tokens.len()); for external_token in grammar.external_tokens { let rule = symbol_replacer.replace_symbols_in_rule(&external_token.rule); if let Rule::Symbol(symbol) = rule { if symbol.is_non_terminal() { - return Err(anyhow!( - "Rule '{}' cannot be used as both an external token and a non-terminal rule", - &variables[symbol.index].name, - )); + Err(ExtractTokensError::ExternalTokenNonTerminal( + variables[symbol.index].name.clone(), + ))?; } if symbol.is_external() { @@ -130,22 +176,59 @@ pub(super) fn extract_tokens( }); } } else { - return Err(anyhow!( - "Non-symbol rules cannot be used as external tokens" - )); + Err(ExtractTokensError::NonSymbolExternalToken)?; } } - let mut word_token = None; - if let Some(token) = grammar.word_token { + let word_token = if let Some(token) = grammar.word_token { let token = symbol_replacer.replace_symbol(token); if token.is_non_terminal() { - return Err(anyhow!( - "Non-terminal symbol '{}' cannot be used as the word token", - &variables[token.index].name - )); + let word_token_variable = &variables[token.index]; + let conflicting_symbol_name = variables + .iter() + .enumerate() + .find(|(i, v)| *i != token.index && v.rule == word_token_variable.rule) + .map(|(_, v)| v.name.clone()); + + Err(ExtractTokensError::WordToken(NonTerminalWordTokenError { + symbol_name: word_token_variable.name.clone(), + conflicting_symbol_name, + }))?; } - word_token = Some(token); + Some(token) + } else { + None + }; + + let mut reserved_word_contexts = Vec::with_capacity(grammar.reserved_word_sets.len()); + for reserved_word_context in grammar.reserved_word_sets { + let mut reserved_words = Vec::with_capacity(reserved_word_contexts.len()); + for reserved_rule in reserved_word_context.reserved_words { + if let Rule::Symbol(symbol) = reserved_rule { + reserved_words.push(symbol_replacer.replace_symbol(symbol)); + } else if let Some(index) = lexical_variables + .iter() + .position(|v| v.rule == reserved_rule) + { + reserved_words.push(Symbol::terminal(index)); + } else { + let rule = if let Rule::Metadata { rule, .. } = &reserved_rule { + rule.as_ref() + } else { + &reserved_rule + }; + let token_name = match rule { + Rule::String(s) => s.clone(), + Rule::Pattern(p, _) => p.clone(), + _ => "unknown".to_string(), + }; + Err(ExtractTokensError::NonTokenReservedWord(token_name))?; + } + } + reserved_word_contexts.push(ReservedWordContext { + name: reserved_word_context.name, + reserved_words, + }); } Ok(( @@ -158,6 +241,7 @@ pub(super) fn extract_tokens( external_tokens, word_token, precedence_orderings: grammar.precedence_orderings, + reserved_word_sets: reserved_word_contexts, }, ExtractedLexicalGrammar { variables: lexical_variables, @@ -183,18 +267,16 @@ impl TokenExtractor { &mut self, is_first: bool, variable: &mut Variable, - ) -> Result<()> { + ) -> ExtractTokensResult<()> { self.current_variable_name.clear(); self.current_variable_name.push_str(&variable.name); self.current_variable_token_count = 0; self.is_first_rule = is_first; - let mut rule = Rule::Blank; - mem::swap(&mut rule, &mut variable.rule); - variable.rule = self.extract_tokens_in_rule(&rule)?; + variable.rule = self.extract_tokens_in_rule(&variable.rule)?; Ok(()) } - fn extract_tokens_in_rule(&mut self, input: &Rule) -> Result { + fn extract_tokens_in_rule(&mut self, input: &Rule) -> ExtractTokensResult { match input { Rule::String(name) => Ok(self.extract_token(input, Some(name))?.into()), Rule::Pattern(..) => Ok(self.extract_token(input, None)?.into()), @@ -203,10 +285,11 @@ impl TokenExtractor { let mut params = params.clone(); params.is_token = false; - let mut string_value = None; - if let Rule::String(value) = rule.as_ref() { - string_value = Some(value); - } + let string_value = if let Rule::String(value) = rule.as_ref() { + Some(value) + } else { + None + }; let rule_to_extract = if params == MetadataParams::default() { rule.as_ref() @@ -229,19 +312,27 @@ impl TokenExtractor { elements .iter() .map(|e| self.extract_tokens_in_rule(e)) - .collect::>>()?, + .collect::>>()?, )), Rule::Choice(elements) => Ok(Rule::Choice( elements .iter() .map(|e| self.extract_tokens_in_rule(e)) - .collect::>>()?, + .collect::>>()?, )), + Rule::Reserved { rule, context_name } => Ok(Rule::Reserved { + rule: Box::new(self.extract_tokens_in_rule(rule)?), + context_name: context_name.clone(), + }), _ => Ok(input.clone()), } } - fn extract_token(&mut self, rule: &Rule, string_value: Option<&String>) -> Result { + fn extract_token( + &mut self, + rule: &Rule, + string_value: Option<&String>, + ) -> ExtractTokensResult { for (i, variable) in self.extracted_variables.iter_mut().enumerate() { if variable.rule == *rule { self.extracted_usage_counts[i] += 1; @@ -252,14 +343,9 @@ impl TokenExtractor { let index = self.extracted_variables.len(); let variable = if let Some(string_value) = string_value { if string_value.is_empty() && !self.is_first_rule { - return Err(anyhow!( - "The rule `{}` contains an empty string. - -Tree-sitter does not support syntactic rules that contain an empty string -unless they are used only as the grammar's start rule. -", - self.current_variable_name - )); + Err(ExtractTokensError::EmptyString( + self.current_variable_name.clone(), + ))?; } Variable { name: string_value.clone(), @@ -271,7 +357,7 @@ unless they are used only as the grammar's start rule. Variable { name: format!( "{}_token{}", - &self.current_variable_name, self.current_variable_token_count + self.current_variable_name, self.current_variable_token_count ), kind: VariableType::Auxiliary, rule: rule.clone(), @@ -305,6 +391,10 @@ impl SymbolReplacer { params: params.clone(), rule: Box::new(self.replace_symbols_in_rule(rule)), }, + Rule::Reserved { rule, context_name } => Rule::Reserved { + rule: Box::new(self.replace_symbols_in_rule(rule)), + context_name: context_name.clone(), + }, _ => rule.clone(), } } @@ -500,14 +590,13 @@ mod test { ]); grammar.external_tokens = vec![Variable::named("rule_1", Rule::non_terminal(1))]; - match extract_tokens(grammar) { - Err(e) => { - assert_eq!(e.to_string(), "Rule 'rule_1' cannot be used as both an external token and a non-terminal rule"); - } - _ => { - panic!("Expected an error but got no error"); - } - } + let result = extract_tokens(grammar); + assert!(result.is_err(), "Expected an error but got no error"); + let err = result.err().unwrap(); + assert_eq!( + err.to_string(), + "Rule 'rule_1' cannot be used as both an external token and a non-terminal rule" + ); } #[test] diff --git a/cli/generate/src/prepare_grammar/flatten_grammar.rs b/crates/generate/src/prepare_grammar/flatten_grammar.rs similarity index 59% rename from cli/generate/src/prepare_grammar/flatten_grammar.rs rename to crates/generate/src/prepare_grammar/flatten_grammar.rs index 86eb0c73..3bec17bf 100644 --- a/cli/generate/src/prepare_grammar/flatten_grammar.rs +++ b/crates/generate/src/prepare_grammar/flatten_grammar.rs @@ -1,48 +1,96 @@ -use anyhow::{anyhow, Result}; -use indoc::indoc; +use std::collections::HashMap; + +use serde::Serialize; +use thiserror::Error; use super::ExtractedSyntaxGrammar; use crate::{ - grammars::{Production, ProductionStep, SyntaxGrammar, SyntaxVariable, Variable}, - rules::{Alias, Associativity, Precedence, Rule, Symbol}, + grammars::{ + Production, ProductionStep, ReservedWordSetId, SyntaxGrammar, SyntaxVariable, Variable, + }, + rules::{Alias, Associativity, Precedence, Rule, Symbol, TokenSet}, }; +pub type FlattenGrammarResult = Result; + +#[derive(Debug, Error, Serialize)] +pub enum FlattenGrammarError { + #[error("No such reserved word set: {0}")] + NoReservedWordSet(String), + #[error( + "The rule `{0}` matches the empty string. + +Tree-sitter does not support syntactic rules that match the empty string +unless they are used only as the grammar's start rule. +" + )] + EmptyString(String), + #[error("Rule `{0}` cannot be inlined because it contains a reference to itself")] + RecursiveInline(String), +} + struct RuleFlattener { production: Production, + reserved_word_set_ids: HashMap, precedence_stack: Vec, associativity_stack: Vec, + reserved_word_stack: Vec, alias_stack: Vec, field_name_stack: Vec, } impl RuleFlattener { - const fn new() -> Self { + const fn new(reserved_word_set_ids: HashMap) -> Self { Self { production: Production { steps: Vec::new(), dynamic_precedence: 0, }, + reserved_word_set_ids, precedence_stack: Vec::new(), associativity_stack: Vec::new(), + reserved_word_stack: Vec::new(), alias_stack: Vec::new(), field_name_stack: Vec::new(), } } - fn flatten(mut self, rule: Rule) -> Production { - self.apply(rule, true); - self.production + fn flatten_variable(&mut self, variable: Variable) -> FlattenGrammarResult { + let choices = extract_choices(variable.rule); + let mut productions = Vec::with_capacity(choices.len()); + for rule in choices { + let production = self.flatten_rule(rule)?; + if !productions.contains(&production) { + productions.push(production); + } + } + Ok(SyntaxVariable { + name: variable.name, + kind: variable.kind, + productions, + }) } - fn apply(&mut self, rule: Rule, at_end: bool) -> bool { + fn flatten_rule(&mut self, rule: Rule) -> FlattenGrammarResult { + self.production = Production::default(); + self.alias_stack.clear(); + self.reserved_word_stack.clear(); + self.precedence_stack.clear(); + self.associativity_stack.clear(); + self.field_name_stack.clear(); + self.apply(rule, true)?; + Ok(self.production.clone()) + } + + fn apply(&mut self, rule: Rule, at_end: bool) -> FlattenGrammarResult { match rule { Rule::Seq(members) => { let mut result = false; let last_index = members.len() - 1; for (i, member) in members.into_iter().enumerate() { - result |= self.apply(member, i == last_index && at_end); + result |= self.apply(member, i == last_index && at_end)?; } - result + Ok(result) } Rule::Metadata { rule, params } => { let mut has_precedence = false; @@ -73,7 +121,7 @@ impl RuleFlattener { self.production.dynamic_precedence = params.dynamic_precedence; } - let did_push = self.apply(*rule, at_end); + let did_push = self.apply(*rule, at_end)?; if has_precedence { self.precedence_stack.pop(); @@ -102,7 +150,20 @@ impl RuleFlattener { self.field_name_stack.pop(); } - did_push + Ok(did_push) + } + Rule::Reserved { rule, context_name } => { + self.reserved_word_stack.push( + self.reserved_word_set_ids + .get(&context_name) + .copied() + .ok_or_else(|| { + FlattenGrammarError::NoReservedWordSet(context_name.clone()) + })?, + ); + let did_push = self.apply(*rule, at_end)?; + self.reserved_word_stack.pop(); + Ok(did_push) } Rule::Symbol(symbol) => { self.production.steps.push(ProductionStep { @@ -113,12 +174,17 @@ impl RuleFlattener { .cloned() .unwrap_or(Precedence::None), associativity: self.associativity_stack.last().copied(), + reserved_word_set_id: self + .reserved_word_stack + .last() + .copied() + .unwrap_or(ReservedWordSetId::default()), alias: self.alias_stack.last().cloned(), field_name: self.field_name_stack.last().cloned(), }); - true + Ok(true) } - _ => false, + _ => Ok(false), } } } @@ -129,7 +195,7 @@ fn extract_choices(rule: Rule) -> Vec { let mut result = vec![Rule::Blank]; for element in elements { let extraction = extract_choices(element); - let mut next_result = Vec::new(); + let mut next_result = Vec::with_capacity(result.len()); for entry in result { for extraction_entry in &extraction { next_result.push(Rule::Seq(vec![entry.clone(), extraction_entry.clone()])); @@ -140,7 +206,7 @@ fn extract_choices(rule: Rule) -> Vec { result } Rule::Choice(elements) => { - let mut result = Vec::new(); + let mut result = Vec::with_capacity(elements.len()); for element in elements { for rule in extract_choices(element) { result.push(rule); @@ -155,25 +221,17 @@ fn extract_choices(rule: Rule) -> Vec { params: params.clone(), }) .collect(), + Rule::Reserved { rule, context_name } => extract_choices(*rule) + .into_iter() + .map(|rule| Rule::Reserved { + rule: Box::new(rule), + context_name: context_name.clone(), + }) + .collect(), _ => vec![rule], } } -fn flatten_variable(variable: Variable) -> SyntaxVariable { - let mut productions = Vec::new(); - for rule in extract_choices(variable.rule) { - let production = RuleFlattener::new().flatten(rule); - if !productions.contains(&production) { - productions.push(production); - } - } - SyntaxVariable { - name: variable.name, - kind: variable.kind, - productions, - } -} - fn symbol_is_used(variables: &[SyntaxVariable], symbol: Symbol) -> bool { for variable in variables { for production in &variable.productions { @@ -187,37 +245,48 @@ fn symbol_is_used(variables: &[SyntaxVariable], symbol: Symbol) -> bool { false } -pub(super) fn flatten_grammar(grammar: ExtractedSyntaxGrammar) -> Result { - let mut variables = Vec::new(); - for variable in grammar.variables { - variables.push(flatten_variable(variable)); +pub(super) fn flatten_grammar( + grammar: ExtractedSyntaxGrammar, +) -> FlattenGrammarResult { + let mut reserved_word_set_ids_by_name = HashMap::new(); + for (ix, set) in grammar.reserved_word_sets.iter().enumerate() { + reserved_word_set_ids_by_name.insert(set.name.clone(), ReservedWordSetId(ix)); } + + let mut flattener = RuleFlattener::new(reserved_word_set_ids_by_name); + let variables = grammar + .variables + .into_iter() + .map(|variable| flattener.flatten_variable(variable)) + .collect::>>()?; + for (i, variable) in variables.iter().enumerate() { let symbol = Symbol::non_terminal(i); + let used = symbol_is_used(&variables, symbol); for production in &variable.productions { - if production.steps.is_empty() && symbol_is_used(&variables, symbol) { - return Err(anyhow!( - indoc! {" - The rule `{}` matches the empty string. - - Tree-sitter does not support syntactic rules that match the empty string - unless they are used only as the grammar's start rule. - "}, - variable.name - )); + if used && production.steps.is_empty() { + Err(FlattenGrammarError::EmptyString(variable.name.clone()))?; } if grammar.variables_to_inline.contains(&symbol) && production.steps.iter().any(|step| step.symbol == symbol) { - return Err(anyhow!( - "Rule `{}` cannot be inlined because it contains a reference to itself.", - variable.name, - )); + Err(FlattenGrammarError::RecursiveInline(variable.name.clone()))?; } } } + let mut reserved_word_sets = grammar + .reserved_word_sets + .into_iter() + .map(|set| set.reserved_words.into_iter().collect()) + .collect::>(); + + // If no default reserved word set is specified, there are no reserved words. + if reserved_word_sets.is_empty() { + reserved_word_sets.push(TokenSet::default()); + } + Ok(SyntaxGrammar { extra_symbols: grammar.extra_symbols, expected_conflicts: grammar.expected_conflicts, @@ -226,6 +295,7 @@ pub(super) fn flatten_grammar(grammar: ExtractedSyntaxGrammar) -> Result Result { +pub type InternSymbolsResult = Result; + +#[derive(Debug, Error, Serialize)] +pub enum InternSymbolsError { + #[error("A grammar's start rule must be visible.")] + HiddenStartRule, + #[error("Undefined symbol `{0}`")] + Undefined(String), + #[error("Undefined symbol `{0}` in grammar's supertypes array")] + UndefinedSupertype(String), + #[error("Undefined symbol `{0}` in grammar's conflicts array")] + UndefinedConflict(String), + #[error("Undefined symbol `{0}` as grammar's word token")] + UndefinedWordToken(String), +} + +pub(super) fn intern_symbols(grammar: &InputGrammar) -> InternSymbolsResult { let interner = Interner { grammar }; if variable_type_for_name(&grammar.variables[0].name) == VariableType::Hidden { - return Err(anyhow!("A grammar's start rule must be visible.")); + Err(InternSymbolsError::HiddenStartRule)?; } let mut variables = Vec::with_capacity(grammar.variables.len()); @@ -41,17 +59,31 @@ pub(super) fn intern_symbols(grammar: &InputGrammar) -> Result let mut supertype_symbols = Vec::with_capacity(grammar.supertype_symbols.len()); for supertype_symbol_name in &grammar.supertype_symbols { supertype_symbols.push(interner.intern_name(supertype_symbol_name).ok_or_else(|| { - anyhow!("Undefined symbol `{supertype_symbol_name}` in grammar's supertypes array") + InternSymbolsError::UndefinedSupertype(supertype_symbol_name.clone()) })?); } - let mut expected_conflicts = Vec::new(); + let mut reserved_words = Vec::with_capacity(grammar.reserved_words.len()); + for reserved_word_set in &grammar.reserved_words { + let mut interned_set = Vec::with_capacity(reserved_word_set.reserved_words.len()); + for rule in &reserved_word_set.reserved_words { + interned_set.push(interner.intern_rule(rule, None)?); + } + reserved_words.push(ReservedWordContext { + name: reserved_word_set.name.clone(), + reserved_words: interned_set, + }); + } + + let mut expected_conflicts = Vec::with_capacity(grammar.expected_conflicts.len()); for conflict in &grammar.expected_conflicts { let mut interned_conflict = Vec::with_capacity(conflict.len()); for name in conflict { - interned_conflict.push(interner.intern_name(name).ok_or_else(|| { - anyhow!("Undefined symbol `{name}` in grammar's conflicts array") - })?); + interned_conflict.push( + interner + .intern_name(name) + .ok_or_else(|| InternSymbolsError::UndefinedConflict(name.clone()))?, + ); } expected_conflicts.push(interned_conflict); } @@ -63,14 +95,15 @@ pub(super) fn intern_symbols(grammar: &InputGrammar) -> Result } } - let mut word_token = None; - if let Some(name) = grammar.word_token.as_ref() { - word_token = Some( + let word_token = if let Some(name) = grammar.word_token.as_ref() { + Some( interner .intern_name(name) - .ok_or_else(|| anyhow!("Undefined symbol `{name}` as grammar's word token"))?, - ); - } + .ok_or_else(|| InternSymbolsError::UndefinedWordToken(name.clone()))?, + ) + } else { + None + }; for (i, variable) in variables.iter_mut().enumerate() { if supertype_symbols.contains(&Symbol::non_terminal(i)) { @@ -87,6 +120,7 @@ pub(super) fn intern_symbols(grammar: &InputGrammar) -> Result supertype_symbols, word_token, precedence_orderings: grammar.precedence_orderings.clone(), + reserved_word_sets: reserved_words, }) } @@ -95,10 +129,10 @@ struct Interner<'a> { } impl Interner<'_> { - fn intern_rule(&self, rule: &Rule, name: Option<&str>) -> Result { + fn intern_rule(&self, rule: &Rule, name: Option<&str>) -> InternSymbolsResult { match rule { Rule::Choice(elements) => { - self.check_single(elements, name); + self.check_single(elements, name, "choice"); let mut result = Vec::with_capacity(elements.len()); for element in elements { result.push(self.intern_rule(element, name)?); @@ -106,7 +140,7 @@ impl Interner<'_> { Ok(Rule::Choice(result)) } Rule::Seq(elements) => { - self.check_single(elements, name); + self.check_single(elements, name, "seq"); let mut result = Vec::with_capacity(elements.len()); for element in elements { result.push(self.intern_rule(element, name)?); @@ -118,8 +152,12 @@ impl Interner<'_> { rule: Box::new(self.intern_rule(rule, name)?), params: params.clone(), }), + Rule::Reserved { rule, context_name } => Ok(Rule::Reserved { + rule: Box::new(self.intern_rule(rule, name)?), + context_name: context_name.clone(), + }), Rule::NamedSymbol(name) => self.intern_name(name).map_or_else( - || Err(anyhow!("Undefined symbol `{name}`")), + || Err(InternSymbolsError::Undefined(name.clone())), |symbol| Ok(Rule::Symbol(symbol)), ), _ => Ok(rule.clone()), @@ -146,10 +184,10 @@ impl Interner<'_> { // In the case of a seq or choice rule of 1 element in a hidden rule, weird // inconsistent behavior with queries can occur. So we should warn the user about it. - fn check_single(&self, elements: &[Rule], name: Option<&str>) { + fn check_single(&self, elements: &[Rule], name: Option<&str>, kind: &str) { if elements.len() == 1 && matches!(elements[0], Rule::String(_) | Rule::Pattern(_, _)) { - eprintln!( - "Warning: rule {} contains a `seq` or `choice` rule with a single element. This is unnecessary.", + warn!( + "rule {} contains a `{kind}` rule with a single element. This is unnecessary.", name.unwrap_or_default() ); } @@ -240,10 +278,9 @@ mod tests { fn test_grammar_with_undefined_symbols() { let result = intern_symbols(&build_grammar(vec![Variable::named("x", Rule::named("y"))])); - match result { - Err(e) => assert_eq!(e.to_string(), "Undefined symbol `y`"), - _ => panic!("Expected an error but got none"), - } + assert!(result.is_err(), "Expected an error but got none"); + let e = result.err().unwrap(); + assert_eq!(e.to_string(), "Undefined symbol `y`"); } fn build_grammar(variables: Vec) -> InputGrammar { diff --git a/cli/generate/src/prepare_grammar/process_inlines.rs b/crates/generate/src/prepare_grammar/process_inlines.rs similarity index 93% rename from cli/generate/src/prepare_grammar/process_inlines.rs rename to crates/generate/src/prepare_grammar/process_inlines.rs index f2acffb6..460d2359 100644 --- a/cli/generate/src/prepare_grammar/process_inlines.rs +++ b/crates/generate/src/prepare_grammar/process_inlines.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; -use anyhow::{anyhow, Result}; +use serde::Serialize; +use thiserror::Error; use crate::{ grammars::{InlinedProductionMap, LexicalGrammar, Production, ProductionStep, SyntaxGrammar}, @@ -69,12 +70,13 @@ impl InlinedProductionMapBuilder { let production_map = production_indices_by_step_id .into_iter() .map(|(step_id, production_indices)| { - let production = step_id.variable_index.map_or_else( - || &productions[step_id.production_index], - |variable_index| { - &grammar.variables[variable_index].productions[step_id.production_index] - }, - ) as *const Production; + let production = + core::ptr::from_ref::(step_id.variable_index.map_or_else( + || &productions[step_id.production_index], + |variable_index| { + &grammar.variables[variable_index].productions[step_id.production_index] + }, + )); ((production, step_id.step_index as u32), production_indices) }) .collect(); @@ -187,29 +189,38 @@ impl InlinedProductionMapBuilder { } } +pub type ProcessInlinesResult = Result; + +#[derive(Debug, Error, Serialize)] +pub enum ProcessInlinesError { + #[error("External token `{0}` cannot be inlined")] + ExternalToken(String), + #[error("Token `{0}` cannot be inlined")] + Token(String), + #[error("Rule `{0}` cannot be inlined because it is the first rule")] + FirstRule(String), +} + pub(super) fn process_inlines( grammar: &SyntaxGrammar, lexical_grammar: &LexicalGrammar, -) -> Result { +) -> ProcessInlinesResult { for symbol in &grammar.variables_to_inline { match symbol.kind { SymbolType::External => { - return Err(anyhow!( - "External token `{}` cannot be inlined", - grammar.external_tokens[symbol.index].name - )) + Err(ProcessInlinesError::ExternalToken( + grammar.external_tokens[symbol.index].name.clone(), + ))?; } SymbolType::Terminal => { - return Err(anyhow!( - "Token `{}` cannot be inlined", - lexical_grammar.variables[symbol.index].name, - )) + Err(ProcessInlinesError::Token( + lexical_grammar.variables[symbol.index].name.clone(), + ))?; } SymbolType::NonTerminal if symbol.index == 0 => { - return Err(anyhow!( - "Rule `{}` cannot be inlined because it is the first rule", - grammar.variables[symbol.index].name, - )) + Err(ProcessInlinesError::FirstRule( + grammar.variables[symbol.index].name.clone(), + ))?; } _ => {} } @@ -538,10 +549,9 @@ mod tests { ..Default::default() }; - if let Err(error) = process_inlines(&grammar, &lexical_grammar) { - assert_eq!(error.to_string(), "Token `something` cannot be inlined"); - } else { - panic!("expected an error, but got none"); - } + let result = process_inlines(&grammar, &lexical_grammar); + assert!(result.is_err(), "expected an error, but got none"); + let err = result.err().unwrap(); + assert_eq!(err.to_string(), "Token `something` cannot be inlined",); } } diff --git a/crates/generate/src/quickjs.rs b/crates/generate/src/quickjs.rs new file mode 100644 index 00000000..d8c71cfe --- /dev/null +++ b/crates/generate/src/quickjs.rs @@ -0,0 +1,477 @@ +use std::{ + collections::HashMap, + path::{Path, PathBuf}, + sync::{LazyLock, Mutex}, +}; + +use log::{error, info, warn}; +use rquickjs::{ + loader::{FileResolver, ScriptLoader}, + Context, Ctx, Function, Module, Object, Runtime, Type, Value, +}; + +use super::{IoError, JSError, JSResult}; + +const DSL: &[u8] = include_bytes!("dsl.js"); + +trait JSResultExt { + fn or_js_error(self, ctx: &Ctx) -> JSResult; +} + +impl JSResultExt for Result { + fn or_js_error(self, ctx: &Ctx) -> JSResult { + match self { + Ok(v) => Ok(v), + Err(rquickjs::Error::Exception) => Err(format_js_exception(ctx.catch())), + Err(e) => Err(JSError::QuickJS(e.to_string())), + } + } +} + +fn format_js_exception(v: Value) -> JSError { + let Some(exception) = v.into_exception() else { + return JSError::QuickJS("Expected a JS exception".to_string()); + }; + + let error_obj = exception.as_object(); + let mut parts = Vec::new(); + + for (key, label) in [("message", "Message"), ("stack", "Stack"), ("name", "Type")] { + if let Ok(value) = error_obj.get::<_, String>(key) { + parts.push(format!("{label}: {value}")); + } + } + + if parts.is_empty() { + JSError::QuickJS(exception.to_string()) + } else { + JSError::QuickJS(parts.join("\n")) + } +} + +static FILE_CACHE: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + +#[rquickjs::function] +fn load_file(path: String) -> rquickjs::Result { + { + let cache = FILE_CACHE.lock().unwrap(); + if let Some(cached) = cache.get(&path) { + return Ok(cached.clone()); + } + } + + let content = std::fs::read_to_string(&path).map_err(|e| { + rquickjs::Error::new_from_js_message("IOError", "FileReadError", e.to_string()) + })?; + + { + let mut cache = FILE_CACHE.lock().unwrap(); + cache.insert(path, content.clone()); + } + + Ok(content) +} + +#[rquickjs::class] +#[derive(rquickjs::class::Trace, rquickjs::JsLifetime, Default)] +pub struct Console {} + +impl Console { + fn format_args(args: &[Value<'_>]) -> String { + args.iter() + .map(|v| match v.type_of() { + Type::Bool => v.as_bool().unwrap().to_string(), + Type::Int => v.as_int().unwrap().to_string(), + Type::Float => v.as_float().unwrap().to_string(), + Type::String => v + .as_string() + .unwrap() + .to_string() + .unwrap_or_else(|_| String::new()), + Type::Null => "null".to_string(), + Type::Undefined => "undefined".to_string(), + Type::Uninitialized => "uninitialized".to_string(), + Type::Module => "module".to_string(), + Type::BigInt => v.get::().unwrap_or_else(|_| "BigInt".to_string()), + Type::Unknown => "unknown".to_string(), + Type::Array => { + let js_vals = v + .as_array() + .unwrap() + .iter::>() + .filter_map(|x| x.ok()) + .map(|x| { + if x.is_string() { + format!("'{}'", Self::format_args(&[x])) + } else { + Self::format_args(&[x]) + } + }) + .collect::>() + .join(", "); + + format!("[ {js_vals} ]") + } + Type::Symbol + | Type::Object + | Type::Proxy + | Type::Function + | Type::Constructor + | Type::Promise + | Type::Exception => "[object Object]".to_string(), + }) + .collect::>() + .join(" ") + } +} + +#[rquickjs::methods] +impl Console { + #[qjs(constructor)] + pub const fn new() -> Self { + Console {} + } + + #[allow(clippy::needless_pass_by_value)] + pub fn log(&self, args: rquickjs::function::Rest>) -> rquickjs::Result<()> { + info!("{}", Self::format_args(&args)); + Ok(()) + } + + #[allow(clippy::needless_pass_by_value)] + pub fn warn(&self, args: rquickjs::function::Rest>) -> rquickjs::Result<()> { + warn!("{}", Self::format_args(&args)); + Ok(()) + } + + #[allow(clippy::needless_pass_by_value)] + pub fn error(&self, args: rquickjs::function::Rest>) -> rquickjs::Result<()> { + error!("Error: {}", Self::format_args(&args)); + Ok(()) + } +} + +fn resolve_module_path(base_path: &Path, module_path: &str) -> rquickjs::Result { + let candidates = if module_path.starts_with("./") || module_path.starts_with("../") { + let target = base_path.join(module_path); + vec![ + target.with_extension("js"), + target.with_extension("json"), + target.clone(), + ] + } else { + let local_target = base_path.join(module_path); + let node_modules_target = Path::new("node_modules").join(module_path); + + vec![ + local_target.with_extension("js"), + local_target.with_extension("json"), + local_target.clone(), + node_modules_target.with_extension("js"), + node_modules_target.with_extension("json"), + node_modules_target, + ] + }; + + for candidate in candidates { + if let Ok(resolved) = try_resolve_path(&candidate) { + return Ok(resolved); + } + } + + Err(rquickjs::Error::new_from_js_message( + "Error", + "ModuleNotFound", + format!("Module not found: {module_path}"), + )) +} + +fn try_resolve_path(path: &Path) -> rquickjs::Result { + let metadata = std::fs::metadata(path).map_err(|_| { + rquickjs::Error::new_from_js_message( + "Error", + "FileNotFound", + format!("Path not found: {}", path.display()), + ) + })?; + + if metadata.is_file() { + return Ok(path.to_path_buf()); + } + + if metadata.is_dir() { + let index_path = path.join("index.js"); + if index_path.exists() { + return Ok(index_path); + } + } + + Err(rquickjs::Error::new_from_js_message( + "Error", + "ResolutionFailed", + format!("Cannot resolve: {}", path.display()), + )) +} + +#[allow(clippy::needless_pass_by_value)] +fn require_from_module<'js>( + ctx: Ctx<'js>, + module_path: String, + from_module: &str, +) -> rquickjs::Result> { + let current_module = PathBuf::from(from_module); + let current_dir = if current_module.is_file() { + current_module.parent().unwrap_or(Path::new(".")) + } else { + current_module.as_path() + }; + + let resolved_path = resolve_module_path(current_dir, &module_path)?; + + let contents = load_file(resolved_path.to_string_lossy().to_string())?; + + load_module_from_content(&ctx, &resolved_path, &contents) +} + +fn load_module_from_content<'js>( + ctx: &Ctx<'js>, + path: &Path, + contents: &str, +) -> rquickjs::Result> { + if path.extension().is_some_and(|ext| ext == "json") { + return ctx.eval::, _>(format!("JSON.parse({contents:?})")); + } + + let exports = Object::new(ctx.clone())?; + let module_obj = Object::new(ctx.clone())?; + module_obj.set("exports", exports.clone())?; + + let filename = path.to_string_lossy().to_string(); + let dirname = path + .parent() + .map_or_else(|| ".".to_string(), |p| p.to_string_lossy().to_string()); + + // Require function specific to *this* module + let module_path = filename.clone(); + let require = Function::new( + ctx.clone(), + move |ctx_inner: Ctx<'js>, target_path: String| -> rquickjs::Result> { + require_from_module(ctx_inner, target_path, &module_path) + }, + )?; + + let wrapper = + format!("(function(exports, require, module, __filename, __dirname) {{ {contents} }})"); + + let module_func = ctx.eval::, _>(wrapper)?; + module_func.call::<_, Value<'js>>((exports, require, module_obj.clone(), filename, dirname))?; + + module_obj.get("exports") +} + +pub fn execute_native_runtime(grammar_path: &Path) -> JSResult { + let runtime = Runtime::new()?; + + runtime.set_memory_limit(64 * 1024 * 1024); // 64MB + runtime.set_max_stack_size(256 * 1024); // 256KB + + let context = Context::full(&runtime)?; + + let resolver = FileResolver::default() + .with_path("./node_modules") + .with_path("./") + .with_pattern("{}.mjs"); + let loader = ScriptLoader::default().with_extension("mjs"); + runtime.set_loader(resolver, loader); + + let cwd = std::env::current_dir().map_err(|e| JSError::IO(IoError::new(&e, None)))?; + let relative_path = pathdiff::diff_paths(grammar_path, &cwd) + .map(|p| p.to_string_lossy().to_string()) + .ok_or(JSError::RelativePath)?; + + context.with(|ctx| -> JSResult { + let globals = ctx.globals(); + + globals.set("native", true).or_js_error(&ctx)?; + globals + .set("__ts_grammar_path", relative_path) + .or_js_error(&ctx)?; + + let console = rquickjs::Class::instance(ctx.clone(), Console::new()).or_js_error(&ctx)?; + globals.set("console", console).or_js_error(&ctx)?; + + let process = Object::new(ctx.clone()).or_js_error(&ctx)?; + let env = Object::new(ctx.clone()).or_js_error(&ctx)?; + for (key, value) in std::env::vars() { + env.set(key, value).or_js_error(&ctx)?; + } + process.set("env", env).or_js_error(&ctx)?; + globals.set("process", process).or_js_error(&ctx)?; + + let module = Object::new(ctx.clone()).or_js_error(&ctx)?; + module + .set("exports", Object::new(ctx.clone()).or_js_error(&ctx)?) + .or_js_error(&ctx)?; + globals.set("module", module).or_js_error(&ctx)?; + + let grammar_path_string = grammar_path.to_string_lossy().to_string(); + let main_require = Function::new( + ctx.clone(), + move |ctx_inner, target_path: String| -> rquickjs::Result { + require_from_module(ctx_inner, target_path, &grammar_path_string) + }, + )?; + globals.set("require", main_require).or_js_error(&ctx)?; + + let promise = Module::evaluate(ctx.clone(), "dsl", DSL).or_js_error(&ctx)?; + promise.finish::<()>().or_js_error(&ctx)?; + + let grammar_json = ctx + .eval::("globalThis.output") + .map(|s| s.to_string()) + .or_js_error(&ctx)? + .or_js_error(&ctx)?; + + let parsed = serde_json::from_str::(&grammar_json)?; + Ok(serde_json::to_string_pretty(&parsed)?) + }) +} + +#[cfg(test)] +mod tests { + use std::{ + fs, + sync::{Arc, Mutex, OnceLock}, + }; + use tempfile::TempDir; + + use super::*; + + static TEST_MUTEX: OnceLock>> = OnceLock::new(); + + fn with_test_lock(test: F) -> R + where + F: FnOnce() -> R, + { + let _guard = TEST_MUTEX.get_or_init(|| Arc::new(Mutex::new(()))).lock(); + let result = test(); + cleanup_runtime_state(); + result + } + + fn cleanup_runtime_state() { + FILE_CACHE.lock().unwrap().clear(); + } + + #[test] + fn test_basic_grammar_execution() { + with_test_lock(|| { + let temp_dir = TempDir::new().unwrap(); + std::env::set_current_dir(temp_dir.path()).unwrap(); + + let grammar_path = temp_dir.path().join("grammar.js"); + fs::write( + &grammar_path, + r" + module.exports = grammar({ + name: 'test', + rules: { source_file: $ => 'hello' } + }); + ", + ) + .unwrap(); + + let json = execute_native_runtime(&grammar_path).expect("Failed to execute grammar"); + assert!(json.contains("\"name\": \"test\"")); + assert!(json.contains("\"hello\"")); + }); + } + + #[test] + fn test_module_imports() { + with_test_lock(|| { + let temp_dir = TempDir::new().unwrap(); + std::env::set_current_dir(temp_dir.path()).unwrap(); + + fs::write( + temp_dir.path().join("common.js"), + r" + module.exports = { identifier: $ => /[a-zA-Z_][a-zA-Z0-9_]*/ }; + ", + ) + .unwrap(); + + fs::write( + temp_dir.path().join("grammar.js"), + r" + const common = require('./common'); + module.exports = grammar({ + name: 'test_import', + rules: { source_file: common.identifier } + }); + ", + ) + .unwrap(); + + let json = execute_native_runtime(&temp_dir.path().join("grammar.js")) + .expect("Failed to execute grammar with imports"); + assert!(json.contains("\"name\": \"test_import\"")); + }); + } + + #[test] + fn test_json_module_loading() { + with_test_lock(|| { + let temp_dir = TempDir::new().unwrap(); + std::env::set_current_dir(temp_dir.path()).unwrap(); + + fs::write( + temp_dir.path().join("package.json"), + r#"{"version": "1.0.0"}"#, + ) + .unwrap(); + fs::write( + temp_dir.path().join("grammar.js"), + r" + const pkg = require('./package.json'); + module.exports = grammar({ + name: 'json_test', + rules: { + source_file: $ => 'version_' + pkg.version.replace(/\./g, '_') + } + }); + ", + ) + .unwrap(); + + let json = execute_native_runtime(&temp_dir.path().join("grammar.js")) + .expect("Failed to execute grammar with JSON import"); + assert!(json.contains("version_1_0_0")); + }); + } + + #[test] + fn test_resource_limits() { + with_test_lock(|| { + let temp_dir = TempDir::new().unwrap(); + std::env::set_current_dir(temp_dir.path()).unwrap(); + + fs::write( + temp_dir.path().join("grammar.js"), + r" + const huge = new Array(10000000).fill('x'.repeat(1000)); + module.exports = grammar({ + name: 'resource_test', + rules: { source_file: $ => 'test' } + }); + ", + ) + .unwrap(); + + let result = execute_native_runtime(&temp_dir.path().join("grammar.js")); + assert!(result.is_err()); + assert!(matches!(result.unwrap_err(), JSError::QuickJS(_))); + }); + } +} diff --git a/cli/generate/src/render.rs b/crates/generate/src/render.rs similarity index 82% rename from cli/generate/src/render.rs rename to crates/generate/src/render.rs index 62993d55..bcfc832e 100644 --- a/cli/generate/src/render.rs +++ b/crates/generate/src/render.rs @@ -1,15 +1,19 @@ use std::{ cmp, - collections::{HashMap, HashSet}, + collections::{BTreeMap, BTreeSet, HashMap, HashSet}, fmt::Write, mem::swap, }; +use crate::LANGUAGE_VERSION; +use indoc::indoc; + use super::{ build_tables::Tables, grammars::{ExternalToken, LexicalGrammar, SyntaxGrammar, VariableType}, nfa::CharacterSet, - rules::{Alias, AliasMap, Symbol, SymbolType}, + node_types::ChildType, + rules::{Alias, AliasMap, Symbol, SymbolType, TokenSet}, tables::{ AdvanceAction, FieldLocation, GotoAction, LexState, LexTable, ParseAction, ParseTable, ParseTableEntry, @@ -17,12 +21,11 @@ use super::{ }; const SMALL_STATE_THRESHOLD: usize = 64; -const ABI_VERSION_MIN: usize = 14; -const ABI_VERSION_MAX: usize = tree_sitter::LANGUAGE_VERSION; -const ABI_VERSION_WITH_METADATA: usize = 15; -const BUILD_VERSION: &str = env!("CARGO_PKG_VERSION"); -const BUILD_SHA: Option<&'static str> = option_env!("BUILD_SHA"); +pub const ABI_VERSION_MIN: usize = 14; +pub const ABI_VERSION_MAX: usize = LANGUAGE_VERSION; +const ABI_VERSION_WITH_RESERVED_WORDS: usize = 15; +#[clippy::format_args] macro_rules! add { ($this: tt, $($arg: tt)*) => {{ $this.buffer.write_fmt(format_args!($($arg)*)).unwrap(); @@ -31,12 +34,15 @@ macro_rules! add { macro_rules! add_whitespace { ($this:tt) => {{ + // 4 bytes per char, 2 spaces per indent level + $this.buffer.reserve(4 * 2 * $this.indent_level); for _ in 0..$this.indent_level { write!(&mut $this.buffer, " ").unwrap(); } }}; } +#[clippy::format_args] macro_rules! add_line { ($this: tt, $($arg: tt)*) => { add_whitespace!($this); @@ -58,6 +64,7 @@ macro_rules! dedent { }; } +#[derive(Default)] struct Generator { buffer: String, indent_level: usize, @@ -68,7 +75,6 @@ struct Generator { large_character_sets: Vec<(Option, CharacterSet)>, large_character_set_info: Vec, large_state_count: usize, - keyword_capture_token: Option, syntax_grammar: SyntaxGrammar, lexical_grammar: LexicalGrammar, default_aliases: AliasMap, @@ -77,10 +83,13 @@ struct Generator { alias_ids: HashMap, unique_aliases: Vec, symbol_map: HashMap, + reserved_word_sets: Vec, + reserved_word_set_ids_by_parse_state: Vec, field_names: Vec, - - #[allow(unused)] + supertype_symbol_map: BTreeMap>, + supertype_map: BTreeMap>, abi_version: usize, + metadata: Option, } struct LargeCharacterSetInfo { @@ -88,6 +97,12 @@ struct LargeCharacterSetInfo { is_used: bool, } +struct Metadata { + major_version: u8, + minor_version: u8, + patch_version: u8, +} + impl Generator { fn generate(mut self) -> String { self.init(); @@ -113,13 +128,17 @@ impl Generator { self.add_non_terminal_alias_map(); self.add_primary_state_id_list(); + if self.abi_version >= ABI_VERSION_WITH_RESERVED_WORDS && !self.supertype_map.is_empty() { + self.add_supertype_map(); + } + let buffer_offset_before_lex_functions = self.buffer.len(); let mut main_lex_table = LexTable::default(); swap(&mut main_lex_table, &mut self.main_lex_table); self.add_lex_function("ts_lex", main_lex_table); - if self.keyword_capture_token.is_some() { + if self.syntax_grammar.word_token.is_some() { let mut keyword_lex_table = LexTable::default(); swap(&mut keyword_lex_table, &mut self.keyword_lex_table); self.add_lex_function("ts_lex_keywords", keyword_lex_table); @@ -135,7 +154,13 @@ impl Generator { } self.buffer.push_str(&lex_functions); - self.add_lex_modes_list(); + self.add_lex_modes(); + + if self.abi_version >= ABI_VERSION_WITH_RESERVED_WORDS && self.reserved_word_sets.len() > 1 + { + self.add_reserved_word_sets(); + } + self.add_parse_table(); if !self.syntax_grammar.external_tokens.is_empty() { @@ -216,33 +241,24 @@ impl Generator { for alias in &production_info.alias_sequence { // Generate a mapping from aliases to C identifiers. if let Some(alias) = &alias { - let existing_symbol = self.parse_table.symbols.iter().copied().find(|symbol| { - self.default_aliases.get(symbol).map_or_else( - || { - let (name, kind) = self.metadata_for_symbol(*symbol); - name == alias.value && kind == alias.kind() - }, - |default_alias| default_alias == alias, - ) - }); - // Some aliases match an existing symbol in the grammar. - let alias_id = if let Some(existing_symbol) = existing_symbol { - self.symbol_ids[&self.symbol_map[&existing_symbol]].clone() - } - // Other aliases don't match any existing symbol, and need their own - // identifiers. - else { - if let Err(i) = self.unique_aliases.binary_search(alias) { - self.unique_aliases.insert(i, alias.clone()); + let alias_id = + if let Some(existing_symbol) = self.symbols_for_alias(alias).first() { + self.symbol_ids[&self.symbol_map[existing_symbol]].clone() } + // Other aliases don't match any existing symbol, and need their own + // identifiers. + else { + if let Err(i) = self.unique_aliases.binary_search(alias) { + self.unique_aliases.insert(i, alias.clone()); + } - if alias.is_named { - format!("alias_sym_{}", self.sanitize_identifier(&alias.value)) - } else { - format!("anon_alias_sym_{}", self.sanitize_identifier(&alias.value)) - } - }; + if alias.is_named { + format!("alias_sym_{}", self.sanitize_identifier(&alias.value)) + } else { + format!("anon_alias_sym_{}", self.sanitize_identifier(&alias.value)) + } + }; self.alias_ids.entry(alias.clone()).or_insert(alias_id); } @@ -266,6 +282,34 @@ impl Generator { }); } + // Assign an id to each unique reserved word set + self.reserved_word_sets.push(TokenSet::new()); + for state in &self.parse_table.states { + let id = if let Some(ix) = self + .reserved_word_sets + .iter() + .position(|set| *set == state.reserved_words) + { + ix + } else { + self.reserved_word_sets.push(state.reserved_words.clone()); + self.reserved_word_sets.len() - 1 + }; + self.reserved_word_set_ids_by_parse_state.push(id); + } + + if self.abi_version >= ABI_VERSION_WITH_RESERVED_WORDS { + for (supertype, subtypes) in &self.supertype_symbol_map { + if let Some(supertype) = self.symbol_ids.get(supertype) { + self.supertype_map + .entry(supertype.clone()) + .or_insert_with(|| subtypes.clone()); + } + } + + self.supertype_symbol_map.clear(); + } + // Determine which states should use the "small state" representation, and which should // use the normal array representation. let threshold = cmp::min(SMALL_STATE_THRESHOLD, self.parse_table.symbols.len() / 2); @@ -281,14 +325,7 @@ impl Generator { } fn add_header(&mut self) { - let version = BUILD_SHA.map_or_else( - || BUILD_VERSION.to_string(), - |build_sha| format!("{BUILD_VERSION} ({build_sha})"), - ); - add_line!( - self, - "/* Automatically generated by tree-sitter v{version} */", - ); + add_line!(self, "/* Automatically @generated by tree-sitter */",); add_line!(self, ""); } @@ -353,7 +390,7 @@ impl Generator { self.parse_table.symbols.len() ); add_line!(self, "#define ALIAS_COUNT {}", self.unique_aliases.len()); - add_line!(self, "#define TOKEN_COUNT {}", token_count); + add_line!(self, "#define TOKEN_COUNT {token_count}"); add_line!( self, "#define EXTERNAL_TOKEN_COUNT {}", @@ -365,11 +402,22 @@ impl Generator { "#define MAX_ALIAS_SEQUENCE_LENGTH {}", self.parse_table.max_aliased_production_length ); + add_line!( + self, + "#define MAX_RESERVED_WORD_SET_SIZE {}", + self.reserved_word_sets + .iter() + .map(TokenSet::len) + .max() + .unwrap() + ); + add_line!( self, "#define PRODUCTION_ID_COUNT {}", self.parse_table.production_infos.len() ); + add_line!(self, "#define SUPERTYPE_COUNT {}", self.supertype_map.len()); add_line!(self, ""); } @@ -631,31 +679,32 @@ impl Generator { &mut next_flat_field_map_index, ); - let mut field_map_ids = Vec::new(); + let mut field_map_ids = Vec::with_capacity(self.parse_table.production_infos.len()); for production_info in &self.parse_table.production_infos { if production_info.field_map.is_empty() { field_map_ids.push((0, 0)); } else { - let mut flat_field_map = Vec::new(); + let mut flat_field_map = Vec::with_capacity(production_info.field_map.len()); for (field_name, locations) in &production_info.field_map { for location in locations { flat_field_map.push((field_name.clone(), *location)); } } + let field_map_len = flat_field_map.len(); field_map_ids.push(( self.get_field_map_id( - flat_field_map.clone(), + flat_field_map, &mut flat_field_maps, &mut next_flat_field_map_index, ), - flat_field_map.len(), + field_map_len, )); } } add_line!( self, - "static const TSFieldMapSlice ts_field_map_slices[PRODUCTION_ID_COUNT] = {{", + "static const TSMapSlice ts_field_map_slices[PRODUCTION_ID_COUNT] = {{", ); indent!(self); for (production_id, (row_id, length)) in field_map_ids.into_iter().enumerate() { @@ -694,6 +743,83 @@ impl Generator { add_line!(self, ""); } + fn add_supertype_map(&mut self) { + add_line!( + self, + "static const TSSymbol ts_supertype_symbols[SUPERTYPE_COUNT] = {{" + ); + indent!(self); + for supertype in self.supertype_map.keys() { + add_line!(self, "{supertype},"); + } + dedent!(self); + add_line!(self, "}};\n"); + + add_line!( + self, + "static const TSMapSlice ts_supertype_map_slices[] = {{", + ); + indent!(self); + let mut row_id = 0; + let mut supertype_ids = vec![0]; + let mut supertype_string_map = BTreeMap::new(); + for (supertype, subtypes) in &self.supertype_map { + supertype_string_map.insert( + supertype, + subtypes + .iter() + .flat_map(|s| match s { + ChildType::Normal(symbol) => vec![self.symbol_ids.get(symbol).cloned()], + ChildType::Aliased(alias) => { + self.alias_ids.get(alias).cloned().map_or_else( + || { + self.symbols_for_alias(alias) + .into_iter() + .map(|s| self.symbol_ids.get(&s).cloned()) + .collect() + }, + |a| vec![Some(a)], + ) + } + }) + .flatten() + .collect::>(), + ); + } + for (supertype, subtypes) in &supertype_string_map { + let length = subtypes.len(); + add_line!( + self, + "[{supertype}] = {{.index = {row_id}, .length = {length}}},", + ); + row_id += length; + supertype_ids.push(row_id); + } + dedent!(self); + add_line!(self, "}};"); + add_line!(self, ""); + + add_line!( + self, + "static const TSSymbol ts_supertype_map_entries[] = {{", + ); + indent!(self); + for (i, (_, subtypes)) in supertype_string_map.iter().enumerate() { + let row_index = supertype_ids[i]; + add_line!(self, "[{row_index}] ="); + indent!(self); + for subtype in subtypes { + add_whitespace!(self); + add!(self, "{subtype},\n"); + } + dedent!(self); + } + + dedent!(self); + add_line!(self, "}};"); + add_line!(self, ""); + } + fn add_lex_function(&mut self, name: &str, lex_table: LexTable) { add_line!( self, @@ -751,7 +877,7 @@ impl Generator { && chars.ranges().all(|r| { let start = *r.start() as u32; let end = *r.end() as u32; - end <= start + 1 && end <= u16::MAX as u32 + end <= start + 1 && u16::try_from(end).is_ok() }) { leading_simple_transition_count += 1; @@ -839,10 +965,7 @@ impl Generator { large_char_set_ix = Some(char_set_ix); } - let mut line_break = "\n".to_string(); - for _ in 0..self.indent_level + 2 { - line_break.push_str(" "); - } + let line_break = format!("\n{}", " ".repeat(self.indent_level + 2)); let has_positive_condition = large_char_set_ix.is_some() || !asserted_chars.is_empty(); let has_negative_condition = !negated_chars.is_empty(); @@ -869,7 +992,7 @@ impl Generator { add!( self, "set_contains({}, {}, lookahead)", - &char_set_info.constant_name, + char_set_info.constant_name, large_set.range_count(), ); if check_eof { @@ -934,7 +1057,6 @@ impl Generator { } self.add_character(end); add!(self, ")"); - continue; } else if end == start { add!(self, "lookahead == "); self.add_character(start); @@ -985,7 +1107,7 @@ impl Generator { add_line!( self, - "static TSCharacterRange {}[] = {{", + "static const TSCharacterRange {}[] = {{", info.constant_name ); @@ -1020,25 +1142,66 @@ impl Generator { } } - fn add_lex_modes_list(&mut self) { + fn add_lex_modes(&mut self) { add_line!( self, - "static const TSLexMode ts_lex_modes[STATE_COUNT] = {{" + "static const {} ts_lex_modes[STATE_COUNT] = {{", + if self.abi_version >= ABI_VERSION_WITH_RESERVED_WORDS { + "TSLexerMode" + } else { + "TSLexMode" + } ); indent!(self); for (i, state) in self.parse_table.states.iter().enumerate() { + add_whitespace!(self); + add!(self, "[{i}] = {{"); if state.is_end_of_non_terminal_extra() { - add_line!(self, "[{i}] = {{(TSStateId)(-1)}},"); - } else if state.external_lex_state_id > 0 { - add_line!( - self, - "[{i}] = {{.lex_state = {}, .external_lex_state = {}}},", - state.lex_state_id, - state.external_lex_state_id - ); + add!(self, "(TSStateId)(-1),"); } else { - add_line!(self, "[{i}] = {{.lex_state = {}}},", state.lex_state_id); + add!(self, ".lex_state = {}", state.lex_state_id); + + if state.external_lex_state_id > 0 { + add!( + self, + ", .external_lex_state = {}", + state.external_lex_state_id + ); + } + + if self.abi_version >= ABI_VERSION_WITH_RESERVED_WORDS { + let reserved_word_set_id = self.reserved_word_set_ids_by_parse_state[i]; + if reserved_word_set_id != 0 { + add!(self, ", .reserved_word_set_id = {reserved_word_set_id}"); + } + } } + + add!(self, "}},\n"); + } + dedent!(self); + add_line!(self, "}};"); + add_line!(self, ""); + } + + fn add_reserved_word_sets(&mut self) { + add_line!( + self, + "static const TSSymbol ts_reserved_words[{}][MAX_RESERVED_WORD_SET_SIZE] = {{", + self.reserved_word_sets.len(), + ); + indent!(self); + for (id, set) in self.reserved_word_sets.iter().enumerate() { + if id == 0 { + continue; + } + add_line!(self, "[{id}] = {{"); + indent!(self); + for token in set.iter() { + add_line!(self, "{},", self.symbol_ids[&token]); + } + dedent!(self); + add_line!(self, "}},"); } dedent!(self); add_line!(self, "}};"); @@ -1092,7 +1255,7 @@ impl Generator { indent!(self); for i in 0..self.parse_table.external_lex_states.len() { if !self.parse_table.external_lex_states[i].is_empty() { - add_line!(self, "[{}] = {{", i); + add_line!(self, "[{i}] = {{"); indent!(self); for token in self.parse_table.external_lex_states[i].iter() { add_line!( @@ -1114,6 +1277,7 @@ impl Generator { let mut parse_table_entries = HashMap::new(); let mut next_parse_action_list_index = 0; + // Parse action lists zero is for the default value, when a symbol is not valid. self.get_parse_action_list_id( &ParseTableEntry { actions: Vec::new(), @@ -1139,7 +1303,7 @@ impl Generator { .enumerate() .take(self.large_state_count) { - add_line!(self, "[{i}] = {{"); + add_line!(self, "[STATE({i})] = {{"); indent!(self); // Ensure the entries are in a deterministic order, since they are @@ -1171,9 +1335,11 @@ impl Generator { ); add_line!(self, "[{}] = ACTIONS({entry_id}),", self.symbol_ids[symbol]); } + dedent!(self); add_line!(self, "}},"); } + dedent!(self); add_line!(self, "}};"); add_line!(self, ""); @@ -1182,11 +1348,16 @@ impl Generator { add_line!(self, "static const uint16_t ts_small_parse_table[] = {{"); indent!(self); - let mut index = 0; - let mut small_state_indices = Vec::new(); + let mut next_table_index = 0; + let mut small_state_indices = Vec::with_capacity( + self.parse_table + .states + .len() + .saturating_sub(self.large_state_count), + ); let mut symbols_by_value = HashMap::<(usize, SymbolType), Vec>::new(); for state in self.parse_table.states.iter().skip(self.large_state_count) { - small_state_indices.push(index); + small_state_indices.push(next_table_index); symbols_by_value.clear(); terminal_entries.clear(); @@ -1225,10 +1396,16 @@ impl Generator { (symbols.len(), *kind, *value, symbols[0]) }); - add_line!(self, "[{index}] = {},", values_with_symbols.len()); + add_line!( + self, + "[{next_table_index}] = {},", + values_with_symbols.len() + ); indent!(self); + next_table_index += 1; for ((value, kind), symbols) in &mut values_with_symbols { + next_table_index += 2 + symbols.len(); if *kind == SymbolType::NonTerminal { add_line!(self, "STATE({value}), {},", symbols.len()); } else { @@ -1244,11 +1421,6 @@ impl Generator { } dedent!(self); - - index += 1 + values_with_symbols - .iter() - .map(|(_, symbols)| 2 + symbols.len()) - .sum::(); } dedent!(self); @@ -1377,7 +1549,7 @@ impl Generator { indent!(self); add_line!(self, "static const TSLanguage language = {{"); indent!(self); - add_line!(self, ".version = LANGUAGE_VERSION,"); + add_line!(self, ".abi_version = LANGUAGE_VERSION,"); // Quantities add_line!(self, ".symbol_count = SYMBOL_COUNT,"); @@ -1387,6 +1559,9 @@ impl Generator { add_line!(self, ".state_count = STATE_COUNT,"); add_line!(self, ".large_state_count = LARGE_STATE_COUNT,"); add_line!(self, ".production_id_count = PRODUCTION_ID_COUNT,"); + if self.abi_version >= ABI_VERSION_WITH_RESERVED_WORDS { + add_line!(self, ".supertype_count = SUPERTYPE_COUNT,"); + } add_line!(self, ".field_count = FIELD_COUNT,"); add_line!( self, @@ -1408,6 +1583,11 @@ impl Generator { add_line!(self, ".field_map_slices = ts_field_map_slices,"); add_line!(self, ".field_map_entries = ts_field_map_entries,"); } + if !self.supertype_map.is_empty() && self.abi_version >= ABI_VERSION_WITH_RESERVED_WORDS { + add_line!(self, ".supertype_map_slices = ts_supertype_map_slices,"); + add_line!(self, ".supertype_map_entries = ts_supertype_map_entries,"); + add_line!(self, ".supertype_symbols = ts_supertype_symbols,"); + } add_line!(self, ".symbol_metadata = ts_symbol_metadata,"); add_line!(self, ".public_symbol_map = ts_symbol_map,"); add_line!(self, ".alias_map = ts_non_terminal_alias_map,"); @@ -1416,9 +1596,9 @@ impl Generator { } // Lexing - add_line!(self, ".lex_modes = ts_lex_modes,"); + add_line!(self, ".lex_modes = (const void*)ts_lex_modes,"); add_line!(self, ".lex_fn = ts_lex,"); - if let Some(keyword_capture_token) = self.keyword_capture_token { + if let Some(keyword_capture_token) = self.syntax_grammar.word_token { add_line!(self, ".keyword_lex_fn = ts_lex_keywords,"); add_line!( self, @@ -1443,8 +1623,40 @@ impl Generator { add_line!(self, ".primary_state_ids = ts_primary_state_ids,"); - if self.abi_version >= ABI_VERSION_WITH_METADATA { + if self.abi_version >= ABI_VERSION_WITH_RESERVED_WORDS { add_line!(self, ".name = \"{}\",", self.language_name); + + if self.reserved_word_sets.len() > 1 { + add_line!(self, ".reserved_words = &ts_reserved_words[0][0],"); + } + + add_line!( + self, + ".max_reserved_word_set_size = {},", + self.reserved_word_sets + .iter() + .map(TokenSet::len) + .max() + .unwrap() + ); + + let Some(metadata) = &self.metadata else { + panic!( + indoc! {" + Metadata is required to generate ABI version {}. + This means that your grammar doesn't have a tree-sitter.json config file with an appropriate version field in the metadata table. + "}, + self.abi_version + ); + }; + + add_line!(self, ".metadata = {{"); + indent!(self); + add_line!(self, ".major_version = {},", metadata.major_version); + add_line!(self, ".minor_version = {},", metadata.minor_version); + add_line!(self, ".patch_version = {},", metadata.patch_version); + dedent!(self); + add_line!(self, "}},"); } dedent!(self); @@ -1546,6 +1758,23 @@ impl Generator { } } + fn symbols_for_alias(&self, alias: &Alias) -> Vec { + self.parse_table + .symbols + .iter() + .copied() + .filter(move |symbol| { + self.default_aliases.get(symbol).map_or_else( + || { + let (name, kind) = self.metadata_for_symbol(*symbol); + name == alias.value && kind == alias.kind() + }, + |default_alias| default_alias == alias, + ) + }) + .collect() + } + fn sanitize_identifier(&self, name: &str) -> String { let mut result = String::with_capacity(name.len()); for c in name.chars() { @@ -1621,11 +1850,11 @@ impl Generator { '\u{007F}' => "DEL", '\u{FEFF}' => "BOM", '\u{0080}'..='\u{FFFF}' => { - result.push_str(&format!("u{:04x}", c as u32)); + write!(result, "u{:04x}", c as u32).unwrap(); break 'special_chars; } '\u{10000}'..='\u{10FFFF}' => { - result.push_str(&format!("U{:08x}", c as u32)); + write!(result, "U{:08x}", c as u32).unwrap(); break 'special_chars; } '0'..='9' | 'a'..='z' | 'A'..='Z' | '_' => unreachable!(), @@ -1656,11 +1885,9 @@ impl Generator { '\r' => result += "\\r", '\t' => result += "\\t", '\0' => result += "\\0", - '\u{0001}'..='\u{001f}' => result += &format!("\\x{:02x}", c as u32), - '\u{007F}'..='\u{FFFF}' => result += &format!("\\u{:04x}", c as u32), - '\u{10000}'..='\u{10FFFF}' => { - result.push_str(&format!("\\U{:08x}", c as u32)); - } + '\u{0001}'..='\u{001f}' => write!(result, "\\x{:02x}", c as u32).unwrap(), + '\u{007F}'..='\u{FFFF}' => write!(result, "\\u{:04x}", c as u32).unwrap(), + '\u{10000}'..='\u{10FFFF}' => write!(result, "\\U{:08x}", c as u32).unwrap(), _ => result.push(c), } } @@ -1713,6 +1940,8 @@ pub fn render_c_code( lexical_grammar: LexicalGrammar, default_aliases: AliasMap, abi_version: usize, + semantic_version: Option<(u8, u8, u8)>, + supertype_symbol_map: BTreeMap>, ) -> String { assert!( (ABI_VERSION_MIN..=ABI_VERSION_MAX).contains(&abi_version), @@ -1720,26 +1949,23 @@ pub fn render_c_code( ); Generator { - buffer: String::new(), - indent_level: 0, language_name: name.to_string(), - large_state_count: 0, parse_table: tables.parse_table, main_lex_table: tables.main_lex_table, keyword_lex_table: tables.keyword_lex_table, - keyword_capture_token: tables.word_token, large_character_sets: tables.large_character_sets, large_character_set_info: Vec::new(), syntax_grammar, lexical_grammar, default_aliases, - symbol_ids: HashMap::new(), - symbol_order: HashMap::new(), - alias_ids: HashMap::new(), - symbol_map: HashMap::new(), - unique_aliases: Vec::new(), - field_names: Vec::new(), abi_version, + metadata: semantic_version.map(|(major_version, minor_version, patch_version)| Metadata { + major_version, + minor_version, + patch_version, + }), + supertype_symbol_map, + ..Default::default() } .generate() } diff --git a/cli/generate/src/rules.rs b/crates/generate/src/rules.rs similarity index 87% rename from cli/generate/src/rules.rs rename to crates/generate/src/rules.rs index 6a922b31..05a0c426 100644 --- a/cli/generate/src/rules.rs +++ b/crates/generate/src/rules.rs @@ -1,10 +1,11 @@ -use std::{collections::HashMap, fmt}; +use std::{collections::BTreeMap, fmt}; +use serde::Serialize; use smallbitvec::SmallBitVec; use super::grammars::VariableType; -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize)] pub enum SymbolType { External, End, @@ -13,19 +14,19 @@ pub enum SymbolType { NonTerminal, } -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize)] pub enum Associativity { Left, Right, } -#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize)] pub struct Alias { pub value: String, pub is_named: bool, } -#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Default)] +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Default, Serialize)] pub enum Precedence { #[default] None, @@ -33,48 +34,50 @@ pub enum Precedence { Name(String), } -pub type AliasMap = HashMap; +pub type AliasMap = BTreeMap; -#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)] +#[derive(Clone, Debug, Default, PartialEq, Eq, Hash, Serialize)] pub struct MetadataParams { pub precedence: Precedence, pub dynamic_precedence: i32, pub associativity: Option, pub is_token: bool, - pub is_string: bool, - pub is_active: bool, pub is_main_token: bool, pub alias: Option, pub field_name: Option, } -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize)] pub struct Symbol { pub kind: SymbolType, pub index: usize, } -#[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize)] pub enum Rule { Blank, String(String), Pattern(String, String), NamedSymbol(String), Symbol(Symbol), - Choice(Vec), + Choice(Vec), Metadata { params: MetadataParams, - rule: Box, + rule: Box, + }, + Repeat(Box), + Seq(Vec), + Reserved { + rule: Box, + context_name: String, }, - Repeat(Box), - Seq(Vec), } // Because tokens are represented as small (~400 max) unsigned integers, // sets of tokens can be efficiently represented as bit vectors with each // index corresponding to a token, and each value representing whether or not // the token is present in the set. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Default, Clone, PartialEq, Eq, Hash)] pub struct TokenSet { terminal_bits: SmallBitVec, external_bits: SmallBitVec, @@ -82,6 +85,32 @@ pub struct TokenSet { end_of_nonterminal_extra: bool, } +impl fmt::Debug for TokenSet { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().entries(self.iter()).finish() + } +} + +impl PartialOrd for TokenSet { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for TokenSet { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.terminal_bits + .iter() + .cmp(other.terminal_bits.iter()) + .then_with(|| self.external_bits.iter().cmp(other.external_bits.iter())) + .then_with(|| self.eof.cmp(&other.eof)) + .then_with(|| { + self.end_of_nonterminal_extra + .cmp(&other.end_of_nonterminal_extra) + }) + } +} + impl Rule { pub fn field(name: String, content: Self) -> Self { add_metadata(content, move |params| { @@ -154,7 +183,9 @@ impl Rule { match self { Self::Blank | Self::Pattern(..) | Self::NamedSymbol(_) | Self::Symbol(_) => false, Self::String(string) => string.is_empty(), - Self::Metadata { rule, .. } | Self::Repeat(rule) => rule.is_empty(), + Self::Metadata { rule, .. } | Self::Repeat(rule) | Self::Reserved { rule, .. } => { + rule.is_empty() + } Self::Choice(rules) => rules.iter().any(Self::is_empty), Self::Seq(rules) => rules.iter().all(Self::is_empty), } @@ -275,7 +306,6 @@ impl Symbol { } impl From for Rule { - #[must_use] fn from(symbol: Symbol) -> Self { Self::Symbol(symbol) } @@ -394,6 +424,9 @@ impl TokenSet { }; if other.index < vec.len() && vec[other.index] { vec.set(other.index, false); + while vec.last() == Some(false) { + vec.pop(); + } return true; } false @@ -406,6 +439,13 @@ impl TokenSet { && !self.external_bits.iter().any(|a| a) } + pub fn len(&self) -> usize { + self.eof as usize + + self.end_of_nonterminal_extra as usize + + self.terminal_bits.iter().filter(|b| *b).count() + + self.external_bits.iter().filter(|b| *b).count() + } + pub fn insert_all_terminals(&mut self, other: &Self) -> bool { let mut result = false; if other.terminal_bits.len() > self.terminal_bits.len() { diff --git a/cli/generate/src/tables.rs b/crates/generate/src/tables.rs similarity index 98% rename from cli/generate/src/tables.rs rename to crates/generate/src/tables.rs index 940fd31a..ef403dc0 100644 --- a/cli/generate/src/tables.rs +++ b/crates/generate/src/tables.rs @@ -47,6 +47,7 @@ pub struct ParseState { pub id: ParseStateId, pub terminal_entries: IndexMap>, pub nonterminal_entries: IndexMap>, + pub reserved_words: TokenSet, pub lex_state_id: usize, pub external_lex_state_id: usize, pub core_id: usize, @@ -64,7 +65,7 @@ pub struct ProductionInfo { pub field_map: BTreeMap>, } -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, Default, PartialEq, Eq)] pub struct ParseTable { pub states: Vec, pub symbols: Vec, diff --git a/cli/generate/src/templates/alloc.h b/crates/generate/src/templates/alloc.h similarity index 100% rename from cli/generate/src/templates/alloc.h rename to crates/generate/src/templates/alloc.h diff --git a/cli/generate/src/templates/array.h b/crates/generate/src/templates/array.h similarity index 100% rename from cli/generate/src/templates/array.h rename to crates/generate/src/templates/array.h diff --git a/highlight/Cargo.toml b/crates/highlight/Cargo.toml similarity index 89% rename from highlight/Cargo.toml rename to crates/highlight/Cargo.toml index c6f0063c..502bb31f 100644 --- a/highlight/Cargo.toml +++ b/crates/highlight/Cargo.toml @@ -11,6 +11,7 @@ rust-version.workspace = true readme = "README.md" homepage.workspace = true repository.workspace = true +documentation = "https://docs.rs/tree-sitter-highlight" license.workspace = true keywords = ["incremental", "parsing", "syntax", "highlighting"] categories = ["parsing", "text-editors"] @@ -19,10 +20,10 @@ categories = ["parsing", "text-editors"] workspace = true [lib] +path = "src/highlight.rs" crate-type = ["lib", "staticlib"] [dependencies] -lazy_static.workspace = true regex.workspace = true thiserror.workspace = true streaming-iterator.workspace = true diff --git a/crates/highlight/LICENSE b/crates/highlight/LICENSE new file mode 100644 index 00000000..971b81f9 --- /dev/null +++ b/crates/highlight/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2018 Max Brunsfeld + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/highlight/README.md b/crates/highlight/README.md similarity index 85% rename from highlight/README.md rename to crates/highlight/README.md index 4ca76d6c..edb89708 100644 --- a/highlight/README.md +++ b/crates/highlight/README.md @@ -12,8 +12,8 @@ to parse, to your `Cargo.toml`: ```toml [dependencies] -tree-sitter-highlight = "0.22.0" -tree-sitter-javascript = "0.21.3" +tree-sitter-highlight = "0.25.4" +tree-sitter-javascript = "0.23.1" ``` Define the list of highlight names that you will recognize: @@ -21,15 +21,23 @@ Define the list of highlight names that you will recognize: ```rust let highlight_names = [ "attribute", + "comment", "constant", - "function.builtin", + "constant.builtin", + "constructor", + "embedded", "function", + "function.builtin", "keyword", + "module", + "number", "operator", "property", + "property.builtin", "punctuation", "punctuation.bracket", "punctuation.delimiter", + "punctuation.special", "string", "string.special", "tag", @@ -55,7 +63,7 @@ Load some highlighting queries from the `queries` directory of the language repo ```rust use tree_sitter_highlight::HighlightConfiguration; -let javascript_language = tree_sitter_javascript::language(); +let javascript_language = tree_sitter_javascript::LANGUAGE.into(); let mut javascript_config = HighlightConfiguration::new( javascript_language, @@ -87,10 +95,10 @@ let highlights = highlighter.highlight( for event in highlights { match event.unwrap() { HighlightEvent::Source {start, end} => { - eprintln!("source: {}-{}", start, end); + eprintln!("source: {start}-{end}"); }, HighlightEvent::HighlightStart(s) => { - eprintln!("highlight style started: {:?}", s); + eprintln!("highlight style started: {s:?}"); }, HighlightEvent::HighlightEnd => { eprintln!("highlight style ended"); diff --git a/highlight/include/tree_sitter/highlight.h b/crates/highlight/include/tree_sitter/highlight.h similarity index 100% rename from highlight/include/tree_sitter/highlight.h rename to crates/highlight/include/tree_sitter/highlight.h diff --git a/highlight/src/c_lib.rs b/crates/highlight/src/c_lib.rs similarity index 96% rename from highlight/src/c_lib.rs rename to crates/highlight/src/c_lib.rs index bf291c98..9cc5c073 100644 --- a/highlight/src/c_lib.rs +++ b/crates/highlight/src/c_lib.rs @@ -9,10 +9,10 @@ use tree_sitter::Language; use super::{Error, Highlight, HighlightConfiguration, Highlighter, HtmlRenderer}; pub struct TSHighlighter { - languages: HashMap, HighlightConfiguration)>, - attribute_strings: Vec<&'static [u8]>, - highlight_names: Vec, - carriage_return_index: Option, + pub languages: HashMap, HighlightConfiguration)>, + pub attribute_strings: Vec<&'static [u8]>, + pub highlight_names: Vec, + pub carriage_return_index: Option, } pub struct TSHighlightBuffer { @@ -304,9 +304,9 @@ impl TSHighlighter { output .renderer .set_carriage_return_highlight(self.carriage_return_index.map(Highlight)); - let result = output - .renderer - .render(highlights, source_code, &|s| self.attribute_strings[s.0]); + let result = output.renderer.render(highlights, source_code, &|s, out| { + out.extend(self.attribute_strings[s.0]); + }); match result { Err(Error::Cancelled | Error::Unknown) => ErrorCode::Timeout, Err(Error::InvalidLanguage) => ErrorCode::InvalidLanguage, diff --git a/highlight/src/lib.rs b/crates/highlight/src/highlight.rs similarity index 93% rename from highlight/src/lib.rs rename to crates/highlight/src/highlight.rs index abd9fb5e..d38351f6 100644 --- a/highlight/src/lib.rs +++ b/crates/highlight/src/highlight.rs @@ -1,4 +1,4 @@ -#![doc = include_str!("../README.md")] +#![cfg_attr(not(any(test, doctest)), doc = include_str!("../README.md"))] pub mod c_lib; use core::slice; @@ -7,12 +7,15 @@ use std::{ iter, marker::PhantomData, mem::{self, MaybeUninit}, - ops, str, - sync::atomic::{AtomicUsize, Ordering}, + ops::{self, ControlFlow}, + str, + sync::{ + atomic::{AtomicUsize, Ordering}, + LazyLock, + }, }; pub use c_lib as c; -use lazy_static::lazy_static; use streaming_iterator::StreamingIterator; use thiserror::Error; use tree_sitter::{ @@ -24,8 +27,8 @@ const CANCELLATION_CHECK_INTERVAL: usize = 100; const BUFFER_HTML_RESERVE_CAPACITY: usize = 10 * 1024; const BUFFER_LINES_RESERVE_CAPACITY: usize = 1000; -lazy_static! { - static ref STANDARD_CAPTURE_NAMES: HashSet<&'static str> = vec![ +static STANDARD_CAPTURE_NAMES: LazyLock> = LazyLock::new(|| { + vec![ "attribute", "boolean", "carriage-return", @@ -80,8 +83,8 @@ lazy_static! { "variable.parameter", ] .into_iter() - .collect(); -} + .collect() +}); /// Indicates which highlight should be applied to a region of source code. #[derive(Copy, Clone, Debug, PartialEq, Eq)] @@ -141,6 +144,8 @@ pub struct HtmlRenderer { pub html: Vec, pub line_offsets: Vec, carriage_return_highlight: Option, + // The offset in `self.html` of the last carriage return. + last_carriage_return: Option, } #[derive(Debug)] @@ -184,7 +189,7 @@ struct HighlightIterLayer<'a> { depth: usize, } -pub struct _QueryCaptures<'query, 'tree: 'query, T: TextProvider, I: AsRef<[u8]>> { +pub struct _QueryCaptures<'query, 'tree, T: TextProvider, I: AsRef<[u8]>> { ptr: *mut ffi::TSQueryCursor, query: &'query Query, text_provider: T, @@ -220,7 +225,7 @@ impl<'tree> _QueryMatch<'_, 'tree> { } } -impl<'query, 'tree: 'query, T: TextProvider, I: AsRef<[u8]>> Iterator +impl<'query, 'tree, T: TextProvider, I: AsRef<[u8]>> Iterator for _QueryCaptures<'query, 'tree, T, I> { type Item = (QueryMatch<'query, 'tree>, usize); @@ -271,7 +276,7 @@ impl Highlighter { } } - pub fn parser(&mut self) -> &mut Parser { + pub const fn parser(&mut self) -> &mut Parser { &mut self.parser } @@ -339,11 +344,13 @@ impl HighlightConfiguration { locals_query: &str, ) -> Result { // Concatenate the query strings, keeping track of the start offset of each section. - let mut query_source = String::new(); + let mut query_source = String::with_capacity( + injection_query.len() + locals_query.len() + highlights_query.len(), + ); query_source.push_str(injection_query); - let locals_query_offset = query_source.len(); + let locals_query_offset = injection_query.len(); query_source.push_str(locals_query); - let highlights_query_offset = query_source.len(); + let highlights_query_offset = injection_query.len() + locals_query.len(); query_source.push_str(highlights_query); // Construct a single query by concatenating the three query strings, but record the @@ -534,9 +541,13 @@ impl<'a> HighlightIterLayer<'a> { None, Some(ParseOptions::new().progress_callback(&mut |_| { if let Some(cancellation_flag) = cancellation_flag { - cancellation_flag.load(Ordering::SeqCst) != 0 + if cancellation_flag.load(Ordering::SeqCst) != 0 { + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } } else { - false + ControlFlow::Continue(()) } })), ) @@ -583,6 +594,7 @@ impl<'a> HighlightIterLayer<'a> { } } + // SAFETY: // The `captures` iterator borrows the `Tree` and the `QueryCursor`, which // prevents them from being moved. But both of these values are really just // pointers, so it's actually ok to move them. @@ -951,7 +963,7 @@ where for prop in layer.config.query.property_settings(match_.pattern_index) { if prop.key.as_ref() == "local.scope-inherits" { scope.inherits = - prop.value.as_ref().map_or(true, |r| r.as_ref() == "true"); + prop.value.as_ref().is_none_or(|r| r.as_ref() == "true"); } } layer.scope_stack.push(scope); @@ -1088,12 +1100,13 @@ impl HtmlRenderer { html: Vec::with_capacity(BUFFER_HTML_RESERVE_CAPACITY), line_offsets: Vec::with_capacity(BUFFER_LINES_RESERVE_CAPACITY), carriage_return_highlight: None, + last_carriage_return: None, }; result.line_offsets.push(0); result } - pub fn set_carriage_return_highlight(&mut self, highlight: Option) { + pub const fn set_carriage_return_highlight(&mut self, highlight: Option) { self.carriage_return_highlight = highlight; } @@ -1103,32 +1116,35 @@ impl HtmlRenderer { self.line_offsets.push(0); } - pub fn render<'a, F>( + pub fn render( &mut self, highlighter: impl Iterator>, - source: &'a [u8], + source: &[u8], attribute_callback: &F, ) -> Result<(), Error> where - F: Fn(Highlight) -> &'a [u8], + F: Fn(Highlight, &mut Vec), { let mut highlights = Vec::new(); for event in highlighter { match event { Ok(HighlightEvent::HighlightStart(s)) => { highlights.push(s); - self.start_highlight(s, attribute_callback); + self.start_highlight(s, &attribute_callback); } Ok(HighlightEvent::HighlightEnd) => { highlights.pop(); self.end_highlight(); } Ok(HighlightEvent::Source { start, end }) => { - self.add_text(&source[start..end], &highlights, attribute_callback); + self.add_text(&source[start..end], &highlights, &attribute_callback); } Err(a) => return Err(a), } } + if let Some(offset) = self.last_carriage_return.take() { + self.add_carriage_return(offset, attribute_callback); + } if self.html.last() != Some(&b'\n') { self.html.push(b'\n'); } @@ -1153,30 +1169,30 @@ impl HtmlRenderer { }) } - fn add_carriage_return<'a, F>(&mut self, attribute_callback: &F) + fn add_carriage_return(&mut self, offset: usize, attribute_callback: &F) where - F: Fn(Highlight) -> &'a [u8], + F: Fn(Highlight, &mut Vec), { if let Some(highlight) = self.carriage_return_highlight { - let attribute_string = (attribute_callback)(highlight); - if !attribute_string.is_empty() { - self.html.extend(b""); - } + // If a CR is the last character in a `HighlightEvent::Source` + // region, then we don't know until the next `Source` event or EOF + // whether it is part of CRLF or on its own. To avoid unbounded + // lookahead, save the offset of the CR and insert there now that we + // know. + let rest = self.html.split_off(offset); + self.html.extend(b""); + self.html.extend(rest); } } - fn start_highlight<'a, F>(&mut self, h: Highlight, attribute_callback: &F) + fn start_highlight(&mut self, h: Highlight, attribute_callback: &F) where - F: Fn(Highlight) -> &'a [u8], + F: Fn(Highlight, &mut Vec), { - let attribute_string = (attribute_callback)(h); - self.html.extend(b""); } @@ -1184,9 +1200,9 @@ impl HtmlRenderer { self.html.extend(b""); } - fn add_text<'a, F>(&mut self, src: &[u8], highlights: &[Highlight], attribute_callback: &F) + fn add_text(&mut self, src: &[u8], highlights: &[Highlight], attribute_callback: &F) where - F: Fn(Highlight) -> &'a [u8], + F: Fn(Highlight, &mut Vec), { pub const fn html_escape(c: u8) -> Option<&'static [u8]> { match c as char { @@ -1199,29 +1215,29 @@ impl HtmlRenderer { } } - let mut last_char_was_cr = false; for c in LossyUtf8::new(src).flat_map(|p| p.bytes()) { // Don't render carriage return characters, but allow lone carriage returns (not // followed by line feeds) to be styled via the attribute callback. if c == b'\r' { - last_char_was_cr = true; + self.last_carriage_return = Some(self.html.len()); continue; } - if last_char_was_cr { + if let Some(offset) = self.last_carriage_return.take() { if c != b'\n' { - self.add_carriage_return(attribute_callback); + self.add_carriage_return(offset, attribute_callback); } - last_char_was_cr = false; } // At line boundaries, close and re-open all of the open tags. if c == b'\n' { - highlights.iter().for_each(|_| self.end_highlight()); + for _ in highlights { + self.end_highlight(); + } self.html.push(c); self.line_offsets.push(self.html.len() as u32); - highlights - .iter() - .for_each(|scope| self.start_highlight(*scope, attribute_callback)); + for scope in highlights { + self.start_highlight(*scope, attribute_callback); + } } else if let Some(escape) = html_escape(c) { self.html.extend_from_slice(escape); } else { diff --git a/lib/language/Cargo.toml b/crates/language/Cargo.toml similarity index 70% rename from lib/language/Cargo.toml rename to crates/language/Cargo.toml index 7de3dca5..b6f5cdf8 100644 --- a/lib/language/Cargo.toml +++ b/crates/language/Cargo.toml @@ -1,19 +1,23 @@ [package] name = "tree-sitter-language" description = "The tree-sitter Language type, used by the library and by language implementations" -version = "0.1.2" +version = "0.1.7" authors.workspace = true edition.workspace = true -rust-version.workspace = true +rust-version = "1.77" readme = "README.md" homepage.workspace = true repository.workspace = true +documentation = "https://docs.rs/tree-sitter-language" license.workspace = true keywords.workspace = true categories = ["api-bindings", "development-tools::ffi", "parsing"] +build = "build.rs" +links = "tree-sitter-language" + [lints] workspace = true [lib] -path = "language.rs" +path = "src/language.rs" diff --git a/crates/language/LICENSE b/crates/language/LICENSE new file mode 100644 index 00000000..971b81f9 --- /dev/null +++ b/crates/language/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2018 Max Brunsfeld + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/lib/language/README.md b/crates/language/README.md similarity index 100% rename from lib/language/README.md rename to crates/language/README.md diff --git a/crates/language/build.rs b/crates/language/build.rs new file mode 100644 index 00000000..930b3dff --- /dev/null +++ b/crates/language/build.rs @@ -0,0 +1,13 @@ +fn main() { + if std::env::var("TARGET") + .unwrap_or_default() + .starts_with("wasm32-unknown") + { + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap(); + let wasm_headers = std::path::Path::new(&manifest_dir).join("wasm/include"); + let wasm_src = std::path::Path::new(&manifest_dir).join("wasm/src"); + + println!("cargo::metadata=wasm-headers={}", wasm_headers.display()); + println!("cargo::metadata=wasm-src={}", wasm_src.display()); + } +} diff --git a/lib/language/language.rs b/crates/language/src/language.rs similarity index 96% rename from lib/language/language.rs rename to crates/language/src/language.rs index 504c9374..cb46b8a9 100644 --- a/lib/language/language.rs +++ b/crates/language/src/language.rs @@ -1,5 +1,5 @@ #![no_std] -/// `LanguageFn` wraps a C function that returns a pointer to a tree-sitter grammer. +/// `LanguageFn` wraps a C function that returns a pointer to a tree-sitter grammar. #[repr(transparent)] #[derive(Clone, Copy)] pub struct LanguageFn(unsafe extern "C" fn() -> *const ()); diff --git a/crates/language/wasm/include/assert.h b/crates/language/wasm/include/assert.h new file mode 100644 index 00000000..e981a20e --- /dev/null +++ b/crates/language/wasm/include/assert.h @@ -0,0 +1,14 @@ +#ifndef TREE_SITTER_WASM_ASSERT_H_ +#define TREE_SITTER_WASM_ASSERT_H_ + +#ifdef NDEBUG +#define assert(e) ((void)0) +#else +__attribute__((noreturn)) void __assert_fail(const char *assertion, const char *file, unsigned line, const char *function) { + __builtin_trap(); +} +#define assert(expression) \ + ((expression) ? (void)0 : __assert_fail(#expression, __FILE__, __LINE__, __func__)) +#endif + +#endif // TREE_SITTER_WASM_ASSERT_H_ diff --git a/crates/language/wasm/include/ctype.h b/crates/language/wasm/include/ctype.h new file mode 100644 index 00000000..cea32970 --- /dev/null +++ b/crates/language/wasm/include/ctype.h @@ -0,0 +1,8 @@ +#ifndef TREE_SITTER_WASM_CTYPE_H_ +#define TREE_SITTER_WASM_CTYPE_H_ + +static inline int isprint(int c) { + return c >= 0x20 && c <= 0x7E; +} + +#endif // TREE_SITTER_WASM_CTYPE_H_ diff --git a/crates/language/wasm/include/endian.h b/crates/language/wasm/include/endian.h new file mode 100644 index 00000000..f35a5962 --- /dev/null +++ b/crates/language/wasm/include/endian.h @@ -0,0 +1,12 @@ +#ifndef TREE_SITTER_WASM_ENDIAN_H_ +#define TREE_SITTER_WASM_ENDIAN_H_ + +#define be16toh(x) __builtin_bswap16(x) +#define be32toh(x) __builtin_bswap32(x) +#define be64toh(x) __builtin_bswap64(x) +#define le16toh(x) (x) +#define le32toh(x) (x) +#define le64toh(x) (x) + + +#endif // TREE_SITTER_WASM_ENDIAN_H_ diff --git a/crates/language/wasm/include/inttypes.h b/crates/language/wasm/include/inttypes.h new file mode 100644 index 00000000..f5cccd07 --- /dev/null +++ b/crates/language/wasm/include/inttypes.h @@ -0,0 +1,8 @@ +#ifndef TREE_SITTER_WASM_INTTYPES_H_ +#define TREE_SITTER_WASM_INTTYPES_H_ + +// https://github.com/llvm/llvm-project/blob/0c3cf200f5b918fb5c1114e9f1764c2d54d1779b/libc/include/llvm-libc-macros/inttypes-macros.h#L209 + +#define PRId32 "d" + +#endif // TREE_SITTER_WASM_INTTYPES_H_ diff --git a/crates/language/wasm/include/stdint.h b/crates/language/wasm/include/stdint.h new file mode 100644 index 00000000..10cc35dc --- /dev/null +++ b/crates/language/wasm/include/stdint.h @@ -0,0 +1,46 @@ +#ifndef TREE_SITTER_WASM_STDINT_H_ +#define TREE_SITTER_WASM_STDINT_H_ + +// https://github.com/llvm/llvm-project/blob/0c3cf200f5b918fb5c1114e9f1764c2d54d1779b/clang/test/Preprocessor/init.c#L1672 + +typedef signed char int8_t; + +typedef short int16_t; + +typedef int int32_t; + +typedef long long int int64_t; + +typedef unsigned char uint8_t; + +typedef unsigned short uint16_t; + +typedef unsigned int uint32_t; + +typedef long long unsigned int uint64_t; + +typedef long unsigned int size_t; + +typedef long unsigned int uintptr_t; + +#define INT8_MAX 127 +#define INT16_MAX 32767 +#define INT32_MAX 2147483647L +#define INT64_MAX 9223372036854775807LL + +#define UINT8_MAX 255 +#define UINT16_MAX 65535 +#define UINT32_MAX 4294967295U +#define UINT64_MAX 18446744073709551615ULL + +#if defined(__wasm32__) + +#define SIZE_MAX 4294967295UL + +#elif defined(__wasm64__) + +#define SIZE_MAX 18446744073709551615UL + +#endif + +#endif // TREE_SITTER_WASM_STDINT_H_ diff --git a/crates/language/wasm/include/stdio.h b/crates/language/wasm/include/stdio.h new file mode 100644 index 00000000..4089cccc --- /dev/null +++ b/crates/language/wasm/include/stdio.h @@ -0,0 +1,36 @@ +#ifndef TREE_SITTER_WASM_STDIO_H_ +#define TREE_SITTER_WASM_STDIO_H_ + +#include +#include + +typedef struct FILE FILE; + +typedef __builtin_va_list va_list; +#define va_start(ap, last) __builtin_va_start(ap, last) +#define va_end(ap) __builtin_va_end(ap) +#define va_arg(ap, type) __builtin_va_arg(ap, type) + +#define stdout ((FILE *)0) + +#define stderr ((FILE *)1) + +#define stdin ((FILE *)2) + +int fclose(FILE *stream); + +FILE *fdopen(int fd, const char *mode); + +int fputc(int c, FILE *stream); + +int fputs(const char *restrict s, FILE *restrict stream); + +size_t fwrite(const void *restrict buffer, size_t size, size_t nmemb, FILE *restrict stream); + +int fprintf(FILE *restrict stream, const char *restrict format, ...); + +int snprintf(char *restrict buffer, size_t buffsz, const char *restrict format, ...); + +int vsnprintf(char *restrict buffer, size_t buffsz, const char *restrict format, va_list vlist); + +#endif // TREE_SITTER_WASM_STDIO_H_ diff --git a/crates/language/wasm/include/stdlib.h b/crates/language/wasm/include/stdlib.h new file mode 100644 index 00000000..2da313ab --- /dev/null +++ b/crates/language/wasm/include/stdlib.h @@ -0,0 +1,15 @@ +#ifndef TREE_SITTER_WASM_STDLIB_H_ +#define TREE_SITTER_WASM_STDLIB_H_ + +#include + +#define NULL ((void*)0) + +void* malloc(size_t); +void* calloc(size_t, size_t); +void free(void*); +void* realloc(void*, size_t); + +__attribute__((noreturn)) void abort(void); + +#endif // TREE_SITTER_WASM_STDLIB_H_ diff --git a/crates/language/wasm/include/string.h b/crates/language/wasm/include/string.h new file mode 100644 index 00000000..10f11958 --- /dev/null +++ b/crates/language/wasm/include/string.h @@ -0,0 +1,18 @@ +#ifndef TREE_SITTER_WASM_STRING_H_ +#define TREE_SITTER_WASM_STRING_H_ + +#include + +int memcmp(const void *lhs, const void *rhs, size_t count); + +void *memcpy(void *restrict dst, const void *restrict src, size_t size); + +void *memmove(void *dst, const void *src, size_t count); + +void *memset(void *dst, int value, size_t count); + +int strncmp(const char *left, const char *right, size_t n); + +size_t strlen(const char *str); + +#endif // TREE_SITTER_WASM_STRING_H_ diff --git a/crates/language/wasm/include/wctype.h b/crates/language/wasm/include/wctype.h new file mode 100644 index 00000000..8d1e8c82 --- /dev/null +++ b/crates/language/wasm/include/wctype.h @@ -0,0 +1,168 @@ +#ifndef TREE_SITTER_WASM_WCTYPE_H_ +#define TREE_SITTER_WASM_WCTYPE_H_ + +typedef int wint_t; + +static inline bool iswalpha(wint_t wch) { + switch (wch) { + case L'a': + case L'b': + case L'c': + case L'd': + case L'e': + case L'f': + case L'g': + case L'h': + case L'i': + case L'j': + case L'k': + case L'l': + case L'm': + case L'n': + case L'o': + case L'p': + case L'q': + case L'r': + case L's': + case L't': + case L'u': + case L'v': + case L'w': + case L'x': + case L'y': + case L'z': + case L'A': + case L'B': + case L'C': + case L'D': + case L'E': + case L'F': + case L'G': + case L'H': + case L'I': + case L'J': + case L'K': + case L'L': + case L'M': + case L'N': + case L'O': + case L'P': + case L'Q': + case L'R': + case L'S': + case L'T': + case L'U': + case L'V': + case L'W': + case L'X': + case L'Y': + case L'Z': + return true; + default: + return false; + } +} + +static inline bool iswdigit(wint_t wch) { + switch (wch) { + case L'0': + case L'1': + case L'2': + case L'3': + case L'4': + case L'5': + case L'6': + case L'7': + case L'8': + case L'9': + return true; + default: + return false; + } +} + +static inline bool iswalnum(wint_t wch) { + switch (wch) { + case L'a': + case L'b': + case L'c': + case L'd': + case L'e': + case L'f': + case L'g': + case L'h': + case L'i': + case L'j': + case L'k': + case L'l': + case L'm': + case L'n': + case L'o': + case L'p': + case L'q': + case L'r': + case L's': + case L't': + case L'u': + case L'v': + case L'w': + case L'x': + case L'y': + case L'z': + case L'A': + case L'B': + case L'C': + case L'D': + case L'E': + case L'F': + case L'G': + case L'H': + case L'I': + case L'J': + case L'K': + case L'L': + case L'M': + case L'N': + case L'O': + case L'P': + case L'Q': + case L'R': + case L'S': + case L'T': + case L'U': + case L'V': + case L'W': + case L'X': + case L'Y': + case L'Z': + case L'0': + case L'1': + case L'2': + case L'3': + case L'4': + case L'5': + case L'6': + case L'7': + case L'8': + case L'9': + return true; + default: + return false; + } +} + +static inline bool iswspace(wint_t wch) { + switch (wch) { + case L' ': + case L'\t': + case L'\n': + case L'\v': + case L'\f': + case L'\r': + return true; + default: + return false; + } +} + +#endif // TREE_SITTER_WASM_WCTYPE_H_ diff --git a/crates/language/wasm/src/stdio.c b/crates/language/wasm/src/stdio.c new file mode 100644 index 00000000..470c1ecc --- /dev/null +++ b/crates/language/wasm/src/stdio.c @@ -0,0 +1,299 @@ +#include +#include + +typedef struct { + bool left_justify; // - + bool zero_pad; // 0 + bool show_sign; // + + bool space_prefix; // ' ' + bool alternate_form; // # +} format_flags_t; + +static const char* parse_format_spec( + const char *format, + int *width, + int *precision, + format_flags_t *flags +) { + *width = 0; + *precision = -1; + flags->left_justify = false; + flags->zero_pad = false; + flags->show_sign = false; + flags->space_prefix = false; + flags->alternate_form = false; + + const char *p = format; + + // Parse flags + while (*p == '-' || *p == '+' || *p == ' ' || *p == '#' || *p == '0') { + switch (*p) { + case '-': flags->left_justify = true; break; + case '0': flags->zero_pad = true; break; + case '+': flags->show_sign = true; break; + case ' ': flags->space_prefix = true; break; + case '#': flags->alternate_form = true; break; + } + p++; + } + + // width + while (*p >= '0' && *p <= '9') { + *width = (*width * 10) + (*p - '0'); + p++; + } + + // precision + if (*p == '.') { + p++; + *precision = 0; + while (*p >= '0' && *p <= '9') { + *precision = (*precision * 10) + (*p - '0'); + p++; + } + } + + return p; +} + +static int int_to_str( + long long value, + char *buffer, + int base, + bool is_signed, + bool uppercase +) { + if (base < 2 || base > 16) return 0; + + const char *digits = uppercase ? "0123456789ABCDEF" : "0123456789abcdef"; + char temp[32]; + int i = 0, len = 0; + bool is_negative = false; + + if (value == 0) { + buffer[0] = '0'; + buffer[1] = '\0'; + return 1; + } + + if (is_signed && value < 0 && base == 10) { + is_negative = true; + value = -value; + } + + unsigned long long uval = (unsigned long long)value; + while (uval > 0) { + temp[i++] = digits[uval % base]; + uval /= base; + } + + if (is_negative) { + buffer[len++] = '-'; + } + + while (i > 0) { + buffer[len++] = temp[--i]; + } + + buffer[len] = '\0'; + return len; +} + +static int ptr_to_str(void *ptr, char *buffer) { + buffer[0] = '0'; + buffer[1] = 'x'; + int len = int_to_str((uintptr_t)ptr, buffer + 2, 16, 0, 0); + return 2 + len; +} + +char *strncpy(char *dest, const char *src, size_t n) { + char *d = dest; + const char *s = src; + while (n-- && (*d++ = *s++)); + if (n == (size_t)-1) *d = '\0'; + return dest; +} + +static int write_formatted_to_buffer( + char *buffer, + size_t buffer_size, + size_t *pos, + const char *str, + int width, + const format_flags_t *flags +) { + int len = strlen(str); + int written = 0; + int pad_len = (width > len) ? (width - len) : 0; + int zero_pad = flags->zero_pad && !flags->left_justify; + + if (!flags->left_justify && pad_len > 0) { + char pad_char = zero_pad ? '0' : ' '; + for (int i = 0; i < pad_len && *pos < buffer_size - 1; i++) { + buffer[(*pos)++] = pad_char; + written++; + } + } + + for (int i = 0; i < len && *pos < buffer_size - 1; i++) { + buffer[(*pos)++] = str[i]; + written++; + } + + if (flags->left_justify && pad_len > 0) { + for (int i = 0; i < pad_len && *pos < buffer_size - 1; i++) { + buffer[(*pos)++] = ' '; + written++; + } + } + + return written; +} + +static int vsnprintf_impl(char *buffer, size_t buffsz, const char *format, va_list args) { + if (!buffer || buffsz == 0 || !format) return -1; + + size_t pos = 0; + int total_chars = 0; + const char *p = format; + + while (*p) { + if (*p == '%') { + p++; + if (*p == '%') { + if (pos < buffsz - 1) buffer[pos++] = '%'; + total_chars++; + p++; + continue; + } + + int width, precision; + format_flags_t flags; + p = parse_format_spec(p, &width, &precision, &flags); + + char temp_buf[64]; + const char *output_str = temp_buf; + + switch (*p) { + case 's': { + const char *str = va_arg(args, const char*); + if (!str) str = "(null)"; + + int str_len = strlen(str); + if (precision >= 0 && str_len > precision) { + strncpy(temp_buf, str, precision); + temp_buf[precision] = '\0'; + output_str = temp_buf; + } else { + output_str = str; + } + break; + } + case 'd': + case 'i': { + int value = va_arg(args, int); + int_to_str(value, temp_buf, 10, true, false); + break; + } + case 'u': { + unsigned int value = va_arg(args, unsigned int); + int_to_str(value, temp_buf, 10, false, false); + break; + } + case 'x': { + unsigned int value = va_arg(args, unsigned int); + int_to_str(value, temp_buf, 16, false, false); + break; + } + case 'X': { + unsigned int value = va_arg(args, unsigned int); + int_to_str(value, temp_buf, 16, false, true); + break; + } + case 'p': { + void *ptr = va_arg(args, void*); + ptr_to_str(ptr, temp_buf); + break; + } + case 'c': { + int c = va_arg(args, int); + temp_buf[0] = (char)c; + temp_buf[1] = '\0'; + break; + } + case 'z': { + if (*(p + 1) == 'u') { + size_t value = va_arg(args, size_t); + int_to_str(value, temp_buf, 10, false, false); + p++; + } else { + temp_buf[0] = 'z'; + temp_buf[1] = '\0'; + } + break; + } + default: + temp_buf[0] = '%'; + temp_buf[1] = *p; + temp_buf[2] = '\0'; + break; + } + + int str_len = strlen(output_str); + int formatted_len = (width > str_len) ? width : str_len; + total_chars += formatted_len; + + if (pos < buffsz - 1) { + write_formatted_to_buffer(buffer, buffsz, &pos, output_str, width, &flags); + } + + } else { + if (pos < buffsz - 1) buffer[pos++] = *p; + total_chars++; + } + p++; + } + + if (buffsz > 0) buffer[pos < buffsz ? pos : buffsz - 1] = '\0'; + + return total_chars; +} + +int snprintf(char *restrict buffer, size_t buffsz, const char *restrict format, ...) { + if (!buffer || buffsz == 0 || !format) return -1; + + va_list args; + va_start(args, format); + int result = vsnprintf_impl(buffer, buffsz, format, args); + va_end(args); + + return result; +} + +int vsnprintf(char *restrict buffer, size_t buffsz, const char *restrict format, va_list vlist) { + return vsnprintf_impl(buffer, buffsz, format, vlist); +} + +int fclose(FILE *stream) { + return 0; +} + +FILE* fdopen(int fd, const char *mode) { + return 0; +} + +int fputc(int c, FILE *stream) { + return c; +} + +int fputs(const char *restrict str, FILE *restrict stream) { + return 0; +} + +size_t fwrite(const void *restrict buffer, size_t size, size_t nmemb, FILE *restrict stream) { + return size * nmemb; +} + +int fprintf(FILE *restrict stream, const char *restrict format, ...) { + return 0; +} diff --git a/lib/src/wasm/stdlib.c b/crates/language/wasm/src/stdlib.c similarity index 72% rename from lib/src/wasm/stdlib.c rename to crates/language/wasm/src/stdlib.c index cfe2e4b3..f50e1da9 100644 --- a/lib/src/wasm/stdlib.c +++ b/crates/language/wasm/src/stdlib.c @@ -1,10 +1,12 @@ // This file implements a very simple allocator for external scanners running -// in WASM. Allocation is just bumping a static pointer and growing the heap -// as needed, and freeing is mostly a noop. But in the special case of freeing -// the last-allocated pointer, we'll reuse that pointer again. +// in Wasm. Allocation is just bumping a static pointer and growing the heap +// as needed, and freeing is just adding the freed region to a free list. +// When additional memory is allocated, the free list is searched first. +// If there is not a suitable region in the free list, the heap is +// grown as necessary, and the allocation is made at the end of the heap. +// When the heap is reset, all allocated memory is considered freed. -#include -#include +#include #include #include @@ -15,12 +17,14 @@ extern void tree_sitter_debug_message(const char *, size_t); typedef struct { size_t size; + struct Region *next; char data[0]; } Region; static Region *heap_end = NULL; static Region *heap_start = NULL; static Region *next = NULL; +static Region *free_list = NULL; // Get the region metadata for the given heap pointer. static inline Region *region_for_ptr(void *ptr) { @@ -49,9 +53,27 @@ void reset_heap(void *new_heap_start) { heap_start = new_heap_start; next = new_heap_start; heap_end = get_heap_end(); + free_list = NULL; } void *malloc(size_t size) { + if (size == 0) return NULL; + + Region *prev = NULL; + Region *curr = free_list; + while (curr != NULL) { + if (curr->size >= size) { + if (prev == NULL) { + free_list = curr->next; + } else { + prev->next = curr->next; + } + return &curr->data; + } + prev = curr; + curr = curr->next; + } + Region *region_end = region_after(next, size); if (region_end > heap_end) { @@ -79,6 +101,9 @@ void free(void *ptr) { // pointer for the next allocation. if (region_end == next) { next = region; + } else { + region->next = free_list; + free_list = region; } } @@ -107,3 +132,7 @@ void *realloc(void *ptr, size_t new_size) { memcpy(result, ®ion->data, region->size); return result; } + +__attribute__((noreturn)) void abort(void) { + __builtin_trap(); +} diff --git a/crates/language/wasm/src/string.c b/crates/language/wasm/src/string.c new file mode 100644 index 00000000..2d0d9096 --- /dev/null +++ b/crates/language/wasm/src/string.c @@ -0,0 +1,66 @@ +#include + +int memcmp(const void *lhs, const void *rhs, size_t count) { + const unsigned char *l = lhs; + const unsigned char *r = rhs; + while (count--) { + if (*l != *r) { + return *l - *r; + } + l++; + r++; + } + return 0; +} + +void *memcpy(void *restrict dst, const void *restrict src, size_t size) { + unsigned char *d = dst; + const unsigned char *s = src; + while (size--) { + *d++ = *s++; + } + return dst; +} + +void *memmove(void *dst, const void *src, size_t count) { + unsigned char *d = dst; + const unsigned char *s = src; + if (d < s) { + while (count--) { + *d++ = *s++; + } + } else if (d > s) { + d += count; + s += count; + while (count--) { + *(--d) = *(--s); + } + } + return dst; +} + +void *memset(void *dst, int value, size_t count) { + unsigned char *p = dst; + while (count--) { + *p++ = (unsigned char)value; + } + return dst; +} + +int strncmp(const char *left, const char *right, size_t n) { + while (n-- > 0) { + if (*left != *right) { + return *(unsigned char *)left - *(unsigned char *)right; + } + if (*left == '\0') break; + left++; + right++; + } + return 0; +} + +size_t strlen(const char *str) { + const char *s = str; + while (*s) s++; + return s - str; +} diff --git a/cli/loader/Cargo.toml b/crates/loader/Cargo.toml similarity index 79% rename from cli/loader/Cargo.toml rename to crates/loader/Cargo.toml index 845070d9..cecae218 100644 --- a/cli/loader/Cargo.toml +++ b/crates/loader/Cargo.toml @@ -8,6 +8,7 @@ rust-version.workspace = true readme = "README.md" homepage.workspace = true repository.workspace = true +documentation = "https://docs.rs/tree-sitter-loader" license.workspace = true keywords.workspace = true categories.workspace = true @@ -16,31 +17,30 @@ categories.workspace = true all-features = true rustdoc-args = ["--cfg", "docsrs"] +[lib] +path = "src/loader.rs" + [lints] workspace = true [features] wasm = ["tree-sitter/wasm"] -# TODO: For backward compatibility these must be enabled by default, -# consider removing for the next semver incompatible release default = ["tree-sitter-highlight", "tree-sitter-tags"] [dependencies] -anyhow.workspace = true cc.workspace = true -dirs.workspace = true +etcetera.workspace = true fs4.workspace = true indoc.workspace = true -lazy_static.workspace = true libloading.workspace = true +log.workspace = true once_cell.workspace = true -path-slash.workspace = true regex.workspace = true semver.workspace = true serde.workspace = true serde_json.workspace = true tempfile.workspace = true -url.workspace = true +thiserror.workspace = true tree-sitter = { workspace = true } tree-sitter-highlight = { workspace = true, optional = true } diff --git a/crates/loader/LICENSE b/crates/loader/LICENSE new file mode 100644 index 00000000..971b81f9 --- /dev/null +++ b/crates/loader/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2018 Max Brunsfeld + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cli/loader/README.md b/crates/loader/README.md similarity index 100% rename from cli/loader/README.md rename to crates/loader/README.md diff --git a/cli/loader/build.rs b/crates/loader/build.rs similarity index 58% rename from cli/loader/build.rs rename to crates/loader/build.rs index b01f01bb..5ffc889a 100644 --- a/cli/loader/build.rs +++ b/crates/loader/build.rs @@ -7,7 +7,4 @@ fn main() { "cargo:rustc-env=BUILD_HOST={}", std::env::var("HOST").unwrap() ); - - let emscripten_version = std::fs::read_to_string("emscripten-version").unwrap(); - println!("cargo:rustc-env=EMSCRIPTEN_VERSION={emscripten_version}"); } diff --git a/crates/loader/emscripten-version b/crates/loader/emscripten-version new file mode 100644 index 00000000..af253c16 --- /dev/null +++ b/crates/loader/emscripten-version @@ -0,0 +1 @@ +4.0.15 diff --git a/cli/loader/src/lib.rs b/crates/loader/src/loader.rs similarity index 53% rename from cli/loader/src/lib.rs rename to crates/loader/src/loader.rs index 4e3effed..e7584378 100644 --- a/cli/loader/src/lib.rs +++ b/crates/loader/src/loader.rs @@ -1,4 +1,4 @@ -#![doc = include_str!("../README.md")] +#![cfg_attr(not(any(test, doctest)), doc = include_str!("../README.md"))] #![cfg_attr(docsrs, feature(doc_cfg))] #[cfg(any(feature = "tree-sitter-highlight", feature = "tree-sitter-tags"))] @@ -7,44 +7,254 @@ use std::ops::Range; use std::sync::Mutex; use std::{ collections::HashMap, - env, - ffi::{OsStr, OsString}, - fs, + env, fs, + hash::{Hash as _, Hasher as _}, io::{BufRead, BufReader}, + marker::PhantomData, mem, path::{Path, PathBuf}, process::Command, - time::SystemTime, + sync::LazyLock, + time::{SystemTime, SystemTimeError}, }; -#[cfg(any(feature = "tree-sitter-highlight", feature = "tree-sitter-tags"))] -use anyhow::Error; -use anyhow::{anyhow, Context, Result}; +use etcetera::BaseStrategy as _; use fs4::fs_std::FileExt; -use indoc::indoc; -use lazy_static::lazy_static; use libloading::{Library, Symbol}; +use log::{error, info, warn}; use once_cell::unsync::OnceCell; -use path_slash::PathBufExt as _; use regex::{Regex, RegexBuilder}; use semver::Version; use serde::{Deserialize, Deserializer, Serialize}; +use thiserror::Error; use tree_sitter::Language; #[cfg(any(feature = "tree-sitter-highlight", feature = "tree-sitter-tags"))] use tree_sitter::QueryError; #[cfg(feature = "tree-sitter-highlight")] use tree_sitter::QueryErrorKind; +#[cfg(feature = "wasm")] +use tree_sitter::WasmError; #[cfg(feature = "tree-sitter-highlight")] use tree_sitter_highlight::HighlightConfiguration; #[cfg(feature = "tree-sitter-tags")] use tree_sitter_tags::{Error as TagsError, TagsConfiguration}; -use url::Url; -lazy_static! { - static ref GRAMMAR_NAME_REGEX: Regex = Regex::new(r#""name":\s*"(.*?)""#).unwrap(); +static GRAMMAR_NAME_REGEX: LazyLock = + LazyLock::new(|| Regex::new(r#""name":\s*"(.*?)""#).unwrap()); + +const WASI_SDK_VERSION: &str = include_str!("../wasi-sdk-version").trim_ascii(); + +pub type LoaderResult = Result; + +#[derive(Debug, Error)] +pub enum LoaderError { + #[error(transparent)] + Compiler(CompilerError), + #[error("Parser compilation failed.\nStdout: {0}\nStderr: {1}")] + Compilation(String, String), + #[error("Failed to execute curl for {0} -- {1}")] + Curl(String, std::io::Error), + #[error("Failed to load language in current directory:\n{0}")] + CurrentDirectoryLoad(Box), + #[error("External file path {0} is outside of parser directory {1}")] + ExternalFile(String, String), + #[error("Failed to extract archive {0} to {1}")] + Extraction(String, String), + #[error("Failed to load language for file name {0}:\n{1}")] + FileNameLoad(String, Box), + #[error("Failed to parse the language name from grammar.json at {0}")] + GrammarJSON(String), + #[error(transparent)] + HomeDir(#[from] etcetera::HomeDirError), + #[error(transparent)] + IO(IoError), + #[error(transparent)] + Library(LibraryError), + #[error("Failed to compare binary and source timestamps:\n{0}")] + ModifiedTime(Box), + #[error("No language found")] + NoLanguage, + #[error(transparent)] + Query(LoaderQueryError), + #[error(transparent)] + ScannerSymbols(ScannerSymbolError), + #[error("Failed to load language for scope '{0}':\n{1}")] + ScopeLoad(String, Box), + #[error(transparent)] + Serialization(#[from] serde_json::Error), + #[error(transparent)] + Symbol(SymbolError), + #[error(transparent)] + Tags(#[from] TagsError), + #[error("Failed to execute tar for {0} -- {1}")] + Tar(String, std::io::Error), + #[error(transparent)] + Time(#[from] SystemTimeError), + #[error("Unknown scope '{0}'")] + UnknownScope(String), + #[error("Failed to download wasi-sdk from {0}")] + WasiSDKDownload(String), + #[error(transparent)] + WasiSDKClang(#[from] WasiSDKClangError), + #[error("Unsupported platform for wasi-sdk")] + WasiSDKPlatform, + #[cfg(feature = "wasm")] + #[error(transparent)] + Wasm(#[from] WasmError), + #[error("Failed to run wasi-sdk clang -- {0}")] + WasmCompiler(std::io::Error), + #[error("wasi-sdk clang command failed: {0}")] + WasmCompilation(String), } -pub const EMSCRIPTEN_TAG: &str = concat!("docker.io/emscripten/emsdk:", env!("EMSCRIPTEN_VERSION")); +#[derive(Debug, Error)] +pub struct CompilerError { + pub error: std::io::Error, + pub command: Box, +} + +impl std::fmt::Display for CompilerError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Failed to execute the C compiler with the following command:\n{:?}\nError: {}", + *self.command, self.error + )?; + Ok(()) + } +} + +#[derive(Debug, Error)] +pub struct IoError { + pub error: std::io::Error, + pub path: Option, +} + +impl IoError { + fn new(error: std::io::Error, path: Option<&Path>) -> Self { + Self { + error, + path: path.map(|p| p.to_string_lossy().to_string()), + } + } +} + +impl std::fmt::Display for IoError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.error)?; + if let Some(ref path) = self.path { + write!(f, " ({path})")?; + } + Ok(()) + } +} + +#[derive(Debug, Error)] +pub struct LibraryError { + pub error: libloading::Error, + pub path: String, +} + +impl std::fmt::Display for LibraryError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Error opening dynamic library {} -- {}", + self.path, self.error + )?; + Ok(()) + } +} + +#[derive(Debug, Error)] +pub struct LoaderQueryError { + pub error: QueryError, + pub file: Option, +} + +impl std::fmt::Display for LoaderQueryError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if let Some(ref path) = self.file { + writeln!(f, "Error in query file {path}:")?; + } + write!(f, "{}", self.error)?; + Ok(()) + } +} + +#[derive(Debug, Error)] +pub struct SymbolError { + pub error: libloading::Error, + pub symbol_name: String, + pub path: String, +} + +impl std::fmt::Display for SymbolError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Failed to load symbol {} from {} -- {}", + self.symbol_name, self.path, self.error + )?; + Ok(()) + } +} + +#[derive(Debug, Error)] +pub struct ScannerSymbolError { + pub missing: Vec, +} + +impl std::fmt::Display for ScannerSymbolError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!( + f, + "Missing required functions in the external scanner, parsing won't work without these!\n" + )?; + for symbol in &self.missing { + writeln!(f, " `{symbol}`")?; + } + writeln!( + f, + "You can read more about this at https://tree-sitter.github.io/tree-sitter/creating-parsers/4-external-scanners\n" + )?; + Ok(()) + } +} + +#[derive(Debug, Error)] +pub struct WasiSDKClangError { + pub wasi_sdk_dir: String, + pub possible_executables: Vec<&'static str>, + pub download: bool, +} + +impl std::fmt::Display for WasiSDKClangError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.download { + write!( + f, + "Failed to find clang executable in downloaded wasi-sdk at '{}'.", + self.wasi_sdk_dir + )?; + } else { + write!(f, "TREE_SITTER_WASI_SDK_PATH is set to '{}', but no clang executable found in 'bin/' directory.", self.wasi_sdk_dir)?; + } + + let possible_exes = self.possible_executables.join(", "); + write!(f, " Looked for: {possible_exes}.")?; + + Ok(()) + } +} + +pub const DEFAULT_HIGHLIGHTS_QUERY_FILE_NAME: &str = "highlights.scm"; + +pub const DEFAULT_INJECTIONS_QUERY_FILE_NAME: &str = "injections.scm"; + +pub const DEFAULT_LOCALS_QUERY_FILE_NAME: &str = "locals.scm"; + +pub const DEFAULT_TAGS_QUERY_FILE_NAME: &str = "tags.scm"; #[derive(Default, Deserialize, Serialize)] pub struct Config { @@ -61,12 +271,12 @@ pub struct Config { pub enum PathsJSON { #[default] Empty, - Single(String), - Multiple(Vec), + Single(PathBuf), + Multiple(Vec), } impl PathsJSON { - fn into_vec(self) -> Option> { + fn into_vec(self) -> Option> { match self { Self::Empty => None, Self::Single(s) => Some(vec![s]), @@ -77,6 +287,17 @@ impl PathsJSON { const fn is_empty(&self) -> bool { matches!(self, Self::Empty) } + + /// Represent this set of paths as a string that can be included in templates + #[must_use] + pub fn to_variable_value<'a>(&'a self, default: &'a PathBuf) -> &'a str { + match self { + Self::Empty => Some(default), + Self::Single(path_buf) => Some(path_buf), + Self::Multiple(paths) => paths.first(), + } + .map_or("", |path| path.as_os_str().to_str().unwrap_or("")) + } } #[derive(Serialize, Deserialize, Clone)] @@ -149,9 +370,10 @@ pub struct TreeSitterJSON { } impl TreeSitterJSON { - pub fn from_file(path: &Path) -> Result { - Ok(serde_json::from_str(&fs::read_to_string( - path.join("tree-sitter.json"), + pub fn from_file(path: &Path) -> LoaderResult { + let path = path.join("tree-sitter.json"); + Ok(serde_json::from_str(&fs::read_to_string(&path).map_err( + |e| LoaderError::IO(IoError::new(e, Some(path.as_path()))), )?)?) } @@ -167,6 +389,8 @@ pub struct Grammar { pub name: String, #[serde(skip_serializing_if = "Option::is_none")] pub camelcase: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub title: Option, pub scope: String, #[serde(skip_serializing_if = "Option::is_none")] pub path: Option, @@ -187,6 +411,8 @@ pub struct Grammar { pub first_line_regex: Option, #[serde(skip_serializing_if = "Option::is_none")] pub content_regex: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub class_name: Option, } #[derive(Serialize, Deserialize)] @@ -215,17 +441,16 @@ pub struct Author { #[derive(Serialize, Deserialize)] pub struct Links { - pub repository: Url, + pub repository: String, #[serde(skip_serializing_if = "Option::is_none")] - pub homepage: Option, + pub funding: Option, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Clone)] #[serde(default)] pub struct Bindings { pub c: bool, pub go: bool, - #[serde(skip)] pub java: bool, #[serde(skip)] pub kotlin: bool, @@ -233,6 +458,62 @@ pub struct Bindings { pub python: bool, pub rust: bool, pub swift: bool, + pub zig: bool, +} + +impl Bindings { + /// return available languages and its default enabled state. + #[must_use] + pub const fn languages(&self) -> [(&'static str, bool); 8] { + [ + ("c", true), + ("go", true), + ("java", false), + // Comment out Kotlin until the bindings are actually available. + // ("kotlin", false), + ("node", true), + ("python", true), + ("rust", true), + ("swift", true), + ("zig", false), + ] + } + + /// construct Bindings from a language list. If a language isn't supported, its name will be put on the error part. + pub fn with_enabled_languages<'a, I>(languages: I) -> Result + where + I: Iterator, + { + let mut out = Self { + c: false, + go: false, + java: false, + kotlin: false, + node: false, + python: false, + rust: false, + swift: false, + zig: false, + }; + + for v in languages { + match v { + "c" => out.c = true, + "go" => out.go = true, + "java" => out.java = true, + // Comment out Kotlin until the bindings are actually available. + // "kotlin" => out.kotlin = true, + "node" => out.node = true, + "python" => out.python = true, + "rust" => out.rust = true, + "swift" => out.swift = true, + "zig" => out.zig = true, + unsupported => return Err(unsupported), + } + } + + Ok(out) + } } impl Default for Bindings { @@ -246,6 +527,7 @@ impl Default for Bindings { python: true, rust: true, swift: true, + zig: false, } } } @@ -258,7 +540,7 @@ where D: Deserializer<'de>, { let paths = Vec::::deserialize(deserializer)?; - let Some(home) = dirs::home_dir() else { + let Ok(home) = etcetera::home_dir() else { return Ok(paths); }; let standardized = paths @@ -281,7 +563,7 @@ fn standardize_path(path: PathBuf, home: &Path) -> PathBuf { impl Config { #[must_use] pub fn initial() -> Self { - let home_dir = dirs::home_dir().expect("Cannot determine home directory"); + let home_dir = etcetera::home_dir().expect("Cannot determine home directory"); Self { parser_directories: vec![ home_dir.join("github"), @@ -296,7 +578,6 @@ impl Config { } const BUILD_TARGET: &str = env!("BUILD_TARGET"); -const BUILD_HOST: &str = env!("BUILD_HOST"); pub struct LanguageConfiguration<'a> { pub scope: Option, @@ -305,10 +586,10 @@ pub struct LanguageConfiguration<'a> { pub injection_regex: Option, pub file_types: Vec, pub root_path: PathBuf, - pub highlights_filenames: Option>, - pub injections_filenames: Option>, - pub locals_filenames: Option>, - pub tags_filenames: Option>, + pub highlights_filenames: Option>, + pub injections_filenames: Option>, + pub locals_filenames: Option>, + pub tags_filenames: Option>, pub language_name: String, language_id: usize, #[cfg(feature = "tree-sitter-highlight")] @@ -319,6 +600,7 @@ pub struct LanguageConfiguration<'a> { highlight_names: &'a Mutex>, #[cfg(feature = "tree-sitter-highlight")] use_all_highlight_names: bool, + _phantom: PhantomData<&'a ()>, } pub struct Loader { @@ -376,13 +658,25 @@ impl<'a> CompileConfig<'a> { unsafe impl Sync for Loader {} impl Loader { - pub fn new() -> Result { - let parser_lib_path = match env::var("TREE_SITTER_LIBDIR") { - Ok(path) => PathBuf::from(path), - _ => dirs::cache_dir() - .ok_or_else(|| anyhow!("Cannot determine cache directory"))? + pub fn new() -> LoaderResult { + let parser_lib_path = if let Ok(path) = env::var("TREE_SITTER_LIBDIR") { + PathBuf::from(path) + } else { + if cfg!(target_os = "macos") { + let legacy_apple_path = etcetera::base_strategy::Apple::new()? + .cache_dir() // `$HOME/Library/Caches/` + .join("tree-sitter"); + if legacy_apple_path.exists() && legacy_apple_path.is_dir() { + std::fs::remove_dir_all(&legacy_apple_path).map_err(|e| { + LoaderError::IO(IoError::new(e, Some(legacy_apple_path.as_path()))) + })?; + } + } + + etcetera::choose_base_strategy()? + .cache_dir() .join("tree-sitter") - .join("lib"), + .join("lib") }; Ok(Self::with_parser_lib_path(parser_lib_path)) } @@ -425,17 +719,19 @@ impl Loader { self.highlight_names.lock().unwrap().clone() } - pub fn find_all_languages(&mut self, config: &Config) -> Result<()> { + pub fn find_all_languages(&mut self, config: &Config) -> LoaderResult<()> { if config.parser_directories.is_empty() { - eprintln!("Warning: You have not configured any parser directories!"); - eprintln!("Please run `tree-sitter init-config` and edit the resulting"); - eprintln!("configuration file to indicate where we should look for"); - eprintln!("language grammars.\n"); + warn!(concat!( + "You have not configured any parser directories!\n", + "Please run `tree-sitter init-config` and edit the resulting\n", + "configuration file to indicate where we should look for\n", + "language grammars.\n" + )); } for parser_container_dir in &config.parser_directories { if let Ok(entries) = fs::read_dir(parser_container_dir) { for entry in entries { - let entry = entry?; + let entry = entry.map_err(|e| LoaderError::IO(IoError::new(e, None)))?; if let Some(parser_dir_name) = entry.file_name().to_str() { if parser_dir_name.starts_with("tree-sitter-") { self.find_language_configurations_at_path( @@ -451,7 +747,7 @@ impl Loader { Ok(()) } - pub fn languages_at_path(&mut self, path: &Path) -> Result> { + pub fn languages_at_path(&mut self, path: &Path) -> LoaderResult> { if let Ok(configurations) = self.find_language_configurations_at_path(path, true) { let mut language_ids = configurations .iter() @@ -462,14 +758,14 @@ impl Loader { language_ids .into_iter() .map(|(id, name)| Ok((self.language_for_id(id)?, name))) - .collect::>>() + .collect::>>() } else { Ok(Vec::new()) } } #[must_use] - pub fn get_all_language_configurations(&self) -> Vec<(&LanguageConfiguration, &Path)> { + pub fn get_all_language_configurations(&self) -> Vec<(&LanguageConfiguration<'static>, &Path)> { self.language_configurations .iter() .map(|c| (c, self.languages_by_id[c.language_id].0.as_ref())) @@ -479,7 +775,7 @@ impl Loader { pub fn language_configuration_for_scope( &self, scope: &str, - ) -> Result> { + ) -> LoaderResult)>> { for configuration in &self.language_configurations { if configuration.scope.as_ref().is_some_and(|s| s == scope) { let language = self.language_for_id(configuration.language_id)?; @@ -492,14 +788,19 @@ impl Loader { pub fn language_configuration_for_first_line_regex( &self, path: &Path, - ) -> Result> { + ) -> LoaderResult)>> { self.language_configuration_ids_by_first_line_regex .iter() .try_fold(None, |_, (regex, ids)| { if let Some(regex) = Self::regex(Some(regex)) { - let file = fs::File::open(path)?; + let file = fs::File::open(path) + .map_err(|e| LoaderError::IO(IoError::new(e, Some(path))))?; let reader = BufReader::new(file); - let first_line = reader.lines().next().transpose()?; + let first_line = reader + .lines() + .next() + .transpose() + .map_err(|e| LoaderError::IO(IoError::new(e, Some(path))))?; if let Some(first_line) = first_line { if regex.is_match(&first_line) && !ids.is_empty() { let configuration = &self.language_configurations[ids[0]]; @@ -516,7 +817,7 @@ impl Loader { pub fn language_configuration_for_file_name( &self, path: &Path, - ) -> Result> { + ) -> LoaderResult)>> { // Find all the language configurations that match this file name // or a suffix of the file name. let configuration_ids = path @@ -544,7 +845,7 @@ impl Loader { // one to use by applying the configurations' content regexes. else { let file_contents = - fs::read(path).with_context(|| format!("Failed to read path {path:?}"))?; + fs::read(path).map_err(|e| LoaderError::IO(IoError::new(e, Some(path))))?; let file_contents = String::from_utf8_lossy(&file_contents); let mut best_score = -2isize; let mut best_configuration_id = None; @@ -588,7 +889,7 @@ impl Loader { pub fn language_configuration_for_injection_string( &self, string: &str, - ) -> Result> { + ) -> LoaderResult)>> { let mut best_match_length = 0; let mut best_match_position = None; for (i, configuration) in self.language_configurations.iter().enumerate() { @@ -615,11 +916,11 @@ impl Loader { pub fn language_for_configuration( &self, configuration: &LanguageConfiguration, - ) -> Result { + ) -> LoaderResult { self.language_for_id(configuration.language_id) } - fn language_for_id(&self, id: usize) -> Result { + fn language_for_id(&self, id: usize) -> LoaderResult { let (path, language, externals) = &self.languages_by_id[id]; language .get_or_try_init(|| { @@ -638,25 +939,25 @@ impl Loader { grammar_path: &Path, output_path: PathBuf, flags: &[&str], - ) -> Result<()> { + ) -> LoaderResult<()> { let src_path = grammar_path.join("src"); let mut config = CompileConfig::new(&src_path, None, Some(output_path)); config.flags = flags; self.load_language_at_path(config).map(|_| ()) } - pub fn load_language_at_path(&self, mut config: CompileConfig) -> Result { + pub fn load_language_at_path(&self, mut config: CompileConfig) -> LoaderResult { let grammar_path = config.src_path.join("grammar.json"); config.name = Self::grammar_json_name(&grammar_path)?; self.load_language_at_path_with_name(config) } - pub fn load_language_at_path_with_name(&self, mut config: CompileConfig) -> Result { - let mut lib_name = config.name.to_string(); - let language_fn_name = format!( - "tree_sitter_{}", - replace_dashes_with_underscores(&config.name) - ); + pub fn load_language_at_path_with_name( + &self, + mut config: CompileConfig, + ) -> LoaderResult { + let mut lib_name = config.name.clone(); + let language_fn_name = format!("tree_sitter_{}", config.name.replace('-', "_")); if self.debug_build { lib_name.push_str(".debug._"); } @@ -667,7 +968,9 @@ impl Loader { } if config.output_path.is_none() { - fs::create_dir_all(&self.parser_lib_path)?; + fs::create_dir_all(&self.parser_lib_path).map_err(|e| { + LoaderError::IO(IoError::new(e, Some(self.parser_lib_path.as_path()))) + })?; } let mut recompile = self.force_rebuild || config.output_path.is_some(); // if specified, always recompile @@ -701,8 +1004,7 @@ impl Loader { ); if !recompile { - recompile = needs_recompile(&output_path, &paths_to_check) - .with_context(|| "Failed to compare source and binary timestamps")?; + recompile = needs_recompile(&output_path, &paths_to_check)?; } #[cfg(feature = "wasm")] @@ -710,68 +1012,82 @@ impl Loader { if recompile { self.compile_parser_to_wasm( &config.name, - None, config.src_path, config .scanner_path .as_ref() .and_then(|p| p.strip_prefix(config.src_path).ok()), &output_path, - false, )?; } - let wasm_bytes = fs::read(&output_path)?; + let wasm_bytes = fs::read(&output_path) + .map_err(|e| LoaderError::IO(IoError::new(e, Some(output_path.as_path()))))?; return Ok(wasm_store.load_language(&config.name, &wasm_bytes)?); } + // Create a unique lock path based on the output path hash to prevent + // interference when multiple processes build the same grammar (by name) + // to different output locations + let lock_hash = { + let mut hasher = std::hash::DefaultHasher::new(); + output_path.hash(&mut hasher); + format!("{:x}", hasher.finish()) + }; + let lock_path = if env::var("CROSS_RUNNER").is_ok() { tempfile::tempdir() - .unwrap() + .expect("create a temp dir") .path() - .join("tree-sitter") - .join("lock") - .join(format!("{}.lock", config.name)) + .to_path_buf() } else { - dirs::cache_dir() - .ok_or_else(|| anyhow!("Cannot determine cache directory"))? - .join("tree-sitter") - .join("lock") - .join(format!("{}.lock", config.name)) - }; + etcetera::choose_base_strategy()?.cache_dir() + } + .join("tree-sitter") + .join("lock") + .join(format!("{}-{lock_hash}.lock", config.name)); if let Ok(lock_file) = fs::OpenOptions::new().write(true).open(&lock_path) { recompile = false; if lock_file.try_lock_exclusive().is_err() { // if we can't acquire the lock, another process is compiling the parser, wait for // it and don't recompile - lock_file.lock_exclusive()?; + lock_file + .lock_exclusive() + .map_err(|e| LoaderError::IO(IoError::new(e, Some(lock_path.as_path()))))?; recompile = false; } else { // if we can acquire the lock, check if the lock file is older than 30 seconds, a // run that was interrupted and left the lock file behind should not block // subsequent runs - let time = lock_file.metadata()?.modified()?.elapsed()?.as_secs(); + let time = lock_file + .metadata() + .map_err(|e| LoaderError::IO(IoError::new(e, Some(lock_path.as_path()))))? + .modified() + .map_err(|e| LoaderError::IO(IoError::new(e, Some(lock_path.as_path()))))? + .elapsed()? + .as_secs(); if time > 30 { - fs::remove_file(&lock_path)?; + fs::remove_file(&lock_path) + .map_err(|e| LoaderError::IO(IoError::new(e, Some(lock_path.as_path()))))?; recompile = true; } } } if recompile { - fs::create_dir_all(lock_path.parent().unwrap()).with_context(|| { - format!( - "Failed to create directory {:?}", - lock_path.parent().unwrap() - ) - })?; + let parent_path = lock_path.parent().unwrap(); + fs::create_dir_all(parent_path) + .map_err(|e| LoaderError::IO(IoError::new(e, Some(parent_path))))?; let lock_file = fs::OpenOptions::new() .create(true) .truncate(true) .write(true) - .open(&lock_path)?; - lock_file.lock_exclusive()?; + .open(&lock_path) + .map_err(|e| LoaderError::IO(IoError::new(e, Some(lock_path.as_path()))))?; + lock_file + .lock_exclusive() + .map_err(|e| LoaderError::IO(IoError::new(e, Some(lock_path.as_path()))))?; self.compile_parser_to_dylib(&config, &lock_file, &lock_path)?; @@ -780,12 +1096,46 @@ impl Loader { } } - let library = unsafe { Library::new(&output_path) } - .with_context(|| format!("Error opening dynamic library {output_path:?}"))?; + // Ensure the dynamic library exists before trying to load it. This can + // happen in race conditions where we couldn't acquire the lock because + // another process was compiling but it still hasn't finished by the + // time we reach this point, so the output file still doesn't exist. + // + // Instead of allowing the `load_language` call below to fail, return a + // clearer error to the user here. + if !output_path.exists() { + let msg = format!( + "Dynamic library `{}` not found after build attempt. \ + Are you running multiple processes building to the same output location?", + output_path.display() + ); + + Err(LoaderError::IO(IoError::new( + std::io::Error::new(std::io::ErrorKind::NotFound, msg), + Some(output_path.as_path()), + )))?; + } + + Self::load_language(&output_path, &language_fn_name) + } + + pub fn load_language(path: &Path, function_name: &str) -> LoaderResult { + let library = unsafe { Library::new(path) }.map_err(|e| { + LoaderError::Library(LibraryError { + error: e, + path: path.to_string_lossy().to_string(), + }) + })?; let language = unsafe { let language_fn = library - .get:: Language>>(language_fn_name.as_bytes()) - .with_context(|| format!("Failed to load symbol {language_fn_name}"))?; + .get:: Language>>(function_name.as_bytes()) + .map_err(|e| { + LoaderError::Symbol(SymbolError { + error: e, + symbol_name: function_name.to_string(), + path: path.to_string_lossy().to_string(), + }) + })?; language_fn() }; mem::forget(library); @@ -797,13 +1147,16 @@ impl Loader { config: &CompileConfig, lock_file: &fs::File, lock_path: &Path, - ) -> Result<(), Error> { + ) -> LoaderResult<()> { let mut cc_config = cc::Build::new(); cc_config .cargo_metadata(false) .cargo_warnings(false) .target(BUILD_TARGET) - .host(BUILD_HOST) + // BUILD_TARGET from the build environment becomes a runtime host for cc. + // Otherwise, when cross compiled, cc will keep looking for a cross-compiler + // on the target system instead of the native compiler. + .host(BUILD_TARGET) .debug(self.debug_build) .file(&config.parser_path) .includes(&config.header_paths) @@ -832,12 +1185,27 @@ impl Loader { let output_path = config.output_path.as_ref().unwrap(); - if compiler.is_like_msvc() { + let temp_dir = if compiler.is_like_msvc() { let out = format!("-out:{}", output_path.to_str().unwrap()); command.arg(if self.debug_build { "-LDd" } else { "-LD" }); command.arg("-utf-8"); + + // Windows creates intermediate files when compiling (.exp, .lib, .obj), which causes + // issues when multiple processes are compiling in the same directory. This creates a + // temporary directory for those files to go into, which is deleted after compilation. + let temp_dir = output_path.parent().unwrap().join(format!( + "tmp_{}_{:?}", + std::process::id(), + std::thread::current().id() + )); + std::fs::create_dir_all(&temp_dir).unwrap(); + + command.arg(format!("/Fo{}\\", temp_dir.display())); command.args(cc_config.get_files()); command.arg("-link").arg(out); + command.arg(format!("/IMPLIB:{}.lib", temp_dir.join("temp").display())); + + Some(temp_dir) } else { command.arg("-Werror=implicit-function-declaration"); if cfg!(any(target_os = "macos", target_os = "ios")) { @@ -849,33 +1217,48 @@ impl Loader { } command.args(cc_config.get_files()); command.arg("-o").arg(output_path); - } - let output = command.output().with_context(|| { - format!("Failed to execute the C compiler with the following command:\n{command:?}") + None + }; + + let output = command.output().map_err(|e| { + LoaderError::Compiler(CompilerError { + error: e, + command: Box::new(command), + }) })?; - FileExt::unlock(lock_file)?; - fs::remove_file(lock_path)?; + if let Some(temp_dir) = temp_dir { + let _ = fs::remove_dir_all(temp_dir); + } + + FileExt::unlock(lock_file) + .map_err(|e| LoaderError::IO(IoError::new(e, Some(lock_path))))?; + fs::remove_file(lock_path) + .map_err(|e| LoaderError::IO(IoError::new(e, Some(lock_path))))?; if output.status.success() { Ok(()) } else { - Err(anyhow!( - "Parser compilation failed.\nStdout: {}\nStderr: {}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) + Err(LoaderError::Compilation( + String::from_utf8_lossy(&output.stdout).to_string(), + String::from_utf8_lossy(&output.stderr).to_string(), )) } } #[cfg(unix)] - fn check_external_scanner(&self, name: &str, library_path: &Path) -> Result<()> { + fn check_external_scanner(&self, name: &str, library_path: &Path) -> LoaderResult<()> { let prefix = if cfg!(any(target_os = "macos", target_os = "ios")) { "_" } else { "" }; + let section = if cfg!(all(target_arch = "powerpc64", target_os = "linux")) { + " D " + } else { + " T " + }; let mut must_have = vec![ format!("{prefix}tree_sitter_{name}_external_scanner_create"), format!("{prefix}tree_sitter_{name}_external_scanner_destroy"), @@ -884,25 +1267,25 @@ impl Loader { format!("{prefix}tree_sitter_{name}_external_scanner_scan"), ]; - let command = Command::new("nm") - .arg("-W") - .arg("-U") + let nm_cmd = env::var("NM").unwrap_or_else(|_| "nm".to_owned()); + let command = Command::new(nm_cmd) + .arg("--defined-only") .arg(library_path) .output(); if let Ok(output) = command { if output.status.success() { let mut found_non_static = false; for line in String::from_utf8_lossy(&output.stdout).lines() { - if line.contains(" T ") { + if line.contains(section) { if let Some(function_name) = line.split_whitespace().collect::>().get(2) { if !line.contains("tree_sitter_") { if !found_non_static { found_non_static = true; - eprintln!("Warning: Found non-static non-tree-sitter functions in the external scannner"); + warn!("Found non-static non-tree-sitter functions in the external scanner"); } - eprintln!(" `{function_name}`"); + warn!(" `{function_name}`"); } else { must_have.retain(|f| f != function_name); } @@ -910,35 +1293,30 @@ impl Loader { } } if found_non_static { - eprintln!("Consider making these functions static, they can cause conflicts when another tree-sitter project uses the same function name"); + warn!(concat!( + "Consider making these functions static, they can cause conflicts ", + "when another tree-sitter project uses the same function name." + )); } if !must_have.is_empty() { - let missing = must_have - .iter() - .map(|f| format!(" `{f}`")) - .collect::>() - .join("\n"); - - return Err(anyhow!(format!( - indoc! {" - Missing required functions in the external scanner, parsing won't work without these! - - {} - - You can read more about this at https://tree-sitter.github.io/tree-sitter/creating-parsers#external-scanners - "}, - missing, - ))); + return Err(LoaderError::ScannerSymbols(ScannerSymbolError { + missing: must_have, + })); } } + } else { + warn!( + "Failed to run `nm` to verify symbols in {}", + library_path.display() + ); } Ok(()) } #[cfg(windows)] - fn check_external_scanner(&self, _name: &str, _library_path: &Path) -> Result<()> { + fn check_external_scanner(&self, _name: &str, _library_path: &Path) -> LoaderResult<()> { // TODO: there's no nm command on windows, whoever wants to implement this can and should :) // let mut must_have = vec![ @@ -955,143 +1333,184 @@ impl Loader { pub fn compile_parser_to_wasm( &self, language_name: &str, - root_path: Option<&Path>, src_path: &Path, scanner_filename: Option<&Path>, output_path: &Path, - force_docker: bool, - ) -> Result<(), Error> { - #[derive(PartialEq, Eq)] - enum EmccSource { - Native, - Docker, - Podman, - } + ) -> LoaderResult<()> { + let clang_executable = self.ensure_wasi_sdk_exists()?; - let root_path = root_path.unwrap_or(src_path); - let emcc_name = if cfg!(windows) { "emcc.bat" } else { "emcc" }; - - // Order of preference: emscripten > docker > podman > error - let source = if !force_docker && Command::new(emcc_name).output().is_ok() { - EmccSource::Native - } else if Command::new("docker") - .arg("info") - .output() - .is_ok_and(|out| out.status.success()) - { - EmccSource::Docker - } else if Command::new("podman") - .arg("--version") - .output() - .is_ok_and(|out| out.status.success()) - { - EmccSource::Podman - } else { - return Err(anyhow!( - "You must have either emcc, docker, or podman on your PATH to run this command" - )); - }; - - let mut command = match source { - EmccSource::Native => { - let mut command = Command::new(emcc_name); - command.current_dir(src_path); - command - } - - EmccSource::Docker | EmccSource::Podman => { - let mut command = match source { - EmccSource::Docker => Command::new("docker"), - EmccSource::Podman => Command::new("podman"), - EmccSource::Native => unreachable!(), - }; - command.args(["run", "--rm"]); - - // The working directory is the directory containing the parser itself - let workdir = if root_path == src_path { - PathBuf::from("/src") - } else { - let mut path = PathBuf::from("/src"); - path.push(src_path.strip_prefix(root_path).unwrap()); - path - }; - command.args(["--workdir", &workdir.to_slash_lossy()]); - - // Mount the root directory as a volume, which is the repo root - let mut volume_string = OsString::from(&root_path); - volume_string.push(":/src:Z"); - command.args([OsStr::new("--volume"), &volume_string]); - - // In case `docker` is an alias to `podman`, ensure that podman - // mounts the current directory as writable by the container - // user which has the same uid as the host user. Setting the - // podman-specific variable is more reliable than attempting to - // detect whether `docker` is an alias for `podman`. - // see https://docs.podman.io/en/latest/markdown/podman-run.1.html#userns-mode - command.env("PODMAN_USERNS", "keep-id"); - - // Get the current user id so that files created in the docker container will have - // the same owner. - #[cfg(unix)] - { - #[link(name = "c")] - extern "C" { - fn getuid() -> u32; - } - // don't need to set user for podman since PODMAN_USERNS=keep-id is already set - if source == EmccSource::Docker { - let user_id = unsafe { getuid() }; - command.args(["--user", &user_id.to_string()]); - } - }; - - // Run `emcc` in a container using the `emscripten-slim` image - command.args([EMSCRIPTEN_TAG, "emcc"]); - command - } - }; - - let output_name = "output.wasm"; - - command.args([ + let mut command = Command::new(&clang_executable); + command.current_dir(src_path).args([ "-o", - output_name, - "-Os", - "-s", - "WASM=1", - "-s", - "SIDE_MODULE=2", - "-s", - "TOTAL_MEMORY=33554432", - "-s", - "NODEJS_CATCH_EXIT=0", - "-s", - &format!("EXPORTED_FUNCTIONS=[\"_tree_sitter_{language_name}\"]"), + output_path.to_str().unwrap(), + "-fPIC", + "-shared", + if self.debug_build { "-g" } else { "-Os" }, + format!("-Wl,--export=tree_sitter_{language_name}").as_str(), + "-Wl,--allow-undefined", + "-Wl,--no-entry", + "-nostdlib", "-fno-exceptions", "-fvisibility=hidden", "-I", ".", + "parser.c", ]); if let Some(scanner_filename) = scanner_filename { command.arg(scanner_filename); } - command.arg("parser.c"); - let status = command - .spawn() - .with_context(|| "Failed to run emcc command")? - .wait()?; - if !status.success() { - return Err(anyhow!("emcc command failed")); + let output = command.output().map_err(LoaderError::WasmCompiler)?; + + if !output.status.success() { + return Err(LoaderError::WasmCompilation( + String::from_utf8_lossy(&output.stderr).to_string(), + )); } - fs::rename(src_path.join(output_name), output_path) - .context("failed to rename wasm output file")?; + Ok(()) + } + + /// Extracts a tar.gz archive with `tar`, stripping the first path component. + fn extract_tar_gz_with_strip( + &self, + archive_path: &Path, + destination: &Path, + ) -> LoaderResult<()> { + let status = Command::new("tar") + .arg("-xzf") + .arg(archive_path) + .arg("--strip-components=1") + .arg("-C") + .arg(destination) + .status() + .map_err(|e| LoaderError::Tar(archive_path.to_string_lossy().to_string(), e))?; + + if !status.success() { + return Err(LoaderError::Extraction( + archive_path.to_string_lossy().to_string(), + destination.to_string_lossy().to_string(), + )); + } Ok(()) } + /// This ensures that the wasi-sdk is available, downloading and extracting it if necessary, + /// and returns the path to the `clang` executable. + /// + /// If `TREE_SITTER_WASI_SDK_PATH` is set, it will use that path to look for the clang executable. + fn ensure_wasi_sdk_exists(&self) -> LoaderResult { + let possible_executables = if cfg!(windows) { + vec![ + "clang.exe", + "wasm32-unknown-wasi-clang.exe", + "wasm32-wasi-clang.exe", + ] + } else { + vec!["clang", "wasm32-unknown-wasi-clang", "wasm32-wasi-clang"] + }; + + if let Ok(wasi_sdk_path) = std::env::var("TREE_SITTER_WASI_SDK_PATH") { + let wasi_sdk_dir = PathBuf::from(wasi_sdk_path); + + for exe in &possible_executables { + let clang_exe = wasi_sdk_dir.join("bin").join(exe); + if clang_exe.exists() { + return Ok(clang_exe); + } + } + + return Err(LoaderError::WasiSDKClang(WasiSDKClangError { + wasi_sdk_dir: wasi_sdk_dir.to_string_lossy().to_string(), + possible_executables, + download: false, + })); + } + + let cache_dir = etcetera::choose_base_strategy()? + .cache_dir() + .join("tree-sitter"); + fs::create_dir_all(&cache_dir) + .map_err(|e| LoaderError::IO(IoError::new(e, Some(cache_dir.as_path()))))?; + + let wasi_sdk_dir = cache_dir.join("wasi-sdk"); + + for exe in &possible_executables { + let clang_exe = wasi_sdk_dir.join("bin").join(exe); + if clang_exe.exists() { + return Ok(clang_exe); + } + } + + fs::create_dir_all(&wasi_sdk_dir) + .map_err(|e| LoaderError::IO(IoError::new(e, Some(wasi_sdk_dir.as_path()))))?; + + let arch_os = if cfg!(target_os = "macos") { + if cfg!(target_arch = "aarch64") { + "arm64-macos" + } else { + "x86_64-macos" + } + } else if cfg!(target_os = "windows") { + if cfg!(target_arch = "aarch64") { + "arm64-windows" + } else { + "x86_64-windows" + } + } else if cfg!(target_os = "linux") { + if cfg!(target_arch = "aarch64") { + "arm64-linux" + } else { + "x86_64-linux" + } + } else { + return Err(LoaderError::WasiSDKPlatform); + }; + + let sdk_filename = format!("wasi-sdk-{WASI_SDK_VERSION}-{arch_os}.tar.gz"); + let wasi_sdk_major_version = WASI_SDK_VERSION + .trim_end_matches(char::is_numeric) // trim minor version... + .trim_end_matches('.'); // ...and '.' separator + let sdk_url = format!( + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-{wasi_sdk_major_version}/{sdk_filename}", + ); + + info!("Downloading wasi-sdk from {sdk_url}..."); + let temp_tar_path = cache_dir.join(sdk_filename); + + let status = Command::new("curl") + .arg("-f") + .arg("-L") + .arg("-o") + .arg(&temp_tar_path) + .arg(&sdk_url) + .status() + .map_err(|e| LoaderError::Curl(sdk_url.clone(), e))?; + + if !status.success() { + return Err(LoaderError::WasiSDKDownload(sdk_url)); + } + + info!("Extracting wasi-sdk to {}...", wasi_sdk_dir.display()); + self.extract_tar_gz_with_strip(&temp_tar_path, &wasi_sdk_dir)?; + + fs::remove_file(temp_tar_path).ok(); + for exe in &possible_executables { + let clang_exe = wasi_sdk_dir.join("bin").join(exe); + if clang_exe.exists() { + return Ok(clang_exe); + } + } + + Err(LoaderError::WasiSDKClang(WasiSDKClangError { + wasi_sdk_dir: wasi_sdk_dir.to_string_lossy().to_string(), + possible_executables, + download: true, + })) + } + #[must_use] #[cfg(feature = "tree-sitter-highlight")] pub fn highlight_config_for_injection_string<'a>( @@ -1100,15 +1519,15 @@ impl Loader { ) -> Option<&'a HighlightConfiguration> { match self.language_configuration_for_injection_string(string) { Err(e) => { - eprintln!("Failed to load language for injection string '{string}': {e}",); + error!("Failed to load language for injection string '{string}': {e}",); None } Ok(None) => None, Ok(Some((language, configuration))) => { match configuration.highlight_config(language, None) { Err(e) => { - eprintln!( - "Failed to load property sheet for injection string '{string}': {e}", + error!( + "Failed to load higlight config for injection string '{string}': {e}" ); None } @@ -1119,116 +1538,133 @@ impl Loader { } } + #[must_use] + pub fn get_language_configuration_in_current_path( + &self, + ) -> Option<&LanguageConfiguration<'static>> { + self.language_configuration_in_current_path + .map(|i| &self.language_configurations[i]) + } + pub fn find_language_configurations_at_path( &mut self, parser_path: &Path, set_current_path_config: bool, - ) -> Result<&[LanguageConfiguration]> { + ) -> LoaderResult<&[LanguageConfiguration<'static>]> { let initial_language_configuration_count = self.language_configurations.len(); - let ts_json = TreeSitterJSON::from_file(parser_path); - if let Ok(config) = ts_json { - let language_count = self.languages_by_id.len(); - for grammar in config.grammars { - // Determine the path to the parser directory. This can be specified in - // the tree-sitter.json, but defaults to the directory containing the - // tree-sitter.json. - let language_path = parser_path.join(grammar.path.unwrap_or(PathBuf::from("."))); + match TreeSitterJSON::from_file(parser_path) { + Ok(config) => { + let language_count = self.languages_by_id.len(); + for grammar in config.grammars { + // Determine the path to the parser directory. This can be specified in + // the tree-sitter.json, but defaults to the directory containing the + // tree-sitter.json. + let language_path = + parser_path.join(grammar.path.unwrap_or(PathBuf::from("."))); - // Determine if a previous language configuration in this package.json file - // already uses the same language. - let mut language_id = None; - for (id, (path, _, _)) in - self.languages_by_id.iter().enumerate().skip(language_count) - { - if language_path == *path { - language_id = Some(id); + // Determine if a previous language configuration in this package.json file + // already uses the same language. + let mut language_id = None; + for (id, (path, _, _)) in + self.languages_by_id.iter().enumerate().skip(language_count) + { + if language_path == *path { + language_id = Some(id); + } } - } - // If not, add a new language path to the list. - let language_id = if let Some(language_id) = language_id { - language_id - } else { - self.languages_by_id.push(( + // If not, add a new language path to the list. + let language_id = if let Some(language_id) = language_id { + language_id + } else { + self.languages_by_id.push(( language_path, OnceCell::new(), - grammar.external_files.clone().into_vec().map(|files| { - files.into_iter() - .map(|path| { - let path = parser_path.join(path); - // prevent p being above/outside of parser_path - if path.starts_with(parser_path) { - Ok(path) - } else { - Err(anyhow!("External file path {path:?} is outside of parser directory {parser_path:?}")) - } - }) - .collect::>>() - }).transpose()?, + grammar + .external_files + .clone() + .into_vec() + .map(|files| { + files + .into_iter() + .map(|path| { + let path = parser_path.join(path); + // prevent p being above/outside of parser_path + if path.starts_with(parser_path) { + Ok(path) + } else { + Err(LoaderError::ExternalFile( + path.to_string_lossy().to_string(), + parser_path.to_string_lossy().to_string(), + )) + } + }) + .collect::>>() + }) + .transpose()?, )); - self.languages_by_id.len() - 1 - }; + self.languages_by_id.len() - 1 + }; - let configuration = LanguageConfiguration { - root_path: parser_path.to_path_buf(), - language_name: grammar.name, - scope: Some(grammar.scope), - language_id, - file_types: grammar.file_types.unwrap_or_default(), - content_regex: Self::regex(grammar.content_regex.as_deref()), - first_line_regex: Self::regex(grammar.first_line_regex.as_deref()), - injection_regex: Self::regex(grammar.injection_regex.as_deref()), - injections_filenames: grammar.injections.into_vec(), - locals_filenames: grammar.locals.into_vec(), - tags_filenames: grammar.tags.into_vec(), - highlights_filenames: grammar.highlights.into_vec(), - #[cfg(feature = "tree-sitter-highlight")] - highlight_config: OnceCell::new(), - #[cfg(feature = "tree-sitter-tags")] - tags_config: OnceCell::new(), - #[cfg(feature = "tree-sitter-highlight")] - highlight_names: &self.highlight_names, - #[cfg(feature = "tree-sitter-highlight")] - use_all_highlight_names: self.use_all_highlight_names, - }; + let configuration = LanguageConfiguration { + root_path: parser_path.to_path_buf(), + language_name: grammar.name, + scope: Some(grammar.scope), + language_id, + file_types: grammar.file_types.unwrap_or_default(), + content_regex: Self::regex(grammar.content_regex.as_deref()), + first_line_regex: Self::regex(grammar.first_line_regex.as_deref()), + injection_regex: Self::regex(grammar.injection_regex.as_deref()), + injections_filenames: grammar.injections.into_vec(), + locals_filenames: grammar.locals.into_vec(), + tags_filenames: grammar.tags.into_vec(), + highlights_filenames: grammar.highlights.into_vec(), + #[cfg(feature = "tree-sitter-highlight")] + highlight_config: OnceCell::new(), + #[cfg(feature = "tree-sitter-tags")] + tags_config: OnceCell::new(), + #[cfg(feature = "tree-sitter-highlight")] + highlight_names: &self.highlight_names, + #[cfg(feature = "tree-sitter-highlight")] + use_all_highlight_names: self.use_all_highlight_names, + _phantom: PhantomData, + }; - for file_type in &configuration.file_types { - self.language_configuration_ids_by_file_type - .entry(file_type.to_string()) - .or_default() - .push(self.language_configurations.len()); - } - if let Some(first_line_regex) = &configuration.first_line_regex { - self.language_configuration_ids_by_first_line_regex - .entry(first_line_regex.to_string()) - .or_default() - .push(self.language_configurations.len()); - } + for file_type in &configuration.file_types { + self.language_configuration_ids_by_file_type + .entry(file_type.clone()) + .or_default() + .push(self.language_configurations.len()); + } + if let Some(first_line_regex) = &configuration.first_line_regex { + self.language_configuration_ids_by_first_line_regex + .entry(first_line_regex.to_string()) + .or_default() + .push(self.language_configurations.len()); + } - self.language_configurations.push(unsafe { - mem::transmute::, LanguageConfiguration<'static>>( - configuration, - ) - }); + self.language_configurations.push(unsafe { + mem::transmute::, LanguageConfiguration<'static>>( + configuration, + ) + }); - if set_current_path_config && self.language_configuration_in_current_path.is_none() - { - self.language_configuration_in_current_path = - Some(self.language_configurations.len() - 1); + if set_current_path_config + && self.language_configuration_in_current_path.is_none() + { + self.language_configuration_in_current_path = + Some(self.language_configurations.len() - 1); + } } } - } else if let Err(e) = ts_json { - match e.downcast_ref::() { - // This is noisy, and not really an issue. - Some(e) if e.kind() == std::io::ErrorKind::NotFound => {} - _ => { - eprintln!( - "Warning: Failed to parse {} -- {e}", - parser_path.join("tree-sitter.json").display() - ); - } + Err(LoaderError::Serialization(e)) => { + warn!( + "Failed to parse {} -- {e}", + parser_path.join("tree-sitter.json").display() + ); } + _ => {} } // If we didn't find any language configurations in the tree-sitter.json file, @@ -1260,6 +1696,7 @@ impl Loader { highlight_names: &self.highlight_names, #[cfg(feature = "tree-sitter-highlight")] use_all_highlight_names: self.use_all_highlight_names, + _phantom: PhantomData, }; self.language_configurations.push(unsafe { mem::transmute::, LanguageConfiguration<'static>>( @@ -1277,86 +1714,86 @@ impl Loader { pattern.and_then(|r| RegexBuilder::new(r).multi_line(true).build().ok()) } - fn grammar_json_name(grammar_path: &Path) -> Result { - let file = fs::File::open(grammar_path).with_context(|| { - format!("Failed to open grammar.json at {}", grammar_path.display()) - })?; + fn grammar_json_name(grammar_path: &Path) -> LoaderResult { + let file = fs::File::open(grammar_path) + .map_err(|e| LoaderError::IO(IoError::new(e, Some(grammar_path))))?; let first_three_lines = BufReader::new(file) .lines() .take(3) - .collect::, _>>() - .with_context(|| { - format!( - "Failed to read the first three lines of grammar.json at {}", - grammar_path.display() - ) - })? + .collect::, std::io::Error>>() + .map_err(|_| LoaderError::GrammarJSON(grammar_path.to_string_lossy().to_string()))? .join("\n"); let name = GRAMMAR_NAME_REGEX .captures(&first_three_lines) .and_then(|c| c.get(1)) - .ok_or_else(|| { - anyhow!( - "Failed to parse the language name from grammar.json at {}", - grammar_path.display() - ) - })?; + .ok_or_else(|| LoaderError::GrammarJSON(grammar_path.to_string_lossy().to_string()))?; Ok(name.as_str().to_string()) } pub fn select_language( &mut self, - path: &Path, + path: Option<&Path>, current_dir: &Path, scope: Option<&str>, - ) -> Result { - if let Some(scope) = scope { + // path to dynamic library, name of language + lib_info: Option<&(PathBuf, &str)>, + ) -> LoaderResult { + if let Some((ref lib_path, language_name)) = lib_info { + let language_fn_name = format!("tree_sitter_{}", language_name.replace('-', "_")); + Self::load_language(lib_path, &language_fn_name) + } else if let Some(scope) = scope { if let Some(config) = self .language_configuration_for_scope(scope) - .with_context(|| format!("Failed to load language for scope '{scope}'"))? + .map_err(|e| LoaderError::ScopeLoad(scope.to_string(), Box::new(e)))? { Ok(config.0) } else { - Err(anyhow!("Unknown scope '{scope}'")) + Err(LoaderError::UnknownScope(scope.to_string())) } - } else if let Some((lang, _)) = self - .language_configuration_for_file_name(path) - .with_context(|| { - format!( - "Failed to load language for file name {}", - &path.file_name().unwrap().to_string_lossy() - ) - })? - { + } else if let Some((lang, _)) = if let Some(path) = path { + self.language_configuration_for_file_name(path) + .map_err(|e| { + LoaderError::FileNameLoad( + path.file_name().unwrap().to_string_lossy().to_string(), + Box::new(e), + ) + })? + } else { + None + } { Ok(lang) } else if let Some(id) = self.language_configuration_in_current_path { Ok(self.language_for_id(self.language_configurations[id].language_id)?) } else if let Some(lang) = self .languages_at_path(current_dir) - .with_context(|| "Failed to load language in current directory")? + .map_err(|e| LoaderError::CurrentDirectoryLoad(Box::new(e)))? .first() .cloned() { Ok(lang.0) - } else if let Some(lang) = self.language_configuration_for_first_line_regex(path)? { + } else if let Some(lang) = if let Some(path) = path { + self.language_configuration_for_first_line_regex(path)? + } else { + None + } { Ok(lang.0) } else { - Err(anyhow!("No language found")) + Err(LoaderError::NoLanguage) } } - pub fn debug_build(&mut self, flag: bool) { + pub const fn debug_build(&mut self, flag: bool) { self.debug_build = flag; } - pub fn sanitize_build(&mut self, flag: bool) { + pub const fn sanitize_build(&mut self, flag: bool) { self.sanitize_build = flag; } - pub fn force_rebuild(&mut self, rebuild: bool) { + pub const fn force_rebuild(&mut self, rebuild: bool) { self.force_rebuild = rebuild; } @@ -1378,28 +1815,28 @@ impl LanguageConfiguration<'_> { pub fn highlight_config( &self, language: Language, - paths: Option<&[String]>, - ) -> Result> { + paths: Option<&[PathBuf]>, + ) -> LoaderResult> { let (highlights_filenames, injections_filenames, locals_filenames) = match paths { Some(paths) => ( Some( paths .iter() - .filter(|p| p.ends_with("tree-sitter-highlights.scm")) + .filter(|p| p.ends_with(DEFAULT_HIGHLIGHTS_QUERY_FILE_NAME)) .cloned() .collect::>(), ), Some( paths .iter() - .filter(|p| p.ends_with("tree-sitter-tags.scm")) + .filter(|p| p.ends_with(DEFAULT_TAGS_QUERY_FILE_NAME)) .cloned() .collect::>(), ), Some( paths .iter() - .filter(|p| p.ends_with("locals.scm")) + .filter(|p| p.ends_with(DEFAULT_LOCALS_QUERY_FILE_NAME)) .cloned() .collect::>(), ), @@ -1414,7 +1851,7 @@ impl LanguageConfiguration<'_> { } else { self.highlights_filenames.as_deref() }, - "tree-sitter-highlights.scm", + DEFAULT_HIGHLIGHTS_QUERY_FILE_NAME, )?; let (injections_query, injection_ranges) = self.read_queries( if injections_filenames.is_some() { @@ -1422,7 +1859,7 @@ impl LanguageConfiguration<'_> { } else { self.injections_filenames.as_deref() }, - "injections.scm", + DEFAULT_INJECTIONS_QUERY_FILE_NAME, )?; let (locals_query, locals_ranges) = self.read_queries( if locals_filenames.is_some() { @@ -1430,7 +1867,7 @@ impl LanguageConfiguration<'_> { } else { self.locals_filenames.as_deref() }, - "locals.scm", + DEFAULT_LOCALS_QUERY_FILE_NAME, )?; if highlights_query.is_empty() { @@ -1444,7 +1881,9 @@ impl LanguageConfiguration<'_> { &locals_query, ) .map_err(|error| match error.kind { - QueryErrorKind::Language => Error::from(error), + QueryErrorKind::Language => { + LoaderError::Query(LoaderQueryError { error, file: None }) + } _ => { if error.offset < injections_query.len() { Self::include_path_in_query_error( @@ -1487,13 +1926,15 @@ impl LanguageConfiguration<'_> { } #[cfg(feature = "tree-sitter-tags")] - pub fn tags_config(&self, language: Language) -> Result> { + pub fn tags_config(&self, language: Language) -> LoaderResult> { self.tags_config .get_or_try_init(|| { - let (tags_query, tags_ranges) = - self.read_queries(self.tags_filenames.as_deref(), "tree-sitter-tags.scm")?; - let (locals_query, locals_ranges) = - self.read_queries(self.locals_filenames.as_deref(), "locals.scm")?; + let (tags_query, tags_ranges) = self + .read_queries(self.tags_filenames.as_deref(), DEFAULT_TAGS_QUERY_FILE_NAME)?; + let (locals_query, locals_ranges) = self.read_queries( + self.locals_filenames.as_deref(), + DEFAULT_LOCALS_QUERY_FILE_NAME, + )?; if tags_query.is_empty() { Ok(None) } else { @@ -1528,10 +1969,10 @@ impl LanguageConfiguration<'_> { #[cfg(any(feature = "tree-sitter-highlight", feature = "tree-sitter-tags"))] fn include_path_in_query_error( mut error: QueryError, - ranges: &[(String, Range)], + ranges: &[(PathBuf, Range)], source: &str, start_offset: usize, - ) -> Error { + ) -> LoaderError { let offset_within_section = error.offset - start_offset; let (path, range) = ranges .iter() @@ -1541,16 +1982,19 @@ impl LanguageConfiguration<'_> { error.row = source[range.start..offset_within_section] .matches('\n') .count(); - Error::from(error).context(format!("Error in query file {path:?}")) + LoaderError::Query(LoaderQueryError { + error, + file: Some(path.to_string_lossy().to_string()), + }) } #[allow(clippy::type_complexity)] #[cfg(any(feature = "tree-sitter-highlight", feature = "tree-sitter-tags"))] fn read_queries( &self, - paths: Option<&[String]>, + paths: Option<&[PathBuf]>, default_path: &str, - ) -> Result<(String, Vec<(String, Range)>)> { + ) -> LoaderResult<(String, Vec<(PathBuf, Range)>)> { let mut query = String::new(); let mut path_ranges = Vec::new(); if let Some(paths) = paths { @@ -1558,28 +2002,30 @@ impl LanguageConfiguration<'_> { let abs_path = self.root_path.join(path); let prev_query_len = query.len(); query += &fs::read_to_string(&abs_path) - .with_context(|| format!("Failed to read query file {path:?}"))?; + .map_err(|e| LoaderError::IO(IoError::new(e, Some(abs_path.as_path()))))?; path_ranges.push((path.clone(), prev_query_len..query.len())); } } else { // highlights.scm is needed to test highlights, and tags.scm to test tags - if default_path == "tree-sitter-highlights.scm" - || default_path == "tree-sitter-tags.scm" + if default_path == DEFAULT_HIGHLIGHTS_QUERY_FILE_NAME + || default_path == DEFAULT_TAGS_QUERY_FILE_NAME { - eprintln!( - indoc! {" - Warning: you should add a `{}` entry pointing to the highlights path in `tree-sitter` language list in the grammar's package.json - See more here: https://tree-sitter.github.io/tree-sitter/syntax-highlighting#query-paths - "}, - default_path.replace(".scm", "") + warn!( + concat!( + "You should add a `{}` entry pointing to the {} path in the `tree-sitter` ", + "object in the grammar's tree-sitter.json file. See more here: ", + "https://tree-sitter.github.io/tree-sitter/3-syntax-highlighting#query-paths" + ), + default_path.replace(".scm", ""), + default_path ); } let queries_path = self.root_path.join("queries"); let path = queries_path.join(default_path); if path.exists() { query = fs::read_to_string(&path) - .with_context(|| format!("Failed to read query file {path:?}"))?; - path_ranges.push((default_path.to_string(), 0..query.len())); + .map_err(|e| LoaderError::IO(IoError::new(e, Some(path.as_path()))))?; + path_ranges.push((PathBuf::from(default_path), 0..query.len())); } } @@ -1587,32 +2033,22 @@ impl LanguageConfiguration<'_> { } } -fn needs_recompile(lib_path: &Path, paths_to_check: &[PathBuf]) -> Result { +fn needs_recompile(lib_path: &Path, paths_to_check: &[PathBuf]) -> LoaderResult { if !lib_path.exists() { return Ok(true); } - let lib_mtime = - mtime(lib_path).with_context(|| format!("Failed to read mtime of {lib_path:?}"))?; + let lib_mtime = mtime(lib_path).map_err(|e| LoaderError::ModifiedTime(Box::new(e)))?; for path in paths_to_check { - if mtime(path)? > lib_mtime { + if mtime(path).map_err(|e| LoaderError::ModifiedTime(Box::new(e)))? > lib_mtime { return Ok(true); } } Ok(false) } -fn mtime(path: &Path) -> Result { - Ok(fs::metadata(path)?.modified()?) -} - -fn replace_dashes_with_underscores(name: &str) -> String { - let mut result = String::with_capacity(name.len()); - for c in name.chars() { - if c == '-' { - result.push('_'); - } else { - result.push(c); - } - } - result +fn mtime(path: &Path) -> LoaderResult { + fs::metadata(path) + .map_err(|e| LoaderError::IO(IoError::new(e, Some(path))))? + .modified() + .map_err(|e| LoaderError::IO(IoError::new(e, Some(path)))) } diff --git a/crates/loader/wasi-sdk-version b/crates/loader/wasi-sdk-version new file mode 100644 index 00000000..231f5c77 --- /dev/null +++ b/crates/loader/wasi-sdk-version @@ -0,0 +1 @@ +29.0 diff --git a/tags/Cargo.toml b/crates/tags/Cargo.toml similarity index 90% rename from tags/Cargo.toml rename to crates/tags/Cargo.toml index 9db54306..b82647e9 100644 --- a/tags/Cargo.toml +++ b/crates/tags/Cargo.toml @@ -11,6 +11,7 @@ rust-version.workspace = true readme = "README.md" homepage.workspace = true repository.workspace = true +documentation = "https://docs.rs/tree-sitter-tags" license.workspace = true keywords = ["incremental", "parsing", "syntax", "tagging"] categories = ["parsing", "text-editors"] @@ -19,6 +20,7 @@ categories = ["parsing", "text-editors"] workspace = true [lib] +path = "src/tags.rs" crate-type = ["lib", "staticlib"] [dependencies] diff --git a/crates/tags/LICENSE b/crates/tags/LICENSE new file mode 100644 index 00000000..971b81f9 --- /dev/null +++ b/crates/tags/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2018 Max Brunsfeld + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/tags/README.md b/crates/tags/README.md similarity index 100% rename from tags/README.md rename to crates/tags/README.md diff --git a/tags/include/tree_sitter/tags.h b/crates/tags/include/tree_sitter/tags.h similarity index 100% rename from tags/include/tree_sitter/tags.h rename to crates/tags/include/tree_sitter/tags.h diff --git a/tags/src/c_lib.rs b/crates/tags/src/c_lib.rs similarity index 98% rename from tags/src/c_lib.rs rename to crates/tags/src/c_lib.rs index 6041642d..63c47a01 100644 --- a/tags/src/c_lib.rs +++ b/crates/tags/src/c_lib.rs @@ -40,8 +40,8 @@ pub struct TSTag { pub line_end_byte: u32, pub start_point: TSPoint, pub end_point: TSPoint, - pub utf16_start_colum: u32, - pub utf16_end_colum: u32, + pub utf16_start_column: u32, + pub utf16_end_column: u32, pub docs_start_byte: u32, pub docs_end_byte: u32, pub syntax_type_id: u32, @@ -198,8 +198,8 @@ pub unsafe extern "C" fn ts_tagger_tag( row: tag.span.end.row as u32, column: tag.span.end.column as u32, }, - utf16_start_colum: tag.utf16_column_range.start as u32, - utf16_end_colum: tag.utf16_column_range.end as u32, + utf16_start_column: tag.utf16_column_range.start as u32, + utf16_end_column: tag.utf16_column_range.end as u32, docs_start_byte: prev_docs_len as u32, docs_end_byte: buffer.docs.len() as u32, syntax_type_id: tag.syntax_type_id, diff --git a/tags/src/lib.rs b/crates/tags/src/tags.rs similarity index 97% rename from tags/src/lib.rs rename to crates/tags/src/tags.rs index 00debaaf..c6654876 100644 --- a/tags/src/lib.rs +++ b/crates/tags/src/tags.rs @@ -1,4 +1,4 @@ -#![doc = include_str!("../README.md")] +#![cfg_attr(not(any(test, doctest)), doc = include_str!("../README.md"))] pub mod c_lib; @@ -7,7 +7,7 @@ use std::{ collections::HashMap, ffi::{CStr, CString}, mem, - ops::Range, + ops::{ControlFlow, Range}, os::raw::c_char, str, sync::atomic::{AtomicUsize, Ordering}, @@ -43,6 +43,9 @@ pub struct TagsConfiguration { pattern_info: Vec, } +unsafe impl Send for TagsConfiguration {} +unsafe impl Sync for TagsConfiguration {} + #[derive(Debug)] pub struct NamedCapture { pub syntax_type_id: u32, @@ -149,7 +152,7 @@ impl TagsConfiguration { "doc" => doc_capture_index = Some(i as u32), "local.scope" => local_scope_capture_index = Some(i as u32), "local.definition" => local_definition_capture_index = Some(i as u32), - "local.reference" | "" => continue, + "local.reference" | "" => {} _ => { let mut is_definition = false; @@ -271,7 +274,7 @@ impl TagsContext { } } - pub fn parser(&mut self) -> &mut Parser { + pub const fn parser(&mut self) -> &mut Parser { &mut self.parser } @@ -298,14 +301,19 @@ impl TagsContext { None, Some(ParseOptions::new().progress_callback(&mut |_| { if let Some(cancellation_flag) = cancellation_flag { - cancellation_flag.load(Ordering::SeqCst) != 0 + if cancellation_flag.load(Ordering::SeqCst) != 0 { + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } } else { - false + ControlFlow::Continue(()) } })), ) .ok_or(Error::Cancelled)?; + // SAFETY: // The `matches` iterator borrows the `Tree`, which prevents it from being // moved. But the tree is really just a pointer, so it's actually ok to // move it. diff --git a/xtask/Cargo.toml b/crates/xtask/Cargo.toml similarity index 68% rename from xtask/Cargo.toml rename to crates/xtask/Cargo.toml index 83bb3008..17972317 100644 --- a/xtask/Cargo.toml +++ b/crates/xtask/Cargo.toml @@ -17,13 +17,15 @@ workspace = true [dependencies] anstyle.workspace = true anyhow.workspace = true -bindgen = { version = "0.70.1" } -cc.workspace = true +bindgen = { version = "0.72.0" } clap.workspace = true -git2.workspace = true +etcetera.workspace = true indoc.workspace = true -toml.workspace = true regex.workspace = true +schemars.workspace = true semver.workspace = true -serde.workspace = true serde_json.workspace = true +tree-sitter-cli = { path = "../cli/" } +tree-sitter-loader = { path = "../loader/" } +notify = "8.2.0" +notify-debouncer-full = "0.6.0" diff --git a/xtask/src/benchmark.rs b/crates/xtask/src/benchmark.rs similarity index 100% rename from xtask/src/benchmark.rs rename to crates/xtask/src/benchmark.rs diff --git a/crates/xtask/src/build_wasm.rs b/crates/xtask/src/build_wasm.rs new file mode 100644 index 00000000..fbb231ce --- /dev/null +++ b/crates/xtask/src/build_wasm.rs @@ -0,0 +1,513 @@ +use std::{ + collections::HashSet, + ffi::{OsStr, OsString}, + fmt::Write, + fs, + path::{Path, PathBuf}, + process::Command, + time::Duration, +}; + +use anyhow::{anyhow, Result}; +use etcetera::BaseStrategy as _; +use indoc::indoc; +use notify::{ + event::{AccessKind, AccessMode}, + EventKind, RecursiveMode, +}; +use notify_debouncer_full::new_debouncer; +use tree_sitter_loader::{IoError, LoaderError, WasiSDKClangError}; + +use crate::{ + bail_on_err, embed_sources::embed_sources_in_map, watch_wasm, BuildWasm, EMSCRIPTEN_TAG, +}; + +#[derive(PartialEq, Eq)] +enum EmccSource { + Native, + Docker, + Podman, +} + +const EXPORTED_RUNTIME_METHODS: [&str; 20] = [ + "AsciiToString", + "stringToUTF8", + "UTF8ToString", + "lengthBytesUTF8", + "stringToUTF16", + "loadWebAssemblyModule", + "getValue", + "setValue", + "HEAPF32", + "HEAPF64", + "HEAP_DATA_VIEW", + "HEAP8", + "HEAPU8", + "HEAP16", + "HEAPU16", + "HEAP32", + "HEAPU32", + "HEAP64", + "HEAPU64", + "LE_HEAP_STORE_I64", +]; + +const WASI_SDK_VERSION: &str = include_str!("../../loader/wasi-sdk-version").trim_ascii(); + +pub fn run_wasm(args: &BuildWasm) -> Result<()> { + let mut emscripten_flags = if args.debug { + vec!["-O0", "--minify", "0"] + } else { + vec!["-O3", "--minify", "0"] + }; + + if args.debug { + emscripten_flags.extend(["-s", "ASSERTIONS=1", "-s", "SAFE_HEAP=1", "-g"]); + } + + if args.verbose { + emscripten_flags.extend(["-s", "VERBOSE=1", "-v"]); + } + + let emcc_name = if cfg!(windows) { "emcc.bat" } else { "emcc" }; + + // Order of preference: emscripten > docker > podman > error + let source = if !args.docker && Command::new(emcc_name).output().is_ok() { + EmccSource::Native + } else if Command::new("docker") + .output() + .is_ok_and(|out| out.status.success()) + { + EmccSource::Docker + } else if Command::new("podman") + .arg("--version") + .output() + .is_ok_and(|out| out.status.success()) + { + EmccSource::Podman + } else { + return Err(anyhow!( + "You must have either emcc, docker, or podman on your PATH to run this command" + )); + }; + + let mut command = match source { + EmccSource::Native => Command::new(emcc_name), + EmccSource::Docker | EmccSource::Podman => { + let mut command = match source { + EmccSource::Docker => Command::new("docker"), + EmccSource::Podman => Command::new("podman"), + _ => unreachable!(), + }; + command.args(["run", "--rm"]); + + // Mount the root directory as a volume, which is the repo root + let mut volume_string = OsString::from(std::env::current_dir().unwrap()); + volume_string.push(":/src:Z"); + command.args([OsStr::new("--volume"), &volume_string]); + + // In case `docker` is an alias to `podman`, ensure that podman + // mounts the current directory as writable by the container + // user which has the same uid as the host user. Setting the + // podman-specific variable is more reliable than attempting to + // detect whether `docker` is an alias for `podman`. + // see https://docs.podman.io/en/latest/markdown/podman-run.1.html#userns-mode + command.env("PODMAN_USERNS", "keep-id"); + + // Get the current user id so that files created in the docker container will have + // the same owner. + #[cfg(unix)] + { + #[link(name = "c")] + extern "C" { + fn getuid() -> u32; + } + // don't need to set user for podman since PODMAN_USERNS=keep-id is already set + if source == EmccSource::Docker { + let user_id = unsafe { getuid() }; + command.args(["--user", &user_id.to_string()]); + } + }; + + // Run `emcc` in a container using the `emscripten-slim` image + command.args([EMSCRIPTEN_TAG, "emcc"]); + command + } + }; + + fs::create_dir_all("target/scratch").unwrap(); + + let exported_functions = format!( + "{}{}", + fs::read_to_string("lib/src/wasm/stdlib-symbols.txt")?, + fs::read_to_string("lib/binding_web/lib/exports.txt")? + ) + .replace('"', "") + .lines() + .fold(String::new(), |mut output, line| { + let _ = write!(output, "_{line}"); + output + }) + .trim_end_matches(',') + .to_string(); + + let exported_functions = format!("EXPORTED_FUNCTIONS={exported_functions}"); + let exported_runtime_methods = format!( + "EXPORTED_RUNTIME_METHODS={}", + EXPORTED_RUNTIME_METHODS.join(",") + ); + + // Clean up old files from prior runs + for file in [ + "web-tree-sitter.mjs", + "web-tree-sitter.cjs", + "web-tree-sitter.wasm", + "web-tree-sitter.wasm.map", + ] { + fs::remove_file(PathBuf::from("lib/binding_web/lib").join(file)).ok(); + } + + if !args.cjs { + emscripten_flags.extend(["-s", "EXPORT_ES6=1"]); + } + + macro_rules! binding_file { + ($ext:literal) => { + concat!("lib/binding_web/lib/web-tree-sitter", $ext) + }; + } + + #[rustfmt::skip] + emscripten_flags.extend([ + "-gsource-map=inline", + "-fno-exceptions", + "-std=c11", + "-s", "WASM=1", + "-s", "MODULARIZE=1", + "-s", "INITIAL_MEMORY=33554432", + "-s", "ALLOW_MEMORY_GROWTH=1", + "-s", "SUPPORT_BIG_ENDIAN=1", + "-s", "WASM_BIGINT=1", + "-s", "MAIN_MODULE=2", + "-s", "FILESYSTEM=0", + "-s", "NODEJS_CATCH_EXIT=0", + "-s", "NODEJS_CATCH_REJECTION=0", + "-s", &exported_functions, + "-s", &exported_runtime_methods, + "-D", "fprintf(...)=", + "-D", "printf(...)=", + "-D", "NDEBUG=", + "-D", "_POSIX_C_SOURCE=200112L", + "-D", "_DEFAULT_SOURCE=", + "-D", "_BSD_SOURCE=", + "-D", "_DARWIN_C_SOURCE=", + "-I", "lib/src", + "-I", "lib/include", + "--js-library", "lib/binding_web/lib/imports.js", + "--pre-js", "lib/binding_web/lib/prefix.js", + "-o", if args.cjs { binding_file!(".cjs") } else { binding_file!(".mjs") }, + "lib/src/lib.c", + "lib/binding_web/lib/tree-sitter.c", + ]); + if args.emit_tsd { + emscripten_flags.extend(["--emit-tsd", "web-tree-sitter.d.ts"]); + } + + let command = command.args(&emscripten_flags); + + if args.watch { + watch_wasm!(|| build_wasm(command, args.emit_tsd)); + } else { + build_wasm(command, args.emit_tsd)?; + } + + Ok(()) +} + +fn build_wasm(cmd: &mut Command, edit_tsd: bool) -> Result<()> { + bail_on_err( + &cmd.spawn()?.wait_with_output()?, + "Failed to compile the Tree-sitter Wasm library", + )?; + + if edit_tsd { + let file = "lib/binding_web/lib/web-tree-sitter.d.ts"; + let content = fs::read_to_string(file)? + .replace("Automatically generated", "Automatically @generated") + .replace( + "AsciiToString(ptr: any): string", + "AsciiToString(ptr: number): string", + ) + .replace( + "stringToUTF8(str: any, outPtr: any, maxBytesToWrite: any): any", + "stringToUTF8(str: string, outPtr: number, maxBytesToWrite: number): number", + ) + .replace( + "UTF8ToString(ptr: number, maxBytesToRead?: number | undefined): string", + "UTF8ToString(ptr: number, maxBytesToRead?: number): string", + ) + .replace( + "lengthBytesUTF8(str: any): number", + "lengthBytesUTF8(str: string): number", + ) + .replace( + "stringToUTF16(str: any, outPtr: any, maxBytesToWrite: any): number", + "stringToUTF16(str: string, outPtr: number, maxBytesToWrite: number): number", + ) + .replace( + concat!( + "loadWebAssemblyModule(binary: any, flags: any, libName?: string | ", + "undefined, localScope?: any | undefined, handle?: number | undefined): any" + ), + concat!( + "loadWebAssemblyModule(binary: Uint8Array | WebAssembly.Module, flags: Record,", + " libName?: string, localScope?: Record, handle?: number):", + " Promise number>>" + ), + ) + .replace( + "getValue(ptr: number, type?: string): any", + "getValue(ptr: number, type?: string): number", + ) + .replace("HEAPF32: any", "HEAPF32: Float32Array") + .replace("HEAPF64: any", "HEAPF64: Float64Array") + .replace("HEAP_DATA_VIEW: any", "HEAP_DATA_VIEW: DataView") + .replace("HEAP8: any", "HEAP8: Int8Array") + .replace("HEAPU8: any", "HEAPU8: Uint8Array") + .replace("HEAP16: any", "HEAP16: Int16Array") + .replace("HEAPU16: any", "HEAPU16: Uint16Array") + .replace("HEAP32: any", "HEAP32: Int32Array") + .replace("HEAPU32: any", "HEAPU32: Uint32Array") + .replace("HEAP64: any", "HEAP64: BigInt64Array") + .replace("HEAPU64: any", "HEAPU64: BigUint64Array") + .replace("BigInt;", "bigint;") + .replace("BigInt)", "bigint)") + .replace( + "WasmModule & typeof RuntimeExports;", + indoc! {" + WasmModule & typeof RuntimeExports & { + currentParseCallback: ((index: number, position: {row: number, column: number}) => string | undefined) | null; + currentLogCallback: ((message: string, isLex: boolean) => void) | null; + currentProgressCallback: ((state: {currentOffset: number, hasError: boolean}) => void) | null; + currentQueryProgressCallback: ((state: {currentOffset: number}) => void) | null; + }; + "}, + ) + .replace( + "MainModuleFactory (options?: unknown): Promise", + "MainModuleFactory(options?: Partial): Promise", + ); + fs::write(file, content)?; + } + + // Post-process the source map to embed source content for optimized builds + let map_path = Path::new("lib") + .join("binding_web") + .join("lib") + .join("web-tree-sitter.wasm.map"); + if map_path.exists() { + if let Err(e) = embed_sources_in_map(&map_path) { + eprintln!("Warning: Failed to embed sources in source map: {e}"); + } + } + + Ok(()) +} + +/// This ensures that the wasi-sdk is available, downloading and extracting it if necessary, +/// and returns the path to the `clang` executable. +/// +/// If `TREE_SITTER_WASI_SDK_PATH` is set, it will use that path to look for the clang executable. +/// +/// Note that this is just a minimially modified version of +/// `tree_sitter_loader::ensure_wasi_sdk_exists`. In the loader, this functionality is implemented +/// as a private method of `Loader`. Rather than add this to the public API, we just +/// re-implement it. Any fixes and/or modifications made to the loader's copy should be reflected +/// here. +pub fn ensure_wasi_sdk_exists() -> Result { + let possible_executables = if cfg!(windows) { + vec![ + "clang.exe", + "wasm32-unknown-wasi-clang.exe", + "wasm32-wasi-clang.exe", + ] + } else { + vec!["clang", "wasm32-unknown-wasi-clang", "wasm32-wasi-clang"] + }; + + if let Ok(wasi_sdk_path) = std::env::var("TREE_SITTER_WASI_SDK_PATH") { + let wasi_sdk_dir = PathBuf::from(wasi_sdk_path); + + for exe in &possible_executables { + let clang_exe = wasi_sdk_dir.join("bin").join(exe); + if clang_exe.exists() { + return Ok(clang_exe); + } + } + + Err(LoaderError::WasiSDKClang(WasiSDKClangError { + wasi_sdk_dir: wasi_sdk_dir.to_string_lossy().to_string(), + possible_executables: possible_executables.clone(), + download: false, + }))?; + } + + let cache_dir = etcetera::choose_base_strategy()? + .cache_dir() + .join("tree-sitter"); + fs::create_dir_all(&cache_dir).map_err(|error| { + LoaderError::IO(IoError { + error, + path: Some(cache_dir.to_string_lossy().to_string()), + }) + })?; + + let wasi_sdk_dir = cache_dir.join("wasi-sdk"); + + for exe in &possible_executables { + let clang_exe = wasi_sdk_dir.join("bin").join(exe); + if clang_exe.exists() { + return Ok(clang_exe); + } + } + + fs::create_dir_all(&wasi_sdk_dir).map_err(|error| { + LoaderError::IO(IoError { + error, + path: Some(wasi_sdk_dir.to_string_lossy().to_string()), + }) + })?; + + let arch_os = if cfg!(target_os = "macos") { + if cfg!(target_arch = "aarch64") { + "arm64-macos" + } else { + "x86_64-macos" + } + } else if cfg!(target_os = "windows") { + if cfg!(target_arch = "aarch64") { + "arm64-windows" + } else { + "x86_64-windows" + } + } else if cfg!(target_os = "linux") { + if cfg!(target_arch = "aarch64") { + "arm64-linux" + } else { + "x86_64-linux" + } + } else { + Err(LoaderError::WasiSDKPlatform)? + }; + + let sdk_filename = format!("wasi-sdk-{WASI_SDK_VERSION}-{arch_os}.tar.gz"); + let wasi_sdk_major_version = WASI_SDK_VERSION + .trim_end_matches(char::is_numeric) // trim minor version... + .trim_end_matches('.'); // ...and '.' separator + let sdk_url = format!( + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-{wasi_sdk_major_version}/{sdk_filename}", + ); + + eprintln!("Downloading wasi-sdk from {sdk_url}..."); + let temp_tar_path = cache_dir.join(sdk_filename); + + let status = Command::new("curl") + .arg("-f") + .arg("-L") + .arg("-o") + .arg(&temp_tar_path) + .arg(&sdk_url) + .status() + .map_err(|e| LoaderError::Curl(sdk_url.clone(), e))?; + + if !status.success() { + Err(LoaderError::WasiSDKDownload(sdk_url))?; + } + + eprintln!("Extracting wasi-sdk to {}...", wasi_sdk_dir.display()); + extract_tar_gz_with_strip(&temp_tar_path, &wasi_sdk_dir)?; + + fs::remove_file(temp_tar_path).ok(); + for exe in &possible_executables { + let clang_exe = wasi_sdk_dir.join("bin").join(exe); + if clang_exe.exists() { + return Ok(clang_exe); + } + } + + Err(LoaderError::WasiSDKClang(WasiSDKClangError { + wasi_sdk_dir: wasi_sdk_dir.to_string_lossy().to_string(), + possible_executables, + download: true, + }))? +} + +/// Extracts a tar.gz archive with `tar`, stripping the first path component. +fn extract_tar_gz_with_strip(archive_path: &Path, destination: &Path) -> Result<()> { + let status = Command::new("tar") + .arg("-xzf") + .arg(archive_path) + .arg("--strip-components=1") + .arg("-C") + .arg(destination) + .status() + .map_err(|e| LoaderError::Tar(archive_path.to_string_lossy().to_string(), e))?; + + if !status.success() { + Err(LoaderError::Extraction( + archive_path.to_string_lossy().to_string(), + destination.to_string_lossy().to_string(), + ))?; + } + + Ok(()) +} + +pub fn run_wasm_stdlib() -> Result<()> { + let export_flags = include_str!("../../../lib/src/wasm/stdlib-symbols.txt") + .lines() + .map(|line| format!("-Wl,--export={}", &line[1..line.len() - 2])) + .collect::>(); + + let clang_exe = ensure_wasi_sdk_exists()?; + + let output = Command::new(&clang_exe) + .args([ + "-o", + "stdlib.wasm", + "-Os", + "-fPIC", + "-DTREE_SITTER_FEATURE_WASM", + "-Wl,--no-entry", + "-Wl,--stack-first", + "-Wl,-z", + "-Wl,stack-size=65536", + "-Wl,--import-undefined", + "-Wl,--import-memory", + "-Wl,--import-table", + "-Wl,--strip-debug", + "-Wl,--export=__wasm_call_ctors", + "-Wl,--export=__stack_pointer", + "-Wl,--export=reset_heap", + ]) + .args(&export_flags) + .arg("crates/language/wasm/src/stdlib.c") + .output()?; + + bail_on_err(&output, "Failed to compile the Tree-sitter Wasm stdlib")?; + + let xxd = Command::new("xxd") + .args(["-C", "-i", "stdlib.wasm"]) + .output()?; + + bail_on_err( + &xxd, + "Failed to run xxd on the compiled Tree-sitter Wasm stdlib", + )?; + + fs::write("lib/src/wasm/wasm-stdlib.h", xxd.stdout)?; + + fs::rename("stdlib.wasm", "target/stdlib.wasm")?; + + Ok(()) +} diff --git a/crates/xtask/src/bump.rs b/crates/xtask/src/bump.rs new file mode 100644 index 00000000..02254274 --- /dev/null +++ b/crates/xtask/src/bump.rs @@ -0,0 +1,295 @@ +use std::{cmp::Ordering, path::Path}; + +use anyhow::{anyhow, Context, Result}; +use indoc::indoc; +use semver::{Prerelease, Version}; + +use crate::{create_commit, BumpVersion}; + +pub fn get_latest_tag() -> Result { + let output = std::process::Command::new("git") + .args(["tag", "-l"]) + .output()?; + if !output.status.success() { + anyhow::bail!( + "Failed to list tags: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + let mut tags = String::from_utf8(output.stdout)? + .lines() + .filter_map(|tag| Version::parse(tag.strip_prefix('v').unwrap_or(tag)).ok()) + .collect::>(); + + tags.sort_by( + |a, b| match (a.pre != Prerelease::EMPTY, b.pre != Prerelease::EMPTY) { + (true, true) | (false, false) => a.cmp(b), + (true, false) => Ordering::Less, + (false, true) => Ordering::Greater, + }, + ); + + tags.last() + .map(std::string::ToString::to_string) + .ok_or_else(|| anyhow!("No tags found")) +} + +pub fn run(args: BumpVersion) -> Result<()> { + let latest_tag = get_latest_tag()?; + let current_version = Version::parse(&latest_tag)?; + + let output = std::process::Command::new("git") + .args(["rev-parse", &format!("v{latest_tag}")]) + .output()?; + if !output.status.success() { + anyhow::bail!( + "Failed to get tag SHA: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + let workspace_toml_version = Version::parse(&fetch_workspace_version()?)?; + + if current_version.major != workspace_toml_version.major + && current_version.minor != workspace_toml_version.minor + { + eprintln!( + indoc! {" + Seems like the workspace Cargo.toml ({}) version does not match up with the latest git tag ({}). + Please ensure you don't change that yourself, this subcommand will handle this for you. + "}, + workspace_toml_version, latest_tag + ); + return Ok(()); + } + + let next_version = args.version; + + println!("Bumping from {current_version} to {next_version}"); + update_crates(¤t_version, &next_version)?; + update_makefile(&next_version)?; + update_cmake(&next_version)?; + update_nix(&next_version)?; + update_npm(&next_version)?; + update_zig(&next_version)?; + tag_next_version(&next_version)?; + + Ok(()) +} + +fn tag_next_version(next_version: &Version) -> Result<()> { + let commit_sha = create_commit( + &format!("{next_version}"), + &[ + "Cargo.lock", + "Cargo.toml", + "Makefile", + "build.zig.zon", + "flake.nix", + "crates/cli/Cargo.toml", + "crates/cli/npm/package.json", + "crates/cli/npm/package-lock.json", + "crates/config/Cargo.toml", + "crates/highlight/Cargo.toml", + "crates/loader/Cargo.toml", + "crates/tags/Cargo.toml", + "CMakeLists.txt", + "lib/Cargo.toml", + "lib/binding_web/package.json", + "lib/binding_web/package-lock.json", + ], + )?; + + // Create tag + let output = std::process::Command::new("git") + .args([ + "tag", + "-a", + &format!("v{next_version}"), + "-m", + &format!("v{next_version}"), + &commit_sha, + ]) + .output()?; + if !output.status.success() { + anyhow::bail!( + "Failed to create tag: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + println!("Tagged commit {commit_sha} with tag v{next_version}"); + + Ok(()) +} + +fn update_makefile(next_version: &Version) -> Result<()> { + let makefile = std::fs::read_to_string("Makefile")?; + let makefile = makefile + .lines() + .map(|line| { + if line.starts_with("VERSION") { + format!("VERSION := {next_version}") + } else { + line.to_string() + } + }) + .collect::>() + .join("\n") + + "\n"; + + std::fs::write("Makefile", makefile)?; + + Ok(()) +} + +fn update_cmake(next_version: &Version) -> Result<()> { + let cmake = std::fs::read_to_string("CMakeLists.txt")?; + let cmake = cmake + .lines() + .map(|line| { + if line.contains(" VERSION") { + let start_quote = line.find('"').unwrap(); + let end_quote = line.rfind('"').unwrap(); + format!( + "{}{next_version}{}", + &line[..=start_quote], + &line[end_quote..] + ) + } else { + line.to_string() + } + }) + .collect::>() + .join("\n") + + "\n"; + + std::fs::write("CMakeLists.txt", cmake)?; + + Ok(()) +} + +fn update_nix(next_version: &Version) -> Result<()> { + let nix = std::fs::read_to_string("flake.nix")?; + let nix = nix + .lines() + .map(|line| { + if line.trim_start().starts_with("version =") { + format!(" version = \"{next_version}\";") + } else { + line.to_string() + } + }) + .collect::>() + .join("\n") + + "\n"; + + std::fs::write("flake.nix", nix)?; + + Ok(()) +} + +fn update_crates(current_version: &Version, next_version: &Version) -> Result<()> { + let mut cmd = std::process::Command::new("cargo"); + cmd.arg("workspaces").arg("version"); + + if next_version.minor > current_version.minor { + cmd.arg("minor"); + } else { + cmd.arg("patch"); + } + + cmd.arg("--no-git-commit") + .arg("--yes") + .arg("--force") + .arg("tree-sitter{,-cli,-config,-generate,-loader,-highlight,-tags}") + .arg("--ignore-changes") + .arg("crates/language/*"); + + let status = cmd.status()?; + + if !status.success() { + return Err(anyhow!("Failed to update crates")); + } + + Ok(()) +} + +fn update_npm(next_version: &Version) -> Result<()> { + for npm_project in ["lib/binding_web", "crates/cli/npm"] { + let npm_path = Path::new(npm_project); + + let package_json_path = npm_path.join("package.json"); + + let package_json = serde_json::from_str::( + &std::fs::read_to_string(&package_json_path) + .with_context(|| format!("Failed to read {}", package_json_path.display()))?, + )?; + + let mut package_json = package_json + .as_object() + .ok_or_else(|| anyhow!("Invalid package.json"))? + .clone(); + package_json.insert( + "version".to_string(), + serde_json::Value::String(next_version.to_string()), + ); + + let package_json = serde_json::to_string_pretty(&package_json)? + "\n"; + + std::fs::write(package_json_path, package_json)?; + + let Ok(cmd) = std::process::Command::new("npm") + .arg("install") + .arg("--package-lock-only") + .arg("--ignore-scripts") + .current_dir(npm_path) + .output() + else { + return Ok(()); // npm is not `executable`, ignore + }; + + if !cmd.status.success() { + let stderr = String::from_utf8_lossy(&cmd.stderr); + return Err(anyhow!( + "Failed to run `npm install` in {}:\n{stderr}", + npm_path.display() + )); + } + } + + Ok(()) +} + +fn update_zig(next_version: &Version) -> Result<()> { + let zig = std::fs::read_to_string("build.zig.zon")? + .lines() + .map(|line| { + if line.starts_with(" .version") { + format!(" .version = \"{next_version}\",") + } else { + line.to_string() + } + }) + .collect::>() + .join("\n") + + "\n"; + + std::fs::write("build.zig.zon", zig)?; + + Ok(()) +} + +/// read Cargo.toml and get the version +fn fetch_workspace_version() -> Result { + std::fs::read_to_string("Cargo.toml")? + .lines() + .find(|line| line.starts_with("version = ")) + .and_then(|line| { + line.split_terminator('"') + .next_back() + .map(|s| s.to_string()) + }) + .ok_or_else(|| anyhow!("No version found in Cargo.toml")) +} diff --git a/crates/xtask/src/check_wasm_exports.rs b/crates/xtask/src/check_wasm_exports.rs new file mode 100644 index 00000000..c93f7cd3 --- /dev/null +++ b/crates/xtask/src/check_wasm_exports.rs @@ -0,0 +1,135 @@ +use std::{ + collections::HashSet, + env, + io::BufRead, + path::PathBuf, + process::{Command, Stdio}, + time::Duration, +}; + +use anyhow::{anyhow, Result}; +use notify::{ + event::{AccessKind, AccessMode}, + EventKind, RecursiveMode, +}; +use notify_debouncer_full::new_debouncer; + +use crate::{bail_on_err, watch_wasm, CheckWasmExports}; + +const EXCLUDES: [&str; 25] = [ + // Unneeded because the JS side has its own way of implementing it + "ts_node_child_by_field_name", + "ts_node_edit", + // Precomputed and stored in the JS side + "ts_node_type", + "ts_node_grammar_type", + "ts_node_eq", + "ts_tree_cursor_current_field_name", + "ts_lookahead_iterator_current_symbol_name", + // Not used in Wasm + "ts_init", + "ts_set_allocator", + "ts_parser_print_dot_graphs", + "ts_tree_print_dot_graph", + "ts_parser_set_wasm_store", + "ts_parser_take_wasm_store", + "ts_parser_language", + "ts_node_language", + "ts_tree_language", + "ts_lookahead_iterator_language", + "ts_parser_logger", + "ts_parser_parse_string", + "ts_parser_parse_string_encoding", + // Query cursor is not managed by user in web bindings + "ts_query_cursor_delete", + "ts_query_cursor_match_limit", + "ts_query_cursor_remove_match", + "ts_query_cursor_set_point_range", + "ts_query_cursor_set_containing_byte_range", +]; + +pub fn run(args: &CheckWasmExports) -> Result<()> { + if args.watch { + watch_wasm!(check_wasm_exports); + } else { + check_wasm_exports()?; + } + + Ok(()) +} + +fn check_wasm_exports() -> Result<()> { + let mut wasm_exports = std::fs::read_to_string("lib/binding_web/lib/exports.txt")? + .lines() + .map(|s| s.replace("_wasm", "").replace("byte", "index")) + // remove leading and trailing quotes, trailing comma + .map(|s| s[1..s.len() - 2].to_string()) + .collect::>(); + + // Run wasm-objdump to see symbols used internally in binding.c but not exposed in any way. + let wasm_objdump = Command::new("wasm-objdump") + .args([ + "--details", + "lib/binding_web/debug/web-tree-sitter.wasm", + "--section", + "Name", + ]) + .output() + .expect("Failed to run wasm-objdump"); + bail_on_err(&wasm_objdump, "Failed to run wasm-objdump")?; + + wasm_exports.extend( + wasm_objdump + .stdout + .lines() + .map_while(Result::ok) + .skip_while(|line| !line.contains("- func")) + .filter_map(|line| { + if line.contains("func") { + if let Some(function) = line.split_whitespace().nth(2).map(String::from) { + let trimmed = function.trim_start_matches('<').trim_end_matches('>'); + if trimmed.starts_with("ts") && !trimmed.contains("__") { + return Some(trimmed.to_string()); + } + } + } + None + }), + ); + + let nm_cmd = env::var("NM").unwrap_or_else(|_| "nm".to_owned()); + let nm_child = Command::new(nm_cmd) + .arg("-W") + .arg("-U") + .arg("libtree-sitter.so") + .stdout(Stdio::piped()) + .output() + .expect("Failed to run nm"); + bail_on_err(&nm_child, "Failed to run nm")?; + let export_reader = nm_child + .stdout + .lines() + .map_while(Result::ok) + .filter(|line| line.contains(" T ")); + + let exports = export_reader + .filter_map(|line| line.split_whitespace().nth(2).map(String::from)) + .filter(|symbol| !EXCLUDES.contains(&symbol.as_str())) + .collect::>(); + + let mut missing = exports + .iter() + .filter(|&symbol| !wasm_exports.contains(symbol)) + .map(String::as_str) + .collect::>(); + missing.sort_unstable(); + + if !missing.is_empty() { + Err(anyhow!(format!( + "Unmatched Wasm exports:\n{}", + missing.join("\n") + )))?; + } + + Ok(()) +} diff --git a/xtask/src/clippy.rs b/crates/xtask/src/clippy.rs similarity index 93% rename from xtask/src/clippy.rs rename to crates/xtask/src/clippy.rs index c8d33348..664884f5 100644 --- a/xtask/src/clippy.rs +++ b/crates/xtask/src/clippy.rs @@ -6,7 +6,7 @@ use crate::{bail_on_err, Clippy}; pub fn run(args: &Clippy) -> Result<()> { let mut clippy_command = Command::new("cargo"); - clippy_command.arg("+nightly").arg("clippy"); + clippy_command.arg("clippy"); if let Some(package) = args.package.as_ref() { clippy_command.args(["--package", package]); diff --git a/crates/xtask/src/embed_sources.rs b/crates/xtask/src/embed_sources.rs new file mode 100644 index 00000000..ce8ec403 --- /dev/null +++ b/crates/xtask/src/embed_sources.rs @@ -0,0 +1,61 @@ +use anyhow::Result; +use std::fs; +use std::path::Path; + +/// Restores sourcesContent if it was stripped by Binaryen. +/// +/// This is a workaround for Binaryen where `wasm-opt -O2` and higher +/// optimization levels strip the `sourcesContent` field from source maps, +/// even when the source map was generated with `--sources` flag. +/// +/// This is fixed upstream in Binaryen as of Apr 9, 2025, but there hasn't been a release with the fix yet. +/// See: +/// +/// This reads the original source files and embeds them in the +/// source map's `sourcesContent` field, making debugging possible even +/// with optimized builds. +/// +/// TODO: Once Binaryen releases a version with the fix, and emscripten updates to that +/// version, and we update our emscripten version, this function can be removed. +pub fn embed_sources_in_map(map_path: &Path) -> Result<()> { + let map_content = fs::read_to_string(map_path)?; + let mut map: serde_json::Value = serde_json::from_str(&map_content)?; + + if let Some(sources_content) = map.get("sourcesContent") { + if let Some(arr) = sources_content.as_array() { + if !arr.is_empty() && arr.iter().any(|v| !v.is_null()) { + return Ok(()); + } + } + } + + let sources = map["sources"] + .as_array() + .ok_or_else(|| anyhow::anyhow!("No sources array in source map"))?; + + let map_dir = map_path.parent().unwrap_or(Path::new(".")); + let mut sources_content = Vec::new(); + + for source in sources { + let source_path = source.as_str().unwrap_or(""); + let full_path = map_dir.join(source_path); + + let content = if full_path.exists() { + match fs::read_to_string(&full_path) { + Ok(content) => serde_json::Value::String(content), + Err(_) => serde_json::Value::Null, + } + } else { + serde_json::Value::Null + }; + + sources_content.push(content); + } + + map["sourcesContent"] = serde_json::Value::Array(sources_content); + + let output = serde_json::to_string(&map)?; + fs::write(map_path, output)?; + + Ok(()) +} diff --git a/crates/xtask/src/fetch.rs b/crates/xtask/src/fetch.rs new file mode 100644 index 00000000..6fa431c6 --- /dev/null +++ b/crates/xtask/src/fetch.rs @@ -0,0 +1,140 @@ +use crate::{bail_on_err, root_dir, FetchFixtures, EMSCRIPTEN_VERSION}; +use anyhow::Result; +use std::{fs, process::Command}; + +pub fn run_fixtures(args: &FetchFixtures) -> Result<()> { + let fixtures_dir = root_dir().join("test").join("fixtures"); + let grammars_dir = fixtures_dir.join("grammars"); + let fixtures_path = fixtures_dir.join("fixtures.json"); + + // grammar name, tag + let mut fixtures: Vec<(String, String)> = + serde_json::from_str(&fs::read_to_string(&fixtures_path)?)?; + + for (grammar, tag) in &mut fixtures { + let grammar_dir = grammars_dir.join(&grammar); + let grammar_url = format!("https://github.com/tree-sitter/tree-sitter-{grammar}"); + + println!("Fetching the {grammar} grammar..."); + + if !grammar_dir.exists() { + let mut command = Command::new("git"); + command.args([ + "clone", + "--depth", + "1", + "--branch", + tag, + &grammar_url, + &grammar_dir.to_string_lossy(), + ]); + bail_on_err( + &command.spawn()?.wait_with_output()?, + &format!("Failed to clone the {grammar} grammar"), + )?; + } else { + let mut describe_command = Command::new("git"); + describe_command.current_dir(&grammar_dir).args([ + "describe", + "--tags", + "--exact-match", + "HEAD", + ]); + + let output = describe_command.output()?; + let current_tag = String::from_utf8_lossy(&output.stdout); + let current_tag = current_tag.trim(); + + if current_tag != tag { + println!("Updating {grammar} grammar from {current_tag} to {tag}..."); + + let mut fetch_command = Command::new("git"); + fetch_command.current_dir(&grammar_dir).args([ + "fetch", + "origin", + &format!("refs/tags/{tag}:refs/tags/{tag}"), + ]); + bail_on_err( + &fetch_command.spawn()?.wait_with_output()?, + &format!("Failed to fetch tag {tag} for {grammar} grammar"), + )?; + + let mut reset_command = Command::new("git"); + reset_command + .current_dir(&grammar_dir) + .args(["reset", "--hard", "HEAD"]); + bail_on_err( + &reset_command.spawn()?.wait_with_output()?, + &format!("Failed to reset {grammar} grammar working tree"), + )?; + + let mut checkout_command = Command::new("git"); + checkout_command + .current_dir(&grammar_dir) + .args(["checkout", tag]); + bail_on_err( + &checkout_command.spawn()?.wait_with_output()?, + &format!("Failed to checkout tag {tag} for {grammar} grammar"), + )?; + } else { + println!("{grammar} grammar is already at tag {tag}"); + } + } + } + + if args.update { + println!("Updating the fixtures lock file"); + fs::write( + &fixtures_path, + // format the JSON without extra newlines + serde_json::to_string(&fixtures)? + .replace("[[", "[\n [") + .replace("],", "],\n ") + .replace("]]", "]\n]"), + )?; + } + + Ok(()) +} + +pub fn run_emscripten() -> Result<()> { + let emscripten_dir = root_dir().join("target").join("emsdk"); + if emscripten_dir.exists() { + println!("Emscripten SDK already exists"); + return Ok(()); + } + println!("Cloning the Emscripten SDK..."); + + let mut command = Command::new("git"); + command.args([ + "clone", + "https://github.com/emscripten-core/emsdk.git", + &emscripten_dir.to_string_lossy(), + ]); + bail_on_err( + &command.spawn()?.wait_with_output()?, + "Failed to clone the Emscripten SDK", + )?; + + std::env::set_current_dir(&emscripten_dir)?; + + let emsdk = if cfg!(windows) { + "emsdk.bat" + } else { + "./emsdk" + }; + + let mut command = Command::new(emsdk); + command.args(["install", EMSCRIPTEN_VERSION]); + bail_on_err( + &command.spawn()?.wait_with_output()?, + "Failed to install Emscripten", + )?; + + let mut command = Command::new(emsdk); + command.args(["activate", EMSCRIPTEN_VERSION]); + bail_on_err( + &command.spawn()?.wait_with_output()?, + "Failed to activate Emscripten", + ) +} diff --git a/xtask/src/generate.rs b/crates/xtask/src/generate.rs similarity index 84% rename from xtask/src/generate.rs rename to crates/xtask/src/generate.rs index 1af04275..662fc3b1 100644 --- a/xtask/src/generate.rs +++ b/crates/xtask/src/generate.rs @@ -1,6 +1,7 @@ -use std::{collections::BTreeSet, ffi::OsStr, fs, path::Path, process::Command}; +use std::{collections::BTreeSet, ffi::OsStr, fs, path::Path, process::Command, str::FromStr}; use anyhow::{Context, Result}; +use bindgen::RustTarget; use crate::{bail_on_err, GenerateFixtures}; @@ -29,7 +30,7 @@ pub fn run_fixtures(args: &GenerateFixtures) -> Result<()> { println!( "Regenerating {grammar_name} parser{}", - if args.wasm { " to wasm" } else { "" } + if args.wasm { " to Wasm" } else { "" } ); if args.wasm { @@ -41,9 +42,6 @@ pub fn run_fixtures(args: &GenerateFixtures) -> Result<()> { &format!("target/release/tree-sitter-{grammar_name}.wasm"), grammar_dir.to_str().unwrap(), ]); - if args.docker { - cmd.arg("--docker"); - } bail_on_err( &cmd.spawn()?.wait_with_output()?, &format!("Failed to regenerate {grammar_name} parser to wasm"), @@ -67,6 +65,29 @@ pub fn run_fixtures(args: &GenerateFixtures) -> Result<()> { } pub fn run_bindings() -> Result<()> { + let output = Command::new("cargo") + .args(["metadata", "--format-version", "1"]) + .output() + .unwrap(); + + let metadata = serde_json::from_slice::(&output.stdout).unwrap(); + + let Some(rust_version) = metadata + .get("packages") + .and_then(|packages| packages.as_array()) + .and_then(|packages| { + packages.iter().find_map(|package| { + if package["name"] == "tree-sitter" { + package.get("rust_version").and_then(|v| v.as_str()) + } else { + None + } + }) + }) + else { + panic!("Failed to find tree-sitter package in cargo metadata"); + }; + let no_copy = [ "TSInput", "TSLanguage", @@ -91,6 +112,7 @@ pub fn run_bindings() -> Result<()> { .prepend_enum_name(false) .use_core() .clang_arg("-D TREE_SITTER_FEATURE_WASM") + .rust_target(RustTarget::from_str(rust_version).unwrap()) .generate() .expect("Failed to generate bindings"); diff --git a/xtask/src/main.rs b/crates/xtask/src/main.rs similarity index 59% rename from xtask/src/main.rs rename to crates/xtask/src/main.rs index 92f7258d..3814cf66 100644 --- a/xtask/src/main.rs +++ b/crates/xtask/src/main.rs @@ -1,15 +1,20 @@ mod benchmark; mod build_wasm; mod bump; +mod check_wasm_exports; mod clippy; +mod embed_sources; mod fetch; mod generate; mod test; +mod test_schema; mod upgrade_wasmtime; +use std::{path::Path, process::Command}; + use anstyle::{AnsiColor, Color, Style}; use anyhow::Result; -use clap::{crate_authors, Args, Command, FromArgMatches as _, Subcommand}; +use clap::{crate_authors, Args, FromArgMatches as _, Subcommand}; use semver::Version; #[derive(Subcommand)] @@ -17,28 +22,32 @@ use semver::Version; enum Commands { /// Runs `cargo benchmark` with some optional environment variables set. Benchmark(Benchmark), - /// Compile the Tree-sitter WASM library. This will create two files in the - /// `lib/binding_web` directory: `tree-sitter.js` and `tree-sitter.wasm`. + /// Compile the Tree-sitter Wasm library. This will create two files in the + /// `lib/binding_web` directory: `web-tree-sitter.js` and `web-tree-sitter.wasm`. BuildWasm(BuildWasm), - /// Compile the Tree-sitter WASM standard library. + /// Compile the Tree-sitter Wasm standard library. BuildWasmStdlib, /// Bumps the version of the workspace. BumpVersion(BumpVersion), + /// Checks that Wasm exports are synced. + CheckWasmExports(CheckWasmExports), /// Runs `cargo clippy`. Clippy(Clippy), /// Fetches emscripten. FetchEmscripten, /// Fetches the fixtures for testing tree-sitter. - FetchFixtures, + FetchFixtures(FetchFixtures), /// Generate the Rust bindings from the C library. GenerateBindings, /// Generates the fixtures for testing tree-sitter. GenerateFixtures(GenerateFixtures), - /// Generate the list of exports from Tree-sitter WASM files. + /// Generates the JSON schema for the test runner summary. + GenerateTestSchema, + /// Generate the list of exports from Tree-sitter Wasm files. GenerateWasmExports, /// Run the test suite Test(Test), - /// Run the WASM test suite + /// Run the Wasm test suite TestWasm, /// Upgrade the wasmtime dependency. UpgradeWasmtime(UpgradeWasmtime), @@ -73,13 +82,30 @@ struct BuildWasm { /// Run emscripten with verbose output. #[arg(long, short)] verbose: bool, + /// Rebuild when relevant files are changed. + #[arg(long, short)] + watch: bool, + /// Emit TypeScript type definitions for the generated bindings, + /// requires `tsc` to be available. + #[arg(long, short)] + emit_tsd: bool, + /// Generate `CommonJS` modules instead of ES modules. + #[arg(long, short, env = "CJS")] + cjs: bool, } #[derive(Args)] struct BumpVersion { /// The version to bump to. + #[arg(index = 1, required = true)] + version: Version, +} + +#[derive(Args)] +struct CheckWasmExports { + /// Recheck when relevant files are changed. #[arg(long, short)] - version: Option, + watch: bool, } #[derive(Args)] @@ -92,14 +118,18 @@ struct Clippy { package: Option, } +#[derive(Args)] +struct FetchFixtures { + /// Update all fixtures to the latest tag + #[arg(long, short)] + update: bool, +} + #[derive(Args)] struct GenerateFixtures { - /// Generates the parser to WASM + /// Generates the parser to Wasm #[arg(long, short)] wasm: bool, - /// Run emscripten via docker even if it is installed locally. - #[arg(long, short, requires = "wasm")] - docker: bool, } #[derive(Args)] @@ -118,7 +148,7 @@ struct Test { iterations: Option, /// Set the seed used to control random behavior. #[arg(long, short)] - seed: Option, + seed: Option, /// Print parsing log to stderr. #[arg(long, short)] debug: bool, @@ -133,6 +163,9 @@ struct Test { /// Don't capture the output #[arg(long)] nocapture: bool, + /// Enable the Wasm tests. + #[arg(long, short)] + wasm: bool, } #[derive(Args)] @@ -144,11 +177,12 @@ struct UpgradeWasmtime { const BUILD_VERSION: &str = env!("CARGO_PKG_VERSION"); const BUILD_SHA: Option<&str> = option_env!("BUILD_SHA"); -const EMSCRIPTEN_VERSION: &str = include_str!("../../cli/loader/emscripten-version"); +const EMSCRIPTEN_VERSION: &str = include_str!("../../loader/emscripten-version").trim_ascii(); const EMSCRIPTEN_TAG: &str = concat!( "docker.io/emscripten/emsdk:", - include_str!("../../cli/loader/emscripten-version") -); + include_str!("../../loader/emscripten-version") +) +.trim_ascii(); fn main() { let result = run(); @@ -173,7 +207,7 @@ fn run() -> Result<()> { ); let version: &'static str = Box::leak(version.into_boxed_str()); - let cli = Command::new("xtask") + let cli = clap::Command::new("xtask") .help_template( "\ {before-help}{name} {version} @@ -195,13 +229,17 @@ fn run() -> Result<()> { Commands::BuildWasm(build_wasm_options) => build_wasm::run_wasm(&build_wasm_options)?, Commands::BuildWasmStdlib => build_wasm::run_wasm_stdlib()?, Commands::BumpVersion(bump_options) => bump::run(bump_options)?, + Commands::CheckWasmExports(check_options) => check_wasm_exports::run(&check_options)?, Commands::Clippy(clippy_options) => clippy::run(&clippy_options)?, Commands::FetchEmscripten => fetch::run_emscripten()?, - Commands::FetchFixtures => fetch::run_fixtures()?, + Commands::FetchFixtures(fetch_fixture_options) => { + fetch::run_fixtures(&fetch_fixture_options)?; + } Commands::GenerateBindings => generate::run_bindings()?, Commands::GenerateFixtures(generate_fixtures_options) => { generate::run_fixtures(&generate_fixtures_options)?; } + Commands::GenerateTestSchema => test_schema::run_test_schema()?, Commands::GenerateWasmExports => generate::run_wasm_exports()?, Commands::Test(test_options) => test::run(&test_options)?, Commands::TestWasm => test::run_wasm()?, @@ -213,6 +251,14 @@ fn run() -> Result<()> { Ok(()) } +fn root_dir() -> &'static Path { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() +} + fn bail_on_err(output: &std::process::Output, prefix: &str) -> Result<()> { if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); @@ -252,3 +298,89 @@ const fn get_styles() -> clap::builder::Styles { ) .placeholder(Style::new().fg_color(Some(Color::Ansi(AnsiColor::White)))) } + +pub fn create_commit(msg: &str, paths: &[&str]) -> Result { + for path in paths { + let output = Command::new("git").args(["add", path]).output()?; + if !output.status.success() { + anyhow::bail!( + "Failed to add {path}: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + } + + let output = Command::new("git").args(["commit", "-m", msg]).output()?; + if !output.status.success() { + anyhow::bail!( + "Failed to commit: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + let output = Command::new("git").args(["rev-parse", "HEAD"]).output()?; + if !output.status.success() { + anyhow::bail!( + "Failed to get commit SHA: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + Ok(String::from_utf8(output.stdout)?.trim().to_string()) +} + +#[macro_export] +macro_rules! watch_wasm { + ($watch_fn:expr) => { + if let Err(e) = $watch_fn() { + eprintln!("{e}"); + } else { + println!("Build succeeded"); + } + + let watch_files = [ + "lib/tree-sitter.c", + "lib/exports.txt", + "lib/imports.js", + "lib/prefix.js", + ] + .iter() + .map(PathBuf::from) + .collect::>(); + let (tx, rx) = std::sync::mpsc::channel(); + let mut debouncer = new_debouncer(Duration::from_secs(1), None, tx)?; + debouncer.watch("lib/binding_web", RecursiveMode::NonRecursive)?; + + for result in rx { + match result { + Ok(events) => { + for event in events { + if event.kind == EventKind::Access(AccessKind::Close(AccessMode::Write)) + && event + .paths + .iter() + .filter_map(|p| p.file_name()) + .any(|p| watch_files.contains(&PathBuf::from(p))) + { + if let Err(e) = $watch_fn() { + eprintln!("{e}"); + } else { + println!("Build succeeded"); + } + } + } + } + Err(errors) => { + return Err(anyhow!( + "{}", + errors + .into_iter() + .map(|e| e.to_string()) + .collect::>() + .join("\n") + )); + } + } + } + }; +} diff --git a/xtask/src/test.rs b/crates/xtask/src/test.rs similarity index 82% rename from xtask/src/test.rs rename to crates/xtask/src/test.rs index 62abe37c..6b8d8243 100644 --- a/xtask/src/test.rs +++ b/crates/xtask/src/test.rs @@ -65,13 +65,15 @@ pub fn run(args: &Test) -> Result<()> { } if args.g { - let cargo_cmd = Command::new("cargo") + let mut cargo_cmd = Command::new("cargo"); + cargo_cmd .arg("test") - .arg(test_flags) + .arg("--all") + .arg(&test_flags) .arg("--no-run") - .arg("--message-format=json") - .stdout(Stdio::piped()) - .spawn()?; + .arg("--message-format=json"); + + let cargo_cmd = cargo_cmd.stdout(Stdio::piped()).spawn()?; let jq_cmd = Command::new("jq") .arg("-rs") @@ -89,13 +91,20 @@ pub fn run(args: &Test) -> Result<()> { )?; } else { let mut cargo_cmd = Command::new("cargo"); - cargo_cmd.arg("test"); + cargo_cmd.arg("test").arg("--all"); + if args.wasm { + cargo_cmd.arg("--features").arg("wasm"); + } if !test_flags.is_empty() { - cargo_cmd.arg(test_flags); + cargo_cmd.arg(&test_flags); } cargo_cmd.args(&args.args); + if args.nocapture { - cargo_cmd.arg("--").arg("--nocapture"); + #[cfg(not(target_os = "windows"))] + cargo_cmd.arg("--"); + + cargo_cmd.arg("--nocapture"); } bail_on_err( &cargo_cmd.spawn()?.wait_with_output()?, @@ -122,8 +131,15 @@ pub fn run_wasm() -> Result<()> { bail_on_err(&output, "Failed to install test dependencies")?; } - let output = Command::new(npm).arg("test").output()?; - bail_on_err(&output, &format!("Failed to run {npm} test"))?; + let child = Command::new(npm).arg("test").spawn()?; + let output = child.wait_with_output()?; + bail_on_err(&output, &format!("Failed to run `{npm} test`"))?; + + // Display test results + let output = String::from_utf8_lossy(&output.stdout); + for line in output.lines() { + println!("{line}"); + } Ok(()) } diff --git a/crates/xtask/src/test_schema.rs b/crates/xtask/src/test_schema.rs new file mode 100644 index 00000000..a2f65ed2 --- /dev/null +++ b/crates/xtask/src/test_schema.rs @@ -0,0 +1,25 @@ +use std::path::PathBuf; + +use anyhow::Result; +use serde_json::to_writer_pretty; + +use tree_sitter_cli::test::TestSummary; + +pub fn run_test_schema() -> Result<()> { + let schema = schemars::schema_for!(TestSummary); + + let xtask_path: PathBuf = env!("CARGO_MANIFEST_DIR").into(); + let schema_path = xtask_path + .parent() + .unwrap() + .parent() + .unwrap() + .join("docs") + .join("src") + .join("assets") + .join("schemas") + .join("test-summary.schema.json"); + let mut file = std::fs::File::create(schema_path)?; + + Ok(to_writer_pretty(&mut file, &schema)?) +} diff --git a/crates/xtask/src/upgrade_wasmtime.rs b/crates/xtask/src/upgrade_wasmtime.rs new file mode 100644 index 00000000..90e5b0f2 --- /dev/null +++ b/crates/xtask/src/upgrade_wasmtime.rs @@ -0,0 +1,101 @@ +use std::process::Command; + +use anyhow::{Context, Result}; +use semver::Version; + +use crate::{create_commit, UpgradeWasmtime}; + +const WASMTIME_RELEASE_URL: &str = "https://github.com/bytecodealliance/wasmtime/releases/download"; + +fn update_cargo(version: &Version) -> Result<()> { + let file = std::fs::read_to_string("lib/Cargo.toml")?; + let mut old_lines = file.lines(); + let mut new_lines = Vec::with_capacity(old_lines.size_hint().0); + + while let Some(line) = old_lines.next() { + new_lines.push(line.to_string()); + if line == "[dependencies.wasmtime-c-api]" { + let _ = old_lines.next(); + new_lines.push(format!("version = \"{version}\"")); + } + } + + std::fs::write("lib/Cargo.toml", new_lines.join("\n") + "\n")?; + + Command::new("cargo") + .arg("update") + .status() + .map(|_| ()) + .with_context(|| "Failed to execute cargo update") +} + +fn zig_fetch(lines: &mut Vec, version: &Version, url_suffix: &str) -> Result<()> { + let url = &format!("{WASMTIME_RELEASE_URL}/v{version}/wasmtime-v{version}-{url_suffix}"); + println!(" Fetching {url}"); + lines.push(format!(" .url = \"{url}\",")); + + let output = Command::new("zig") + .arg("fetch") + .arg(url) + .output() + .with_context(|| format!("Failed to execute zig fetch {url}"))?; + + let hash = String::from_utf8_lossy(&output.stdout); + lines.push(format!(" .hash = \"{}\",", hash.trim_end())); + + Ok(()) +} + +fn update_zig(version: &Version) -> Result<()> { + let file = std::fs::read_to_string("build.zig.zon")?; + let mut old_lines = file.lines(); + let new_lines = &mut Vec::with_capacity(old_lines.size_hint().0); + + macro_rules! match_wasmtime_zig_dep { + ($line:ident, {$($platform:literal => [$($arch:literal),*]),*,}) => { + match $line { + $($(concat!(" .wasmtime_c_api_", $arch, "_", $platform, " = .{") => { + let (_, _) = (old_lines.next(), old_lines.next()); + let suffix = if $platform == "windows" || $platform == "mingw" { + concat!($arch, "-", $platform, "-c-api.zip") + } else { + concat!($arch, "-", $platform, "-c-api.tar.xz") + }; + zig_fetch(new_lines, version, suffix)?; + })*)* + _ => {} + } + }; + } + + while let Some(line) = old_lines.next() { + new_lines.push(line.to_string()); + match_wasmtime_zig_dep!(line, { + "android" => ["aarch64", "x86_64"], + "linux" => ["aarch64", "armv7", "i686", "riscv64gc", "s390x", "x86_64"], + "macos" => ["aarch64", "x86_64"], + "mingw" => ["x86_64"], + "musl" => ["aarch64", "x86_64"], + "windows" => ["aarch64", "i686", "x86_64"], + }); + } + + std::fs::write("build.zig.zon", new_lines.join("\n") + "\n")?; + + Ok(()) +} + +pub fn run(args: &UpgradeWasmtime) -> Result<()> { + println!("Upgrading wasmtime for Rust"); + update_cargo(&args.version)?; + + println!("Upgrading wasmtime for Zig"); + update_zig(&args.version)?; + + create_commit( + &format!("build(deps): bump wasmtime-c-api to v{}", args.version), + &["lib/Cargo.toml", "Cargo.lock", "build.zig.zon"], + )?; + + Ok(()) +} diff --git a/docs/.gitignore b/docs/.gitignore index 339efff8..7585238e 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1,3 +1 @@ -vendor -_site -.bundle +book diff --git a/docs/Gemfile b/docs/Gemfile deleted file mode 100644 index ee114290..00000000 --- a/docs/Gemfile +++ /dev/null @@ -1,3 +0,0 @@ -source 'https://rubygems.org' -gem 'github-pages', group: :jekyll_plugins -gem "webrick" diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock deleted file mode 100644 index 7204a9a7..00000000 --- a/docs/Gemfile.lock +++ /dev/null @@ -1,273 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - activesupport (7.1.3) - base64 - bigdecimal - concurrent-ruby (~> 1.0, >= 1.0.2) - connection_pool (>= 2.2.5) - drb - i18n (>= 1.6, < 2) - minitest (>= 5.1) - mutex_m - tzinfo (~> 2.0) - addressable (2.8.1) - public_suffix (>= 2.0.2, < 6.0) - base64 (0.2.0) - bigdecimal (3.1.6) - coffee-script (2.4.1) - coffee-script-source - execjs - coffee-script-source (1.11.1) - colorator (1.1.0) - commonmarker (0.23.10) - concurrent-ruby (1.2.3) - connection_pool (2.4.1) - dnsruby (1.61.9) - simpleidn (~> 0.1) - drb (2.2.0) - ruby2_keywords - em-websocket (0.5.3) - eventmachine (>= 0.12.9) - http_parser.rb (~> 0) - ethon (0.16.0) - ffi (>= 1.15.0) - eventmachine (1.2.7) - execjs (2.8.1) - faraday (2.7.4) - faraday-net_http (>= 2.0, < 3.1) - ruby2_keywords (>= 0.0.4) - faraday-net_http (3.0.2) - ffi (1.15.5) - forwardable-extended (2.6.0) - gemoji (3.0.1) - github-pages (228) - github-pages-health-check (= 1.17.9) - jekyll (= 3.9.3) - jekyll-avatar (= 0.7.0) - jekyll-coffeescript (= 1.1.1) - jekyll-commonmark-ghpages (= 0.4.0) - jekyll-default-layout (= 0.1.4) - jekyll-feed (= 0.15.1) - jekyll-gist (= 1.5.0) - jekyll-github-metadata (= 2.13.0) - jekyll-include-cache (= 0.2.1) - jekyll-mentions (= 1.6.0) - jekyll-optional-front-matter (= 0.3.2) - jekyll-paginate (= 1.1.0) - jekyll-readme-index (= 0.3.0) - jekyll-redirect-from (= 0.16.0) - jekyll-relative-links (= 0.6.1) - jekyll-remote-theme (= 0.4.3) - jekyll-sass-converter (= 1.5.2) - jekyll-seo-tag (= 2.8.0) - jekyll-sitemap (= 1.4.0) - jekyll-swiss (= 1.0.0) - jekyll-theme-architect (= 0.2.0) - jekyll-theme-cayman (= 0.2.0) - jekyll-theme-dinky (= 0.2.0) - jekyll-theme-hacker (= 0.2.0) - jekyll-theme-leap-day (= 0.2.0) - jekyll-theme-merlot (= 0.2.0) - jekyll-theme-midnight (= 0.2.0) - jekyll-theme-minimal (= 0.2.0) - jekyll-theme-modernist (= 0.2.0) - jekyll-theme-primer (= 0.6.0) - jekyll-theme-slate (= 0.2.0) - jekyll-theme-tactile (= 0.2.0) - jekyll-theme-time-machine (= 0.2.0) - jekyll-titles-from-headings (= 0.5.3) - jemoji (= 0.12.0) - kramdown (= 2.3.2) - kramdown-parser-gfm (= 1.1.0) - liquid (= 4.0.4) - mercenary (~> 0.3) - minima (= 2.5.1) - nokogiri (>= 1.13.6, < 2.0) - rouge (= 3.26.0) - terminal-table (~> 1.4) - github-pages-health-check (1.17.9) - addressable (~> 2.3) - dnsruby (~> 1.60) - octokit (~> 4.0) - public_suffix (>= 3.0, < 5.0) - typhoeus (~> 1.3) - html-pipeline (2.14.3) - activesupport (>= 2) - nokogiri (>= 1.4) - http_parser.rb (0.8.0) - i18n (1.14.1) - concurrent-ruby (~> 1.0) - jekyll (3.9.3) - addressable (~> 2.4) - colorator (~> 1.0) - em-websocket (~> 0.5) - i18n (>= 0.7, < 2) - jekyll-sass-converter (~> 1.0) - jekyll-watch (~> 2.0) - kramdown (>= 1.17, < 3) - liquid (~> 4.0) - mercenary (~> 0.3.3) - pathutil (~> 0.9) - rouge (>= 1.7, < 4) - safe_yaml (~> 1.0) - jekyll-avatar (0.7.0) - jekyll (>= 3.0, < 5.0) - jekyll-coffeescript (1.1.1) - coffee-script (~> 2.2) - coffee-script-source (~> 1.11.1) - jekyll-commonmark (1.4.0) - commonmarker (~> 0.22) - jekyll-commonmark-ghpages (0.4.0) - commonmarker (~> 0.23.7) - jekyll (~> 3.9.0) - jekyll-commonmark (~> 1.4.0) - rouge (>= 2.0, < 5.0) - jekyll-default-layout (0.1.4) - jekyll (~> 3.0) - jekyll-feed (0.15.1) - jekyll (>= 3.7, < 5.0) - jekyll-gist (1.5.0) - octokit (~> 4.2) - jekyll-github-metadata (2.13.0) - jekyll (>= 3.4, < 5.0) - octokit (~> 4.0, != 4.4.0) - jekyll-include-cache (0.2.1) - jekyll (>= 3.7, < 5.0) - jekyll-mentions (1.6.0) - html-pipeline (~> 2.3) - jekyll (>= 3.7, < 5.0) - jekyll-optional-front-matter (0.3.2) - jekyll (>= 3.0, < 5.0) - jekyll-paginate (1.1.0) - jekyll-readme-index (0.3.0) - jekyll (>= 3.0, < 5.0) - jekyll-redirect-from (0.16.0) - jekyll (>= 3.3, < 5.0) - jekyll-relative-links (0.6.1) - jekyll (>= 3.3, < 5.0) - jekyll-remote-theme (0.4.3) - addressable (~> 2.0) - jekyll (>= 3.5, < 5.0) - jekyll-sass-converter (>= 1.0, <= 3.0.0, != 2.0.0) - rubyzip (>= 1.3.0, < 3.0) - jekyll-sass-converter (1.5.2) - sass (~> 3.4) - jekyll-seo-tag (2.8.0) - jekyll (>= 3.8, < 5.0) - jekyll-sitemap (1.4.0) - jekyll (>= 3.7, < 5.0) - jekyll-swiss (1.0.0) - jekyll-theme-architect (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-cayman (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-dinky (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-hacker (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-leap-day (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-merlot (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-midnight (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-minimal (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-modernist (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-primer (0.6.0) - jekyll (> 3.5, < 5.0) - jekyll-github-metadata (~> 2.9) - jekyll-seo-tag (~> 2.0) - jekyll-theme-slate (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-tactile (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-time-machine (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-titles-from-headings (0.5.3) - jekyll (>= 3.3, < 5.0) - jekyll-watch (2.2.1) - listen (~> 3.0) - jemoji (0.12.0) - gemoji (~> 3.0) - html-pipeline (~> 2.2) - jekyll (>= 3.0, < 5.0) - kramdown (2.3.2) - rexml - kramdown-parser-gfm (1.1.0) - kramdown (~> 2.0) - liquid (4.0.4) - listen (3.8.0) - rb-fsevent (~> 0.10, >= 0.10.3) - rb-inotify (~> 0.9, >= 0.9.10) - mercenary (0.3.6) - minima (2.5.1) - jekyll (>= 3.5, < 5.0) - jekyll-feed (~> 0.9) - jekyll-seo-tag (~> 2.1) - minitest (5.21.2) - mutex_m (0.2.0) - nokogiri (1.16.5-x86_64-linux) - racc (~> 1.4) - octokit (4.25.1) - faraday (>= 1, < 3) - sawyer (~> 0.9) - pathutil (0.16.2) - forwardable-extended (~> 2.6) - public_suffix (4.0.7) - racc (1.7.3) - rb-fsevent (0.11.2) - rb-inotify (0.10.1) - ffi (~> 1.0) - rexml (3.3.3) - strscan - rouge (3.26.0) - ruby2_keywords (0.0.5) - rubyzip (2.3.2) - safe_yaml (1.0.5) - sass (3.7.4) - sass-listen (~> 4.0.0) - sass-listen (4.0.0) - rb-fsevent (~> 0.9, >= 0.9.4) - rb-inotify (~> 0.9, >= 0.9.7) - sawyer (0.9.2) - addressable (>= 2.3.5) - faraday (>= 0.17.3, < 3) - simpleidn (0.2.1) - unf (~> 0.1.4) - strscan (3.1.0) - terminal-table (1.8.0) - unicode-display_width (~> 1.1, >= 1.1.1) - typhoeus (1.4.0) - ethon (>= 0.9.0) - tzinfo (2.0.6) - concurrent-ruby (~> 1.0) - unf (0.1.4) - unf_ext - unf_ext (0.0.8.2) - unicode-display_width (1.8.0) - webrick (1.8.1) - -PLATFORMS - x86_64-linux - -DEPENDENCIES - github-pages - webrick - -BUNDLED WITH - 2.4.8 diff --git a/docs/_config.yml b/docs/_config.yml deleted file mode 100644 index 891551df..00000000 --- a/docs/_config.yml +++ /dev/null @@ -1,2 +0,0 @@ -markdown: kramdown -theme: jekyll-theme-cayman diff --git a/docs/_layouts/default.html b/docs/_layouts/default.html deleted file mode 100644 index 587ab4f0..00000000 --- a/docs/_layouts/default.html +++ /dev/null @@ -1,137 +0,0 @@ - - - - - - - - Tree-sitter|{{ page.title }} - - - - - - - -
- - - - -
- {{ content }} -
-
- - - - - - - - diff --git a/docs/assets/css/style.scss b/docs/assets/css/style.scss deleted file mode 100644 index b838211f..00000000 --- a/docs/assets/css/style.scss +++ /dev/null @@ -1,203 +0,0 @@ ---- ---- - -@import 'jekyll-theme-cayman'; - -$padding: 20px; -$sidebar-width: 300px; -$sidebar-transition: left 0.25s; -$container-width: 1024px; - -body { - overflow: scroll; -} - -a[href^="http"]:after { - content: ""; - display: inline-block; - transform: translate(0px, 2px); - width: .9em; - height: .9em; - margin-left: 3px; - background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' fill='%23777'%3E%3Cpath d='M20 3h-5a1 1 0 1 0 0 2h3L8 14a1 1 0 1 0 2 2l9-10v3a1 1 0 1 0 2 0V4a1 1 0 0 0-1-1zM5 3L3 5v14l2 2h14l2-2v-6a1 1 0 1 0-2 0v6H5V5h6a1 1 0 1 0 0-2H5z'/%3E%3C/svg%3E"); - background-size: cover; -} - -#container { - position: relative; - max-width: $container-width; - margin: 0 auto; -} - -#main-content, #sidebar { - padding: $padding 0; -} - -#main-content code:not(pre code, a code) { - color: #c7254e; - font-size: 0.9em; - background-color: #f8f8f8; - border: 1px solid #eaeaea; - border-radius: 3px; - margin: 0 2px; - padding: 0 5px; -} - -#sidebar { - position: fixed; - background: white; - top: 0; - bottom: 0; - width: $sidebar-width; - overflow-y: auto; - border-right: 1px solid #ccc; - z-index: 1; - - .github-repo { - display: inline-block; - padding-left: 3.75em; - font-size: .85em; - } -} - -#sidebar-toggle-link { - font-size: 24px; - position: fixed; - background-color: white; - opacity: 0.75; - box-shadow: 1px 1px 5px #aaa; - left: $sidebar-width; - padding: 5px 10px; - display: none; - z-index: 100; - text-decoration: none !important; - color: #aaa; -} - -#main-content { - position: relative; - padding: $padding; - padding-left: $sidebar-width + $padding; -} - -.nav-link.active { - text-decoration: underline; -} - -a > span { - text-decoration: inherit; -} - -.table-of-contents-section { - border-bottom: 1px solid #ccc; -} - -.logo { - display: block; -} - -.table-of-contents-section.active { - background-color: #edffcb; -} - -.table-of-contents-section { - padding: 10px 20px; -} - -#table-of-contents { - ul { - padding: 0; - margin: 0; - } - - li { - display: block; - padding: 5px 20px; - } -} - -@media (max-width: 900px) { - #sidebar { - left: 0; - transition: $sidebar-transition; - } - - #sidebar-toggle-link { - display: block; - transition: $sidebar-transition; - } - - #main-content { - left: $sidebar-width; - padding-left: $padding; - transition: $sidebar-transition; - } - - body.sidebar-hidden { - #sidebar { - left: -$sidebar-width; - } - - #main-content { - left: 0; - } - - #sidebar-toggle-link { - left: 0; - } - } -} - -#playground-container { - .CodeMirror { - height: auto; - max-height: 350px; - border: 1px solid #aaa; - } - - .CodeMirror-scroll { - height: auto; - max-height: 350px; - } - - h4, select, .field, label { - display: inline-block; - margin-right: 20px; - } - - #logging-checkbox { - height: 15px; - } - - .CodeMirror div.CodeMirror-cursor { - border-left: 3px solid red; - } - - h4#about { - margin: 10ex 0 0 0; - } -} - -#output-container { - padding: 0 10px; - margin: 0; -} - -#output-container-scroll { - padding: 0; - position: relative; - margin-top: 0; - overflow: auto; - max-height: 350px; - border: 1px solid #aaa; -} - -a.highlighted { - background-color: #ddd; - text-decoration: underline; -} - -.query-error { - text-decoration: underline red dashed; - -webkit-text-decoration: underline red dashed; -} diff --git a/docs/assets/js/playground.js b/docs/assets/js/playground.js deleted file mode 100644 index 5864d979..00000000 --- a/docs/assets/js/playground.js +++ /dev/null @@ -1,459 +0,0 @@ -let tree; - -(async () => { - const CAPTURE_REGEX = /@\s*([\w._-]+)/g; - const COLORS_BY_INDEX = [ - 'blue', - 'chocolate', - 'darkblue', - 'darkcyan', - 'darkgreen', - 'darkred', - 'darkslategray', - 'dimgray', - 'green', - 'indigo', - 'navy', - 'red', - 'sienna', - ]; - - const codeInput = document.getElementById('code-input'); - const languageSelect = document.getElementById('language-select'); - const loggingCheckbox = document.getElementById('logging-checkbox'); - const outputContainer = document.getElementById('output-container'); - const outputContainerScroll = document.getElementById('output-container-scroll'); - const playgroundContainer = document.getElementById('playground-container'); - const queryCheckbox = document.getElementById('query-checkbox'); - const queryContainer = document.getElementById('query-container'); - const queryInput = document.getElementById('query-input'); - const updateTimeSpan = document.getElementById('update-time'); - const languagesByName = {}; - - loadState(); - - await TreeSitter.init(); - - const parser = new TreeSitter(); - const codeEditor = CodeMirror.fromTextArea(codeInput, { - lineNumbers: true, - showCursorWhenSelecting: true - }); - - const queryEditor = CodeMirror.fromTextArea(queryInput, { - lineNumbers: true, - showCursorWhenSelecting: true - }); - - const cluster = new Clusterize({ - rows: [], - noDataText: null, - contentElem: outputContainer, - scrollElem: outputContainerScroll - }); - const renderTreeOnCodeChange = debounce(renderTree, 50); - const saveStateOnChange = debounce(saveState, 2000); - const runTreeQueryOnChange = debounce(runTreeQuery, 50); - - let languageName = languageSelect.value; - let treeRows = null; - let treeRowHighlightedIndex = -1; - let parseCount = 0; - let isRendering = 0; - let query; - - codeEditor.on('changes', handleCodeChange); - codeEditor.on('viewportChange', runTreeQueryOnChange); - codeEditor.on('cursorActivity', debounce(handleCursorMovement, 150)); - queryEditor.on('changes', debounce(handleQueryChange, 150)); - - loggingCheckbox.addEventListener('change', handleLoggingChange); - queryCheckbox.addEventListener('change', handleQueryEnableChange); - languageSelect.addEventListener('change', handleLanguageChange); - outputContainer.addEventListener('click', handleTreeClick); - - handleQueryEnableChange(); - await handleLanguageChange() - - playgroundContainer.style.visibility = 'visible'; - - async function handleLanguageChange() { - const newLanguageName = languageSelect.value; - if (!languagesByName[newLanguageName]) { - const url = `${LANGUAGE_BASE_URL}/tree-sitter-${newLanguageName}.wasm` - languageSelect.disabled = true; - try { - languagesByName[newLanguageName] = await TreeSitter.Language.load(url); - } catch (e) { - console.error(e); - languageSelect.value = languageName; - return - } finally { - languageSelect.disabled = false; - } - } - - tree = null; - languageName = newLanguageName; - parser.setLanguage(languagesByName[newLanguageName]); - handleCodeChange(); - handleQueryChange(); - } - - async function handleCodeChange(_editor, changes) { - const newText = `${codeEditor.getValue()}\n`; - const edits = tree && changes && changes.map(treeEditForEditorChange); - - const start = performance.now(); - if (edits) { - for (const edit of edits) { - tree.edit(edit); - } - } - const newTree = parser.parse(newText, tree); - const duration = (performance.now() - start).toFixed(1); - - updateTimeSpan.innerText = `${duration} ms`; - if (tree) tree.delete(); - tree = newTree; - parseCount++; - renderTreeOnCodeChange(); - runTreeQueryOnChange(); - saveStateOnChange(); - } - - async function renderTree() { - isRendering++; - const cursor = tree.walk(); - - const currentRenderCount = parseCount; - let row = ''; - const rows = []; - let finishedRow = false; - let visitedChildren = false; - let indentLevel = 0; - - for (let i = 0;; i++) { - if (i > 0 && i % 10000 === 0) { - await new Promise(r => setTimeout(r, 0)); - if (parseCount !== currentRenderCount) { - cursor.delete(); - isRendering--; - return; - } - } - - let displayName; - if (cursor.nodeIsMissing) { - displayName = `MISSING ${cursor.nodeType}` - } else if (cursor.nodeIsNamed) { - displayName = cursor.nodeType; - } - - if (visitedChildren) { - if (displayName) { - finishedRow = true; - } - - if (cursor.gotoNextSibling()) { - visitedChildren = false; - } else if (cursor.gotoParent()) { - visitedChildren = true; - indentLevel--; - } else { - break; - } - } else { - if (displayName) { - if (finishedRow) { - row += ''; - rows.push(row); - finishedRow = false; - } - const start = cursor.startPosition; - const end = cursor.endPosition; - const id = cursor.nodeId; - let fieldName = cursor.currentFieldName; - if (fieldName) { - fieldName += ': '; - } else { - fieldName = ''; - } - row = `
${' '.repeat(indentLevel)}${fieldName}${displayName} [${start.row}, ${start.column}] - [${end.row}, ${end.column}]`; - finishedRow = true; - } - - if (cursor.gotoFirstChild()) { - visitedChildren = false; - indentLevel++; - } else { - visitedChildren = true; - } - } - } - if (finishedRow) { - row += '
'; - rows.push(row); - } - - cursor.delete(); - cluster.update(rows); - treeRows = rows; - isRendering--; - handleCursorMovement(); - } - - function runTreeQuery(_, startRow, endRow) { - if (endRow == null) { - const viewport = codeEditor.getViewport(); - startRow = viewport.from; - endRow = viewport.to; - } - - codeEditor.operation(() => { - const marks = codeEditor.getAllMarks(); - marks.forEach(m => m.clear()); - - if (tree && query) { - const captures = query.captures( - tree.rootNode, - {row: startRow, column: 0}, - {row: endRow, column: 0}, - ); - let lastNodeId; - for (const {name, node} of captures) { - if (node.id === lastNodeId) continue; - lastNodeId = node.id; - const {startPosition, endPosition} = node; - codeEditor.markText( - {line: startPosition.row, ch: startPosition.column}, - {line: endPosition.row, ch: endPosition.column}, - { - inclusiveLeft: true, - inclusiveRight: true, - css: `color: ${colorForCaptureName(name)}` - } - ); - } - } - }); - } - - function handleQueryChange() { - if (query) { - query.delete(); - query.deleted = true; - query = null; - } - - queryEditor.operation(() => { - queryEditor.getAllMarks().forEach(m => m.clear()); - if (!queryCheckbox.checked) return; - - const queryText = queryEditor.getValue(); - - try { - query = parser.getLanguage().query(queryText); - let match; - - let row = 0; - queryEditor.eachLine((line) => { - while (match = CAPTURE_REGEX.exec(line.text)) { - queryEditor.markText( - {line: row, ch: match.index}, - {line: row, ch: match.index + match[0].length}, - { - inclusiveLeft: true, - inclusiveRight: true, - css: `color: ${colorForCaptureName(match[1])}` - } - ); - } - row++; - }); - } catch (error) { - const startPosition = queryEditor.posFromIndex(error.index); - const endPosition = { - line: startPosition.line, - ch: startPosition.ch + (error.length || Infinity) - }; - - if (error.index === queryText.length) { - if (startPosition.ch > 0) { - startPosition.ch--; - } else if (startPosition.row > 0) { - startPosition.row--; - startPosition.column = Infinity; - } - } - - queryEditor.markText( - startPosition, - endPosition, - { - className: 'query-error', - inclusiveLeft: true, - inclusiveRight: true, - attributes: {title: error.message} - } - ); - } - }); - - runTreeQuery(); - saveQueryState(); - } - - function handleCursorMovement() { - if (isRendering) return; - - const selection = codeEditor.getDoc().listSelections()[0]; - let start = {row: selection.anchor.line, column: selection.anchor.ch}; - let end = {row: selection.head.line, column: selection.head.ch}; - if ( - start.row > end.row || - ( - start.row === end.row && - start.column > end.column - ) - ) { - const swap = end; - end = start; - start = swap; - } - const node = tree.rootNode.namedDescendantForPosition(start, end); - if (treeRows) { - if (treeRowHighlightedIndex !== -1) { - const row = treeRows[treeRowHighlightedIndex]; - if (row) treeRows[treeRowHighlightedIndex] = row.replace('highlighted', 'plain'); - } - treeRowHighlightedIndex = treeRows.findIndex(row => row.includes(`data-id=${node.id}`)); - if (treeRowHighlightedIndex !== -1) { - const row = treeRows[treeRowHighlightedIndex]; - if (row) treeRows[treeRowHighlightedIndex] = row.replace('plain', 'highlighted'); - } - cluster.update(treeRows); - const lineHeight = cluster.options.item_height; - const scrollTop = outputContainerScroll.scrollTop; - const containerHeight = outputContainerScroll.clientHeight; - const offset = treeRowHighlightedIndex * lineHeight; - if (scrollTop > offset - 20) { - $(outputContainerScroll).animate({scrollTop: offset - 20}, 150); - } else if (scrollTop < offset + lineHeight + 40 - containerHeight) { - $(outputContainerScroll).animate({scrollTop: offset - containerHeight + 40}, 150); - } - } - } - - function handleTreeClick(event) { - if (event.target.tagName === 'A') { - event.preventDefault(); - const [startRow, startColumn, endRow, endColumn] = event - .target - .dataset - .range - .split(',') - .map(n => parseInt(n)); - codeEditor.focus(); - codeEditor.setSelection( - {line: startRow, ch: startColumn}, - {line: endRow, ch: endColumn} - ); - } - } - - function handleLoggingChange() { - if (loggingCheckbox.checked) { - parser.setLogger((message, lexing) => { - if (lexing) { - console.log(" ", message) - } else { - console.log(message) - } - }); - } else { - parser.setLogger(null); - } - } - - function handleQueryEnableChange() { - if (queryCheckbox.checked) { - queryContainer.style.visibility = ''; - queryContainer.style.position = ''; - } else { - queryContainer.style.visibility = 'hidden'; - queryContainer.style.position = 'absolute'; - } - handleQueryChange(); - } - - function treeEditForEditorChange(change) { - const oldLineCount = change.removed.length; - const newLineCount = change.text.length; - const lastLineLength = change.text[newLineCount - 1].length; - - const startPosition = {row: change.from.line, column: change.from.ch}; - const oldEndPosition = {row: change.to.line, column: change.to.ch}; - const newEndPosition = { - row: startPosition.row + newLineCount - 1, - column: newLineCount === 1 - ? startPosition.column + lastLineLength - : lastLineLength - }; - - const startIndex = codeEditor.indexFromPos(change.from); - let newEndIndex = startIndex + newLineCount - 1; - let oldEndIndex = startIndex + oldLineCount - 1; - for (let i = 0; i < newLineCount; i++) newEndIndex += change.text[i].length; - for (let i = 0; i < oldLineCount; i++) oldEndIndex += change.removed[i].length; - - return { - startIndex, oldEndIndex, newEndIndex, - startPosition, oldEndPosition, newEndPosition - }; - } - - function colorForCaptureName(capture) { - const id = query.captureNames.indexOf(capture); - return COLORS_BY_INDEX[id % COLORS_BY_INDEX.length]; - } - - function loadState() { - const language = localStorage.getItem("language"); - const sourceCode = localStorage.getItem("sourceCode"); - const query = localStorage.getItem("query"); - const queryEnabled = localStorage.getItem("queryEnabled"); - if (language != null && sourceCode != null && query != null) { - queryInput.value = query; - codeInput.value = sourceCode; - languageSelect.value = language; - queryCheckbox.checked = (queryEnabled === 'true'); - } - } - - function saveState() { - localStorage.setItem("language", languageSelect.value); - localStorage.setItem("sourceCode", codeEditor.getValue()); - saveQueryState(); - } - - function saveQueryState() { - localStorage.setItem("queryEnabled", queryCheckbox.checked); - localStorage.setItem("query", queryEditor.getValue()); - } - - function debounce(func, wait, immediate) { - let timeout; - return function() { - const context = this, args = arguments; - const later = function() { - timeout = null; - if (!immediate) func.apply(context, args); - }; - const callNow = immediate && !timeout; - clearTimeout(timeout); - timeout = setTimeout(later, wait); - if (callNow) func.apply(context, args); - }; - } -})(); diff --git a/docs/book.toml b/docs/book.toml new file mode 100644 index 00000000..0894c988 --- /dev/null +++ b/docs/book.toml @@ -0,0 +1,32 @@ +[book] +authors = [ + "Max Brunsfeld ", + "Amaan Qureshi ", +] +language = "en" +src = "src" +title = "Tree-sitter" + +[output.html] +additional-css = [ + "src/assets/css/playground.css", + "src/assets/css/mdbook-admonish.css", +] +additional-js = ["src/assets/js/playground.js"] +git-repository-url = "https://github.com/tree-sitter/tree-sitter" +git-repository-icon = "fa-github" +edit-url-template = "https://github.com/tree-sitter/tree-sitter/edit/master/docs/{path}" + +[output.html.search] +limit-results = 20 +use-boolean-and = true +boost-title = 2 +boost-hierarchy = 2 +boost-paragraph = 1 +expand = true + +[preprocessor] + +[preprocessor.admonish] +command = "mdbook-admonish" +assets_version = "3.0.2" # do not edit: managed by `mdbook-admonish install` diff --git a/docs/index.md b/docs/index.md deleted file mode 100644 index 9657b062..00000000 --- a/docs/index.md +++ /dev/null @@ -1,68 +0,0 @@ ---- -title: Introduction ---- - -# Introduction - -Tree-sitter is a parser generator tool and an incremental parsing library. It can build a concrete syntax tree for a source file and efficiently update the syntax tree as the source file is edited. Tree-sitter aims to be: - -* **General** enough to parse any programming language -* **Fast** enough to parse on every keystroke in a text editor -* **Robust** enough to provide useful results even in the presence of syntax errors -* **Dependency-free** so that the runtime library (which is written in pure [C](https://github.com/tree-sitter/tree-sitter/tree/master/lib)) can be embedded in any application - -### Language Bindings - -There are currently bindings that allow Tree-sitter to be used from the following languages: - -#### Official - -* [C#](https://github.com/tree-sitter/csharp-tree-sitter) -* [Go](https://github.com/tree-sitter/go-tree-sitter) -* [Haskell](https://github.com/tree-sitter/haskell-tree-sitter) -* [Java (JDK 22)](https://github.com/tree-sitter/java-tree-sitter) -* [JavaScript (Node.js)](https://github.com/tree-sitter/node-tree-sitter) -* [JavaScript (Wasm)](https://github.com/tree-sitter/tree-sitter/tree/master/lib/binding_web) -* [Kotlin](https://github.com/tree-sitter/kotlin-tree-sitter) -* [Python](https://github.com/tree-sitter/py-tree-sitter) -* [Rust](https://github.com/tree-sitter/tree-sitter/tree/master/lib/binding_rust) - -#### Third-party - -* [Delphi](https://github.com/modersohn/delphi-tree-sitter) -* [ELisp](https://www.gnu.org/software/emacs/manual/html_node/elisp/Parsing-Program-Source.html) -* [Go](https://github.com/smacker/go-tree-sitter) -* [Guile](https://github.com/Z572/guile-ts) -* [Java (Android)](https://github.com/AndroidIDEOfficial/android-tree-sitter) -* [Java (JDK 8+)](https://github.com/bonede/tree-sitter-ng) -* [Java (JDK 11+)](https://github.com/seart-group/java-tree-sitter) -* [Julia](https://github.com/MichaelHatherly/TreeSitter.jl) -* [Lua](https://github.com/euclidianAce/ltreesitter) -* [Lua](https://github.com/xcb-xwii/lua-tree-sitter) -* [OCaml](https://github.com/semgrep/ocaml-tree-sitter-core) -* [Odin](https://github.com/laytan/odin-tree-sitter) -* [Perl](https://metacpan.org/pod/Text::Treesitter) -* [R](https://github.com/DavisVaughan/r-tree-sitter) -* [Ruby](https://github.com/Faveod/ruby-tree-sitter) -* [Swift](https://github.com/ChimeHQ/SwiftTreeSitter) - -### Parsers - -A list of known parsers can be found in the [wiki](https://github.com/tree-sitter/tree-sitter/wiki/List-of-parsers). - -### Talks on Tree-sitter - -* [Strange Loop 2018](https://www.thestrangeloop.com/2018/tree-sitter---a-new-parsing-system-for-programming-tools.html) -* [FOSDEM 2018](https://www.youtube.com/watch?v=0CGzC_iss-8) -* [GitHub Universe 2017](https://www.youtube.com/watch?v=a1rC79DHpmY) - -### Underlying Research - -The design of Tree-sitter was greatly influenced by the following research papers: - -* [Practical Algorithms for Incremental Software Development Environments](https://www2.eecs.berkeley.edu/Pubs/TechRpts/1997/CSD-97-946.pdf) -* [Context Aware Scanning for Parsing Extensible Languages](https://www-users.cse.umn.edu/~evw/pubs/vanwyk07gpce/vanwyk07gpce.pdf) -* [Efficient and Flexible Incremental Parsing](https://harmonia.cs.berkeley.edu/papers/twagner-parsing.pdf) -* [Incremental Analysis of Real Programming Languages](https://harmonia.cs.berkeley.edu/papers/twagner-glr.pdf) -* [Error Detection and Recovery in LR Parsers](https://web.archive.org/web/20240302031213/https://what-when-how.com/compiler-writing/bottom-up-parsing-compiler-writing-part-13/) -* [Error Recovery for LR Parsers](https://apps.dtic.mil/sti/pdfs/ADA043470.pdf) diff --git a/docs/package.nix b/docs/package.nix new file mode 100644 index 00000000..1d07631f --- /dev/null +++ b/docs/package.nix @@ -0,0 +1,33 @@ +{ + stdenv, + lib, + version, + mdbook, + mdbook-admonish, +}: +stdenv.mkDerivation { + inherit version; + + src = ./.; + pname = "tree-sitter-docs"; + + nativeBuildInputs = [ + mdbook + mdbook-admonish + ]; + + buildPhase = '' + mdbook build + ''; + + installPhase = '' + mkdir -p $out/share/doc + cp -r book $out/share/doc/tree-sitter + ''; + + meta = { + description = "Tree-sitter documentation"; + homepage = "https://tree-sitter.github.io/tree-sitter"; + license = lib.licenses.mit; + }; +} diff --git a/docs/section-2-using-parsers.md b/docs/section-2-using-parsers.md deleted file mode 100644 index 46c39616..00000000 --- a/docs/section-2-using-parsers.md +++ /dev/null @@ -1,973 +0,0 @@ ---- -title: Using Parsers -permalink: using-parsers ---- - -# Using Parsers - -All of Tree-sitter's parsing functionality is exposed through C APIs. Applications written in higher-level languages can use Tree-sitter via binding libraries like [node-tree-sitter](https://github.com/tree-sitter/node-tree-sitter) or the [tree-sitter rust crate](https://github.com/tree-sitter/tree-sitter/tree/master/lib/binding_rust), which have their own documentation. - -This document will describe the general concepts of how to use Tree-sitter, which should be relevant regardless of what language you're using. It also goes into some C-specific details that are useful if you're using the C API directly or are building a new binding to a different language. - -All of the API functions shown here are declared and documented in the [`tree_sitter/api.h`](https://github.com/tree-sitter/tree-sitter/blob/master/lib/include/tree_sitter/api.h) header file. You may also want to browse the [online Rust API docs](https://docs.rs/tree-sitter), which correspond to the C APIs closely. - -## Getting Started - -### Building the Library - -To build the library on a POSIX system, just run `make` in the Tree-sitter directory. This will create a static library called `libtree-sitter.a` as well as dynamic libraries. - -Alternatively, you can incorporate the library in a larger project's build system by adding one source file to the build. This source file needs two directories to be in the include path when compiled: - -**source file:** - -- `tree-sitter/lib/src/lib.c` - -**include directories:** - -- `tree-sitter/lib/src` -- `tree-sitter/lib/include` - -### The Basic Objects - -There are four main types of objects involved when using Tree-sitter: languages, parsers, syntax trees, and syntax nodes. In C, these are called `TSLanguage`, `TSParser`, `TSTree`, and `TSNode`. - -- A `TSLanguage` is an opaque object that defines how to parse a particular programming language. The code for each `TSLanguage` is generated by Tree-sitter. Many languages are already available in separate git repositories within the [Tree-sitter GitHub organization](https://github.com/tree-sitter). See [the next page](./creating-parsers) for how to create new languages. -- A `TSParser` is a stateful object that can be assigned a `TSLanguage` and used to produce a `TSTree` based on some source code. -- A `TSTree` represents the syntax tree of an entire source code file. It contains `TSNode` instances that indicate the structure of the source code. It can also be edited and used to produce a new `TSTree` in the event that the source code changes. -- A `TSNode` represents a single node in the syntax tree. It tracks its start and end positions in the source code, as well as its relation to other nodes like its parent, siblings and children. - -### An Example Program - -Here's an example of a simple C program that uses the Tree-sitter [JSON parser](https://github.com/tree-sitter/tree-sitter-json). - -```c -// Filename - test-json-parser.c - -#include -#include -#include -#include - -// Declare the `tree_sitter_json` function, which is -// implemented by the `tree-sitter-json` library. -const TSLanguage *tree_sitter_json(void); - -int main() { - // Create a parser. - TSParser *parser = ts_parser_new(); - - // Set the parser's language (JSON in this case). - ts_parser_set_language(parser, tree_sitter_json()); - - // Build a syntax tree based on source code stored in a string. - const char *source_code = "[1, null]"; - TSTree *tree = ts_parser_parse_string( - parser, - NULL, - source_code, - strlen(source_code) - ); - - // Get the root node of the syntax tree. - TSNode root_node = ts_tree_root_node(tree); - - // Get some child nodes. - TSNode array_node = ts_node_named_child(root_node, 0); - TSNode number_node = ts_node_named_child(array_node, 0); - - // Check that the nodes have the expected types. - assert(strcmp(ts_node_type(root_node), "document") == 0); - assert(strcmp(ts_node_type(array_node), "array") == 0); - assert(strcmp(ts_node_type(number_node), "number") == 0); - - // Check that the nodes have the expected child counts. - assert(ts_node_child_count(root_node) == 1); - assert(ts_node_child_count(array_node) == 5); - assert(ts_node_named_child_count(array_node) == 2); - assert(ts_node_child_count(number_node) == 0); - - // Print the syntax tree as an S-expression. - char *string = ts_node_string(root_node); - printf("Syntax tree: %s\n", string); - - // Free all of the heap-allocated memory. - free(string); - ts_tree_delete(tree); - ts_parser_delete(parser); - return 0; -} -``` - -This program uses the Tree-sitter C API, which is declared in the header file `tree-sitter/api.h`, so we need to add the `tree-sitter/lib/include` directory to the include path. We also need to link `libtree-sitter.a` into the binary. We compile the source code of the JSON language directly into the binary as well. - -```sh -clang \ - -I tree-sitter/lib/include \ - test-json-parser.c \ - tree-sitter-json/src/parser.c \ - tree-sitter/libtree-sitter.a \ - -o test-json-parser - -./test-json-parser -``` - -## Basic Parsing - -### Providing the Code - -In the example above, we parsed source code stored in a simple string using the `ts_parser_parse_string` function: - -```c -TSTree *ts_parser_parse_string( - TSParser *self, - const TSTree *old_tree, - const char *string, - uint32_t length -); -``` - -You may want to parse source code that's stored in a custom data structure, like a [piece table](https://en.wikipedia.org/wiki/Piece_table) or a [rope](). In this case, you can use the more general `ts_parser_parse` function: - -```c -TSTree *ts_parser_parse( - TSParser *self, - const TSTree *old_tree, - TSInput input -); -``` - -The `TSInput` structure lets you provide your own function for reading a chunk of text at a given byte offset and row/column position. The function can return text encoded in either UTF8 or UTF16. This interface allows you to efficiently parse text that is stored in your own data structure. - -```c -typedef struct { - void *payload; - const char *(*read)( - void *payload, - uint32_t byte_offset, - TSPoint position, - uint32_t *bytes_read - ); - TSInputEncoding encoding; - DecodeFunction decode; -} TSInput; -``` - -In the event that you want to decode text that is not encoded in UTF-8 or UTF16, then you can set the `decode` field of the input to your function that will decode text. The signature of the `DecodeFunction` is as follows: - -```c -typedef uint32_t (*DecodeFunction)( - const uint8_t *string, - uint32_t length, - int32_t *code_point -); -``` - -The `string` argument is a pointer to the text to decode, which comes from the `read` function, and the `length` argument is the length of the `string`. The `code_point` argument is a pointer to an integer that represents the decoded code point, and should be written to in your `decode` callback. The function should return the number of bytes decoded. - -### Syntax Nodes - -Tree-sitter provides a [DOM](https://en.wikipedia.org/wiki/Document_Object_Model)-style interface for inspecting syntax trees. A syntax node's _type_ is a string that indicates which grammar rule the node represents. - -```c -const char *ts_node_type(TSNode); -``` - -Syntax nodes store their position in the source code both in terms of raw bytes and row/column coordinates. -In a point, rows and columns are zero-based. The `row` field represents the number of newlines before a given -position, while `column` represents the number of bytes between the position and beginning of the line. - -```c -uint32_t ts_node_start_byte(TSNode); -uint32_t ts_node_end_byte(TSNode); - -typedef struct { - uint32_t row; - uint32_t column; -} TSPoint; - -TSPoint ts_node_start_point(TSNode); -TSPoint ts_node_end_point(TSNode); -``` - -### Retrieving Nodes - -Every tree has a _root node_: - -```c -TSNode ts_tree_root_node(const TSTree *); -``` - -Once you have a node, you can access the node's children: - -```c -uint32_t ts_node_child_count(TSNode); -TSNode ts_node_child(TSNode, uint32_t); -``` - -You can also access its siblings and parent: - -```c -TSNode ts_node_next_sibling(TSNode); -TSNode ts_node_prev_sibling(TSNode); -TSNode ts_node_parent(TSNode); -``` - -These methods may all return a _null node_ to indicate, for example, that a node does not _have_ a next sibling. You can check if a node is null: - -```c -bool ts_node_is_null(TSNode); -``` - -### Named vs Anonymous Nodes - -Tree-sitter produces [_concrete_ syntax trees](https://en.wikipedia.org/wiki/Parse_tree) - trees that contain nodes for every individual token in the source code, including things like commas and parentheses. This is important for use-cases that deal with individual tokens, like [syntax highlighting](https://en.wikipedia.org/wiki/Syntax_highlighting). But some types of code analysis are easier to perform using an [_abstract_ syntax tree](https://en.wikipedia.org/wiki/Abstract_syntax_tree) - a tree in which the less important details have been removed. Tree-sitter's trees support these use cases by making a distinction between _named_ and _anonymous_ nodes. - -Consider a grammar rule like this: - -```js -if_statement: ($) => seq("if", "(", $._expression, ")", $._statement); -``` - -A syntax node representing an `if_statement` in this language would have 5 children: the condition expression, the body statement, as well as the `if`, `(`, and `)` tokens. The expression and the statement would be marked as _named_ nodes, because they have been given explicit names in the grammar. But the `if`, `(`, and `)` nodes would _not_ be named nodes, because they are represented in the grammar as simple strings. - -You can check whether any given node is named: - -```c -bool ts_node_is_named(TSNode); -``` - -When traversing the tree, you can also choose to skip over anonymous nodes by using the `_named_` variants of all of the methods described above: - -```c -TSNode ts_node_named_child(TSNode, uint32_t); -uint32_t ts_node_named_child_count(TSNode); -TSNode ts_node_next_named_sibling(TSNode); -TSNode ts_node_prev_named_sibling(TSNode); -``` - -If you use this group of methods, the syntax tree functions much like an abstract syntax tree. - -### Node Field Names - -To make syntax nodes easier to analyze, many grammars assign unique _field names_ to particular child nodes. The next page [explains](./creating-parsers#using-fields) how to do this on your own grammars. If a syntax node has fields, you can access its children using their field name: - -```c -TSNode ts_node_child_by_field_name( - TSNode self, - const char *field_name, - uint32_t field_name_length -); -``` - -Fields also have numeric ids that you can use, if you want to avoid repeated string comparisons. You can convert between strings and ids using the `TSLanguage`: - -```c -uint32_t ts_language_field_count(const TSLanguage *); -const char *ts_language_field_name_for_id(const TSLanguage *, TSFieldId); -TSFieldId ts_language_field_id_for_name(const TSLanguage *, const char *, uint32_t); -``` - -The field ids can be used in place of the name: - -```c -TSNode ts_node_child_by_field_id(TSNode, TSFieldId); -``` - -## Advanced Parsing - -### Editing - -In applications like text editors, you often need to re-parse a file after its source code has changed. Tree-sitter is designed to support this use case efficiently. There are two steps required. First, you must _edit_ the syntax tree, which adjusts the ranges of its nodes so that they stay in sync with the code. - -```c -typedef struct { - uint32_t start_byte; - uint32_t old_end_byte; - uint32_t new_end_byte; - TSPoint start_point; - TSPoint old_end_point; - TSPoint new_end_point; -} TSInputEdit; - -void ts_tree_edit(TSTree *, const TSInputEdit *); -``` - -Then, you can call `ts_parser_parse` again, passing in the old tree. This will create a new tree that internally shares structure with the old tree. - -When you edit a syntax tree, the positions of its nodes will change. If you have stored any `TSNode` instances outside of the `TSTree`, you must update their positions separately, using the same `TSInput` value, in order to update their cached positions. - -```c -void ts_node_edit(TSNode *, const TSInputEdit *); -``` - -This `ts_node_edit` function is _only_ needed in the case where you have retrieved `TSNode` instances _before_ editing the tree, and then _after_ editing the tree, you want to continue to use those specific node instances. Often, you'll just want to re-fetch nodes from the edited tree, in which case `ts_node_edit` is not needed. - -### Multi-language Documents - -Sometimes, different parts of a file may be written in different languages. For example, templating languages like [EJS](https://ejs.co) and [ERB](https://ruby-doc.org/stdlib-2.5.1/libdoc/erb/rdoc/ERB.html) allow you to generate HTML by writing a mixture of HTML and another language like JavaScript or Ruby. - -Tree-sitter handles these types of documents by allowing you to create a syntax tree based on the text in certain _ranges_ of a file. - -```c -typedef struct { - TSPoint start_point; - TSPoint end_point; - uint32_t start_byte; - uint32_t end_byte; -} TSRange; - -void ts_parser_set_included_ranges( - TSParser *self, - const TSRange *ranges, - uint32_t range_count -); -``` - -For example, consider this ERB document: - -```erb -
    - <% people.each do |person| %> -
  • <%= person.name %>
  • - <% end %> -
-``` - -Conceptually, it can be represented by three syntax trees with overlapping ranges: an ERB syntax tree, a Ruby syntax tree, and an HTML syntax tree. You could generate these syntax trees with the following code: - -```c -#include -#include - -// These functions are each implemented in their own repo. -const TSLanguage *tree_sitter_embedded_template(void); -const TSLanguage *tree_sitter_html(void); -const TSLanguage *tree_sitter_ruby(void); - -int main(int argc, const char **argv) { - const char *text = argv[1]; - unsigned len = strlen(text); - - // Parse the entire text as ERB. - TSParser *parser = ts_parser_new(); - ts_parser_set_language(parser, tree_sitter_embedded_template()); - TSTree *erb_tree = ts_parser_parse_string(parser, NULL, text, len); - TSNode erb_root_node = ts_tree_root_node(erb_tree); - - // In the ERB syntax tree, find the ranges of the `content` nodes, - // which represent the underlying HTML, and the `code` nodes, which - // represent the interpolated Ruby. - TSRange html_ranges[10]; - TSRange ruby_ranges[10]; - unsigned html_range_count = 0; - unsigned ruby_range_count = 0; - unsigned child_count = ts_node_child_count(erb_root_node); - - for (unsigned i = 0; i < child_count; i++) { - TSNode node = ts_node_child(erb_root_node, i); - if (strcmp(ts_node_type(node), "content") == 0) { - html_ranges[html_range_count++] = (TSRange) { - ts_node_start_point(node), - ts_node_end_point(node), - ts_node_start_byte(node), - ts_node_end_byte(node), - }; - } else { - TSNode code_node = ts_node_named_child(node, 0); - ruby_ranges[ruby_range_count++] = (TSRange) { - ts_node_start_point(code_node), - ts_node_end_point(code_node), - ts_node_start_byte(code_node), - ts_node_end_byte(code_node), - }; - } - } - - // Use the HTML ranges to parse the HTML. - ts_parser_set_language(parser, tree_sitter_html()); - ts_parser_set_included_ranges(parser, html_ranges, html_range_count); - TSTree *html_tree = ts_parser_parse_string(parser, NULL, text, len); - TSNode html_root_node = ts_tree_root_node(html_tree); - - // Use the Ruby ranges to parse the Ruby. - ts_parser_set_language(parser, tree_sitter_ruby()); - ts_parser_set_included_ranges(parser, ruby_ranges, ruby_range_count); - TSTree *ruby_tree = ts_parser_parse_string(parser, NULL, text, len); - TSNode ruby_root_node = ts_tree_root_node(ruby_tree); - - // Print all three trees. - char *erb_sexp = ts_node_string(erb_root_node); - char *html_sexp = ts_node_string(html_root_node); - char *ruby_sexp = ts_node_string(ruby_root_node); - printf("ERB: %s\n", erb_sexp); - printf("HTML: %s\n", html_sexp); - printf("Ruby: %s\n", ruby_sexp); - return 0; -} -``` - -This API allows for great flexibility in how languages can be composed. Tree-sitter is not responsible for mediating the interactions between languages. Instead, you are free to do that using arbitrary application-specific logic. - -### Concurrency - -Tree-sitter supports multi-threaded use cases by making syntax trees very cheap to copy. - -```c -TSTree *ts_tree_copy(const TSTree *); -``` - -Internally, copying a syntax tree just entails incrementing an atomic reference count. Conceptually, it provides you a new tree which you can freely query, edit, reparse, or delete on a new thread while continuing to use the original tree on a different thread. Note that individual `TSTree` instances are _not_ thread safe; you must copy a tree if you want to use it on multiple threads simultaneously. - -## Other Tree Operations - -### Walking Trees with Tree Cursors - -You can access every node in a syntax tree using the `TSNode` APIs [described above](#retrieving-nodes), but if you need to access a large number of nodes, the fastest way to do so is with a _tree cursor_. A cursor is a stateful object that allows you to walk a syntax tree with maximum efficiency. - -Note that the given input node is considered the root of the cursor, and the -cursor cannot walk outside this node, so going to the parent or any sibling -of the root node will return `false`. This has no unexpected effects if the given -input node is the actual `root` node of the tree, but is something to keep in mind -when using nodes that are not the `root` node. - -You can initialize a cursor from any node: - -```c -TSTreeCursor ts_tree_cursor_new(TSNode); -``` - -You can move the cursor around the tree: - -```c -bool ts_tree_cursor_goto_first_child(TSTreeCursor *); -bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *); -bool ts_tree_cursor_goto_parent(TSTreeCursor *); -``` - -These methods return `true` if the cursor successfully moved and `false` if there was no node to move to. - -You can always retrieve the cursor's current node, as well as the [field name](#node-field-names) that is associated with the current node. - -```c -TSNode ts_tree_cursor_current_node(const TSTreeCursor *); -const char *ts_tree_cursor_current_field_name(const TSTreeCursor *); -TSFieldId ts_tree_cursor_current_field_id(const TSTreeCursor *); -``` - -## Pattern Matching with Queries - -Many code analysis tasks involve searching for patterns in syntax trees. Tree-sitter provides a small declarative language for expressing these patterns and searching for matches. The language is similar to the format of Tree-sitter's [unit test system](./creating-parsers#command-test). - -### Query Syntax - -A _query_ consists of one or more _patterns_, where each pattern is an [S-expression](https://en.wikipedia.org/wiki/S-expression) that matches a certain set of nodes in a syntax tree. The expression to match a given node consists of a pair of parentheses containing two things: the node's type, and optionally, a series of other S-expressions that match the node's children. For example, this pattern would match any `binary_expression` node whose children are both `number_literal` nodes: - -```scheme -(binary_expression (number_literal) (number_literal)) -``` - -Children can also be omitted. For example, this would match any `binary_expression` where at least _one_ of child is a `string_literal` node: - -```scheme -(binary_expression (string_literal)) -``` - -#### Fields - -In general, it's a good idea to make patterns more specific by specifying [field names](#node-field-names) associated with child nodes. You do this by prefixing a child pattern with a field name followed by a colon. For example, this pattern would match an `assignment_expression` node where the `left` child is a `member_expression` whose `object` is a `call_expression`. - -```scheme -(assignment_expression - left: (member_expression - object: (call_expression))) -``` - -#### Negated Fields - -You can also constrain a pattern so that it only matches nodes that _lack_ a certain field. To do this, add a field name prefixed by a `!` within the parent pattern. For example, this pattern would match a class declaration with no type parameters: - -```scheme -(class_declaration - name: (identifier) @class_name - !type_parameters) -``` - -#### Anonymous Nodes - -The parenthesized syntax for writing nodes only applies to [named nodes](#named-vs-anonymous-nodes). To match specific anonymous nodes, you write their name between double quotes. For example, this pattern would match any `binary_expression` where the operator is `!=` and the right side is `null`: - -```scheme -(binary_expression - operator: "!=" - right: (null)) -``` - -#### Capturing Nodes - -When matching patterns, you may want to process specific nodes within the pattern. Captures allow you to associate names with specific nodes in a pattern, so that you can later refer to those nodes by those names. Capture names are written _after_ the nodes that they refer to, and start with an `@` character. - -For example, this pattern would match any assignment of a `function` to an `identifier`, and it would associate the name `the-function-name` with the identifier: - -```scheme -(assignment_expression - left: (identifier) @the-function-name - right: (function)) -``` - -And this pattern would match all method definitions, associating the name `the-method-name` with the method name, `the-class-name` with the containing class name: - -```scheme -(class_declaration - name: (identifier) @the-class-name - body: (class_body - (method_definition - name: (property_identifier) @the-method-name))) -``` - -#### Quantification Operators - -You can match a repeating sequence of sibling nodes using the postfix `+` and `*` _repetition_ operators, which work analogously to the `+` and `*` operators [in regular expressions](https://en.wikipedia.org/wiki/Regular_expression#Basic_concepts). The `+` operator matches _one or more_ repetitions of a pattern, and the `*` operator matches _zero or more_. - -For example, this pattern would match a sequence of one or more comments: - -```scheme -(comment)+ -``` - -This pattern would match a class declaration, capturing all of the decorators if any were present: - -```scheme -(class_declaration - (decorator)* @the-decorator - name: (identifier) @the-name) -``` - -You can also mark a node as optional using the `?` operator. For example, this pattern would match all function calls, capturing a string argument if one was present: - -```scheme -(call_expression - function: (identifier) @the-function - arguments: (arguments (string)? @the-string-arg)) -``` - -#### Grouping Sibling Nodes - -You can also use parentheses for grouping a sequence of _sibling_ nodes. For example, this pattern would match a comment followed by a function declaration: - -```scheme -( - (comment) - (function_declaration) -) -``` - -Any of the quantification operators mentioned above (`+`, `*`, and `?`) can also be applied to groups. For example, this pattern would match a comma-separated series of numbers: - -```scheme -( - (number) - ("," (number))* -) -``` - -#### Alternations - -An alternation is written as a pair of square brackets (`[]`) containing a list of alternative patterns. -This is similar to _character classes_ from regular expressions (`[abc]` matches either a, b, or c). - -For example, this pattern would match a call to either a variable or an object property. -In the case of a variable, capture it as `@function`, and in the case of a property, capture it as `@method`: - -```scheme -(call_expression - function: [ - (identifier) @function - (member_expression - property: (property_identifier) @method) - ]) -``` - -This pattern would match a set of possible keyword tokens, capturing them as `@keyword`: - -```scheme -[ - "break" - "delete" - "else" - "for" - "function" - "if" - "return" - "try" - "while" -] @keyword -``` - -#### Wildcard Node - -A wildcard node is represented with an underscore (`_`), it matches any node. -This is similar to `.` in regular expressions. -There are two types, `(_)` will match any named node, -and `_` will match any named or anonymous node. - -For example, this pattern would match any node inside a call: - -```scheme -(call (_) @call.inner) -``` - -#### Anchors - -The anchor operator, `.`, is used to constrain the ways in which child patterns are matched. It has different behaviors depending on where it's placed inside a query. - -When `.` is placed before the _first_ child within a parent pattern, the child will only match when it is the first named node in the parent. For example, the below pattern matches a given `array` node at most once, assigning the `@the-element` capture to the first `identifier` node in the parent `array`: - -```scheme -(array . (identifier) @the-element) -``` - -Without this anchor, the pattern would match once for every identifier in the array, with `@the-element` bound to each matched identifier. - -Similarly, an anchor placed after a pattern's _last_ child will cause that child pattern to only match nodes that are the last named child of their parent. The below pattern matches only nodes that are the last named child within a `block`. - -```scheme -(block (_) @last-expression .) -``` - -Finally, an anchor _between_ two child patterns will cause the patterns to only match nodes that are immediate siblings. The pattern below, given a long dotted name like `a.b.c.d`, will only match pairs of consecutive identifiers: `a, b`, `b, c`, and `c, d`. - -```scheme -(dotted_name - (identifier) @prev-id - . - (identifier) @next-id) -``` - -Without the anchor, non-consecutive pairs like `a, c` and `b, d` would also be matched. - -The restrictions placed on a pattern by an anchor operator ignore anonymous nodes. - -#### Predicates - -You can also specify arbitrary metadata and conditions associated with a pattern -by adding _predicate_ S-expressions anywhere within your pattern. Predicate S-expressions -start with a _predicate name_ beginning with a `#` character. After that, they can -contain an arbitrary number of `@`-prefixed capture names or strings. - -Tree-Sitter's CLI supports the following predicates by default: - -##### eq?, not-eq?, any-eq?, any-not-eq? - -This family of predicates allows you to match against a single capture or string -value. - -The first argument must be a capture, but the second can be either a capture to -compare the two captures' text, or a string to compare first capture's text -against. - -The base predicate is "#eq?", but its complement "#not-eq?" can be used to _not_ -match a value. - -Consider the following example targeting C: - -```scheme -((identifier) @variable.builtin - (#eq? @variable.builtin "self")) -``` - -This pattern would match any identifier that is `self`. - -And this pattern would match key-value pairs where the `value` is an identifier -with the same name as the key: - -```scheme -( - (pair - key: (property_identifier) @key-name - value: (identifier) @value-name) - (#eq? @key-name @value-name) -) -``` - -The prefix "any-" is meant for use with quantified captures. Here's -an example finding a segment of empty comments - -```scheme -((comment)+ @comment.empty - (#any-eq? @comment.empty "//")) -``` - -Note that "#any-eq?" will match a quantified capture if -_any_ of the nodes match the predicate, while by default a quantified capture -will only match if _all_ the nodes match the predicate. - -##### match?, not-match?, any-match?, any-not-match? - -These predicates are similar to the eq? predicates, but they use regular expressions -to match against the capture's text. - -The first argument must be a capture, and the second must be a string containing -a regular expression. - -For example, this pattern would match identifier whose name is written in `SCREAMING_SNAKE_CASE`: - -```scheme -((identifier) @constant - (#match? @constant "^[A-Z][A-Z_]+")) -``` - -Here's an example finding potential documentation comments in C - -```scheme -((comment)+ @comment.documentation - (#match? @comment.documentation "^///\\s+.*")) -``` - -Here's another example finding Cgo comments to potentially inject with C - -```scheme -((comment)+ @injection.content - . - (import_declaration - (import_spec path: (interpreted_string_literal) @_import_c)) - (#eq? @_import_c "\"C\"") - (#match? @injection.content "^//")) -``` - -##### any-of?, not-any-of? - -The "any-of?" predicate allows you to match a capture against multiple strings, -and will match if the capture's text is equal to any of the strings. - -Consider this example that targets JavaScript: - -```scheme -((identifier) @variable.builtin - (#any-of? @variable.builtin - "arguments" - "module" - "console" - "window" - "document")) -``` - -This will match any of the builtin variables in JavaScript. - -_Note_ — Predicates are not handled directly by the Tree-sitter C library. -They are just exposed in a structured form so that higher-level code can perform -the filtering. However, higher-level bindings to Tree-sitter like -[the Rust Crate](https://github.com/tree-sitter/tree-sitter/tree/master/lib/binding_rust) -or the [WebAssembly binding](https://github.com/tree-sitter/tree-sitter/tree/master/lib/binding_web) -do implement a few common predicates like the `#eq?`, `#match?`, and `#any-of?` -predicates explained above. - -To recap about the predicates Tree-Sitter's bindings support: - -- `#eq?` checks for a direct match against a capture or string -- `#match?` checks for a match against a regular expression -- `#any-of?` checks for a match against a list of strings -- Adding `not-` to the beginning of any of these predicates will negate the match -- By default, a quantified capture will only match if _all_ of the nodes match the predicate -- Adding `any-` before the `eq` or `match` predicates will instead match if any of the nodes match the predicate - - -### The Query API - -Create a query by specifying a string containing one or more patterns: - -```c -TSQuery *ts_query_new( - const TSLanguage *language, - const char *source, - uint32_t source_len, - uint32_t *error_offset, - TSQueryError *error_type -); -``` - -If there is an error in the query, then the `error_offset` argument will be set to the byte offset of the error, and the `error_type` argument will be set to a value that indicates the type of error: - -```c -typedef enum { - TSQueryErrorNone = 0, - TSQueryErrorSyntax, - TSQueryErrorNodeType, - TSQueryErrorField, - TSQueryErrorCapture, -} TSQueryError; -``` - -The `TSQuery` value is immutable and can be safely shared between threads. To execute the query, create a `TSQueryCursor`, which carries the state needed for processing the queries. The query cursor should not be shared between threads, but can be reused for many query executions. - -```c -TSQueryCursor *ts_query_cursor_new(void); -``` - -You can then execute the query on a given syntax node: - -```c -void ts_query_cursor_exec(TSQueryCursor *, const TSQuery *, TSNode); -``` - -You can then iterate over the matches: - -```c -typedef struct { - TSNode node; - uint32_t index; -} TSQueryCapture; - -typedef struct { - uint32_t id; - uint16_t pattern_index; - uint16_t capture_count; - const TSQueryCapture *captures; -} TSQueryMatch; - -bool ts_query_cursor_next_match(TSQueryCursor *, TSQueryMatch *match); -``` - -This function will return `false` when there are no more matches. Otherwise, it will populate the `match` with data about which pattern matched and which nodes were captured. - -## Static Node Types - -In languages with static typing, it can be helpful for syntax trees to provide specific type information about individual syntax nodes. Tree-sitter makes this information available via a generated file called `node-types.json`. This _node types_ file provides structured data about every possible syntax node in a grammar. - -You can use this data to generate type declarations in statically-typed programming languages. For example, GitHub's [Semantic](https://github.com/github/semantic) uses these node types files to [generate Haskell data types](https://github.com/github/semantic/tree/master/semantic-ast) for every possible syntax node, which allows for code analysis algorithms to be structurally verified by the Haskell type system. - -The node types file contains an array of objects, each of which describes a particular type of syntax node using the following entries: - -#### Basic Info - -Every object in this array has these two entries: - -- `"type"` - A string that indicates which grammar rule the node represents. This corresponds to the `ts_node_type` function described [above](#syntax-nodes). -- `"named"` - A boolean that indicates whether this kind of node corresponds to a rule name in the grammar or just a string literal. See [above](#named-vs-anonymous-nodes) for more info. - -Examples: - -```json -{ - "type": "string_literal", - "named": true -} -{ - "type": "+", - "named": false -} -``` - -Together, these two fields constitute a unique identifier for a node type; no two top-level objects in the `node-types.json` should have the same values for both `"type"` and `"named"`. - -#### Internal Nodes - -Many syntax nodes can have _children_. The node type object describes the possible children that a node can have using the following entries: - -- `"fields"` - An object that describes the possible [fields](#node-field-names) that the node can have. The keys of this object are field names, and the values are _child type_ objects, described below. -- `"children"` - Another _child type_ object that describes all of the node's possible _named_ children _without_ fields. - -A _child type_ object describes a set of child nodes using the following entries: - -- `"required"` - A boolean indicating whether there is always _at least one_ node in this set. -- `"multiple"` - A boolean indicating whether there can be _multiple_ nodes in this set. -- `"types"`- An array of objects that represent the possible types of nodes in this set. Each object has two keys: `"type"` and `"named"`, whose meanings are described above. - -Example with fields: - -```json -{ - "type": "method_definition", - "named": true, - "fields": { - "body": { - "multiple": false, - "required": true, - "types": [{ "type": "statement_block", "named": true }] - }, - "decorator": { - "multiple": true, - "required": false, - "types": [{ "type": "decorator", "named": true }] - }, - "name": { - "multiple": false, - "required": true, - "types": [ - { "type": "computed_property_name", "named": true }, - { "type": "property_identifier", "named": true } - ] - }, - "parameters": { - "multiple": false, - "required": true, - "types": [{ "type": "formal_parameters", "named": true }] - } - } -} -``` - -Example with children: - -```json -{ - "type": "array", - "named": true, - "fields": {}, - "children": { - "multiple": true, - "required": false, - "types": [ - { "type": "_expression", "named": true }, - { "type": "spread_element", "named": true } - ] - } -} -``` - -#### Supertype Nodes - -In Tree-sitter grammars, there are usually certain rules that represent abstract _categories_ of syntax nodes (e.g. "expression", "type", "declaration"). In the `grammar.js` file, these are often written as [hidden rules](./creating-parsers#hiding-rules) whose definition is a simple [`choice`](./creating-parsers#the-grammar-dsl) where each member is just a single symbol. - -Normally, hidden rules are not mentioned in the node types file, since they don't appear in the syntax tree. But if you add a hidden rule to the grammar's [`supertypes` list](./creating-parsers#the-grammar-dsl), then it _will_ show up in the node types file, with the following special entry: - -- `"subtypes"` - An array of objects that specify the _types_ of nodes that this 'supertype' node can wrap. - -Example: - -```json -{ - "type": "_declaration", - "named": true, - "subtypes": [ - { "type": "class_declaration", "named": true }, - { "type": "function_declaration", "named": true }, - { "type": "generator_function_declaration", "named": true }, - { "type": "lexical_declaration", "named": true }, - { "type": "variable_declaration", "named": true } - ] -} -``` - -Supertype nodes will also appear elsewhere in the node types file, as children of other node types, in a way that corresponds with how the supertype rule was used in the grammar. This can make the node types much shorter and easier to read, because a single supertype will take the place of multiple subtypes. - -Example: - -```json -{ - "type": "export_statement", - "named": true, - "fields": { - "declaration": { - "multiple": false, - "required": false, - "types": [{ "type": "_declaration", "named": true }] - }, - "source": { - "multiple": false, - "required": false, - "types": [{ "type": "string", "named": true }] - } - } -} -``` diff --git a/docs/section-3-creating-parsers.md b/docs/section-3-creating-parsers.md deleted file mode 100644 index ee8d6fbf..00000000 --- a/docs/section-3-creating-parsers.md +++ /dev/null @@ -1,1122 +0,0 @@ ---- -title: Creating Parsers -permalink: creating-parsers ---- - -# Creating parsers - -Developing Tree-sitter grammars can have a difficult learning curve, but once you get the hang of it, it can be fun and even zen-like. This document will help you to get started and to develop a useful mental model. - -## Getting Started - -### Dependencies - -In order to develop a Tree-sitter parser, there are two dependencies that you need to install: - -* **Node.js** - Tree-sitter grammars are written in JavaScript, and Tree-sitter uses [Node.js][node.js] to interpret JavaScript files. It requires the `node` command to be in one of the directories in your [`PATH`][path-env]. You'll need Node.js version 6.0 or greater. -* **A C Compiler** - Tree-sitter creates parsers that are written in C. In order to run and test these parsers with the `tree-sitter parse` or `tree-sitter test` commands, you must have a C compiler installed. Tree-sitter will try to look for these compilers in the standard places for each platform. - -### Installation - -To create a Tree-sitter parser, you need to use [the `tree-sitter` CLI][tree-sitter-cli]. You can install the CLI in a few different ways: - -* Build the `tree-sitter-cli` [Rust crate][crate] from source using [`cargo`][cargo], the Rust package manager. This works on any platform. See [the contributing docs](./contributing#developing-tree-sitter) for more information. -* Install the `tree-sitter-cli` [Node.js module][node-module] using [`npm`][npm], the Node package manager. This approach is fast, but is only works on certain platforms, because it relies on pre-built binaries. -* Download a binary for your platform from [the latest GitHub release][releases], and put it into a directory on your `PATH`. - -### Project Setup - -The preferred convention is to name the parser repository "tree-sitter-" followed by the name of the language. - -```sh -mkdir tree-sitter-${YOUR_LANGUAGE_NAME} -cd tree-sitter-${YOUR_LANGUAGE_NAME} -``` - -You can use the `tree-sitter` CLI tool to set up your project, and allows your parser to be used from multiple languages. - -```sh -# This will prompt you for input -tree-sitter init -``` - -Once you have installed the CLI and run through the `init` command's prompts, a file called `grammar.js` should exist with the following contents: - -```js -/// -// @ts-check - -module.exports = grammar({ - name: 'YOUR_LANGUAGE_NAME', - - rules: { - // TODO: add the actual grammar rules - source_file: $ => 'hello' - } -}); -``` - -Now, run the following command: - -```sh -tree-sitter generate -``` - -This will generate the C code required to parse this trivial language, as well as a few files that are needed to compile and load this native parser as a Node.js module. - -You can test this parser by creating a source file with the contents "hello" and parsing it: - -```sh -echo 'hello' > example-file -tree-sitter parse example-file -``` - -Alternatively, in Windows PowerShell: - -```pwsh -"hello" | Out-File example-file -Encoding utf8 -tree-sitter parse example-file -``` - -This should print the following: - -```text -(source_file [0, 0] - [1, 0]) -``` - -You now have a working parser. - -Finally, look back at the [triple-slash][] and [`@ts-check`][ts-check] comments in `grammar.js`; these tell your editor to provide documentation and type information as you edit your grammar. For these to work, you must download Tree-sitter's TypeScript API from npm into a `node_modules` directory in your project: - -```sh -npm install -``` - -## Tool Overview - -Let's go over all of the functionality of the `tree-sitter` command line tool. - -### Command: `init` - -The first command you will likely run is the `init` command. This command sets up an empty repository with everything you need to get going with a grammar repository. -It only has one optional argument, `--update`, which will update outdated generated files, if needed. - -The main file of interest for users to configure is `tree-sitter.json`, which tells the CLI information about your grammar, such as the queries. - -#### Structure of `tree-sitter.json` - -##### The `grammars` field - -This field is an array of objects, you typically only need one object in this array, unless your repo has multiple grammars (e.g. like `Typescript` and `TSX`) - -###### Basics - -These keys specify basic information about the parser: - -* `scope` (required) - A string like `"source.js"` that identifies the language. Currently, we strive to match the scope names used by popular [TextMate grammars](https://macromates.com/manual/en/language_grammars) and by the [Linguist](https://github.com/github/linguist) library. - -* `path` - A relative path from the directory containing `tree-sitter.json` to another directory containing the `src/` folder, which contains the actual generated parser. The default value is `"."` (so that `src/` is in the same folder as `tree-sitter.json`), and this very rarely needs to be overridden. - -* `external-files` - A list of relative paths from the root dir of a -parser to files that should be checked for modifications during recompilation. -This is useful during development to have changes to other files besides scanner.c -be picked up by the cli. - -###### Language Detection - -These keys help to decide whether the language applies to a given file: - -* `file-types` - An array of filename suffix strings. The grammar will be used for files whose names end with one of these suffixes. Note that the suffix may match an *entire* filename. - -* `first-line-regex` - A regex pattern that will be tested against the first line of a file in order to determine whether this language applies to the file. If present, this regex will be used for any file whose language does not match any grammar's `file-types`. - -* `content-regex` - A regex pattern that will be tested against the contents of the file in order to break ties in cases where multiple grammars matched the file using the above two criteria. If the regex matches, this grammar will be preferred over another grammar with no `content-regex`. If the regex does not match, a grammar with no `content-regex` will be preferred over this one. - -* `injection-regex` - A regex pattern that will be tested against a *language name* in order to determine whether this language should be used for a potential *language injection* site. Language injection is described in more detail in [a later section](#language-injection). - -###### Query Paths - -These keys specify relative paths from the directory containing `tree-sitter.json` to the files that control syntax highlighting: - -* `highlights` - Path to a *highlight query*. Default: `queries/highlights.scm` -* `locals` - Path to a *local variable query*. Default: `queries/locals.scm`. -* `injections` - Path to an *injection query*. Default: `queries/injections.scm`. -* `tags` - Path to an *tag query*. Default: `queries/tags.scm`. - -The behaviors of these three files are described in the next section. - -##### The `metadata` field - -This field contains information that tree-sitter will use to populate relevant bindings' files, especially their versions. A future -`bump-version` and `publish` subcommand will leverage this version information as well. Typically, this will all be set up when you -run `tree-sitter init`, but you are welcome to update it as you see fit. - -* `version` (required) - The current version of your grammar, which should follow [semver](https://semver.org) -* `license` - The license of your grammar, which should be a valid [SPDX license](https://spdx.org/licenses) -* `description` - The brief description of your grammar -* `authors` (required) - An array of objects that contain a `name` field, and optionally an `email` and `url` field. Each field is a string -* `links` - An object that contains a `repository` field, and optionally a `homepage` field. Each field is a string -* `namespace` - The namespace for the `Java` and `Kotlin` bindings, defaults to `io.github.tree-sitter` if not provided - -##### The `bindings` field - -This field controls what bindings are generated when the `init` command is run. Each key is a language name, and the value is a boolean. - -* `c` (default: `true`) -* `go` (default: `true`) -* `java` (default: `false`) -* `kotlin` (default: `false`) -* `node` (default: `true`) -* `python` (default: `true`) -* `rust` (default: `true`) -* `swift` (default: `false`) - -### Command: `version` - -The `version` command prints the version of the `tree-sitter` CLI tool that you have installed. - -```sh -tree-sitter version 1.0.0 -``` - -The only argument is the version itself, which is the first positional argument. -This will update the version in several files, if they exist: - -* tree-sitter.json -* Cargo.toml -* package.json -* Makefile -* CMakeLists.txt -* pyproject.toml - -As a grammar author, you should keep the version of your grammar in sync across -different bindings. However, doing so manually is error-prone and tedious, so -this command takes care of the burden. - -### Command: `generate` - -The most important command you'll use is `tree-sitter generate`. This command reads the `grammar.js` file in your current working directory and creates a file called `src/parser.c`, which implements the parser. After making changes to your grammar, just run `tree-sitter generate` again. - -The first time you run `tree-sitter generate`, it will also generate a few other files for bindings for the following languages: - -#### C/C++ - -* `Makefile` - This file tells `make` how to compile your language. -* `bindings/c/tree-sitter-language.h` - This file provides the C interface of your language. -* `bindings/c/tree-sitter-language.pc` - This file provides pkg-config metadata about your language's C library. -* `src/tree_sitter/parser.h` - This file provides some basic C definitions that are used in your generated `parser.c` file. -* `src/tree_sitter/alloc.h` - This file provides some memory allocation macros that are to be used in your external scanner, if you have one. -* `src/tree_sitter/array.h` - This file provides some array macros that are to be used in your external scanner, if you have one. - -#### Go - -* `bindings/go/binding.go` - This file wraps your language in a Go module. -* `bindings/go/binding_test.go` - This file contains a test for the Go package. - -#### Node - -* `binding.gyp` - This file tells Node.js how to compile your language. -* `bindings/node/index.js` - This is the file that Node.js initially loads when using your language. -* `bindings/node/binding.cc` - This file wraps your language in a JavaScript module for Node.js. - -#### Python - -* `pyproject.toml` - This file is the manifest of the Python package. -* `setup.py` - This file tells Python how to compile your language. -* `bindings/python/binding.c` - This file wraps your language in a Python module. -* `bindings/python/tree_sitter_language/__init__.py` - This file tells Python how to load your language. -* `bindings/python/tree_sitter_language/__init__.pyi` - This file provides type hints for your parser when used in Python. -* `bindings/python/tree_sitter_language/py.typed` - This file provides type hints for your parser when used in Python. - -#### Rust - -* `Cargo.toml` - This file is the manifest of the Rust package. -* `bindings/rust/lib.rs` - This file wraps your language in a Rust crate when used in Rust. -* `bindings/rust/build.rs` - This file wraps the building process for the Rust crate. - -#### Swift - -* `Package.swift` - This file tells Swift how to compile your language. -* `bindings/swift/TreeSitterLanguage/language.h` - This file wraps your language in a Swift module when used in Swift. - -If there is an ambiguity or *local ambiguity* in your grammar, Tree-sitter will detect it during parser generation, and it will exit with a `Unresolved conflict` error message. See below for more information on these errors. - -### Command: `build` - -The `build` command compiles your parser into a dynamically-loadable library, either as a shared object (`.so`, `.dylib`, or `.dll`) or as a WASM module. - -You can change the compiler executable via the `CC` environment variable and add extra flags via `CFLAGS`. For macOS or iOS, you can set `MACOSX_DEPLOYMENT_TARGET` or `IPHONEOS_DEPLOYMENT_TARGET` respectively to define the minimum supported version. - -You can specify whether to compile it as a wasm module with the `--wasm`/`-w` flag, and you can opt to use docker or podman to supply emscripten with the `--docker`/`-d` flag. This removes the need to install emscripten on your machine locally. - -You can specify where to output the shared object file (native or WASM) with the `--output`/`-o` flag, which accepts either an absolute path or relative path. Note that if you don't supply this flag, the CLI will attempt to figure out what the language name is based on the parent directory (so building in `tree-sitter-javascript` will resolve to `javascript`) to use for the output file. If it can't figure it out, it will default to `parser`, thus generating `parser.so` or `parser.wasm` in the current working directory. - -Lastly, you can also specify a path to the actual grammar directory, in case you are not currently in one. This is done by providing a path as the first *positional* argument. - -Example: - -```sh -tree-sitter build --wasm --output ./build/parser.wasm tree-sitter-javascript -``` - -Notice how the `tree-sitter-javascript` argument is the first positional argument. - -### Command: `test` - -The `tree-sitter test` command allows you to easily test that your parser is working correctly. - -For each rule that you add to the grammar, you should first create a *test* that describes how the syntax trees should look when parsing that rule. These tests are written using specially-formatted text files in the `test/corpus/` directory within your parser's root folder. - -For example, you might have a file called `test/corpus/statements.txt` that contains a series of entries like this: - -```text -================== -Return statements -================== - -func x() int { - return 1; -} - ---- - -(source_file - (function_definition - (identifier) - (parameter_list) - (primitive_type) - (block - (return_statement (number))))) -``` - -* The **name** of each test is written between two lines containing only `=` (equal sign) characters. -* Then the **input source code** is written, followed by a line containing three or more `-` (dash) characters. -* Then, the **expected output syntax tree** is written as an [S-expression][s-exp]. The exact placement of whitespace in the S-expression doesn't matter, but ideally the syntax tree should be legible. Note that the S-expression does not show syntax nodes like `func`, `(` and `;`, which are expressed as strings and regexes in the grammar. It only shows the *named* nodes, as described in [this section][named-vs-anonymous-nodes-section] of the page on parser usage. - - The expected output section can also *optionally* show the [*field names*][field-names-section] associated with each child node. To include field names in your tests, you write a node's field name followed by a colon, before the node itself in the S-expression: - -```text -(source_file - (function_definition - name: (identifier) - parameters: (parameter_list) - result: (primitive_type) - body: (block - (return_statement (number))))) -``` - -* If your language's syntax conflicts with the `===` and `---` test separators, you can optionally add an arbitrary identical suffix (in the below example, `|||`) to disambiguate them: - -```text -==================||| -Basic module -==================||| - ----- MODULE Test ---- -increment(n) == n + 1 -==== - ----||| - -(source_file - (module (identifier) - (operator (identifier) - (parameter_list (identifier)) - (plus (identifier_ref) (number))))) -``` - -These tests are important. They serve as the parser's API documentation, and they can be run every time you change the grammar to verify that everything still parses correctly. - -By default, the `tree-sitter test` command runs all of the tests in your `test/corpus/` folder. To run a particular test, you can use the `-f` flag: - -```sh -tree-sitter test -f 'Return statements' -``` - -The recommendation is to be comprehensive in adding tests. If it's a visible node, add it to a test file in your `test/corpus` directory. It's typically a good idea to test all of the permutations of each language construct. This increases test coverage, but doubly acquaints readers with a way to examine expected outputs and understand the "edges" of a language. - -#### Attributes - -Tests can be annotated with a few `attributes`. Attributes must be put in the header, below the test name, and start with a `:`. -A couple of attributes also take in a parameter, which require the use of parenthesis. - -**Note**: If you'd like to supply in multiple parameters, e.g. to run tests on multiple platforms or to test multiple languages, you can repeat the attribute on a new line. - -The following attributes are available: - -* `:skip` — This attribute will skip the test when running `tree-sitter test`. - This is useful when you want to temporarily disable running a test without deleting it. -* `:error` — This attribute will assert that the parse tree contains an error. It's useful to just validate that a certain input is invalid without displaying the whole parse tree, as such you should omit the parse tree below the `---` line. -* `:fail-fast` — This attribute will stop the testing additional tests if the test marked with this attribute fails. -* `:language(LANG)` — This attribute will run the tests using the parser for the specified language. This is useful for multi-parser repos, such as XML and DTD, or Typescript and TSX. The default parser used will always be the first entry in the `grammars` field in the `tree-sitter.json` config file, so having a way to pick a second or even third parser is useful. -* `:platform(PLATFORM)` — This attribute specifies the platform on which the test should run. It is useful to test platform-specific behavior (e.g. Windows newlines are different from Unix). This attribute must match up with Rust's [`std::env::consts::OS`](https://doc.rust-lang.org/std/env/consts/constant.OS.html). - -Examples using attributes: - -```text -========================= -Test that will be skipped -:skip -========================= - -int main() {} - -------------------------- - -==================================== -Test that will run on Linux or macOS - -:platform(linux) -:platform(macos) -==================================== - -int main() {} - ------------------------------------- - -======================================================================== -Test that expects an error, and will fail fast if there's no parse error -:fail-fast -:error -======================================================================== - -int main ( {} - ------------------------------------------------------------------------- - -================================================= -Test that will parse with both Typescript and TSX -:language(typescript) -:language(tsx) -================================================= - -console.log('Hello, world!'); - -------------------------------------------------- -``` - -#### Automatic Compilation - -You might notice that the first time you run `tree-sitter test` after regenerating your parser, it takes some extra time. This is because Tree-sitter automatically compiles your C code into a dynamically-loadable library. It recompiles your parser as-needed whenever you update it by re-running `tree-sitter generate`. - -#### Syntax Highlighting Tests - -The `tree-sitter test` command will *also* run any syntax highlighting tests in the `test/highlight` folder, if it exists. For more information about syntax highlighting tests, see [the syntax highlighting page][syntax-highlighting-tests]. - -### Command: `parse` - -You can run your parser on an arbitrary file using `tree-sitter parse`. This will print the resulting the syntax tree, including nodes' ranges and field names, like this: - -```text -(source_file [0, 0] - [3, 0] - (function_declaration [0, 0] - [2, 1] - name: (identifier [0, 5] - [0, 9]) - parameters: (parameter_list [0, 9] - [0, 11]) - result: (type_identifier [0, 12] - [0, 15]) - body: (block [0, 16] - [2, 1] - (return_statement [1, 2] - [1, 10] - (expression_list [1, 9] - [1, 10] - (int_literal [1, 9] - [1, 10])))))) -``` - -You can pass any number of file paths and glob patterns to `tree-sitter parse`, and it will parse all of the given files. The command will exit with a non-zero status code if any parse errors occurred. Passing the `--cst` flag will output a pretty-printed CST instead of the normal S-expression representation. You can also prevent the syntax trees from being printed using the `--quiet` flag. Additionally, the `--stat` flag prints out aggregated parse success/failure information for all processed files. This makes `tree-sitter parse` usable as a secondary testing strategy: you can check that a large number of files parse without error: - -```sh -tree-sitter parse 'examples/**/*.go' --quiet --stat -``` - -### Command: `highlight` - -You can run syntax highlighting on an arbitrary file using `tree-sitter highlight`. This can either output colors directly to your terminal using ansi escape codes, or produce HTML (if the `--html` flag is passed). For more information, see [the syntax highlighting page][syntax-highlighting]. - -### The Grammar DSL - -The following is a complete list of built-in functions you can use in your `grammar.js` to define rules. Use-cases for some of these functions will be explained in more detail in later sections. - -* **Symbols (the `$` object)** - Every grammar rule is written as a JavaScript function that takes a parameter conventionally called `$`. The syntax `$.identifier` is how you refer to another grammar symbol within a rule. Names starting with `$.MISSING` or `$.UNEXPECTED` should be avoided as they have special meaning for the `tree-sitter test` command. -* **String and Regex literals** - The terminal symbols in a grammar are described using JavaScript strings and regular expressions. Of course during parsing, Tree-sitter does not actually use JavaScript's regex engine to evaluate these regexes; it generates its own regex-matching logic as part of each parser. Regex literals are just used as a convenient way of writing regular expressions in your grammar. -* **Regex Limitations** - Currently, only a subset of the Regex engine is actually -supported. This is due to certain features like lookahead and lookaround assertions -not feasible to use in an LR(1) grammar, as well as certain flags being unnecessary -for tree-sitter. However, plenty of features are supported by default: - - * Character classes - * Character ranges - * Character sets - * Quantifiers - * Alternation - * Grouping - * Unicode character escapes - * Unicode property escapes - -* **Sequences : `seq(rule1, rule2, ...)`** - This function creates a rule that matches any number of other rules, one after another. It is analogous to simply writing multiple symbols next to each other in [EBNF notation][ebnf]. -* **Alternatives : `choice(rule1, rule2, ...)`** - This function creates a rule that matches *one* of a set of possible rules. The order of the arguments does not matter. This is analogous to the `|` (pipe) operator in EBNF notation. -* **Repetitions : `repeat(rule)`** - This function creates a rule that matches *zero-or-more* occurrences of a given rule. It is analogous to the `{x}` (curly brace) syntax in EBNF notation. -* **Repetitions : `repeat1(rule)`** - This function creates a rule that matches *one-or-more* occurrences of a given rule. The previous `repeat` rule is implemented in terms of `repeat1` but is included because it is very commonly used. -* **Options : `optional(rule)`** - This function creates a rule that matches *zero or one* occurrence of a given rule. It is analogous to the `[x]` (square bracket) syntax in EBNF notation. -* **Precedence : `prec(number, rule)`** - This function marks the given rule with a numerical precedence which will be used to resolve [*LR(1) Conflicts*][lr-conflict] at parser-generation time. When two rules overlap in a way that represents either a true ambiguity or a *local* ambiguity given one token of lookahead, Tree-sitter will try to resolve the conflict by matching the rule with the higher precedence. The default precedence of all rules is zero. This works similarly to the [precedence directives][yacc-prec] in Yacc grammars. -* **Left Associativity : `prec.left([number], rule)`** - This function marks the given rule as left-associative (and optionally applies a numerical precedence). When an LR(1) conflict arises in which all of the rules have the same numerical precedence, Tree-sitter will consult the rules' associativity. If there is a left-associative rule, Tree-sitter will prefer matching a rule that ends *earlier*. This works similarly to [associativity directives][yacc-prec] in Yacc grammars. -* **Right Associativity : `prec.right([number], rule)`** - This function is like `prec.left`, but it instructs Tree-sitter to prefer matching a rule that ends *later*. -* **Dynamic Precedence : `prec.dynamic(number, rule)`** - This function is similar to `prec`, but the given numerical precedence is applied at *runtime* instead of at parser generation time. This is only necessary when handling a conflict dynamically using the `conflicts` field in the grammar, and when there is a genuine *ambiguity*: multiple rules correctly match a given piece of code. In that event, Tree-sitter compares the total dynamic precedence associated with each rule, and selects the one with the highest total. This is similar to [dynamic precedence directives][bison-dprec] in Bison grammars. -* **Tokens : `token(rule)`** - This function marks the given rule as producing only -a single token. Tree-sitter's default is to treat each String or RegExp literal -in the grammar as a separate token. Each token is matched separately by the lexer -and returned as its own leaf node in the tree. The `token` function allows you to -express a complex rule using the functions described above (rather than as a single -regular expression) but still have Tree-sitter treat it as a single token. -The token function will only accept terminal rules, so `token($.foo)` will not work. -You can think of it as a shortcut for squashing complex rules of strings or regexes -down to a single token. -* **Immediate Tokens : `token.immediate(rule)`** - Usually, whitespace (and any other extras, such as comments) is optional before each token. This function means that the token will only match if there is no whitespace. -* **Aliases : `alias(rule, name)`** - This function causes the given rule to *appear* with an alternative name in the syntax tree. If `name` is a *symbol*, as in `alias($.foo, $.bar)`, then the aliased rule will *appear* as a [named node][named-vs-anonymous-nodes-section] called `bar`. And if `name` is a *string literal*, as in `alias($.foo, 'bar')`, then the aliased rule will appear as an [anonymous node][named-vs-anonymous-nodes-section], as if the rule had been written as the simple string. -* **Field Names : `field(name, rule)`** - This function assigns a *field name* to the child node(s) matched by the given rule. In the resulting syntax tree, you can then use that field name to access specific children. - -In addition to the `name` and `rules` fields, grammars have a few other optional public fields that influence the behavior of the parser. - -* **`extras`** - an array of tokens that may appear *anywhere* in the language. This is often used for whitespace and comments. The default value of `extras` is to accept whitespace. To control whitespace explicitly, specify `extras: $ => []` in your grammar. -* **`inline`** - an array of rule names that should be automatically *removed* from the grammar by replacing all of their usages with a copy of their definition. This is useful for rules that are used in multiple places but for which you *don't* want to create syntax tree nodes at runtime. -* **`conflicts`** - an array of arrays of rule names. Each inner array represents a set of rules that's involved in an *LR(1) conflict* that is *intended to exist* in the grammar. When these conflicts occur at runtime, Tree-sitter will use the GLR algorithm to explore all of the possible interpretations. If *multiple* parses end up succeeding, Tree-sitter will pick the subtree whose corresponding rule has the highest total *dynamic precedence*. -* **`externals`** - an array of token names which can be returned by an [*external scanner*](#external-scanners). External scanners allow you to write custom C code which runs during the lexing process in order to handle lexical rules (e.g. Python's indentation tokens) that cannot be described by regular expressions. -* **`precedences`** - an array of array of strings, where each array of strings defines named precedence levels in descending order. These names can be used in the `prec` functions to define precedence relative only to other names in the array, rather than globally. Can only be used with parse precedence, not lexical precedence. -* **`word`** - the name of a token that will match keywords for the purpose of the [keyword extraction](#keyword-extraction) optimization. -* **`supertypes`** an array of hidden rule names which should be considered to be 'supertypes' in the generated [*node types* file][static-node-types]. - -## Writing the Grammar - -Writing a grammar requires creativity. There are an infinite number of CFGs (context-free grammars) that can be used to describe any given language. In order to produce a good Tree-sitter parser, you need to create a grammar with two important properties: - -1. **An intuitive structure** - Tree-sitter's output is a [concrete syntax tree][cst]; each node in the tree corresponds directly to a [terminal or non-terminal symbol][non-terminal] in the grammar. So in order to produce an easy-to-analyze tree, there should be a direct correspondence between the symbols in your grammar and the recognizable constructs in the language. This might seem obvious, but it is very different from the way that context-free grammars are often written in contexts like [language specifications][language-spec] or [Yacc][yacc]/[Bison][bison] parsers. - -2. **A close adherence to LR(1)** - Tree-sitter is based on the [GLR parsing][glr-parsing] algorithm. This means that while it can handle any context-free grammar, it works most efficiently with a class of context-free grammars called [LR(1) Grammars][lr-grammars]. In this respect, Tree-sitter's grammars are similar to (but less restrictive than) [Yacc][yacc] and [Bison][bison] grammars, but *different* from [ANTLR grammars][antlr], [Parsing Expression Grammars][peg], or the [ambiguous grammars][ambiguous-grammar] commonly used in language specifications. - -It's unlikely that you'll be able to satisfy these two properties just by translating an existing context-free grammar directly into Tree-sitter's grammar format. There are a few kinds of adjustments that are often required. The following sections will explain these adjustments in more depth. - -### The First Few Rules - -It's usually a good idea to find a formal specification for the language you're trying to parse. This specification will most likely contain a context-free grammar. As you read through the rules of this CFG, you will probably discover a complex and cyclic graph of relationships. It might be unclear how you should navigate this graph as you define your grammar. - -Although languages have very different constructs, their constructs can often be categorized in to similar groups like *Declarations*, *Definitions*, *Statements*, *Expressions*, *Types*, and *Patterns*. In writing your grammar, a good first step is to create just enough structure to include all of these basic *groups* of symbols. For a language like Go, you might start with something like this: - -```js -{ - // ... - - rules: { - source_file: $ => repeat($._definition), - - _definition: $ => choice( - $.function_definition - // TODO: other kinds of definitions - ), - - function_definition: $ => seq( - 'func', - $.identifier, - $.parameter_list, - $._type, - $.block - ), - - parameter_list: $ => seq( - '(', - // TODO: parameters - ')' - ), - - _type: $ => choice( - 'bool' - // TODO: other kinds of types - ), - - block: $ => seq( - '{', - repeat($._statement), - '}' - ), - - _statement: $ => choice( - $.return_statement - // TODO: other kinds of statements - ), - - return_statement: $ => seq( - 'return', - $._expression, - ';' - ), - - _expression: $ => choice( - $.identifier, - $.number - // TODO: other kinds of expressions - ), - - identifier: $ => /[a-z]+/, - - number: $ => /\d+/ - } -} -``` - -Some of the details of this grammar will be explained in more depth later on, but if you focus on the `TODO` comments, you can see that the overall strategy is *breadth-first*. Notably, this initial skeleton does not need to directly match an exact subset of the context-free grammar in the language specification. It just needs to touch on the major groupings of rules in as simple and obvious a way as possible. - -With this structure in place, you can now freely decide what part of the grammar to flesh out next. For example, you might decide to start with *types*. One-by-one, you could define the rules for writing basic types and composing them into more complex types: - -```js -{ - // ... - - _type: $ => choice( - $.primitive_type, - $.array_type, - $.pointer_type - ), - - primitive_type: $ => choice( - 'bool', - 'int' - ), - - array_type: $ => seq( - '[', - ']', - $._type - ), - - pointer_type: $ => seq( - '*', - $._type - ) -} -``` - -After developing the *type* sublanguage a bit further, you might decide to switch to working on *statements* or *expressions* instead. It's often useful to check your progress by trying to parse some real code using `tree-sitter parse`. - -**And remember to add tests for each rule in your `test/corpus` folder!** - -### Structuring Rules Well - -Imagine that you were just starting work on the [Tree-sitter JavaScript parser][tree-sitter-javascript]. Naively, you might try to directly mirror the structure of the [ECMAScript Language Spec][ecmascript-spec]. To illustrate the problem with this approach, consider the following line of code: - -```js -return x + y; -``` - -According to the specification, this line is a `ReturnStatement`, the fragment `x + y` is an `AdditiveExpression`, and `x` and `y` are both `IdentifierReferences`. The relationship between these constructs is captured by a complex series of production rules: - -```text -ReturnStatement -> 'return' Expression -Expression -> AssignmentExpression -AssignmentExpression -> ConditionalExpression -ConditionalExpression -> LogicalORExpression -LogicalORExpression -> LogicalANDExpression -LogicalANDExpression -> BitwiseORExpression -BitwiseORExpression -> BitwiseXORExpression -BitwiseXORExpression -> BitwiseANDExpression -BitwiseANDExpression -> EqualityExpression -EqualityExpression -> RelationalExpression -RelationalExpression -> ShiftExpression -ShiftExpression -> AdditiveExpression -AdditiveExpression -> MultiplicativeExpression -MultiplicativeExpression -> ExponentiationExpression -ExponentiationExpression -> UnaryExpression -UnaryExpression -> UpdateExpression -UpdateExpression -> LeftHandSideExpression -LeftHandSideExpression -> NewExpression -NewExpression -> MemberExpression -MemberExpression -> PrimaryExpression -PrimaryExpression -> IdentifierReference -``` - -The language spec encodes the twenty different precedence levels of JavaScript expressions using twenty levels of indirection between `IdentifierReference` and `Expression`. If we were to create a concrete syntax tree representing this statement according to the language spec, it would have twenty levels of nesting, and it would contain nodes with names like `BitwiseXORExpression`, which are unrelated to the actual code. - -### Using Precedence - -To produce a readable syntax tree, we'd like to model JavaScript expressions using a much flatter structure like this: - -```js -{ - // ... - - _expression: $ => choice( - $.identifier, - $.unary_expression, - $.binary_expression, - // ... - ), - - unary_expression: $ => choice( - seq('-', $._expression), - seq('!', $._expression), - // ... - ), - - binary_expression: $ => choice( - seq($._expression, '*', $._expression), - seq($._expression, '+', $._expression), - // ... - ), -} -``` - -Of course, this flat structure is highly ambiguous. If we try to generate a parser, Tree-sitter gives us an error message: - -```text -Error: Unresolved conflict for symbol sequence: - - '-' _expression • '*' … - -Possible interpretations: - - 1: '-' (binary_expression _expression • '*' _expression) - 2: (unary_expression '-' _expression) • '*' … - -Possible resolutions: - - 1: Specify a higher precedence in `binary_expression` than in the other rules. - 2: Specify a higher precedence in `unary_expression` than in the other rules. - 3: Specify a left or right associativity in `unary_expression` - 4: Add a conflict for these rules: `binary_expression` `unary_expression` -``` - -Note: The • character in the error message indicates where exactly during -parsing the conflict occurs, or in other words, where the parser is encountering -ambiguity. - -For an expression like `-a * b`, it's not clear whether the `-` operator applies to the `a * b` or just to the `a`. This is where the `prec` function [described above](#the-grammar-dsl) comes into play. By wrapping a rule with `prec`, we can indicate that certain sequence of symbols should *bind to each other more tightly* than others. For example, the `'-', $._expression` sequence in `unary_expression` should bind more tightly than the `$._expression, '+', $._expression` sequence in `binary_expression`: - -```js -{ - // ... - - unary_expression: $ => prec(2, choice( - seq('-', $._expression), - seq('!', $._expression), - // ... - )) -} -``` - -### Using Associativity - -Applying a higher precedence in `unary_expression` fixes that conflict, but there is still another conflict: - -```text -Error: Unresolved conflict for symbol sequence: - - _expression '*' _expression • '*' … - -Possible interpretations: - - 1: _expression '*' (binary_expression _expression • '*' _expression) - 2: (binary_expression _expression '*' _expression) • '*' … - -Possible resolutions: - - 1: Specify a left or right associativity in `binary_expression` - 2: Add a conflict for these rules: `binary_expression` -``` - -For an expression like `a * b * c`, it's not clear whether we mean `a * (b * c)` or `(a * b) * c`. This is where `prec.left` and `prec.right` come into use. We want to select the second interpretation, so we use `prec.left`. - -```js -{ - // ... - - binary_expression: $ => choice( - prec.left(2, seq($._expression, '*', $._expression)), - prec.left(1, seq($._expression, '+', $._expression)), - // ... - ), -} -``` - -### Hiding Rules - -You may have noticed in the above examples that some of the grammar rule name like `_expression` and `_type` began with an underscore. Starting a rule's name with an underscore causes the rule to be *hidden* in the syntax tree. This is useful for rules like `_expression` in the grammars above, which always just wrap a single child node. If these nodes were not hidden, they would add substantial depth and noise to the syntax tree without making it any easier to understand. - -### Using Fields - -Often, it's easier to analyze a syntax node if you can refer to its children by *name* instead of by their position in an ordered list. Tree-sitter grammars support this using the `field` function. This function allows you to assign unique names to some or all of a node's children: - -```js -function_definition: $ => seq( - 'func', - field('name', $.identifier), - field('parameters', $.parameter_list), - field('return_type', $._type), - field('body', $.block) -) -``` - -Adding fields like this allows you to retrieve nodes using the [field APIs][field-names-section]. - -## Lexical Analysis - -Tree-sitter's parsing process is divided into two phases: parsing (which is described above) and [lexing][lexing] - the process of grouping individual characters into the language's fundamental *tokens*. There are a few important things to know about how Tree-sitter's lexing works. - -### Conflicting Tokens - -Grammars often contain multiple tokens that can match the same characters. For example, a grammar might contain the tokens (`"if"` and `/[a-z]+/`). Tree-sitter differentiates between these conflicting tokens in a few ways. - -1. **Context-aware Lexing** - Tree-sitter performs lexing on-demand, during the parsing process. At any given position in a source document, the lexer only tries to recognize tokens that are *valid* at that position in the document. - -2. **Lexical Precedence** - When the precedence functions described [above](#the-grammar-dsl) are used *within* the `token` function, the given explicit precedence values serve as instructions to the lexer. If there are two valid tokens that match the characters at a given position in the document, Tree-sitter will select the one with the higher precedence. - -3. **Match Length** - If multiple valid tokens with the same precedence match the characters at a given position in a document, Tree-sitter will select the token that matches the [longest sequence of characters][longest-match]. - -4. **Match Specificity** - If there are two valid tokens with the same precedence and which both match the same number of characters, Tree-sitter will prefer a token that is specified in the grammar as a `String` over a token specified as a `RegExp`. - -5. **Rule Order** - If none of the above criteria can be used to select one token over another, Tree-sitter will prefer the token that appears earlier in the grammar. - -If there is an external scanner it may have [an additional impact](#other-external-scanner-details) over regular tokens defined in the grammar. - -### Lexical Precedence vs. Parse Precedence - -One common mistake involves not distinguishing *lexical precedence* from *parse precedence*. Parse precedence determines which rule is chosen to interpret a given sequence of tokens. *Lexical precedence* determines which token is chosen to interpret at a given position of text and it is a lower-level operation that is done first. The above list fully captures Tree-sitter's lexical precedence rules, and you will probably refer back to this section of the documentation more often than any other. Most of the time when you really get stuck, you're dealing with a lexical precedence problem. Pay particular attention to the difference in meaning between using `prec` inside of the `token` function versus outside of it. The *lexical precedence* syntax is `token(prec(N, ...))`. - -### Keywords - -Many languages have a set of *keyword* tokens (e.g. `if`, `for`, `return`), as well as a more general token (e.g. `identifier`) that matches any word, including many of the keyword strings. For example, JavaScript has a keyword `instanceof`, which is used as a binary operator, like this: - -```js -if (a instanceof Something) b(); -``` - -The following, however, is not valid JavaScript: - -```js -if (a instanceofSomething) b(); -``` - -A keyword like `instanceof` cannot be followed immediately by another letter, because then it would be tokenized as an `identifier`, **even though an identifier is not valid at that position**. Because Tree-sitter uses context-aware lexing, as described [above](#conflicting-tokens), it would not normally impose this restriction. By default, Tree-sitter would recognize `instanceofSomething` as two separate tokens: the `instanceof` keyword followed by an `identifier`. - -### Keyword Extraction - -Fortunately, Tree-sitter has a feature that allows you to fix this, so that you can match the behavior of other standard parsers: the `word` token. If you specify a `word` token in your grammar, Tree-sitter will find the set of *keyword* tokens that match strings also matched by the `word` token. Then, during lexing, instead of matching each of these keywords individually, Tree-sitter will match the keywords via a two-step process where it *first* matches the `word` token. - -For example, suppose we added `identifier` as the `word` token in our JavaScript grammar: - -```js -grammar({ - name: 'javascript', - - word: $ => $.identifier, - - rules: { - _expression: $ => choice( - $.identifier, - $.unary_expression, - $.binary_expression - // ... - ), - - binary_expression: $ => choice( - prec.left(1, seq($._expression, 'instanceof', $._expression)) - // ... - ), - - unary_expression: $ => choice( - prec.left(2, seq('typeof', $._expression)) - // ... - ), - - identifier: $ => /[a-z_]+/ - } -}); -``` - -Tree-sitter would identify `typeof` and `instanceof` as keywords. Then, when parsing the invalid code above, rather than scanning for the `instanceof` token individually, it would scan for an `identifier` first, and find `instanceofSomething`. It would then correctly recognize the code as invalid. - -Aside from improving error detection, keyword extraction also has performance benefits. It allows Tree-sitter to generate a smaller, simpler lexing function, which means that **the parser will compile much more quickly**. - -### External Scanners - -Many languages have some tokens whose structure is impossible or inconvenient to describe with a regular expression. Some examples: - -* [Indent and dedent][indent-tokens] tokens in Python -* [Heredocs][heredoc] in Bash and Ruby -* [Percent strings][percent-string] in Ruby - -Tree-sitter allows you to handle these kinds of tokens using *external scanners*. An external scanner is a set of C functions that you, the grammar author, can write by hand in order to add custom logic for recognizing certain tokens. - -To use an external scanner, there are a few steps. First, add an `externals` section to your grammar. This section should list the names of all of your external tokens. These names can then be used elsewhere in your grammar. - -```js -grammar({ - name: 'my_language', - - externals: $ => [ - $.indent, - $.dedent, - $.newline - ], - - // ... -}); -``` - -Then, add another C source file to your project. Currently, its path must be `src/scanner.c` for the CLI to recognize it. Be sure to add this file to the `sources` section of your `binding.gyp` file so that it will be included when your project is compiled by Node.js and uncomment the appropriate block in your `bindings/rust/build.rs` file so that it will be included in your Rust crate. - -In this new source file, define an [`enum`][enum] type containing the names of all of your external tokens. The ordering of this enum must match the order in your grammar's `externals` array; the actual names do not matter. - -```c -#include "tree_sitter/parser.h" -#include "tree_sitter/alloc.h" -#include "tree_sitter/array.h" - -enum TokenType { - INDENT, - DEDENT, - NEWLINE -} -``` - -Finally, you must define five functions with specific names, based on your language's name and five actions: *create*, *destroy*, *serialize*, *deserialize*, and *scan*. - -#### Create - -```c -void *tree_sitter_my_language_external_scanner_create(void) { - // ... -} -``` - -This function should create your scanner object. It will only be called once anytime your language is set on a parser. Often, you will want to allocate memory on the heap and return a pointer to it. If your external scanner doesn't need to maintain any state, it's ok to return `NULL`. - -#### Destroy - -```c -void tree_sitter_my_language_external_scanner_destroy(void *payload) { - // ... -} -``` - -This function should free any memory used by your scanner. It is called once when a parser is deleted or assigned a different language. It receives as an argument the same pointer that was returned from the *create* function. If your *create* function didn't allocate any memory, this function can be a noop. - -#### Serialize - -```c -unsigned tree_sitter_my_language_external_scanner_serialize( - void *payload, - char *buffer -) { - // ... -} -``` - -This function should copy the complete state of your scanner into a given byte buffer, and return the number of bytes written. The function is called every time the external scanner successfully recognizes a token. It receives a pointer to your scanner and a pointer to a buffer. The maximum number of bytes that you can write is given by the `TREE_SITTER_SERIALIZATION_BUFFER_SIZE` constant, defined in the `tree_sitter/parser.h` header file. - -The data that this function writes will ultimately be stored in the syntax tree so that the scanner can be restored to the right state when handling edits or ambiguities. For your parser to work correctly, the `serialize` function must store its entire state, and `deserialize` must restore the entire state. For good performance, you should design your scanner so that its state can be serialized as quickly and compactly as possible. - -#### Deserialize - -```c -void tree_sitter_my_language_external_scanner_deserialize( - void *payload, - const char *buffer, - unsigned length -) { - // ... -} -``` - -This function should *restore* the state of your scanner based the bytes that were previously written by the `serialize` function. It is called with a pointer to your scanner, a pointer to the buffer of bytes, and the number of bytes that should be read. -It is good practice to explicitly erase your scanner state variables at the start of this function, before restoring their values from the byte buffer. - -#### Scan - -```c -bool tree_sitter_my_language_external_scanner_scan( - void *payload, - TSLexer *lexer, - const bool *valid_symbols -) { - // ... -} -``` - -This function is responsible for recognizing external tokens. It should return `true` if a token was recognized, and `false` otherwise. It is called with a "lexer" struct with the following fields: - -* **`int32_t lookahead`** - The current next character in the input stream, represented as a 32-bit unicode code point. -* **`TSSymbol result_symbol`** - The symbol that was recognized. Your scan function should *assign* to this field one of the values from the `TokenType` enum, described above. -* **`void (*advance)(TSLexer *, bool skip)`** - A function for advancing to the next character. If you pass `true` for the second argument, the current character will be treated as whitespace; whitespace won't be included in the text range associated with tokens emitted by the external scanner. -* **`void (*mark_end)(TSLexer *)`** - A function for marking the end of the recognized token. This allows matching tokens that require multiple characters of lookahead. By default (if you don't call `mark_end`), any character that you moved past using the `advance` function will be included in the size of the token. But once you call `mark_end`, then any later calls to `advance` will *not* increase the size of the returned token. You can call `mark_end` multiple times to increase the size of the token. -* **`uint32_t (*get_column)(TSLexer *)`** - A function for querying the current column position of the lexer. It returns the number of codepoints since the start of the current line. The codepoint position is recalculated on every call to this function by reading from the start of the line. -* **`bool (*is_at_included_range_start)(const TSLexer *)`** - A function for checking whether the parser has just skipped some characters in the document. When parsing an embedded document using the `ts_parser_set_included_ranges` function (described in the [multi-language document section][multi-language-section]), the scanner may want to apply some special behavior when moving to a disjoint part of the document. For example, in [EJS documents][ejs], the JavaScript parser uses this function to enable inserting automatic semicolon tokens in between the code directives, delimited by `<%` and `%>`. -* **`bool (*eof)(const TSLexer *)`** - A function for determining whether the lexer is at the end of the file. The value of `lookahead` will be `0` at the end of a file, but this function should be used instead of checking for that value because the `0` or "NUL" value is also a valid character that could be present in the file being parsed. -* **`void (*log)(const TSLexer *, const char * format, ...)`** - A `printf`-like function for logging. The log is viewable through e.g. `tree-sitter parse --debug` or the browser's console after checking the `log` option in the [Playground](./playground). - -The third argument to the `scan` function is an array of booleans that indicates which of external tokens are currently expected by the parser. You should only look for a given token if it is valid according to this array. At the same time, you cannot backtrack, so you may need to combine certain pieces of logic. - -```c -if (valid_symbols[INDENT] || valid_symbols[DEDENT]) { - - // ... logic that is common to both `INDENT` and `DEDENT` - - if (valid_symbols[INDENT]) { - - // ... logic that is specific to `INDENT` - - lexer->result_symbol = INDENT; - return true; - } -} -``` - -#### External Scanner Helpers - -##### Allocator - -Instead of using libc's `malloc`, `calloc`, `realloc`, and `free`, you should use the versions prefixed with `ts_` from `tree_sitter/alloc.h`. -These macros can allow a potential consumer to override the default allocator with their own implementation, but by default will use the libc functions. - -As a consumer of the tree-sitter core library as well as any parser libraries that might use allocations, you can enable overriding the default allocator and have it use the same one as the library allocator, of which you can set with `ts_set_allocator`. -To enable this overriding in scanners, you must compile them with the `TREE_SITTER_REUSE_ALLOCATOR` macro defined, and tree-sitter the library must be linked into your final app dynamically, since it needs to resolve the internal functions at runtime. If you are compiling -an executable binary that uses the core library, but want to load parsers dynamically at runtime, then you will have to use a special linker flag on Unix. For non-Darwin systems, that would be `--dynamic-list` and for Darwin systems, that would be `-exported_symbols_list`. -The CLI does exactly this, so you can use it as a reference (check out `cli/build.rs`). - -For example, assuming you wanted to allocate 100 bytes for your scanner, you'd do so like the following example: - -```c -#include "tree_sitter/parser.h" -#include "tree_sitter/alloc.h" - -// ... - -void *tree_sitter_my_language_external_scanner_create(void) { - return ts_calloc(100, 1); // or ts_malloc(100) -} - -// ... - -``` - -##### Arrays - -If you need to use array-like types in your scanner, such as tracking a stack of indentations or tags, you should use the array macros from `tree_sitter/array.h`. - -There are quite a few of them provided for you, but here's how you could get started tracking some . Check out the header itself for more detailed documentation. - -**NOTE**: Do not use any of the array functions or macros that are prefixed with an underscore and have comments saying that it is not what you are looking for. -These are internal functions used as helpers by other macros that are public. They are not meant to be used directly, nor are they what you want. - -```c -#include "tree_sitter/parser.h" -#include "tree_sitter/array.h" - -enum TokenType { - INDENT, - DEDENT, - NEWLINE, - STRING, -} - -// Create the array in your create function - -void *tree_sitter_my_language_external_scanner_create(void) { - return ts_calloc(1, sizeof(Array(int))); - - // or if you want to zero out the memory yourself - - Array(int) *stack = ts_malloc(sizeof(Array(int))); - array_init(&stack); - return stack; -} - -bool tree_sitter_my_language_external_scanner_scan( - void *payload, - TSLexer *lexer, - const bool *valid_symbols -) { - Array(int) *stack = payload; - if (valid_symbols[INDENT]) { - array_push(stack, lexer->get_column(lexer)); - lexer->result_symbol = INDENT; - return true; - } - if (valid_symbols[DEDENT]) { - array_pop(stack); // this returns the popped element by value, but we don't need it - lexer->result_symbol = DEDENT; - return true; - } - - // we can also use an array on the stack to keep track of a string - - Array(char) next_string = array_new(); - - if (valid_symbols[STRING] && lexer->lookahead == '"') { - lexer->advance(lexer, false); - while (lexer->lookahead != '"' && lexer->lookahead != '\n' && !lexer->eof(lexer)) { - array_push(&next_string, lexer->lookahead); - lexer->advance(lexer, false); - } - - // assume we have some arbitrary constraint of not having more than 100 characters in a string - if (lexer->lookahead == '"' && next_string.size <= 100) { - lexer->advance(lexer, false); - lexer->result_symbol = STRING; - return true; - } - } - - return false; -} - -``` - -#### Other External Scanner Details - -If a token in the `externals` array is valid at a given position in the parse, the external scanner will be called first before anything else is done. This means the external scanner functions as a powerful override of Tree-sitter's lexing behavior, and can be used to solve problems that can't be cracked with ordinary lexical, parse, or dynamic precedence. - -If a syntax error is encountered during regular parsing, Tree-sitter's first action during error recovery will be to call the external scanner's `scan` function with all tokens marked valid. The scanner should detect this case and handle it appropriately. One simple method of detection is to add an unused token to the end of the `externals` array, for example `externals: $ => [$.token1, $.token2, $.error_sentinel]`, then check whether that token is marked valid to determine whether Tree-sitter is in error correction mode. - -If you put terminal keywords in the `externals` array, for example `externals: $ => ['if', 'then', 'else']`, then any time those terminals are present in the grammar they will be tokenized by the external scanner. It is similar to writing `externals: [$.if_keyword, $.then_keyword, $.else_keyword]` then using `alias($.if_keyword, 'if')` in the grammar. - -If in the `externals` array use literal keywords then lexing works in two steps, the external scanner will be called first and if it sets a resulting token and returns `true` then the token considered as recognized and Tree-sitter moves to a next token. But the external scanner may return `false` and in this case Tree-sitter fallbacks to the internal lexing mechanism. - -In case of some keywords defined in the `externals` array in a rule referencing form like `$.if_keyword` and there is no additional definition of that rule in the grammar rules, e.g., `if_keyword: $ => 'if'` then fallback to the internal lexer isn't possible because Tree-sitter doesn't know the actual keyword and it's fully the external scanner resposibilty to recognize such tokens. - -External scanners are a common cause of infinite loops. -Be very careful when emitting zero-width tokens from your external scanner, and if you consume characters in a loop be sure use the `eof` function to check whether you are at the end of the file. - -[ambiguous-grammar]: https://en.wikipedia.org/wiki/Ambiguous_grammar -[antlr]: https://www.antlr.org -[bison-dprec]: https://www.gnu.org/software/bison/manual/html_node/Generalized-LR-Parsing.html -[bison]: https://en.wikipedia.org/wiki/GNU_bison -[cargo]: https://doc.rust-lang.org/cargo/getting-started/installation.html -[crate]: https://crates.io/crates/tree-sitter-cli -[cst]: https://en.wikipedia.org/wiki/Parse_tree -[ebnf]: https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form -[ecmascript-spec]: https://262.ecma-international.org/6.0/ -[ejs]: https://ejs.co -[enum]: https://en.wikipedia.org/wiki/Enumerated_type#C -[glr-parsing]: https://en.wikipedia.org/wiki/GLR_parser -[heredoc]: https://en.wikipedia.org/wiki/Here_document -[indent-tokens]: https://en.wikipedia.org/wiki/Off-side_rule -[language-spec]: https://en.wikipedia.org/wiki/Programming_language_specification -[lexing]: https://en.wikipedia.org/wiki/Lexical_analysis -[longest-match]: https://en.wikipedia.org/wiki/Maximal_munch -[lr-conflict]: https://en.wikipedia.org/wiki/LR_parser#Conflicts_in_the_constructed_tables -[lr-grammars]: https://en.wikipedia.org/wiki/LR_parser -[multi-language-section]: ./using-parsers#multi-language-documents -[named-vs-anonymous-nodes-section]: ./using-parsers#named-vs-anonymous-nodes -[field-names-section]: ./using-parsers#node-field-names -[node-module]: https://www.npmjs.com/package/tree-sitter-cli -[node.js]: https://nodejs.org -[static-node-types]: ./using-parsers#static-node-types -[non-terminal]: https://en.wikipedia.org/wiki/Terminal_and_nonterminal_symbols -[npm]: https://docs.npmjs.com -[path-env]: https://en.wikipedia.org/wiki/PATH_(variable) -[peg]: https://en.wikipedia.org/wiki/Parsing_expression_grammar -[percent-string]: https://docs.ruby-lang.org/en/2.5.0/doc/syntax/literals_rdoc.html#label-Percent+Strings -[releases]: https://github.com/tree-sitter/tree-sitter/releases/latest -[s-exp]: https://en.wikipedia.org/wiki/S-expression -[syntax-highlighting]: ./syntax-highlighting -[syntax-highlighting-tests]: ./syntax-highlighting#unit-testing -[tree-sitter-cli]: https://github.com/tree-sitter/tree-sitter/tree/master/cli -[tree-sitter-javascript]: https://github.com/tree-sitter/tree-sitter-javascript -[triple-slash]: https://www.typescriptlang.org/docs/handbook/triple-slash-directives.html -[ts-check]: https://www.typescriptlang.org/docs/handbook/intro-to-js-ts.html -[yacc-prec]: https://docs.oracle.com/cd/E19504-01/802-5880/6i9k05dh3/index.html -[yacc]: https://en.wikipedia.org/wiki/Yacc diff --git a/docs/section-4-syntax-highlighting.md b/docs/section-4-syntax-highlighting.md deleted file mode 100644 index 67c2bc25..00000000 --- a/docs/section-4-syntax-highlighting.md +++ /dev/null @@ -1,493 +0,0 @@ ---- -title: Syntax Highlighting -permalink: syntax-highlighting ---- - -# Syntax Highlighting - -Syntax highlighting is a very common feature in applications that deal with code. Tree-sitter has built-in support for syntax highlighting, via the [`tree-sitter-highlight`](https://github.com/tree-sitter/tree-sitter/tree/master/highlight) library, which is currently used on GitHub.com for highlighting code written in several languages. You can also perform syntax highlighting at the command line using the `tree-sitter highlight` command. - -This document explains how the Tree-sitter syntax highlighting system works, using the command line interface. If you are using `tree-sitter-highlight` library (either from C or from Rust), all of these concepts are still applicable, but the configuration data is provided using in-memory objects, rather than files. - -## Overview - -All of the files needed to highlight a given language are normally included in the same git repository as the Tree-sitter grammar for that language (for example, [`tree-sitter-javascript`](https://github.com/tree-sitter/tree-sitter-javascript), [`tree-sitter-ruby`](https://github.com/tree-sitter/tree-sitter-ruby)). In order to run syntax highlighting from the command-line, three types of files are needed: - -1. Per-user configuration in `~/.config/tree-sitter/config.json` -2. Language configuration in grammar repositories' `tree-sitter.json` files. -3. Tree queries in the grammars repositories' `queries` folders. - -For an example of the language-specific files, see the [`tree-sitter.json` file](https://github.com/tree-sitter/tree-sitter-ruby/blob/master/tree-sitter.json) and [`queries` directory](https://github.com/tree-sitter/tree-sitter-ruby/tree/master/queries) in the `tree-sitter-ruby` repository. The following sections describe the behavior of each file. - -## Per-user Configuration - -The Tree-sitter CLI automatically creates two directories in your home folder. One holds a JSON configuration file, that lets you customize the behavior of the CLI. The other holds any compiled language parsers that you use. - -These directories are created in the "normal" place for your platform: - -* On Linux, `~/.config/tree-sitter` and `~/.cache/tree-sitter` -* On Mac, `~/Library/Application Support/tree-sitter` and `~/Library/Caches/tree-sitter` -* On Windows, `C:\Users\[username]\AppData\Roaming\tree-sitter` and `C:\Users\[username]\AppData\Local\tree-sitter` - -The CLI will work if there's no config file present, falling back on default values for each configuration option. To create a config file that you can edit, run this command: - -```sh -tree-sitter init-config -``` - -(This will print out the location of the file that it creates so that you can easily find and modify it.) - -### Paths - -The `tree-sitter highlight` command takes one or more file paths, and tries to automatically determine which language should be used to highlight those files. In order to do this, it needs to know *where* to look for Tree-sitter grammars on your filesystem. You can control this using the `"parser-directories"` key in your configuration file: - -```json -{ - "parser-directories": [ - "/Users/my-name/code", - "/Users/my-name/other-code" - ] -} -``` - -Currently, any folder within one of these *parser directories* whose name begins with `tree-sitter-` will be treated as a Tree-sitter grammar repository. - -### Theme - -The Tree-sitter highlighting system works by annotating ranges of source code with logical "highlight names" like `function.method`, `type.builtin`, `keyword`, etc. In order to decide what *color* should be used for rendering each highlight, a *theme* is needed. - -In your config file, the `"theme"` value is an object whose keys are dot-separated highlight names like `function.builtin` or `keyword`, and whose values are JSON expressions that represent text styling parameters. - -### Parse Theme - -The Tree-sitter `parse` command will output a pretty-printed CST when the `--cst` option is used. You can control which colors are used for various parts of the tree in your configuration file. Note that omitting a field will cause the relevant text to be rendered with its default color. - -```json5 -{ - "parse-theme": { - // The color of node kinds - "node-kind": [20, 20, 20], - // The color of text associated with a node - "node-text": [255, 255, 255], - // The color of node fields - "field": [42, 42, 42], - // The color of the range information for unnamed nodes - "row-color": [255, 255, 255], - // The color of the range information for named nodes - "row-color-named": [255, 130, 0], - // The color of extra nodes - "extra": [255, 0, 255], - // The color of ERROR nodes - "error": [255, 0, 0], - // The color of MISSING nodes and their associated text - "missing": [153, 75, 0], - // The color of newline characters - "line-feed": [150, 150, 150], - // The color of backtick characters - "backtick": [0, 200, 0], - // The color of literals - "literal": [0, 0, 200], - } -} -``` - -#### Highlight Names - -A theme can contain multiple keys that share a common subsequence. Examples: - -* `variable` and `variable.parameter` -* `function`, `function.builtin`, and `function.method` - -For a given highlight produced, styling will be determined based on the **longest matching theme key**. For example, the highlight `function.builtin.static` would match the key `function.builtin` rather than `function`. - -#### Styling Values - -Styling values can be any of the following: - -* Integers from 0 to 255, representing ANSI terminal color ids. -* Strings like `"#e45649"` representing hexadecimal RGB colors. -* Strings naming basic ANSI colors like `"red"`, `"black"`, `"purple"`, or `"cyan"`. -* Objects with the following keys: - * `color` - An integer or string as described above. - * `underline` - A boolean indicating whether the text should be underlined. - * `italic` - A boolean indicating whether the text should be italicized. - * `bold` - A boolean indicating whether the text should be bold-face. - -## Language Configuration - -The `tree-sitter.json` file is used by the Tree-sitter CLI. Within this file, the CLI looks for data nested under the top-level `"grammars"` key. This key is expected to contain an array of objects with the following keys: - -### Basics - -These keys specify basic information about the parser: - -* `scope` (required) - A string like `"source.js"` that identifies the language. Currently, we strive to match the scope names used by popular [TextMate grammars](https://macromates.com/manual/en/language_grammars) and by the [Linguist](https://github.com/github/linguist) library. - -* `path` (optional) - A relative path from the directory containing `tree-sitter.json` to another directory containing the `src/` folder, which contains the actual generated parser. The default value is `"."` (so that `src/` is in the same folder as `tree-sitter.json`), and this very rarely needs to be overridden. - -* `external-files` (optional) - A list of relative paths from the root dir of a -parser to files that should be checked for modifications during recompilation. -This is useful during development to have changes to other files besides scanner.c -be picked up by the cli. - -### Language Detection - -These keys help to decide whether the language applies to a given file: - -* `file-types` - An array of filename suffix strings. The grammar will be used for files whose names end with one of these suffixes. Note that the suffix may match an *entire* filename. - -* `first-line-regex` - A regex pattern that will be tested against the first line of a file in order to determine whether this language applies to the file. If present, this regex will be used for any file whose language does not match any grammar's `file-types`. - -* `content-regex` - A regex pattern that will be tested against the contents of the file in order to break ties in cases where multiple grammars matched the file using the above two criteria. If the regex matches, this grammar will be preferred over another grammar with no `content-regex`. If the regex does not match, a grammar with no `content-regex` will be preferred over this one. - -* `injection-regex` - A regex pattern that will be tested against a *language name* in order to determine whether this language should be used for a potential *language injection* site. Language injection is described in more detail in [a later section](#language-injection). - -### Query Paths - -These keys specify relative paths from the directory containing `tree-sitter.json` to the files that control syntax highlighting: - -* `highlights` - Path to a *highlight query*. Default: `queries/highlights.scm` -* `locals` - Path to a *local variable query*. Default: `queries/locals.scm`. -* `injections` - Path to an *injection query*. Default: `queries/injections.scm`. - -The behaviors of these three files are described in the next section. - -### Example - -Typically, the `"tree-sitter"` array only needs to contain one object, which only needs to specify a few keys: - -```json -{ - "tree-sitter": [ - { - "scope": "source.ruby", - "file-types": [ - "rb", - "gemspec", - "Gemfile", - "Rakefile" - ], - "first-line-regex": "#!.*\\bruby$" - } - ] -} -``` - -## Queries - -Tree-sitter's syntax highlighting system is based on *tree queries*, which are a general system for pattern-matching on Tree-sitter's syntax trees. See [this section](./using-parsers#pattern-matching-with-queries) of the documentation for more information about tree queries. - -Syntax highlighting is controlled by *three* different types of query files that are usually included in the `queries` folder. The default names for the query files use the `.scm` file. We chose this extension because it commonly used for files written in [Scheme](https://en.wikipedia.org/wiki/Scheme_%28programming_language%29), a popular dialect of Lisp, and these query files use a Lisp-like syntax. - -Alternatively, you can think of `.scm` as an acronym for "Source Code Matching". - -### Highlights - -The most important query is called the highlights query. The highlights query uses *captures* to assign arbitrary *highlight names* to different nodes in the tree. Each highlight name can then be mapped to a color (as described [above](#theme)). Commonly used highlight names include `keyword`, `function`, `type`, `property`, and `string`. Names can also be dot-separated like `function.builtin`. - -#### Example Input - -For example, consider the following Go code: - -```go -func increment(a int) int { - return a + 1 -} -``` - -With this syntax tree: - -```scheme -(source_file - (function_declaration - name: (identifier) - parameters: (parameter_list - (parameter_declaration - name: (identifier) - type: (type_identifier))) - result: (type_identifier) - body: (block - (return_statement - (expression_list - (binary_expression - left: (identifier) - right: (int_literal))))))) -``` - -#### Example Query - -Suppose we wanted to render this code with the following colors: - -* keywords `func` and `return` in purple -* function `increment` in blue -* type `int` in green -* number `5` brown - -We can assign each of these categories a *highlight name* using a query like this: - -```scheme -; highlights.scm - -"func" @keyword -"return" @keyword -(type_identifier) @type -(int_literal) @number -(function_declaration name: (identifier) @function) -``` - -Then, in our config file, we could map each of these highlight names to a color: - -```json -{ - "theme": { - "keyword": "purple", - "function": "blue", - "type": "green", - "number": "brown" - } -} -``` - -#### Result - -Running `tree-sitter highlight` on this Go file would produce output like this: - -
-func increment(a int) int {
-    return a + 1
-}
-
- -### Local Variables - -Good syntax highlighting helps the reader to quickly distinguish between the different types of *entities* in their code. Ideally, if a given entity appears in *multiple* places, it should be colored the same in each place. The Tree-sitter syntax highlighting system can help you to achieve this by keeping track of local scopes and variables. - -The *local variables* query is different from the highlights query in that, while the highlights query uses *arbitrary* capture names which can then be mapped to colors, the locals variable query uses a fixed set of capture names, each of which has a special meaning. - -The capture names are as follows: - -* `@local.scope` - indicates that a syntax node introduces a new local scope. -* `@local.definition` - indicates that a syntax node contains the *name* of a definition within the current local scope. -* `@local.reference` - indicates that a syntax node contains the *name* which *may* refer to an earlier definition within some enclosing scope. - -When highlighting a file, Tree-sitter will keep track of the set of scopes that contains any given position, and the set of definitions within each scope. When processing a syntax node that is captured as a `local.reference`, Tree-sitter will try to find a definition for a name that matches the node's text. If it finds a match, Tree-sitter will ensure that the *reference* and the *definition* are colored the same. - -The information produced by this query can also be *used* by the highlights query. You can *disable* a pattern for nodes which have been identified as local variables by adding the predicate `(#is-not? local)` to the pattern. This is used in the example below: - -#### Example Input - -Consider this Ruby code: - -```ruby -def process_list(list) - context = current_context - list.map do |item| - process_item(item, context) - end -end - -item = 5 -list = [item] -``` - -With this syntax tree: - -```scheme -(program - (method - name: (identifier) - parameters: (method_parameters - (identifier)) - (assignment - left: (identifier) - right: (identifier)) - (method_call - method: (call - receiver: (identifier) - method: (identifier)) - block: (do_block - (block_parameters - (identifier)) - (method_call - method: (identifier) - arguments: (argument_list - (identifier) - (identifier)))))) - (assignment - left: (identifier) - right: (integer)) - (assignment - left: (identifier) - right: (array - (identifier)))) -``` - -There are several different types of names within this method: - -* `process_list` is a method. -* Within this method, `list` is a formal parameter -* `context` is a local variable. -* `current_context` is *not* a local variable, so it must be a method. -* Within the `do` block, `item` is a formal parameter -* Later on, `item` and `list` are both local variables (not formal parameters). - -#### Example Queries - -Let's write some queries that let us clearly distinguish between these types of names. First, set up the highlighting query, as described in the previous section. We'll assign distinct colors to method calls, method definitions, and formal parameters: - -```scheme -; highlights.scm - -(call method: (identifier) @function.method) -(method_call method: (identifier) @function.method) - -(method name: (identifier) @function.method) - -(method_parameters (identifier) @variable.parameter) -(block_parameters (identifier) @variable.parameter) - -((identifier) @function.method - (#is-not? local)) -``` - -Then, we'll set up a local variable query to keep track of the variables and scopes. Here, we're indicating that methods and blocks create local *scopes*, parameters and assignments create *definitions*, and other identifiers should be considered *references*: - -```scheme -; locals.scm - -(method) @local.scope -(do_block) @local.scope - -(method_parameters (identifier) @local.definition) -(block_parameters (identifier) @local.definition) - -(assignment left:(identifier) @local.definition) - -(identifier) @local.reference -``` - -#### Result - -Running `tree-sitter highlight` on this ruby file would produce output like this: - -
-def process_list(list)
-  context = current_context
-  list.map do |item|
-    process_item(item, context)
-  end
-end
-
-item = 5
-list = [item]
-
- -### Language Injection - -Some source files contain code written in multiple different languages. Examples include: - -* HTML files, which can contain JavaScript inside of ` - -{% if jekyll.environment == "development" %} - - -{% else %} - - -{% endif %} - - - diff --git a/docs/section-8-code-navigation-systems.md b/docs/section-8-code-navigation-systems.md deleted file mode 100644 index 04346e46..00000000 --- a/docs/section-8-code-navigation-systems.md +++ /dev/null @@ -1,120 +0,0 @@ ---- -title: Code Navigation Systems -permalink: code-navigation-systems ---- - -# Code Navigation Systems - -Tree-sitter can be used in conjunction with its [tree query language](https://tree-sitter.github.io/tree-sitter/using-parsers#pattern-matching-with-queries) as a part of code navigation systems. An example of such a system can be seen in the `tree-sitter tags` command, which emits a textual dump of the interesting syntactic nodes in its file argument. A notable application of this is GitHub's support for [search-based code navigation](https://docs.github.com/en/repositories/working-with-files/using-files/navigating-code-on-github#precise-and-search-based-navigation). This document exists to describe how to integrate with such systems, and how to extend this functionality to any language with a Tree-sitter grammar. - -## Tagging and captures - -_Tagging_ is the act of identifying the entities that can be named in a program. We use Tree-sitter queries to find those entities. Having found them, you use a syntax capture to label the entity and its name. - -The essence of a given tag lies in two pieces of data: the _role_ of the entity that is matched (i.e. whether it is a definition or a reference) and the _kind_ of that entity, which describes how the entity is used (i.e. whether it's a class definition, function call, variable reference, and so on). Our convention is to use a syntax capture following the `@role.kind` capture name format, and another inner capture, always called `@name`, that pulls out the name of a given identifier. - -You may optionally include a capture named `@doc` to bind a docstring. For convenience purposes, the tagging system provides two built-in functions, `#select-adjacent!` and `#strip!` that are convenient for removing comment syntax from a docstring. `#strip!` takes a capture as its first argument and a regular expression as its second, expressed as a quoted string. Any text patterns matched by the regular expression will be removed from the text associated with the passed capture. `#select-adjacent!`, when passed two capture names, filters the text associated with the first capture so that only nodes adjacent to the second capture are preserved. This can be useful when writing queries that would otherwise include too much information in matched comments. - -## Examples - -This [query](https://github.com/tree-sitter/tree-sitter-python/blob/78c4e9b6b2f08e1be23b541ffced47b15e2972ad/queries/tags.scm#L4-L5) recognizes Python function definitions and captures their declared name. The `function_definition` syntax node is defined in the [Python Tree-sitter grammar](https://github.com/tree-sitter/tree-sitter-python/blob/78c4e9b6b2f08e1be23b541ffced47b15e2972ad/grammar.js#L354). - -```scheme -(function_definition - name: (identifier) @name) @definition.function -``` - -A more sophisticated query can be found in the [JavaScript Tree-sitter repository](https://github.com/tree-sitter/tree-sitter-javascript/blob/fdeb68ac8d2bd5a78b943528bb68ceda3aade2eb/queries/tags.scm#L63-L70): - -```scheme -(assignment_expression - left: [ - (identifier) @name - (member_expression - property: (property_identifier) @name) - ] - right: [(arrow_function) (function)] -) @definition.function -``` - -An even more sophisticated query is in the [Ruby Tree-sitter repository](https://github.com/tree-sitter/tree-sitter-ruby/blob/1ebfdb288842dae5a9233e2509a135949023dd82/queries/tags.scm#L24-L43), which uses built-in functions to strip the Ruby comment character (`#`) from the docstrings associated with a class or singleton-class declaration, then selects only the docstrings adjacent to the node matched as `@definition.class`. - -```scheme -( - (comment)* @doc - . - [ - (class - name: [ - (constant) @name - (scope_resolution - name: (_) @name) - ]) @definition.class - (singleton_class - value: [ - (constant) @name - (scope_resolution - name: (_) @name) - ]) @definition.class - ] - (#strip! @doc "^#\\s*") - (#select-adjacent! @doc @definition.class) -) -``` - -The below table describes a standard vocabulary for kinds and roles during the tagging process. New applications may extend (or only recognize a subset of) these capture names, but it is desirable to standardize on the names below. - -| Category | Tag | -|--------------------------|-----------------------------| -| Class definitions | `@definition.class` | -| Function definitions | `@definition.function` | -| Interface definitions | `@definition.interface` | -| Method definitions | `@definition.method` | -| Module definitions | `@definition.module` | -| Function/method calls | `@reference.call` | -| Class reference | `@reference.class` | -| Interface implementation | `@reference.implementation` | - -## Command-line invocation - -You can use the `tree-sitter tags` command to test out a tags query file, passing as arguments one or more files to tag. We can run this tool from within the Tree-sitter Ruby repository, over code in a file called `test.rb`: - -```ruby -module Foo - class Bar - # won't be included - - # is adjacent, will be - def baz - end - end -end -``` - -Invoking `tree-sitter tags test.rb` produces the following console output, representing matched entities' name, role, location, first line, and docstring: - -```text - test.rb - Foo | module def (0, 7) - (0, 10) `module Foo` - Bar | class def (1, 8) - (1, 11) `class Bar` - baz | method def (2, 8) - (2, 11) `def baz` "is adjacent, will be" -``` - -It is expected that tag queries for a given language are located at `queries/tags.scm` in that language's repository. - -## Unit Testing - -Tags queries may be tested with `tree-sitter test`. Files under `test/tags/` are checked using the same comment system as [highlights queries](https://tree-sitter.github.io/tree-sitter/syntax-highlighting#unit-testing). For example, the above Ruby tags can be tested with these comments: - -```ruby -module Foo - # ^ definition.module - class Bar - # ^ definition.class - - def baz - # ^ definition.method - end - end -end -``` diff --git a/docs/src/3-syntax-highlighting.md b/docs/src/3-syntax-highlighting.md new file mode 100644 index 00000000..c6356fbb --- /dev/null +++ b/docs/src/3-syntax-highlighting.md @@ -0,0 +1,447 @@ +# Syntax Highlighting + +Syntax highlighting is a very common feature in applications that deal with code. Tree-sitter has built-in support for +syntax highlighting via the [`tree-sitter-highlight`][highlight crate] library, which is now used on GitHub.com for highlighting +code written in several languages. You can also perform syntax highlighting at the command line using the +`tree-sitter highlight` command. + +This document explains how the Tree-sitter syntax highlighting system works, using the command line interface. If you are +using `tree-sitter-highlight` library (either from C or from Rust), all of these concepts are still applicable, but the +configuration data is provided using in-memory objects, rather than files. + +## Overview + +All the files needed to highlight a given language are normally included in the same git repository as the Tree-sitter +grammar for that language (for example, [`tree-sitter-javascript`][js grammar], [`tree-sitter-ruby`][ruby grammar]). +To run syntax highlighting from the command-line, three types of files are needed: + +1. Per-user configuration in `~/.config/tree-sitter/config.json` (see the [init-config][init-config] page for more info). +2. Language configuration in grammar repositories' `tree-sitter.json` files (see the [init][init] page for more info). +3. Tree queries in the grammars repositories' `queries` folders. + +For an example of the language-specific files, see the [`tree-sitter.json` file][ts json] and [`queries` directory][queries] +in the `tree-sitter-ruby` repository. The following sections describe the behavior of each file. + +## Language Configuration + +The `tree-sitter.json` file is used by the Tree-sitter CLI. Within this file, the CLI looks for data nested under the +top-level `"grammars"` key. This key is expected to contain an array of objects with the following keys: + +### Basics + +These keys specify basic information about the parser: + +- `scope` (required) — A string like `"source.js"` that identifies the language. We strive to match the scope names used +by popular [TextMate grammars][textmate] and by the [Linguist][linguist] library. + +- `path` (optional) — A relative path from the directory containing `tree-sitter.json` to another directory containing +the `src/` folder, which contains the actual generated parser. The default value is `"."` (so that `src/` is in the same +folder as `tree-sitter.json`), and this very rarely needs to be overridden. + +- `external-files` (optional) — A list of relative paths from the root dir of a +parser to files that should be checked for modifications during recompilation. +This is useful during development to have changes to other files besides scanner.c +be picked up by the cli. + +### Language Detection + +These keys help to decide whether the language applies to a given file: + +- `file-types` — An array of filename suffix strings. The grammar will be used for files whose names end with one of these +suffixes. Note that the suffix may match an *entire* filename. + +- `first-line-regex` — A regex pattern that will be tested against the first line of a file to determine whether this language +applies to the file. If present, this regex will be used for any file whose language does not match any grammar's `file-types`. + +- `content-regex` — A regex pattern that will be tested against the contents of the file to break ties in cases where +multiple grammars matched the file using the above two criteria. If the regex matches, this grammar will be preferred over +another grammar with no `content-regex`. If the regex does not match, a grammar with no `content-regex` will be preferred +over this one. + +- `injection-regex` — A regex pattern that will be tested against a *language name* ito determine whether this language +should be used for a potential *language injection* site. Language injection is described in more detail in [a later section](#language-injection). + +### Query Paths + +These keys specify relative paths from the directory containing `tree-sitter.json` to the files that control syntax highlighting: + +- `highlights` — Path to a *highlight query*. Default: `queries/highlights.scm` +- `locals` — Path to a *local variable query*. Default: `queries/locals.scm`. +- `injections` — Path to an *injection query*. Default: `queries/injections.scm`. + +The behaviors of these three files are described in the next section. + +## Queries + +Tree-sitter's syntax highlighting system is based on *tree queries*, which are a general system for pattern-matching on +Tree-sitter's syntax trees. See [this section][pattern matching] of the documentation for more information about tree queries. + +Syntax highlighting is controlled by *three* different types of query files that are usually included in the `queries` folder. +The default names for the query files use the `.scm` file. We chose this extension because it commonly used for files written +in [Scheme][scheme], a popular dialect of Lisp, and these query files use a Lisp-like syntax. + +### Highlights + +The most important query is called the highlights query. The highlights query uses *captures* to assign arbitrary +*highlight names* to different nodes in the tree. Each highlight name can then be mapped to a color +(as described in the [init-config command][theme]). Commonly used highlight names include +`keyword`, `function`, `type`, `property`, and `string`. Names can also be dot-separated like `function.builtin`. + +#### Example Go Snippet + +For example, consider the following Go code: + +```go +func increment(a int) int { + return a + 1 +} +``` + +With this syntax tree: + +```scheme +(source_file + (function_declaration + name: (identifier) + parameters: (parameter_list + (parameter_declaration + name: (identifier) + type: (type_identifier))) + result: (type_identifier) + body: (block + (return_statement + (expression_list + (binary_expression + left: (identifier) + right: (int_literal))))))) +``` + +#### Example Query + +Suppose we wanted to render this code with the following colors: + +- keywords `func` and `return` in purple +- function `increment` in blue +- type `int` in green +- number `5` brown + +We can assign each of these categories a *highlight name* using a query like this: + +```scheme +; highlights.scm + +"func" @keyword +"return" @keyword +(type_identifier) @type +(int_literal) @number +(function_declaration name: (identifier) @function) +``` + +Then, in our config file, we could map each of these highlight names to a color: + +```json +{ + "theme": { + "keyword": "purple", + "function": "blue", + "type": "green", + "number": "brown" + } +} +``` + +#### Highlights Result + +Running `tree-sitter highlight` on this Go file would produce output like this: + +```admonish example collapsible=true, title='Output' +
+func increment(a int) int {
+    return a + 1
+}
+
+``` + +### Local Variables + +Good syntax highlighting helps the reader to quickly distinguish between the different types of *entities* in their code. +Ideally, if a given entity appears in *multiple* places, it should be colored the same in each place. The Tree-sitter syntax +highlighting system can help you to achieve this by keeping track of local scopes and variables. + +The *local variables* query is different from the highlights query in that, while the highlights query uses *arbitrary* +capture names, which can then be mapped to colors, the locals variable query uses a fixed set of capture names, each of +which has a special meaning. + +The capture names are as follows: + +- `@local.scope` — indicates that a syntax node introduces a new local scope. +- `@local.definition` — indicates that a syntax node contains the *name* of a definition within the current local scope. +- `@local.reference` — indicates that a syntax node contains the *name*, which *may* refer to an earlier definition within +some enclosing scope. + +Additionally, to ignore certain nodes from being tagged, you can use the `@ignore` capture. This is useful if you want to +exclude a subset of nodes from being tagged. When writing a query leveraging this, you should ensure this pattern comes +before any other patterns that would be used for tagging, for example: + +```scheme +(expression (identifier) @ignore) + +(identifier) @local.reference +``` + +When highlighting a file, Tree-sitter will keep track of the set of scopes that contains any given position, and the set +of definitions within each scope. When processing a syntax node that is captured as a `local.reference`, Tree-sitter will +try to find a definition for a name that matches the node's text. If it finds a match, Tree-sitter will ensure that the +*reference*, and the *definition* are colored the same. + +The information produced by this query can also be *used* by the highlights query. You can *disable* a pattern for nodes, +which have been identified as local variables by adding the predicate `(#is-not? local)` to the pattern. This is used in +the example below: + +#### Example Ruby Snippet + +Consider this Ruby code: + +```ruby +def process_list(list) + context = current_context + list.map do |item| + process_item(item, context) + end +end + +item = 5 +list = [item] +``` + +With this syntax tree: + +```scheme +(program + (method + name: (identifier) + parameters: (method_parameters + (identifier)) + (assignment + left: (identifier) + right: (identifier)) + (method_call + method: (call + receiver: (identifier) + method: (identifier)) + block: (do_block + (block_parameters + (identifier)) + (method_call + method: (identifier) + arguments: (argument_list + (identifier) + (identifier)))))) + (assignment + left: (identifier) + right: (integer)) + (assignment + left: (identifier) + right: (array + (identifier)))) +``` + +There are several types of names within this method: + +- `process_list` is a method. +- Within this method, `list` is a formal parameter +- `context` is a local variable. +- `current_context` is *not* a local variable, so it must be a method. +- Within the `do` block, `item` is a formal parameter +- Later on, `item` and `list` are both local variables (not formal parameters). + +#### Example Queries + +Let's write some queries that let us clearly distinguish between these types of names. First, set up the highlighting query, +as described in the previous section. We'll assign distinct colors to method calls, method definitions, and formal parameters: + +```scheme +; highlights.scm + +(call method: (identifier) @function.method) +(method_call method: (identifier) @function.method) + +(method name: (identifier) @function.method) + +(method_parameters (identifier) @variable.parameter) +(block_parameters (identifier) @variable.parameter) + +((identifier) @function.method + (#is-not? local)) +``` + +Then, we'll set up a local variable query to keep track of the variables and scopes. Here, we're indicating that methods +and blocks create local *scopes*, parameters and assignments create *definitions*, and other identifiers should be considered +*references*: + +```scheme +; locals.scm + +(method) @local.scope +(do_block) @local.scope + +(method_parameters (identifier) @local.definition) +(block_parameters (identifier) @local.definition) + +(assignment left:(identifier) @local.definition) + +(identifier) @local.reference +``` + +#### Locals Result + +Running `tree-sitter highlight` on this ruby file would produce output like this: + +```admonish example collapsible=true, title='Output' +
+def process_list(list)
+  context = current_context
+  list.map do |item|
+    process_item(item, context)
+  end
+end
+
+item = 5
+list = [item]
+
+``` + +### Language Injection + +Some source files contain code written in multiple different languages. Examples include: + +- HTML files, which can contain JavaScript inside ` + + + + + diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md new file mode 100644 index 00000000..231085ae --- /dev/null +++ b/docs/src/SUMMARY.md @@ -0,0 +1,48 @@ +# Summary + +[Introduction](./index.md) + +# User Guide + +- [Using Parsers](./using-parsers/index.md) + - [Getting Started](./using-parsers/1-getting-started.md) + - [Basic Parsing](./using-parsers/2-basic-parsing.md) + - [Advanced Parsing](./using-parsers/3-advanced-parsing.md) + - [Walking Trees](./using-parsers/4-walking-trees.md) + - [Queries](./using-parsers/queries/index.md) + - [Basic Syntax](./using-parsers/queries/1-syntax.md) + - [Operators](./using-parsers/queries/2-operators.md) + - [Predicates and Directives](./using-parsers/queries/3-predicates-and-directives.md) + - [API](./using-parsers/queries/4-api.md) + - [Static Node Types](./using-parsers/6-static-node-types.md) + - [ABI versions](./using-parsers/7-abi-versions.md) +- [Creating Parsers](./creating-parsers/index.md) + - [Getting Started](./creating-parsers/1-getting-started.md) + - [The Grammar DSL](./creating-parsers/2-the-grammar-dsl.md) + - [Writing the Grammar](./creating-parsers/3-writing-the-grammar.md) + - [External Scanners](./creating-parsers/4-external-scanners.md) + - [Writing Tests](./creating-parsers/5-writing-tests.md) + - [Publishing Parsers](./creating-parsers/6-publishing.md) +- [Syntax Highlighting](./3-syntax-highlighting.md) +- [Code Navigation](./4-code-navigation.md) +- [Implementation](./5-implementation.md) +- [Contributing](./6-contributing.md) +- [Playground](./7-playground.md) + +# Reference Guide + +- [Command Line Interface](./cli/index.md) + - [Init Config](./cli/init-config.md) + - [Init](./cli/init.md) + - [Generate](./cli/generate.md) + - [Build](./cli/build.md) + - [Parse](./cli/parse.md) + - [Test](./cli/test.md) + - [Version](./cli/version.md) + - [Fuzz](./cli/fuzz.md) + - [Query](./cli/query.md) + - [Highlight](./cli/highlight.md) + - [Tags](./cli/tags.md) + - [Playground](./cli/playground.md) + - [Dump Languages](./cli/dump-languages.md) + - [Complete](./cli/complete.md) diff --git a/docs/src/assets/css/mdbook-admonish.css b/docs/src/assets/css/mdbook-admonish.css new file mode 100644 index 00000000..45aeff05 --- /dev/null +++ b/docs/src/assets/css/mdbook-admonish.css @@ -0,0 +1,348 @@ +@charset "UTF-8"; +:is(.admonition) { + display: flow-root; + margin: 1.5625em 0; + padding: 0 1.2rem; + color: var(--fg); + page-break-inside: avoid; + background-color: var(--bg); + border: 0 solid black; + border-inline-start-width: 0.4rem; + border-radius: 0.2rem; + box-shadow: 0 0.2rem 1rem rgba(0, 0, 0, 0.05), 0 0 0.1rem rgba(0, 0, 0, 0.1); +} +@media print { + :is(.admonition) { + box-shadow: none; + } +} +:is(.admonition) > * { + box-sizing: border-box; +} +:is(.admonition) :is(.admonition) { + margin-top: 1em; + margin-bottom: 1em; +} +:is(.admonition) > .tabbed-set:only-child { + margin-top: 0; +} +html :is(.admonition) > :last-child { + margin-bottom: 1.2rem; +} + +a.admonition-anchor-link { + display: none; + position: absolute; + left: -1.2rem; + padding-right: 1rem; +} +a.admonition-anchor-link:link, a.admonition-anchor-link:visited { + color: var(--fg); +} +a.admonition-anchor-link:link:hover, a.admonition-anchor-link:visited:hover { + text-decoration: none; +} +a.admonition-anchor-link::before { + content: "§"; +} + +:is(.admonition-title, summary.admonition-title) { + position: relative; + min-height: 4rem; + margin-block: 0; + margin-inline: -1.6rem -1.2rem; + padding-block: 0.8rem; + padding-inline: 4.4rem 1.2rem; + font-weight: 700; + background-color: rgba(68, 138, 255, 0.1); + print-color-adjust: exact; + -webkit-print-color-adjust: exact; + display: flex; +} +:is(.admonition-title, summary.admonition-title) p { + margin: 0; +} +html :is(.admonition-title, summary.admonition-title):last-child { + margin-bottom: 0; +} +:is(.admonition-title, summary.admonition-title)::before { + position: absolute; + top: 0.625em; + inset-inline-start: 1.6rem; + width: 2rem; + height: 2rem; + background-color: #448aff; + print-color-adjust: exact; + -webkit-print-color-adjust: exact; + mask-image: url('data:image/svg+xml;charset=utf-8,'); + -webkit-mask-image: url('data:image/svg+xml;charset=utf-8,'); + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-size: contain; + -webkit-mask-size: contain; + content: ""; +} +:is(.admonition-title, summary.admonition-title):hover a.admonition-anchor-link { + display: initial; +} + +details.admonition > summary.admonition-title::after { + position: absolute; + top: 0.625em; + inset-inline-end: 1.6rem; + height: 2rem; + width: 2rem; + background-color: currentcolor; + mask-image: var(--md-details-icon); + -webkit-mask-image: var(--md-details-icon); + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-size: contain; + -webkit-mask-size: contain; + content: ""; + transform: rotate(0deg); + transition: transform 0.25s; +} +details[open].admonition > summary.admonition-title::after { + transform: rotate(90deg); +} + +:root { + --md-details-icon: url("data:image/svg+xml;charset=utf-8,"); +} + +:root { + --md-admonition-icon--admonish-note: url("data:image/svg+xml;charset=utf-8,"); + --md-admonition-icon--admonish-abstract: url("data:image/svg+xml;charset=utf-8,"); + --md-admonition-icon--admonish-info: url("data:image/svg+xml;charset=utf-8,"); + --md-admonition-icon--admonish-tip: url("data:image/svg+xml;charset=utf-8,"); + --md-admonition-icon--admonish-success: url("data:image/svg+xml;charset=utf-8,"); + --md-admonition-icon--admonish-question: url("data:image/svg+xml;charset=utf-8,"); + --md-admonition-icon--admonish-warning: url("data:image/svg+xml;charset=utf-8,"); + --md-admonition-icon--admonish-failure: url("data:image/svg+xml;charset=utf-8,"); + --md-admonition-icon--admonish-danger: url("data:image/svg+xml;charset=utf-8,"); + --md-admonition-icon--admonish-bug: url("data:image/svg+xml;charset=utf-8,"); + --md-admonition-icon--admonish-example: url("data:image/svg+xml;charset=utf-8,"); + --md-admonition-icon--admonish-quote: url("data:image/svg+xml;charset=utf-8,"); +} + +:is(.admonition):is(.admonish-note) { + border-color: #448aff; +} + +:is(.admonish-note) > :is(.admonition-title, summary.admonition-title) { + background-color: rgba(68, 138, 255, 0.1); +} +:is(.admonish-note) > :is(.admonition-title, summary.admonition-title)::before { + background-color: #448aff; + mask-image: var(--md-admonition-icon--admonish-note); + -webkit-mask-image: var(--md-admonition-icon--admonish-note); + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-size: contain; + -webkit-mask-repeat: no-repeat; +} + +:is(.admonition):is(.admonish-abstract, .admonish-summary, .admonish-tldr) { + border-color: #00b0ff; +} + +:is(.admonish-abstract, .admonish-summary, .admonish-tldr) > :is(.admonition-title, summary.admonition-title) { + background-color: rgba(0, 176, 255, 0.1); +} +:is(.admonish-abstract, .admonish-summary, .admonish-tldr) > :is(.admonition-title, summary.admonition-title)::before { + background-color: #00b0ff; + mask-image: var(--md-admonition-icon--admonish-abstract); + -webkit-mask-image: var(--md-admonition-icon--admonish-abstract); + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-size: contain; + -webkit-mask-repeat: no-repeat; +} + +:is(.admonition):is(.admonish-info, .admonish-todo) { + border-color: #00b8d4; +} + +:is(.admonish-info, .admonish-todo) > :is(.admonition-title, summary.admonition-title) { + background-color: rgba(0, 184, 212, 0.1); +} +:is(.admonish-info, .admonish-todo) > :is(.admonition-title, summary.admonition-title)::before { + background-color: #00b8d4; + mask-image: var(--md-admonition-icon--admonish-info); + -webkit-mask-image: var(--md-admonition-icon--admonish-info); + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-size: contain; + -webkit-mask-repeat: no-repeat; +} + +:is(.admonition):is(.admonish-tip, .admonish-hint, .admonish-important) { + border-color: #00bfa5; +} + +:is(.admonish-tip, .admonish-hint, .admonish-important) > :is(.admonition-title, summary.admonition-title) { + background-color: rgba(0, 191, 165, 0.1); +} +:is(.admonish-tip, .admonish-hint, .admonish-important) > :is(.admonition-title, summary.admonition-title)::before { + background-color: #00bfa5; + mask-image: var(--md-admonition-icon--admonish-tip); + -webkit-mask-image: var(--md-admonition-icon--admonish-tip); + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-size: contain; + -webkit-mask-repeat: no-repeat; +} + +:is(.admonition):is(.admonish-success, .admonish-check, .admonish-done) { + border-color: #00c853; +} + +:is(.admonish-success, .admonish-check, .admonish-done) > :is(.admonition-title, summary.admonition-title) { + background-color: rgba(0, 200, 83, 0.1); +} +:is(.admonish-success, .admonish-check, .admonish-done) > :is(.admonition-title, summary.admonition-title)::before { + background-color: #00c853; + mask-image: var(--md-admonition-icon--admonish-success); + -webkit-mask-image: var(--md-admonition-icon--admonish-success); + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-size: contain; + -webkit-mask-repeat: no-repeat; +} + +:is(.admonition):is(.admonish-question, .admonish-help, .admonish-faq) { + border-color: #64dd17; +} + +:is(.admonish-question, .admonish-help, .admonish-faq) > :is(.admonition-title, summary.admonition-title) { + background-color: rgba(100, 221, 23, 0.1); +} +:is(.admonish-question, .admonish-help, .admonish-faq) > :is(.admonition-title, summary.admonition-title)::before { + background-color: #64dd17; + mask-image: var(--md-admonition-icon--admonish-question); + -webkit-mask-image: var(--md-admonition-icon--admonish-question); + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-size: contain; + -webkit-mask-repeat: no-repeat; +} + +:is(.admonition):is(.admonish-warning, .admonish-caution, .admonish-attention) { + border-color: #ff9100; +} + +:is(.admonish-warning, .admonish-caution, .admonish-attention) > :is(.admonition-title, summary.admonition-title) { + background-color: rgba(255, 145, 0, 0.1); +} +:is(.admonish-warning, .admonish-caution, .admonish-attention) > :is(.admonition-title, summary.admonition-title)::before { + background-color: #ff9100; + mask-image: var(--md-admonition-icon--admonish-warning); + -webkit-mask-image: var(--md-admonition-icon--admonish-warning); + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-size: contain; + -webkit-mask-repeat: no-repeat; +} + +:is(.admonition):is(.admonish-failure, .admonish-fail, .admonish-missing) { + border-color: #ff5252; +} + +:is(.admonish-failure, .admonish-fail, .admonish-missing) > :is(.admonition-title, summary.admonition-title) { + background-color: rgba(255, 82, 82, 0.1); +} +:is(.admonish-failure, .admonish-fail, .admonish-missing) > :is(.admonition-title, summary.admonition-title)::before { + background-color: #ff5252; + mask-image: var(--md-admonition-icon--admonish-failure); + -webkit-mask-image: var(--md-admonition-icon--admonish-failure); + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-size: contain; + -webkit-mask-repeat: no-repeat; +} + +:is(.admonition):is(.admonish-danger, .admonish-error) { + border-color: #ff1744; +} + +:is(.admonish-danger, .admonish-error) > :is(.admonition-title, summary.admonition-title) { + background-color: rgba(255, 23, 68, 0.1); +} +:is(.admonish-danger, .admonish-error) > :is(.admonition-title, summary.admonition-title)::before { + background-color: #ff1744; + mask-image: var(--md-admonition-icon--admonish-danger); + -webkit-mask-image: var(--md-admonition-icon--admonish-danger); + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-size: contain; + -webkit-mask-repeat: no-repeat; +} + +:is(.admonition):is(.admonish-bug) { + border-color: #f50057; +} + +:is(.admonish-bug) > :is(.admonition-title, summary.admonition-title) { + background-color: rgba(245, 0, 87, 0.1); +} +:is(.admonish-bug) > :is(.admonition-title, summary.admonition-title)::before { + background-color: #f50057; + mask-image: var(--md-admonition-icon--admonish-bug); + -webkit-mask-image: var(--md-admonition-icon--admonish-bug); + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-size: contain; + -webkit-mask-repeat: no-repeat; +} + +:is(.admonition):is(.admonish-example) { + border-color: #7c4dff; +} + +:is(.admonish-example) > :is(.admonition-title, summary.admonition-title) { + background-color: rgba(124, 77, 255, 0.1); +} +:is(.admonish-example) > :is(.admonition-title, summary.admonition-title)::before { + background-color: #7c4dff; + mask-image: var(--md-admonition-icon--admonish-example); + -webkit-mask-image: var(--md-admonition-icon--admonish-example); + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-size: contain; + -webkit-mask-repeat: no-repeat; +} + +:is(.admonition):is(.admonish-quote, .admonish-cite) { + border-color: #9e9e9e; +} + +:is(.admonish-quote, .admonish-cite) > :is(.admonition-title, summary.admonition-title) { + background-color: rgba(158, 158, 158, 0.1); +} +:is(.admonish-quote, .admonish-cite) > :is(.admonition-title, summary.admonition-title)::before { + background-color: #9e9e9e; + mask-image: var(--md-admonition-icon--admonish-quote); + -webkit-mask-image: var(--md-admonition-icon--admonish-quote); + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-size: contain; + -webkit-mask-repeat: no-repeat; +} + +.navy :is(.admonition) { + background-color: var(--sidebar-bg); +} + +.ayu :is(.admonition), +.coal :is(.admonition) { + background-color: var(--theme-hover); +} + +.rust :is(.admonition) { + background-color: var(--sidebar-bg); + color: var(--sidebar-fg); +} +.rust .admonition-anchor-link:link, .rust .admonition-anchor-link:visited { + color: var(--sidebar-fg); +} diff --git a/docs/src/assets/css/playground.css b/docs/src/assets/css/playground.css new file mode 100644 index 00000000..ab59ebec --- /dev/null +++ b/docs/src/assets/css/playground.css @@ -0,0 +1,468 @@ +/* Base Variables */ +:root { + --light-bg: #f9f9f9; + --light-border: #e0e0e0; + --light-text: #333; + --light-hover-border: #c1c1c1; + --light-scrollbar-track: #f1f1f1; + --light-scrollbar-thumb: #c1c1c1; + --light-scrollbar-thumb-hover: #a8a8a8; + + --dark-bg: #1d1f21; + --dark-border: #2d2d2d; + --dark-text: #c5c8c6; + --dark-scrollbar-track: #25282c; + --dark-scrollbar-thumb: #4a4d51; + --dark-scrollbar-thumb-hover: #5a5d61; + + --primary-color: #0550ae; + --primary-color-alpha: rgba(5, 80, 174, 0.1); + --primary-color-alpha-dark: rgba(121, 192, 255, 0.1); + --selection-color: rgba(39, 95, 255, 0.3); +} + +/* Common Scrollbar Styles */ +::-webkit-scrollbar { + width: 8px; + height: 8px; +} + +::-webkit-scrollbar-track { + border-radius: 4px; +} + +::-webkit-scrollbar-thumb { + border-radius: 4px; +} + +/* Base Light Theme Scrollbars */ +::-webkit-scrollbar-track { + background: var(--light-scrollbar-track); +} + +::-webkit-scrollbar-thumb { + background: var(--light-scrollbar-thumb); +} + +::-webkit-scrollbar-thumb:hover { + background: var(--light-scrollbar-thumb-hover); +} + +/* Dropdown Styling */ +.custom-select { + position: relative; + display: inline-block; +} + +.language-container { + display: flex; + align-items: center; + gap: 16px; + margin-bottom: 16px; +} + +#language-version { + color: var(--light-text); + font-size: 14px; + font-weight: 500; + padding: 4px 8px; + background: var(--light-bg); + border-radius: 4px; + border: 1px solid var(--light-border); +} + +#language-select { + background-color: var(--light-bg); + border: 1px solid var(--light-border); + border-radius: 4px; + padding: 4px 24px 4px 8px; + font-size: 14px; + color: var(--light-text); + cursor: pointer; + min-width: 120px; + appearance: none; + background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' viewBox='0 0 24 24' fill='none' stroke='%23666' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpolyline points='6 9 12 15 18 9'%3E%3C/polyline%3E%3C/svg%3E"); + background-repeat: no-repeat; + background-position: right 8px center; +} + +#copy-button { + background: none; + border: 1px solid var(--light-border); + border-radius: 4px; + padding: 6px; + cursor: pointer; + color: var(--light-text); + display: inline-flex; + align-items: center; + justify-content: center; + margin-left: 8px; +} + +#copy-button:hover { + background-color: var(--primary-color-alpha); + border-color: var(--light-hover-border); +} + +#copy-button:focus { + outline: none; + border-color: var(--primary-color); + box-shadow: 0 0 0 2px var(--primary-color-alpha); +} + +.toast { + position: fixed; + bottom: 20px; + right: 20px; + background-color: var(--lighbt-bg); + color: var(--light-text); + padding: 12px 16px; + border-radius: 6px; + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.3); + font-size: 14px; + font-weight: 500; + opacity: 0; + transform: translateY(20px); + transition: all 0.3s ease; + z-index: 1000; + pointer-events: none; +} + +.toast.show { + opacity: 1; + transform: translateY(0); +} + +.select-button { + background-color: var(--light-bg); + border: 1px solid var(--light-border); + border-radius: 4px; + padding: 4px 8px; + font-size: 14px; + color: var(--light-text); + cursor: pointer; + min-width: 120px; + display: flex; + align-items: center; + justify-content: space-between; +} + +#language-select:hover, +.select-button:hover { + border-color: var(--light-hover-border); +} + +#language-select:focus, +.select-button:focus { + outline: none; + border-color: var(--primary-color); + box-shadow: 0 0 0 2px var(--primary-color-alpha); +} + +/* Custom Checkbox Styling */ +input[type="checkbox"] { + appearance: none; + width: 16px; + height: 16px; + border: 1px solid var(--light-border); + border-radius: 3px; + margin-right: 6px; + position: relative; + cursor: pointer; + vertical-align: middle; +} + +input[type="checkbox"]:checked { + background-color: var(--primary-color); + border-color: var(--primary-color); +} + +input[type="checkbox"]:checked::after { + content: ''; + position: absolute; + left: 5px; + top: 2px; + width: 4px; + height: 8px; + border: solid white; + border-width: 0 2px 2px 0; + transform: rotate(45deg); +} + +input[type="checkbox"]:hover { + border-color: var(--light-hover-border); +} + +input[type="checkbox"]:focus { + outline: none; + border-color: var(--primary-color); + box-shadow: 0 0 0 2px var(--primary-color-alpha); +} + +/* Select Dropdown */ +.select-dropdown { + position: absolute; + top: 100%; + left: 0; + right: 0; + background-color: var(--light-bg); + border: 1px solid var(--light-border); + border-radius: 4px; + margin-top: 4px; + box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); + display: none; + z-index: 1000; + max-height: 300px; + overflow-y: auto; +} + +.select-dropdown.show { + display: block; +} + +.option { + padding: 8px 12px; + cursor: pointer; +} + +.option:hover { + background-color: var(--primary-color-alpha); +} + +.option.selected { + background-color: var(--primary-color-alpha); +} + +/* CodeMirror Base Styles */ +.ts-playground .CodeMirror { + border-radius: 6px; + background-color: var(--light-bg) !important; + color: #080808 !important; +} + +.ts-playground .CodeMirror-scroll { + padding: 8px; + border: 1px solid var(--light-border); + border-radius: 6px; + box-shadow: 0 1px 2px rgba(0, 0, 0, 0.1); +} + +.ayu .ts-playground .CodeMirror-scroll, +.coal .ts-playground .CodeMirror-scroll, +.navy .ts-playground .CodeMirror-scroll { + border-color: var(--dark-border); +} + +.ts-playground .CodeMirror-gutters { + background: #ebebeb !important; + border-right: 1px solid #e8e8e8 !important; +} + +.ts-playground .CodeMirror-cursor { + border-left: 2px solid #000 !important; +} + +.ts-playground .CodeMirror-selected { + background: var(--selection-color) !important; +} + +.ts-playground .CodeMirror-activeline-background { + background: rgba(36, 99, 180, 0.12) !important; +} + +.query-error { + text-decoration: underline red dashed; + -webkit-text-decoration: underline red dashed; +} + +/* Output Container Styles */ +#output-container { + color: #080808; + background-color: var(--light-bg); + margin: 0; + white-space: pre; + font-family: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, "Liberation Mono", monospace; +} + +#output-container-scroll { + max-height: 400px; + overflow: auto; + padding: 8px; + border: 1px solid var(--light-border); + border-radius: 6px; + box-shadow: 0 1px 2px rgba(0, 0, 0, 0.1); + background-color: var(--light-bg); +} + +#output-container a { + color: var(--primary-color); + text-decoration: none; +} + +#output-container a:hover { + text-decoration: underline; +} + +#output-container a.node-link.anonymous { + color: #116329; +} + +#output-container a.node-link.anonymous:before { + content: '"'; +} + +#output-container a.node-link.anonymous:after { + content: '"'; +} + +#output-container a.node-link.error { + color: #cf222e; +} + +#output-container a.highlighted { + background-color: var(--selection-color); +} + +/* Dark Theme Overrides */ +.ayu, +.coal, +.navy { + + & #language-version, + & #language-select, + & #copy-button, + & .select-button { + background-color: var(--dark-bg); + border-color: var(--dark-border); + color: var(--dark-text); + } + + & #copy-button:hover, + & #language-select:hover, + & .select-button:hover { + border-color: var(--dark-border); + background-color: var(--primary-color-alpha-dark); + } + + & .toast { + background-color: var(--dark-bg); + color: var(--dark-text); + } + + #language-select:focus, + & .select-button:focus { + border-color: #79c0ff; + box-shadow: 0 0 0 2px var(--primary-color-alpha-dark); + } + + & input[type="checkbox"] { + border-color: var(--dark-border); + background-color: var(--dark-bg); + } + + & input[type="checkbox"]:checked { + background-color: #79c0ff; + border-color: #79c0ff; + } + + & label { + color: var(--dark-text); + } + + & .select-dropdown { + background-color: var(--dark-bg); + border-color: var(--dark-border); + box-shadow: 0 2px 4px rgba(0, 0, 0, 0.3); + } + + & .option:hover { + background-color: var(--primary-color-alpha-dark); + } + + & .option.selected { + background-color: var(--primary-color-alpha-dark); + } + + & .ts-playground .CodeMirror { + background-color: var(--dark-bg) !important; + color: var(--dark-text) !important; + } + + & .ts-playground .CodeMirror-gutters { + background: var(--dark-scrollbar-track) !important; + border-right-color: var(--dark-border) !important; + } + + & .ts-playground .CodeMirror-cursor { + border-left-color: #aeafad !important; + } + + & .ts-playground .CodeMirror-selected { + background: #373b41 !important; + } + + & .ts-playground .CodeMirror-activeline-background { + background: #282a2e !important; + } + + & #output-container { + color: var(--dark-text); + background-color: var(--dark-bg); + } + + & #output-container-scroll { + background-color: var(--dark-bg); + border-color: var(--dark-border); + } + + & #output-container a { + color: #79c0ff; + } + + & #output-container a.node-link.anonymous { + color: #7ee787; + } + + & #output-container a.node-link.error { + color: #ff7b72; + } + + & #output-container a.highlighted { + background-color: #373b41; + } + + /* Dark Theme Scrollbars */ + & ::-webkit-scrollbar-track { + background: var(--dark-scrollbar-track) !important; + } + + & ::-webkit-scrollbar-thumb { + background: var(--dark-scrollbar-thumb) !important; + } + + & ::-webkit-scrollbar-thumb:hover { + background: var(--dark-scrollbar-thumb-hover) !important; + } + + & * { + scrollbar-width: thin !important; + scrollbar-color: var(--dark-scrollbar-thumb) var(--dark-scrollbar-track) !important; + } +} + +/* Spacing Utilities */ +#language-select, +input[type="checkbox"], +label { + margin: 0 4px; +} + +#language-select { + margin-right: 16px; +} + +label { + font-size: 14px; + margin-right: 16px; + cursor: pointer; +} diff --git a/docs/assets/images/favicon-16x16.png b/docs/src/assets/images/favicon-16x16.png similarity index 100% rename from docs/assets/images/favicon-16x16.png rename to docs/src/assets/images/favicon-16x16.png diff --git a/docs/assets/images/favicon-32x32.png b/docs/src/assets/images/favicon-32x32.png similarity index 100% rename from docs/assets/images/favicon-32x32.png rename to docs/src/assets/images/favicon-32x32.png diff --git a/docs/assets/images/tree-sitter-small.png b/docs/src/assets/images/tree-sitter-small.png similarity index 100% rename from docs/assets/images/tree-sitter-small.png rename to docs/src/assets/images/tree-sitter-small.png diff --git a/docs/src/assets/js/playground.js b/docs/src/assets/js/playground.js new file mode 100644 index 00000000..ef65c371 --- /dev/null +++ b/docs/src/assets/js/playground.js @@ -0,0 +1,663 @@ +function initializeLocalTheme() { + const themeToggle = document.getElementById('theme-toggle'); + if (!themeToggle) return; + + // Load saved theme or use system preference + const savedTheme = localStorage.getItem('theme'); + const prefersDark = window.matchMedia('(prefers-color-scheme: dark)').matches; + const initialTheme = savedTheme || (prefersDark ? 'dark' : 'light'); + + // Set initial theme + document.documentElement.setAttribute('data-theme', initialTheme); + + themeToggle.addEventListener('click', () => { + const currentTheme = document.documentElement.getAttribute('data-theme'); + const newTheme = currentTheme === 'light' ? 'dark' : 'light'; + document.documentElement.setAttribute('data-theme', newTheme); + localStorage.setItem('theme', newTheme); + }); +} + +function initializeCustomSelect({ initialValue = null, addListeners = false }) { + const button = document.getElementById('language-button'); + const select = document.getElementById('language-select'); + if (!button || !select) return; + + const dropdown = button.nextElementSibling; + const selectedValue = button.querySelector('.selected-value'); + + if (initialValue) { + select.value = initialValue; + } + if (select.selectedIndex >= 0 && select.options[select.selectedIndex]) { + selectedValue.textContent = select.options[select.selectedIndex].text; + } else { + selectedValue.textContent = 'JavaScript'; + } + + if (addListeners) { + button.addEventListener('click', (e) => { + e.preventDefault(); // Prevent form submission + dropdown.classList.toggle('show'); + }); + + document.addEventListener('click', (e) => { + if (!button.contains(e.target)) { + dropdown.classList.remove('show'); + } + }); + + dropdown.querySelectorAll('.option').forEach(option => { + option.addEventListener('click', () => { + selectedValue.textContent = option.textContent; + select.value = option.dataset.value; + dropdown.classList.remove('show'); + + const event = new Event('change'); + select.dispatchEvent(event); + }); + }); + } +} + +window.initializePlayground = async (opts) => { + const { Parser, Language, Query } = window.TreeSitter; + + const { local } = opts; + if (local) { + initializeLocalTheme(); + } + initializeCustomSelect({ addListeners: true }); + + let tree; + + const CAPTURE_REGEX = /@\s*([\w\._-]+)/g; + const LIGHT_COLORS = [ + "#0550ae", // blue + "#ab5000", // rust brown + "#116329", // forest green + "#844708", // warm brown + "#6639ba", // purple + "#7d4e00", // orange brown + "#0969da", // bright blue + "#1a7f37", // green + "#cf222e", // red + "#8250df", // violet + "#6e7781", // gray + "#953800", // dark orange + "#1b7c83" // teal + ]; + + const DARK_COLORS = [ + "#79c0ff", // light blue + "#ffa657", // orange + "#7ee787", // light green + "#ff7b72", // salmon + "#d2a8ff", // light purple + "#ffa198", // pink + "#a5d6ff", // pale blue + "#56d364", // bright green + "#ff9492", // light red + "#e0b8ff", // pale purple + "#9ca3af", // gray + "#ffb757", // yellow orange + "#80cbc4" // light teal + ]; + + const codeInput = document.getElementById("code-input"); + const languageSelect = document.getElementById("language-select"); + const languageVersion = document.getElementById('language-version'); + const loggingCheckbox = document.getElementById("logging-checkbox"); + const anonymousNodes = document.getElementById('anonymous-nodes-checkbox'); + const outputContainer = document.getElementById("output-container"); + const outputContainerScroll = document.getElementById( + "output-container-scroll", + ); + const playgroundContainer = document.getElementById("playground-container"); + const queryCheckbox = document.getElementById("query-checkbox"); + const queryContainer = document.getElementById("query-container"); + const queryInput = document.getElementById("query-input"); + const accessibilityCheckbox = document.getElementById("accessibility-checkbox"); + const copyButton = document.getElementById("copy-button"); + const updateTimeSpan = document.getElementById("update-time"); + const languagesByName = {}; + + loadState(); + + await Parser.init(); + + const parser = new Parser(); + + const codeEditor = CodeMirror.fromTextArea(codeInput, { + lineNumbers: true, + showCursorWhenSelecting: true + }); + + codeEditor.on('keydown', (_, event) => { + const key = event.key; + if (key === 'ArrowLeft' || key === 'ArrowRight' || key === '?') { + event.stopPropagation(); // Prevent mdBook from going back/forward, or showing help + } + }); + + const queryEditor = CodeMirror.fromTextArea(queryInput, { + lineNumbers: true, + showCursorWhenSelecting: true, + }); + + queryEditor.on('keydown', (_, event) => { + const key = event.key; + if (key === 'ArrowLeft' || key === 'ArrowRight' || key === '?') { + event.stopPropagation(); // Prevent mdBook from going back/forward, or showing help + } + }); + + const cluster = new Clusterize({ + rows: [], + noDataText: null, + contentElem: outputContainer, + scrollElem: outputContainerScroll, + }); + const renderTreeOnCodeChange = debounce(renderTree, 50); + const saveStateOnChange = debounce(saveState, 2000); + const runTreeQueryOnChange = debounce(runTreeQuery, 50); + + let languageName = languageSelect.value; + let treeRows = null; + let treeRowHighlightedIndex = -1; + let parseCount = 0; + let isRendering = 0; + let query; + + codeEditor.on("changes", handleCodeChange); + codeEditor.on("viewportChange", runTreeQueryOnChange); + codeEditor.on("cursorActivity", debounce(handleCursorMovement, 150)); + queryEditor.on("changes", debounce(handleQueryChange, 150)); + + loggingCheckbox.addEventListener("change", handleLoggingChange); + anonymousNodes.addEventListener("change", renderTree); + queryCheckbox.addEventListener("change", handleQueryEnableChange); + accessibilityCheckbox.addEventListener("change", handleQueryChange); + languageSelect.addEventListener("change", handleLanguageChange); + outputContainer.addEventListener("click", handleTreeClick); + copyButton?.addEventListener("click", handleCopy); + + handleQueryEnableChange(); + await handleLanguageChange(); + + playgroundContainer.style.visibility = "visible"; + + async function handleLanguageChange() { + const newLanguageName = languageSelect.value; + if (!languagesByName[newLanguageName]) { + const url = `${LANGUAGE_BASE_URL}/tree-sitter-${newLanguageName}.wasm`; + languageSelect.disabled = true; + try { + languagesByName[newLanguageName] = await Language.load(url); + } catch (e) { + console.error(e); + languageSelect.value = languageName; + return; + } finally { + languageSelect.disabled = false; + } + } + + tree = null; + languageName = newLanguageName; + + const metadata = languagesByName[languageName].metadata; + if (languageVersion && metadata) { + languageVersion.textContent = `v${metadata.major_version}.${metadata.minor_version}.${metadata.patch_version}`; + languageVersion.style.visibility = 'visible'; + } else if (languageVersion) { + languageVersion.style.visibility = 'hidden'; + } + + parser.setLanguage(languagesByName[newLanguageName]); + handleCodeChange(); + handleQueryChange(); + } + + async function handleCodeChange(editor, changes) { + const newText = codeEditor.getValue() + "\n"; + const edits = tree && changes && changes.map(treeEditForEditorChange); + + const start = performance.now(); + if (edits) { + for (const edit of edits) { + tree.edit(edit); + } + } + const newTree = parser.parse(newText, tree); + const duration = (performance.now() - start).toFixed(1); + + updateTimeSpan.innerText = `${duration} ms`; + if (tree) tree.delete(); + tree = newTree; + parseCount++; + renderTreeOnCodeChange(); + runTreeQueryOnChange(); + saveStateOnChange(); + } + + async function renderTree() { + isRendering++; + const cursor = tree.walk(); + + let currentRenderCount = parseCount; + let row = ""; + let rows = []; + let finishedRow = false; + let visitedChildren = false; + let indentLevel = 0; + + for (let i = 0; ; i++) { + if (i > 0 && i % 10000 === 0) { + await new Promise((r) => setTimeout(r, 0)); + if (parseCount !== currentRenderCount) { + cursor.delete(); + isRendering--; + return; + } + } + + let displayName; + if (cursor.nodeIsMissing) { + const nodeTypeText = cursor.nodeIsNamed ? cursor.nodeType : `"${cursor.nodeType}"`; + displayName = `MISSING ${nodeTypeText}`; + } else if (cursor.nodeIsNamed) { + displayName = cursor.nodeType; + } else if (anonymousNodes.checked) { + displayName = cursor.nodeType + } + + if (visitedChildren) { + if (displayName) { + finishedRow = true; + } + + if (cursor.gotoNextSibling()) { + visitedChildren = false; + } else if (cursor.gotoParent()) { + visitedChildren = true; + indentLevel--; + } else { + break; + } + } else { + if (displayName) { + if (finishedRow) { + row += ""; + rows.push(row); + finishedRow = false; + } + const start = cursor.startPosition; + const end = cursor.endPosition; + const id = cursor.nodeId; + let fieldName = cursor.currentFieldName; + if (fieldName) { + fieldName += ": "; + } else { + fieldName = ""; + } + + const nodeClass = + displayName === 'ERROR' || displayName.startsWith('MISSING') + ? 'node-link error plain' + : cursor.nodeIsNamed + ? 'node-link named plain' + : 'node-link anonymous plain'; + + row = `
${" ".repeat(indentLevel)}${fieldName}` + + `` + + `${displayName} ` + + `[${start.row}, ${start.column}] - [${end.row}, ${end.column}]`; + finishedRow = true; + } + + if (cursor.gotoFirstChild()) { + visitedChildren = false; + indentLevel++; + } else { + visitedChildren = true; + } + } + } + if (finishedRow) { + row += "
"; + rows.push(row); + } + + cursor.delete(); + cluster.update(rows); + treeRows = rows; + isRendering--; + handleCursorMovement(); + } + + function getCaptureCSS(name) { + if (accessibilityCheckbox.checked) { + return `color: white; background-color: ${colorForCaptureName(name)}`; + } else { + return `color: ${colorForCaptureName(name)}`; + } + } + + function runTreeQuery(_, startRow, endRow) { + if (endRow == null) { + const viewport = codeEditor.getViewport(); + startRow = viewport.from; + endRow = viewport.to; + } + + codeEditor.operation(() => { + const marks = codeEditor.getAllMarks(); + marks.forEach((m) => m.clear()); + + if (tree && query) { + const captures = query.captures(tree.rootNode, { + startPosition: { row: startRow, column: 0 }, + endPosition: { row: endRow, column: 0 }, + }); + let lastNodeId; + for (const { name, node } of captures) { + if (node.id === lastNodeId) continue; + lastNodeId = node.id; + const { startPosition, endPosition } = node; + codeEditor.markText( + { line: startPosition.row, ch: startPosition.column }, + { line: endPosition.row, ch: endPosition.column }, + { + inclusiveLeft: true, + inclusiveRight: true, + css: getCaptureCSS(name), + }, + ); + } + } + }); + } + + // When we change from a dark theme to a light theme (and vice versa), the colors of the + // captures need to be updated. + const observer = new MutationObserver((mutations) => { + mutations.forEach((mutation) => { + if (mutation.attributeName === 'class') { + handleQueryChange(); + } + }); + }); + + observer.observe(document.documentElement, { + attributes: true, + attributeFilter: ['class'] + }); + + function handleQueryChange() { + if (query) { + query.delete(); + query.deleted = true; + query = null; + } + + queryEditor.operation(() => { + queryEditor.getAllMarks().forEach((m) => m.clear()); + if (!queryCheckbox.checked) return; + + const queryText = queryEditor.getValue(); + + try { + query = new Query(parser.language, queryText); + let match; + + let row = 0; + queryEditor.eachLine((line) => { + while ((match = CAPTURE_REGEX.exec(line.text))) { + queryEditor.markText( + { line: row, ch: match.index }, + { line: row, ch: match.index + match[0].length }, + { + inclusiveLeft: true, + inclusiveRight: true, + css: `color: ${colorForCaptureName(match[1])}`, + }, + ); + } + row++; + }); + } catch (error) { + const startPosition = queryEditor.posFromIndex(error.index); + const endPosition = { + line: startPosition.line, + ch: startPosition.ch + (error.length || Infinity), + }; + + if (error.index === queryText.length) { + if (startPosition.ch > 0) { + startPosition.ch--; + } else if (startPosition.row > 0) { + startPosition.row--; + startPosition.column = Infinity; + } + } + + queryEditor.markText(startPosition, endPosition, { + className: "query-error", + inclusiveLeft: true, + inclusiveRight: true, + attributes: { title: error.message }, + }); + } + }); + + runTreeQuery(); + saveQueryState(); + } + + function handleCursorMovement() { + if (isRendering) return; + + const selection = codeEditor.getDoc().listSelections()[0]; + let start = { row: selection.anchor.line, column: selection.anchor.ch }; + let end = { row: selection.head.line, column: selection.head.ch }; + if ( + start.row > end.row || + (start.row === end.row && start.column > end.column) + ) { + let swap = end; + end = start; + start = swap; + } + const node = tree.rootNode.namedDescendantForPosition(start, end); + if (treeRows) { + if (treeRowHighlightedIndex !== -1) { + const row = treeRows[treeRowHighlightedIndex]; + if (row) + treeRows[treeRowHighlightedIndex] = row.replace( + "highlighted", + "plain", + ); + } + treeRowHighlightedIndex = treeRows.findIndex((row) => + row.includes(`data-id=${node.id}`), + ); + if (treeRowHighlightedIndex !== -1) { + const row = treeRows[treeRowHighlightedIndex]; + if (row) + treeRows[treeRowHighlightedIndex] = row.replace( + "plain", + "highlighted", + ); + } + cluster.update(treeRows); + const lineHeight = cluster.options.item_height; + const scrollTop = outputContainerScroll.scrollTop; + const containerHeight = outputContainerScroll.clientHeight; + const offset = treeRowHighlightedIndex * lineHeight; + if (scrollTop > offset - 20) { + outputContainerScroll.scrollTo({ top: offset - 20, behavior: 'smooth' }); + } else if (scrollTop < offset + lineHeight + 40 - containerHeight) { + outputContainerScroll.scrollTo({ + top: offset - containerHeight + 40, + behavior: 'smooth' + }); + } + } + } + + function handleCopy() { + const selection = window.getSelection(); + selection.removeAllRanges(); + const range = document.createRange(); + range.selectNodeContents(outputContainer); + selection.addRange(range); + navigator.clipboard.writeText(selection.toString()); + selection.removeRange(range); + showToast('Tree copied to clipboard!'); + } + + function handleTreeClick(event) { + if (event.target.tagName === "A") { + event.preventDefault(); + const [startRow, startColumn, endRow, endColumn] = + event.target.dataset.range.split(",").map((n) => parseInt(n)); + codeEditor.focus(); + codeEditor.setSelection( + { line: startRow, ch: startColumn }, + { line: endRow, ch: endColumn }, + ); + } + } + + function handleLoggingChange() { + if (loggingCheckbox.checked) { + parser.setLogger((message, lexing) => { + if (lexing) { + console.log(" ", message); + } else { + console.log(message); + } + }); + } else { + parser.setLogger(null); + } + } + + function handleQueryEnableChange() { + if (queryCheckbox.checked) { + queryContainer.style.visibility = ""; + queryContainer.style.position = ""; + } else { + queryContainer.style.visibility = "hidden"; + queryContainer.style.position = "absolute"; + } + handleQueryChange(); + } + + function treeEditForEditorChange(change) { + const oldLineCount = change.removed.length; + const newLineCount = change.text.length; + const lastLineLength = change.text[newLineCount - 1].length; + + const startPosition = { row: change.from.line, column: change.from.ch }; + const oldEndPosition = { row: change.to.line, column: change.to.ch }; + const newEndPosition = { + row: startPosition.row + newLineCount - 1, + column: + newLineCount === 1 + ? startPosition.column + lastLineLength + : lastLineLength, + }; + + const startIndex = codeEditor.indexFromPos(change.from); + let newEndIndex = startIndex + newLineCount - 1; + let oldEndIndex = startIndex + oldLineCount - 1; + for (let i = 0; i < newLineCount; i++) newEndIndex += change.text[i].length; + for (let i = 0; i < oldLineCount; i++) + oldEndIndex += change.removed[i].length; + + return { + startIndex, + oldEndIndex, + newEndIndex, + startPosition, + oldEndPosition, + newEndPosition, + }; + } + + function colorForCaptureName(capture) { + const id = query.captureNames.indexOf(capture); + const isDark = document.querySelector('html').classList.contains('ayu') || + document.querySelector('html').classList.contains('coal') || + document.querySelector('html').classList.contains('navy'); + + const colors = isDark ? DARK_COLORS : LIGHT_COLORS; + return colors[id % colors.length]; + } + + function loadState() { + const language = localStorage.getItem("language"); + const sourceCode = localStorage.getItem("sourceCode"); + const anonNodes = localStorage.getItem("anonymousNodes"); + const query = localStorage.getItem("query"); + const queryEnabled = localStorage.getItem("queryEnabled"); + if (language != null && sourceCode != null && query != null) { + queryInput.value = query; + codeInput.value = sourceCode; + languageSelect.value = language; + initializeCustomSelect({ initialValue: language }); + anonymousNodes.checked = anonNodes === "true"; + queryCheckbox.checked = queryEnabled === "true"; + } + } + + function saveState() { + localStorage.setItem("language", languageSelect.value); + localStorage.setItem("sourceCode", codeEditor.getValue()); + localStorage.setItem("anonymousNodes", anonymousNodes.checked); + saveQueryState(); + } + + function saveQueryState() { + localStorage.setItem("queryEnabled", queryCheckbox.checked); + localStorage.setItem("query", queryEditor.getValue()); + } + + function debounce(func, wait, immediate) { + var timeout; + return function () { + var context = this, + args = arguments; + var later = function () { + timeout = null; + if (!immediate) func.apply(context, args); + }; + var callNow = immediate && !timeout; + clearTimeout(timeout); + timeout = setTimeout(later, wait); + if (callNow) func.apply(context, args); + }; + } + + function showToast(message) { + const existingToast = document.querySelector('.toast'); + if (existingToast) { + existingToast.remove(); + } + + const toast = document.createElement('div'); + toast.className = 'toast'; + toast.textContent = message; + document.body.appendChild(toast); + + setTimeout(() => toast.classList.add('show'), 50); + + setTimeout(() => { + toast.classList.remove('show'); + setTimeout(() => toast.remove(), 200); + }, 1000); + } +}; diff --git a/docs/assets/schemas/config.schema.json b/docs/src/assets/schemas/config.schema.json similarity index 91% rename from docs/assets/schemas/config.schema.json rename to docs/src/assets/schemas/config.schema.json index 98760d07..92453f37 100644 --- a/docs/assets/schemas/config.schema.json +++ b/docs/src/assets/schemas/config.schema.json @@ -22,13 +22,20 @@ "examples": [ "Rust", "HTML" - ], - "$comment": "This is used in the description and the class names." + ] + }, + "title": { + "type": "string", + "description": "The title of the language.", + "examples": [ + "Rust", + "HTML" + ] }, "scope": { "type": "string", "description": "The TextMate scope that represents this language.", - "pattern": "^(source|text)(\\.\\w+)+$", + "pattern": "^(source|text)(\\.[\\w\\-]+)+$", "examples": [ "source.rust", "text.html" @@ -133,6 +140,11 @@ "type": "string", "format": "regex", "description": "A regex pattern that will be tested against the contents of the file in order to break ties in cases where multiple grammars matched the file." + }, + "class-name": { + "type": "string", + "pattern": "^TreeSitter\\w+$", + "description": "The class name for the Swift, Java & Kotlin bindings" } }, "additionalProperties": false, @@ -172,10 +184,10 @@ "format": "uri", "description": "The project's repository." }, - "homepage": { + "funding": { "type": "string", "format": "uri", - "description": "The project's homepage." + "description": "The project's funding link." } }, "additionalProperties": false, @@ -227,9 +239,7 @@ "properties": { "c": { "type": "boolean", - "default": true, - "const": true, - "$comment": "Always generated" + "default": true }, "go": { "type": "boolean", @@ -245,9 +255,7 @@ }, "node": { "type": "boolean", - "default": true, - "const": true, - "$comment": "Always generated (for now)" + "default": true }, "python": { "type": "boolean", @@ -255,13 +263,15 @@ }, "rust": { "type": "boolean", - "default": true, - "const": true, - "$comment": "Always generated" + "default": true }, "swift": { "type": "boolean", "default": true + }, + "zig": { + "type": "boolean", + "default": false } }, "additionalProperties": false diff --git a/docs/assets/schemas/grammar.schema.json b/docs/src/assets/schemas/grammar.schema.json similarity index 89% rename from docs/assets/schemas/grammar.schema.json rename to docs/src/assets/schemas/grammar.schema.json index 442beb36..e30c7ba0 100644 --- a/docs/assets/schemas/grammar.schema.json +++ b/docs/src/assets/schemas/grammar.schema.json @@ -57,6 +57,20 @@ } }, + "reserved": { + "type": "object", + "patternProperties": { + "^[a-zA-Z_]\\w*$": { + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/rule" + } + } + }, + "additionalProperties": false + }, + "externals": { "type": "array", "uniqueItems": true, @@ -93,7 +107,7 @@ }, "supertypes": { - "description": "A list of hidden rule names that should be considered supertypes in the generated node types file. See https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types.", + "description": "A list of hidden rule names that should be considered supertypes in the generated node types file. See https://tree-sitter.github.io/tree-sitter/using-parsers/6-static-node-types.", "type": "array", "uniqueItems": true, "items": { @@ -232,6 +246,21 @@ "required": ["type", "content"] }, + "reserved-rule": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "RESERVED" + }, + "context_name": { "type": "string" }, + "content": { + "$ref": "#/definitions/rule" + } + }, + "required": ["type", "context_name", "content"] + }, + "token-rule": { "type": "object", "properties": { @@ -299,6 +328,7 @@ { "$ref": "#/definitions/choice-rule" }, { "$ref": "#/definitions/repeat1-rule" }, { "$ref": "#/definitions/repeat-rule" }, + { "$ref": "#/definitions/reserved-rule" }, { "$ref": "#/definitions/token-rule" }, { "$ref": "#/definitions/field-rule" }, { "$ref": "#/definitions/prec-rule" } diff --git a/docs/src/assets/schemas/node-types.schema.json b/docs/src/assets/schemas/node-types.schema.json new file mode 100644 index 00000000..7ea8a5af --- /dev/null +++ b/docs/src/assets/schemas/node-types.schema.json @@ -0,0 +1,108 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Tree-sitter node types specification", + "type": "array", + "items": { + "$ref": "#/definitions/NodeInfo" + }, + "definitions": { + "NodeInfo": { + "type": "object", + "required": [ + "type", + "named" + ], + "properties": { + "type": { + "type": "string" + }, + "named": { + "type": "boolean" + }, + "root": { + "type": "boolean", + "default": false + }, + "extra": { + "type": "boolean", + "default": false + }, + "fields": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/FieldInfo" + } + }, + "children": { + "$ref": "#/definitions/FieldInfo" + }, + "subtypes": { + "type": "array", + "items": { + "$ref": "#/definitions/NodeType" + } + } + }, + "oneOf": [ + { + "description": "Regular node", + "properties": { + "subtypes": false + } + }, + { + "description": "Supertype node", + "required": [ + "subtypes" + ], + "properties": { + "children": false, + "fields": false + } + } + ] + }, + "NodeType": { + "type": "object", + "required": [ + "type", + "named" + ], + "properties": { + "type": { + "type": "string", + "description": "The kind of node type" + }, + "named": { + "type": "boolean", + "description": "Whether the node type is named" + } + } + }, + "FieldInfo": { + "type": "object", + "required": [ + "multiple", + "required", + "types" + ], + "properties": { + "multiple": { + "type": "boolean", + "default": false + }, + "required": { + "type": "boolean", + "default": true + }, + "types": { + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/NodeType" + } + } + } + } + } +} diff --git a/docs/src/assets/schemas/test-summary.schema.json b/docs/src/assets/schemas/test-summary.schema.json new file mode 100644 index 00000000..6211d60c --- /dev/null +++ b/docs/src/assets/schemas/test-summary.schema.json @@ -0,0 +1,247 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "TestSummary", + "description": "A stateful object used to collect results from running a grammar's test suite", + "type": "object", + "properties": { + "parse_results": { + "type": "array", + "items": { + "$ref": "#/$defs/TestResult" + } + }, + "parse_failures": { + "type": "array", + "items": { + "$ref": "#/$defs/TestFailure" + } + }, + "parse_stats": { + "$ref": "#/$defs/Stats" + }, + "highlight_results": { + "type": "array", + "items": { + "$ref": "#/$defs/TestResult" + } + }, + "tag_results": { + "type": "array", + "items": { + "$ref": "#/$defs/TestResult" + } + }, + "query_results": { + "type": "array", + "items": { + "$ref": "#/$defs/TestResult" + } + } + }, + "required": [ + "parse_results", + "parse_failures", + "parse_stats", + "highlight_results", + "tag_results", + "query_results" + ], + "$defs": { + "TestResult": { + "type": "object", + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ], + "anyOf": [ + { + "type": "object", + "properties": { + "children": { + "type": "array", + "items": { + "$ref": "#/$defs/TestResult" + } + } + }, + "required": [ + "children" + ] + }, + { + "type": "object", + "properties": { + "outcome": { + "$ref": "#/$defs/TestOutcome" + }, + "parse_rate": { + "type": [ + "number", + "null" + ], + "format": "double" + }, + "test_num": { + "type": "integer", + "format": "uint", + "minimum": 0 + } + }, + "required": [ + "outcome", + "parse_rate", + "test_num" + ] + }, + { + "type": "object", + "properties": { + "outcome": { + "$ref": "#/$defs/TestOutcome" + }, + "test_num": { + "type": "integer", + "format": "uint", + "minimum": 0 + } + }, + "required": [ + "outcome", + "test_num" + ] + } + ] + }, + "TestOutcome": { + "oneOf": [ + { + "type": "string", + "enum": [ + "Passed", + "Failed", + "Updated", + "Skipped", + "Platform" + ] + }, + { + "type": "object", + "properties": { + "AssertionPassed": { + "type": "object", + "properties": { + "assertion_count": { + "type": "integer", + "format": "uint", + "minimum": 0 + } + }, + "required": [ + "assertion_count" + ] + } + }, + "required": [ + "AssertionPassed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "AssertionFailed": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ] + } + }, + "required": [ + "AssertionFailed" + ], + "additionalProperties": false + } + ] + }, + "TestFailure": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "actual": { + "type": "string" + }, + "expected": { + "type": "string" + }, + "is_cst": { + "type": "boolean" + } + }, + "required": [ + "name", + "actual", + "expected", + "is_cst" + ] + }, + "Stats": { + "type": "object", + "properties": { + "successful_parses": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "total_parses": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "total_bytes": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "total_duration": { + "$ref": "#/$defs/Duration" + } + }, + "required": [ + "successful_parses", + "total_parses", + "total_bytes", + "total_duration" + ] + }, + "Duration": { + "type": "object", + "properties": { + "secs": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "nanos": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "secs", + "nanos" + ] + } + } +} \ No newline at end of file diff --git a/docs/src/cli/build.md b/docs/src/cli/build.md new file mode 100644 index 00000000..44ee8271 --- /dev/null +++ b/docs/src/cli/build.md @@ -0,0 +1,45 @@ +# `tree-sitter build` + +The `build` command compiles your parser into a dynamically-loadable library, +either as a shared object (`.so`, `.dylib`, or `.dll`) or as a Wasm module. + +```bash +tree-sitter build [OPTIONS] [PATH] # Aliases: b +``` + +You can change the compiler executable via the `CC` environment variable and add extra flags via `CFLAGS`. +For macOS or iOS, you can set `MACOSX_DEPLOYMENT_TARGET` or `IPHONEOS_DEPLOYMENT_TARGET` respectively to define the +minimum supported version. + +The path argument allows you to specify the directory of the parser to build. If you don't supply this argument, the CLI +will attempt to build the parser in the current working directory. + +## Options + +### `-w/--wasm` + +Compile the parser as a Wasm module. This command looks for the [Wasi SDK][wasi_sdk] indicated by the `TREE_SITTER_WASI_SDK_PATH` +environment variable. If you don't have the binary, the CLI will attempt to download it for you to `/tree-sitter/wasi-sdk/`, +where `` is resolved according to the [XDG base directory][XDG] or Window's [Known_Folder_Locations][Known_Folder]. + +### `-o/--output` + +Specify where to output the shared object file (native or Wasm). This flag accepts either an absolute path or a relative +path. If you don't supply this flag, the CLI will attempt to figure out what the language name is based on the parent +directory name to use for the output file. If the CLI can't figure it out, it will default to `parser`, thus generating +`parser.so` or `parser.wasm` in the current working directory. + +### `--reuse-allocator` + +Reuse the allocator that's set in the core library for the parser's external scanner. This is useful in applications +where the author overrides the default allocator with their own, and wants to ensure every parser that allocates memory +in the external scanner does so using their allocator. + +### `-0/--debug` + +Compile the parser with debug flags enabled. This is useful when debugging issues that require a debugger like `gdb` or +`lldb`. + +[Known_Folder]: https://learn.microsoft.com/en-us/windows/win32/shell/knownfolderid +[wasi_sdk]: https://github.com/WebAssembly/wasi-sdk +[XDG]: https://specifications.freedesktop.org/basedir/latest/ diff --git a/docs/src/cli/complete.md b/docs/src/cli/complete.md new file mode 100644 index 00000000..4c9aabfc --- /dev/null +++ b/docs/src/cli/complete.md @@ -0,0 +1,16 @@ +# `tree-sitter complete` + +The `complete` command generates a completion script for your shell. +This script can be used to enable autocompletion for the `tree-sitter` CLI. + +```bash +tree-sitter complete --shell # Aliases: comp +``` + +## Options + +### `--shell ` + +The shell for which to generate the completion script. + +Supported values: `bash`, `elvish`, `fish`, `power-shell`, `zsh`, and `nushell`. diff --git a/docs/src/cli/dump-languages.md b/docs/src/cli/dump-languages.md new file mode 100644 index 00000000..f29daa57 --- /dev/null +++ b/docs/src/cli/dump-languages.md @@ -0,0 +1,18 @@ +# `tree-sitter dump-languages` + +The `dump-languages` command prints out a list of all the languages that the CLI knows about. This can be useful for debugging +purposes, or for scripting. The paths to search comes from the config file's [`parser-directories`][parser-directories] +object. + +```bash +tree-sitter dump-languages [OPTIONS] # Aliases: langs +``` + +## Options + +### `--config-path` + +The path to the configuration file. Ordinarily, the CLI will use the default location as explained in the [init-config](./init-config.md) +command. This flag allows you to explicitly override that default, and use a config defined elsewhere. + +[parser-directories]: ./init-config.md#parser-directories diff --git a/docs/src/cli/fuzz.md b/docs/src/cli/fuzz.md new file mode 100644 index 00000000..7f97f9ba --- /dev/null +++ b/docs/src/cli/fuzz.md @@ -0,0 +1,61 @@ +# `tree-sitter fuzz` + +The `fuzz` command is used to fuzz a parser by performing random edits and ensuring that undoing these edits results in +consistent parse trees. It will fail if the parse trees are not equal, or if the changed ranges are inconsistent. + +```bash +tree-sitter fuzz [OPTIONS] # Aliases: f +``` + +## Options + +### `-s/--skip ` + +A list of test names to skip fuzzing. + +### `--subdir ` + +The directory containing the parser. This is primarily useful in multi-language repositories. + +### `-p/--grammar-path` + +The path to the directory containing the grammar. + +### `--lib-path` + +The path to the parser's dynamic library. This is used instead of the cached or automatically generated dynamic library. + +### `--lang-name` + +If `--lib-path` is used, the name of the language used to extract the library's language function + +### `--edits ` + +The maximum number of edits to perform. The default is 3. + +### `--iterations ` + +The number of iterations to run. The default is 10. + +### `-i/--include ` + +Only run tests whose names match this regex. + +### `-e/--exclude ` + +Skip tests whose names match this regex. + +### `--log-graphs` + +Outputs logs of the graphs of the stack and parse trees during parsing, as well as the actual parsing and lexing message. +The graphs are constructed with [graphviz dot][dot], and the output is written to `log.html`. + +### `-l/--log` + +Outputs parsing and lexing logs. This logs to stderr. + +### `-r/--rebuild` + +Force a rebuild of the parser before running the fuzzer. + +[dot]: https://graphviz.org/doc/info/lang.html diff --git a/docs/src/cli/generate.md b/docs/src/cli/generate.md new file mode 100644 index 00000000..df9111f0 --- /dev/null +++ b/docs/src/cli/generate.md @@ -0,0 +1,71 @@ +# `tree-sitter generate` + +The most important command for grammar development is `tree-sitter generate`, which reads the grammar in structured form +and outputs C files that can be compiled into a shared or static library (e.g., using the [`build`](./build.md) command). + +```bash +tree-sitter generate [OPTIONS] [GRAMMAR_PATH] # Aliases: gen, g +``` + +The optional `GRAMMAR_PATH` argument should point to the structured grammar, in one of two forms: +- `grammar.js` a (ESM or CJS) JavaScript file; if the argument is omitted, it defaults to `./grammar.js`. +- `grammar.json` a structured representation of the grammar that is created as a byproduct of `generate`; this can be used +to regenerate a missing `parser.c` without requiring a JavaScript runtime (useful when distributing parsers to consumers). + +If there is an ambiguity or *local ambiguity* in your grammar, Tree-sitter will detect it during parser generation, and +it will exit with a `Unresolved conflict` error message. To learn more about conflicts and how to handle them, see +the section on [`Structuring Rules Well`](../creating-parsers/3-writing-the-grammar.md#structuring-rules-well) +in the user guide. + +## Generated files + +- `src/parser.c` implements the parser logic specified in the grammar. +- `src/tree_sitter/parser.h` provides basic C definitions that are used in the generated `parser.c` file. +- `src/tree_sitter/alloc.h` provides memory allocation macros that can be used in an external scanner. +- `src/tree_sitter/array.h` provides array macros that can be used in an external scanner. +- `src/grammar.json` contains a structured representation of the grammar; can be used to regenerate the parser without having +to re-evaluate the `grammar.js`. +- `src/node-types.json` provides type information about individual syntax nodes; see the section on [`Static Node Types`](../using-parsers/6-static-node-types.md). + + +## Options + +### `-l/--log` + +Print the log of the parser generation process. This includes information such as what tokens are included in the error +recovery state, what keywords were extracted, what states were split and why, and the entry point state. + +### `--abi ` + +The ABI to use for parser generation. The default is ABI 15, with ABI 14 being a supported target. + +### `--no-parser` + +Only generate `grammar.json` and `node-types.json` + +### `-o/--output` + +The directory to place the generated parser in. The default is `src/` in the current directory. + +### `--report-states-for-rule ` + +Print the overview of states from the given rule. This is useful for debugging and understanding the generated parser's +item sets for all given states in a given rule. To solely view state count numbers for rules, pass in `-` for the rule argument. +To view the overview of states for every rule, pass in `*` for the rule argument. + +### `--json-summary` + +Report conflicts in a JSON format. + +### `--js-runtime ` + +The path to the JavaScript runtime executable to use when generating the parser. The default is `node`. +Note that you can also set this with `TREE_SITTER_JS_RUNTIME`. Starting from version 0.26, you can +also pass in `native` to use the experimental native QuickJS runtime that comes bundled with the CLI. +This avoids the dependency on a JavaScript runtime entirely. The native QuickJS runtime is compatible +with ESM as well as with CommonJS in strict mode. If your grammar depends on `npm` to install dependencies such as base +grammars, the native runtime can be used *after* running `npm install`. + +### `--disable-optimization` + +Disable optimizations when generating the parser. Currently, this only affects the merging of compatible parse states. diff --git a/docs/src/cli/highlight.md b/docs/src/cli/highlight.md new file mode 100644 index 00000000..1a4ed1f6 --- /dev/null +++ b/docs/src/cli/highlight.md @@ -0,0 +1,64 @@ +# `tree-sitter highlight` + +You can run syntax highlighting on an arbitrary file using `tree-sitter highlight`. This can either output colors directly +to your terminal using ANSI escape codes, or produce HTML (if the `--html` flag is passed). For more information, see +[the syntax highlighting page](../3-syntax-highlighting.md). + +```bash +tree-sitter highlight [OPTIONS] [PATHS]... # Aliases: hi +``` + +## Options + +### `-H/--html` + +Output an HTML document with syntax highlighting. + +### `--css-classes` + +Output HTML with CSS classes instead of inline styles. + +### `--check` + +Check that the highlighting captures conform strictly to the standards. + +### `--captures-path ` + +The path to a file with captures. These captures would be considered the "standard" captures to compare against. + +### `--query-paths ` + +The paths to query files to use for syntax highlighting. These should end in `highlights.scm`. + +### `--scope ` + +The language scope to use for syntax highlighting. This is useful when the language is ambiguous. + +### `-t/--time` + +Print the time taken to highlight the file. + +### `-q/--quiet` + +Suppress main output. + +### `--paths ` + +The path to a file that contains paths to source files to highlight + +### `-p/--grammar-path ` + +The path to the directory containing the grammar. + +### `--config-path ` + +The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more +information. + +### `-n/--test-number ` + +Highlight the contents of a specific test. + +### `-r/--rebuild` + +Force a rebuild of the parser before running the fuzzer. diff --git a/docs/src/cli/index.md b/docs/src/cli/index.md new file mode 100644 index 00000000..042c0196 --- /dev/null +++ b/docs/src/cli/index.md @@ -0,0 +1,8 @@ +# CLI Overview + +The `tree-sitter` command-line interface is used to create, manage, test, and build tree-sitter parsers. It is controlled +by + +- a personal `tree-sitter/config.json` config file generated by [`tree-sitter init-config`](./init-config.md) +- a parser `tree-sitter.json` config file generated by [`tree-sitter init`](./init.md). + diff --git a/docs/src/cli/init-config.md b/docs/src/cli/init-config.md new file mode 100644 index 00000000..77aacd66 --- /dev/null +++ b/docs/src/cli/init-config.md @@ -0,0 +1,153 @@ +# `tree-sitter init-config` + +This command initializes a configuration file for the Tree-sitter CLI. + +```bash +tree-sitter init-config +``` + +These directories are created in the "default" location for your platform: + +* On Unix, `$XDG_CONFIG_HOME/tree-sitter` or `$HOME/.config/tree-sitter` +* On Windows, `%APPDATA%\tree-sitter` or `$HOME\AppData\Roaming\tree-sitter` + +```admonish info +The CLI will work if there's no config file present, falling back on default values for each configuration option. +``` + +When you run the `init-config` command, it will print out the location of the file that it creates so that you can easily +find and modify it. + +The configuration file is a JSON file that contains the following fields: + +## `parser-directories` + +The [`tree-sitter highlight`](./highlight.md) command takes one or more file paths, and tries to automatically determine, +which language should be used to highlight those files. To do this, it needs to know *where* to look for Tree-sitter grammars +on your filesystem. You can control this using the `"parser-directories"` key in your configuration file: + +```json +{ + "parser-directories": [ + "/Users/my-name/code", + "~/other-code", + "$HOME/another-code" + ] +} +``` + +Any folder within one of these *parser directories* whose name begins with `tree-sitter-` will be treated as a Tree-sitter +grammar repository. + +## `theme` + +The [Tree-sitter highlighting system](../3-syntax-highlighting.md) works by annotating ranges of source code with logical +"highlight names" like `function.method`, `type.builtin`, `keyword`, etc. To decide what *color* should be used for rendering +each highlight, a *theme* is needed. + +In your config file, the `"theme"` value is an object whose keys are dot-separated highlight names like +`function.builtin` or `keyword`, and whose values are JSON expressions that represent text styling parameters. + +### Highlight Names + +A theme can contain multiple keys that share a common subsequence. Examples: + +* `variable` and `variable.parameter` +* `function`, `function.builtin`, and `function.method` + +For a given highlight produced, styling will be determined based on the **longest matching theme key**. For example, the +highlight `function.builtin.static` would match the key `function.builtin` rather than `function`. + +### Styling Values + +Styling values can be any of the following: + +* Integers from 0 to 255, representing ANSI terminal color ids. +* Strings like `"#e45649"` representing hexadecimal RGB colors. +* Strings naming basic ANSI colors like `"red"`, `"black"`, `"purple"`, or `"cyan"`. +* Objects with the following keys: + * `color` — An integer or string as described above. + * `underline` — A boolean indicating whether the text should be underlined. + * `italic` — A boolean indicating whether the text should be italicized. + * `bold` — A boolean indicating whether the text should be bold-face. + +An example theme can be seen below: + +```json +{ + "function": 26, + "operator": { + "bold": true, + "color": 239 + }, + "variable.builtin": { + "bold": true + }, + "variable.parameter": { + "underline": true + }, + "type.builtin": { + "color": 23, + "bold": true + }, + "keyword": 56, + "type": 23, + "number": { + "bold": true, + "color": 94 + }, + "constant": 94, + "attribute": { + "color": 124, + "italic": true + }, + "comment": { + "color": 245, + "italic": true + }, + "constant.builtin": { + "color": 94, + "bold": true + }, +} +``` + +## `parse-theme` + +The [`tree-sitter parse`](./parse.md) command will output a pretty-printed CST when the `-c/--cst` option is used. You can +control what colors are used for various parts of the tree in your configuration file. + +```admonish note +Omitting a field will cause the relevant text to be rendered with its default color. +``` + +An example parse theme can be seen below: + +```json +{ + "parse-theme": { + // The color of node kinds + "node-kind": [20, 20, 20], + // The color of text associated with a node + "node-text": [255, 255, 255], + // The color of node fields + "field": [42, 42, 42], + // The color of the range information for unnamed nodes + "row-color": [255, 255, 255], + // The color of the range information for named nodes + "row-color-named": [255, 130, 0], + // The color of extra nodes + "extra": [255, 0, 255], + // The color of ERROR nodes + "error": [255, 0, 0], + // The color of MISSING nodes and their associated text + "missing": [153, 75, 0], + // The color of newline characters + "line-feed": [150, 150, 150], + // The color of backtick characters + "backtick": [0, 200, 0], + // The color of literals + "literal": [0, 0, 200], + } +} +``` diff --git a/docs/src/cli/init.md b/docs/src/cli/init.md new file mode 100644 index 00000000..568bc98f --- /dev/null +++ b/docs/src/cli/init.md @@ -0,0 +1,209 @@ +# `tree-sitter init` + +The `init` command is your starting point for creating a new grammar. When you run it, it sets up a repository with all +the essential files and structure needed for grammar development. Since the command includes git-related files by default, +we recommend using git for version control of your grammar. + +```bash +tree-sitter init [OPTIONS] # Aliases: i +``` + +## Generated files + +### Required files + +The following required files are always created if missing: + +- `tree-sitter.json` - The main configuration file that determines how `tree-sitter` interacts with the grammar. If missing, +the `init` command will prompt the user for the required fields. See [below](./init.md#structure-of-tree-sitterjson) for +the full documentation of the structure of this file. +- `package.json` - The `npm` manifest for the parser. This file is required for some `tree-sitter` subcommands, and if the +grammar has dependencies (e.g., another published base grammar that this grammar extends). +- `grammar.js` - An empty template for the main grammar file; see [the section on creating parsers](../2-creating-parser). + +### Language bindings + +Language bindings are files that allow your parser to be directly used by projects written in the respective language. +The following bindings are created if enabled in `tree-sitter.json`: + +#### C/C++ + +- `Makefile` — This file tells [`make`][make] how to compile your language. +- `CMakeLists.txt` — This file tells [`cmake`][cmake] how to compile your language. +- `bindings/c/tree_sitter/tree-sitter-language.h` — This file provides the C interface of your language. +- `bindings/c/tree-sitter-language.pc` — This file provides [pkg-config][pkg-config] metadata about your language's C library. + +#### Go + +- `go.mod` — This file is the manifest of the Go module. +- `bindings/go/binding.go` — This file wraps your language in a Go module. +- `bindings/go/binding_test.go` — This file contains a test for the Go package. + +#### Node + +- `binding.gyp` — This file tells Node.js how to compile your language. +- `bindings/node/binding.cc` — This file wraps your language in a JavaScript module for Node.js. +- `bindings/node/index.js` — This is the file that Node.js initially loads when using your language. +- `bindings/node/index.d.ts` — This file provides type hints for your parser when used in TypeScript. +- `bindings/node/binding_test.js` — This file contains a test for the Node.js package. + +#### Java + +- `pom.xml` - This file is the manifest of the Maven package. +- `bindings/java/main/namespace/language/TreeSitterLanguage.java` - This file wraps your language in a Java class. +- `bindings/java/test/TreeSitterLanguageTest.java` - This file contains a test for the Java package. + +#### Python + +- `pyproject.toml` — This file is the manifest of the Python package. +- `setup.py` — This file tells Python how to compile your language. +- `bindings/python/tree_sitter_language/binding.c` — This file wraps your language in a Python module. +- `bindings/python/tree_sitter_language/__init__.py` — This file tells Python how to load your language. +- `bindings/python/tree_sitter_language/__init__.pyi` — This file provides type hints for your parser when used in Python. +- `bindings/python/tree_sitter_language/py.typed` — This file provides type hints for your parser when used in Python. +- `bindings/python/tests/test_binding.py` — This file contains a test for the Python package. + +#### Rust + +- `Cargo.toml` — This file is the manifest of the Rust package. +- `bindings/rust/build.rs` — This file tells Rust how to compile your language. +- `bindings/rust/lib.rs` — This file wraps your language in a Rust crate when used in Rust. + +#### Swift + +- `Package.swift` — This file tells Swift how to compile your language. +- `bindings/swift/TreeSitterLanguage/language.h` — This file wraps your language in a Swift module when used in Swift. +- `bindings/swift/TreeSitterLanguageTests/TreeSitterLanguageTests.swift` — This file contains a test for the Swift package. + +#### Zig + +- `build.zig` - This file tells Zig how to compile your language. +- `build.zig.zon` - This file is the manifest of the Zig package. +- `bindings/zig/root.zig` - This file wraps your language in a Zig module. +- `bindings/zig/test.zig` - This file contains a test for the Zig package. + +### Additional files + +In addition, the following files are created that aim to improve the development experience: + +- `.editorconfig` — This file tells your editor how to format your code. More information about this file can be found [here][editorconfig]. +- `.gitattributes` — This file tells Git how to handle line endings and tells GitHub which files are generated. +- `.gitignore` — This file tells Git which files to ignore when committing changes. + +## Structure of `tree-sitter.json` + +### The `grammars` field + +This field is an array of objects, though you typically only need one object in this array unless your repo has +multiple grammars (for example, `Typescript` and `TSX`), e.g., +```json +{ + "tree-sitter": [ + { + "scope": "source.ruby", + "file-types": [ + "rb", + "gemspec", + "Gemfile", + "Rakefile" + ], + "first-line-regex": "#!.*\\bruby$" + } + ] +} +``` + +#### Basic fields + +These keys specify basic information about the parser: + +- `scope` (required) — A string like `"source.js"` that identifies the language. +We strive to match the scope names used by popular [TextMate grammars][textmate] and by the [Linguist][linguist] library. + +- `path` — A relative path from the directory containing `tree-sitter.json` to another directory containing the `src/` +folder, which contains the actual generated parser. The default value is `"."` +(so that `src/` is in the same folder as `tree-sitter.json`), and this very rarely needs to be overridden. + +- `external-files` — A list of relative paths from the root dir of a +parser to files that should be checked for modifications during recompilation. +This is useful during development to have changes to other files besides scanner.c +be picked up by the cli. + +#### Language detection + +These keys help to decide whether the language applies to a given file: + +- `file-types` — An array of filename suffix strings (not including the dot). The grammar will be used for files whose names +end with one of these suffixes. Note that the suffix may match an *entire* filename. + +- `first-line-regex` — A regex pattern that will be tested against the first line of a file +to determine whether this language applies to the file. If present, this regex will be used for any file whose +language does not match any grammar's `file-types`. + +- `content-regex` — A regex pattern that will be tested against the contents of the file +to break ties in cases where multiple grammars matched the file using the above two criteria. If the regex matches, +this grammar will be preferred over another grammar with no `content-regex`. If the regex does not match, a grammar with +no `content-regex` will be preferred over this one. + +- `injection-regex` — A regex pattern that will be tested against a *language name* to determine whether this language +should be used for a potential *language injection* site. +Language injection is described in more detail in [the relevant section](../3-syntax-highlighting.md#language-injection). + +#### Query paths + +These keys specify relative paths from the directory containing `tree-sitter.json` to the files that control syntax highlighting: + +- `highlights` — Path to a *highlight query*. Default: `queries/highlights.scm` +- `locals` — Path to a *local variable query*. Default: `queries/locals.scm`. +- `injections` — Path to an *injection query*. Default: `queries/injections.scm`. +- `tags` — Path to a *tag query*. Default: `queries/tags.scm`. + +### The `metadata` field + +This field contains information that tree-sitter will use to populate relevant bindings' files, especially their versions. +Typically, this will all be set up when you run `tree-sitter init`, but you are welcome to update it as you see fit. + +- `version` (required) — The current version of your grammar, which should follow [semver][semver] +- `license` — The license of your grammar, which should be a valid [SPDX license][spdx] +- `description` — The brief description of your grammar +- `authors` (required) — An array of objects that contain a `name` field, and optionally an `email` and `url` field. +Each field is a string +- `links` — An object that contains a `repository` field, and optionally a `funding` field. Each field is a string +- `namespace` — The namespace for the `Java` and `Kotlin` bindings, defaults to `io.github.tree-sitter` if not provided + +### The `bindings` field + +This field controls what bindings are generated when the `init` command is run. +Each key is a language name, and the value is a boolean. + +- `c` (default: `true`) +- `go` (default: `true`) +- `java` (default: `false`) +- `node` (default: `true`) +- `python` (default: `true`) +- `rust` (default: `true`) +- `swift` (default: `false`) +- `zig` (default: `false`) + +## Options + +### `-u/--update` + +Update outdated generated files, if possible. + +**Note:** Existing files that may have been edited manually are _not_ updated in general. To force an update to such files, +remove them and call `tree-sitter init -u` again. + +### `-p/--grammar-path ` + +The path to the directory containing the grammar. + + +[cmake]: https://cmake.org/cmake/help/latest +[editorconfig]: https://editorconfig.org +[linguist]: https://github.com/github/linguist +[make]: https://www.gnu.org/software/make/manual/make.html +[pkg-config]: https://www.freedesktop.org/wiki/Software/pkg-config +[semver]: https://semver.org +[spdx]: https://spdx.org/licenses +[textmate]: https://macromates.com/manual/en/language_grammars diff --git a/docs/src/cli/parse.md b/docs/src/cli/parse.md new file mode 100644 index 00000000..2e9bc835 --- /dev/null +++ b/docs/src/cli/parse.md @@ -0,0 +1,115 @@ +# `tree-sitter parse` + +The `parse` command parses source files using a Tree-sitter parser. You can pass any number of file paths and glob patterns +to `tree-sitter parse`, and it will parse all the given files. If no paths are provided, input will be parsed from stdin. +The command will exit with a non-zero status code if any parse errors occurred. + +```bash +tree-sitter parse [OPTIONS] [PATHS]... # Aliases: p +``` + +## Options + +### `--paths ` + +The path to a file that contains paths to source files to parse. + +### `-p/--grammar-path ` + +The path to the directory containing the grammar. + +### `-l/--lib-path` + +The path to the parser's dynamic library. This is used instead of the cached or automatically generated dynamic library. + +### `--lang-name` + +If `--lib-path` is used, the name of the language used to extract the library's language function + +### `--scope ` + +The language scope to use for parsing. This is useful when the language is ambiguous. + +### `-d/--debug` + +Outputs parsing and lexing logs. This logs to stderr. + +### `-0/--debug-build` + +Compile the parser with debug flags enabled. This is useful when debugging issues that require a debugger like `gdb` or `lldb`. + +### `-D/--debug-graph` + +Outputs logs of the graphs of the stack and parse trees during parsing, as well as the actual parsing and lexing message. +The graphs are constructed with [graphviz dot][dot], and the output is written to `log.html`. + +### `--wasm` + +Compile and run the parser as a Wasm module (only if the tree-sitter CLI was built with `--features=wasm`). + +### `--dot` + +Output the parse tree with [graphviz dot][dot]. + +### `-x/--xml` + +Output the parse tree in XML format. + +### `-c/--cst` + +Output the parse tree in a pretty-printed CST format. + +### `-s/--stat` + +Show parsing statistics. + +### `--timeout ` + +Set the timeout for parsing a single file, in microseconds. + +### `-t/--time` + +Print the time taken to parse the file. If edits are provided, this will also print the time taken to parse the file after +each edit. + +### `-q/--quiet` + +Suppress main output. + +### `--edits ...` + +Apply edits after parsing the file. Edits are in the form of `row,col|position delcount insert_text` where row and col, +or position are 0-indexed. + +### `--encoding ` + +Set the encoding of the input file. By default, the CLI will look for the [`BOM`][bom] to determine if the file is encoded +in `UTF-16BE` or `UTF-16LE`. If no `BOM` is present, `UTF-8` is the default. One of `utf8`, `utf16-le`, `utf16-be`. + +### `--open-log` + +When using the `--debug-graph` option, open the log file in the default browser. + +### `-j/--json-summary` + +Output parsing results in a JSON format. + +### `--config-path ` + +The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more +information. + +### `-n/--test-number ` + +Parse a specific test in the corpus. The test number is the same number that appears in the output of `tree-sitter test`. + +### `-r/--rebuild` + +Force a rebuild of the parser before running tests. + +### `--no-ranges` + +Omit the node's ranges from the default parse output. This is useful when copying S-Expressions to a test file. + +[dot]: https://graphviz.org/doc/info/lang.html +[bom]: https://en.wikipedia.org/wiki/Byte_order_mark diff --git a/docs/src/cli/playground.md b/docs/src/cli/playground.md new file mode 100644 index 00000000..c0bfb495 --- /dev/null +++ b/docs/src/cli/playground.md @@ -0,0 +1,26 @@ +# `tree-sitter playground` + +The `playground` command allows you to start a local playground to test your parser interactively. + +```bash +tree-sitter playground [OPTIONS] # Aliases: play, pg, web-ui +``` + +```admonish note +For this to work, you must have already built the parser as a Wasm module. This can be done with the [`build`](./build.md) +subcommand (`tree-sitter build --wasm`). +``` + +## Options + +### `-q/--quiet` + +Don't automatically open the playground in the default browser. + +### `--grammar-path ` + +The path to the directory containing the grammar and wasm files. + +### `-e/--export ` + +Export static playground files to the specified directory instead of serving them. diff --git a/docs/src/cli/query.md b/docs/src/cli/query.md new file mode 100644 index 00000000..fbd6dafd --- /dev/null +++ b/docs/src/cli/query.md @@ -0,0 +1,76 @@ +# `tree-sitter query` + +The `query` command is used to run a query on a parser, and view the results. + +```bash +tree-sitter query [OPTIONS] [PATHS]... # Aliases: q +``` + +## Options + +### `-p/--grammar-path ` + +The path to the directory containing the grammar. + +### `--lib-path` + +The path to the parser's dynamic library. This is used instead of the cached or automatically generated dynamic library. + +### `--lang-name` + +If `--lib-path` is used, the name of the language used to extract the library's language function + +### `-t/--time` + +Print the time taken to execute the query on the file. + +### `-q/--quiet` + +Suppress main output. + +### `--paths ` + +The path to a file that contains paths to source files in which the query will be executed. + +### `--byte-range ` + +The range of byte offsets in which the query will be executed. The format is `start_byte:end_byte`. + +### `--containing-byte-range ` + +The range of byte offsets in which the query will be executed. Only the matches that are fully contained within the provided +byte range will be returned. + +### `--row-range ` + +The range of rows in which the query will be executed. The format is `start_row:end_row`. + +### `--containing-row-range ` + +The range of rows in which the query will be executed. Only the matches that are fully contained within the provided row +range will be returned. + +### `--scope ` + +The language scope to use for parsing and querying. This is useful when the language is ambiguous. + +### `-c/--captures` + +Order the query results by captures instead of matches. + +### `--test` + +Whether to run query tests or not. + +### `--config-path ` + +The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more +information. + +### `-n/--test-number ` + +Query the contents of a specific test. + +### `-r/--rebuild` + +Force a rebuild of the parser before executing the query. diff --git a/docs/src/cli/tags.md b/docs/src/cli/tags.md new file mode 100644 index 00000000..8275237e --- /dev/null +++ b/docs/src/cli/tags.md @@ -0,0 +1,43 @@ +# `tree-sitter tags` + +You can run symbol tagging on an arbitrary file using `tree-sitter tags`. This will output a list of tags. +For more information, see [the code navigation page](../4-code-navigation.md#tagging-and-captures). + +```bash +tree-sitter tags [OPTIONS] [PATHS]... +``` + +## Options + +### `--scope ` + +The language scope to use for symbol tagging. This is useful when the language is ambiguous. + +### `-t/--time` + +Print the time taken to generate tags for the file. + +### `-q/--quiet` + +Suppress main output. + +### `--paths ` + +The path to a file that contains paths to source files to tag. + +### `-p/--grammar-path ` + +The path to the directory containing the grammar. + +### `--config-path ` + +The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more +information. + +### `-n/--test-number ` + +Generate tags from the contents of a specific test. + +### `-r/--rebuild` + +Force a rebuild of the parser before running the tags. diff --git a/docs/src/cli/test.md b/docs/src/cli/test.md new file mode 100644 index 00000000..7e662f60 --- /dev/null +++ b/docs/src/cli/test.md @@ -0,0 +1,93 @@ +# `tree-sitter test` + +The `test` command is used to run the test suite for a parser. + +```bash +tree-sitter test [OPTIONS] # Aliases: t +``` + +## Options + +### `-i/--include ` + +Only run tests whose names match this regex. + +### `-e/--exclude ` + +Skip tests whose names match this regex. + +### `--file-name ` + +Only run tests from the given filename in the corpus. + +### `-p/--grammar-path ` + +The path to the directory containing the grammar. + +### `--lib-path` + +The path to the parser's dynamic library. This is used instead of the cached or automatically generated dynamic library. + +### `--lang-name` + +If `--lib-path` is used, the name of the language used to extract the library's language function + +### `-u/--update` + +Update the expected output of tests. + +```admonish info +Tests containing `ERROR` nodes or `MISSING` nodes will not be updated. +``` + +### `-d/--debug` + +Outputs parsing and lexing logs. This logs to stderr. + +### `-0/--debug-build` + +Compile the parser with debug flags enabled. This is useful when debugging issues that require a debugger like `gdb` or `lldb`. + +### `-D/--debug-graph` + +Outputs logs of the graphs of the stack and parse trees during parsing, as well as the actual parsing and lexing message. +The graphs are constructed with [graphviz dot][dot], and the output is written to `log.html`. + +### `--wasm` + +Compile and run the parser as a Wasm module (only if the tree-sitter CLI was built with `--features=wasm`). + +### `--open-log` + +When using the `--debug-graph` option, open the log file in the default browser. + +### `--config-path ` + +The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more +information. + +### `--show-fields` + +Force showing fields in test diffs. + +### `--stat ` + +Show parsing statistics when tests are being run. One of `all`, `outliers-and-total`, or `total-only`. + +- `all`: Show statistics for every test. + +- `outliers-and-total`: Show statistics only for outliers, and total statistics. + +- `total-only`: Show only total statistics. + +### `-r/--rebuild` + +Force a rebuild of the parser before running tests. + +### `--overview-only` + +Only show the overview of the test results, and not the diff. + +### `--json-summary` + +Output the test summary in a JSON format. diff --git a/docs/src/cli/version.md b/docs/src/cli/version.md new file mode 100644 index 00000000..39cda84a --- /dev/null +++ b/docs/src/cli/version.md @@ -0,0 +1,55 @@ +# `tree-sitter version` + +The `version` command upgrades the version of your grammar. + +```bash +tree-sitter version # Aliases: publish +``` + +This will update the version in several files, if they exist: + +* tree-sitter.json +* Cargo.toml +* Cargo.lock +* package.json +* package-lock.json +* Makefile +* CMakeLists.txt +* pyproject.toml + +Alternative forms can use the version in `tree-sitter.json` to bump automatically: + +```bash +tree-sitter version --bump patch # patch bump +tree-sitter version --bump minor # minor bump +tree-sitter version --bump major # major bump +``` + +As a grammar author, you should keep the version of your grammar in sync across different bindings. However, doing so manually +is error-prone and tedious, so this command takes care of the burden. If you are using a version control system, it is recommended +to commit the changes made by this command, and to tag the commit with the new version. + +To print the current version without bumping it, use: + +```bash +tree-sitter version +``` + +Note that some of the binding updates require access to external tooling: + +* Updating Cargo.toml and Cargo.lock bindings requires that `cargo` is installed. +* Updating package-lock.json requires that `npm` is installed. + +## Options + +### `-p/--grammar-path ` + +The path to the directory containing the grammar. + +### `--bump` + +Automatically bump the version. Possible values are: + +- `patch`: Bump the patch version. +- `minor`: Bump the minor version. +- `major`: Bump the major version. diff --git a/docs/src/creating-parsers/1-getting-started.md b/docs/src/creating-parsers/1-getting-started.md new file mode 100644 index 00000000..5095eb75 --- /dev/null +++ b/docs/src/creating-parsers/1-getting-started.md @@ -0,0 +1,136 @@ +# Getting Started + +## Dependencies + +To develop a Tree-sitter parser, there are two dependencies that you need to install: + +- **A JavaScript runtime** — Tree-sitter grammars are written in JavaScript, and Tree-sitter uses a JavaScript runtime +(the default being [Node.js][node.js]) to interpret JavaScript files. It requires this runtime command (default: `node`) +to be in one of the directories in your [`PATH`][path-env]. + +- **A C Compiler** — Tree-sitter creates parsers that are written in C. To run and test these parsers with the +`tree-sitter parse` or `tree-sitter test` commands, you must have a C/C++ compiler installed. Tree-sitter will try to look +for these compilers in the standard places for each platform. + +## Installation + +To create a Tree-sitter parser, you need to use [the `tree-sitter` CLI][tree-sitter-cli]. You can install the CLI in a few +different ways: + +- Build the `tree-sitter-cli` [Rust crate][crate] from source using [`cargo`][cargo], the Rust package manager. This works +on any platform. See [the contributing docs](../6-contributing.md#developing-tree-sitter) for more information. + +- Install the `tree-sitter-cli` [Rust crate][crate] from [crates.io][crates.io] using [`cargo`][cargo]. You can do so by +running the following command: `cargo install tree-sitter-cli --locked` + +- Install the `tree-sitter-cli` [Node.js module][node-module] using [`npm`][npm], the Node package manager. This approach +is fast, but it only works on certain platforms, because it relies on pre-built binaries. + +- Download a binary for your platform from [the latest GitHub release][releases], and put it into a directory on your `PATH`. + +## Project Setup + +The preferred convention is to name the parser repository "tree-sitter-" followed by the name of the language, in lowercase. + +```sh +mkdir tree-sitter-${LOWER_PARSER_NAME} +cd tree-sitter-${LOWER_PARSER_NAME} +``` + +```admonish note +The `LOWER_` prefix here means the "lowercase" name of the language. +``` + +### Init + +Once you've installed the `tree-sitter` CLI tool, you can start setting up your project, which will allow your parser to +be used from multiple languages. + +```sh +# This will prompt you for input +tree-sitter init +``` + +The `init` command will create a bunch of files in the project. +There should be a file called `grammar.js` with the following contents: + +```js +/** + * @file PARSER_DESCRIPTION + * @author PARSER_AUTHOR_NAME PARSER_AUTHOR_EMAIL + * @license PARSER_LICENSE + */ + +/// +// @ts-check + +export default grammar({ + name: 'LOWER_PARSER_NAME', + + rules: { + // TODO: add the actual grammar rules + source_file: $ => 'hello' + } +}); +``` + +```admonish info +The placeholders shown above would be replaced with the corresponding data you provided in the `init` sub-command's +prompts. +``` + +To learn more about this command, check the [reference page](../cli/init.md). + +### Generate + +Next, run the following command: + +```sh +tree-sitter generate +``` + +This will generate the C code required to parse this trivial language. + +You can test this parser by creating a source file with the contents "hello" and parsing it: + +```sh +echo 'hello' > example-file +tree-sitter parse example-file +``` + +Alternatively, in Windows PowerShell: + +```pwsh +"hello" | Out-File example-file -Encoding utf8 +tree-sitter parse example-file +``` + +This should print the following: + +```text +(source_file [0, 0] - [1, 0]) +``` + +You now have a working parser. + +Finally, look back at the [triple-slash][] and [`@ts-check`][ts-check] comments in `grammar.js`; these tell your editor +to provide documentation and type information as you edit your grammar. For these to work, you must download Tree-sitter's +TypeScript API from npm into a `node_modules` directory in your project: + +```sh +npm install # or your package manager of choice +``` + +To learn more about this command, check the [reference page](../cli/generate.md). + +[cargo]: https://doc.rust-lang.org/cargo/getting-started/installation.html +[crate]: https://crates.io/crates/tree-sitter-cli +[crates.io]: https://crates.io/crates/tree-sitter-cli +[node-module]: https://www.npmjs.com/package/tree-sitter-cli +[node.js]: https://nodejs.org +[npm]: https://docs.npmjs.com +[path-env]: https://en.wikipedia.org/wiki/PATH_(variable) +[releases]: https://github.com/tree-sitter/tree-sitter/releases/latest +[tree-sitter-cli]: https://github.com/tree-sitter/tree-sitter/tree/master/crates/cli +[triple-slash]: https://www.typescriptlang.org/docs/handbook/triple-slash-directives.html +[ts-check]: https://www.typescriptlang.org/docs/handbook/intro-to-js-ts.html diff --git a/docs/src/creating-parsers/2-the-grammar-dsl.md b/docs/src/creating-parsers/2-the-grammar-dsl.md new file mode 100644 index 00000000..0592c6bd --- /dev/null +++ b/docs/src/creating-parsers/2-the-grammar-dsl.md @@ -0,0 +1,159 @@ +# The Grammar DSL + +The following is a complete list of built-in functions you can use in your `grammar.js` to define rules. Use-cases for some +of these functions will be explained in more detail in later sections. + +- **Symbols (the `$` object)** — Every grammar rule is written as a JavaScript function that takes a parameter conventionally +called `$`. The syntax `$.identifier` is how you refer to another grammar symbol within a rule. Names starting with `$.MISSING` +or `$.UNEXPECTED` should be avoided as they have special meaning for the `tree-sitter test` command. +- **String and Regex literals** — The terminal symbols in a grammar are described using JavaScript strings and regular +expressions. Of course during parsing, Tree-sitter does not actually use JavaScript's regex engine to evaluate these regexes; +it generates its own regex-matching logic based on the Rust regex syntax as part of each parser. Regex literals are just +used as a convenient way of writing regular expressions in your grammar. You can use Rust regular expressions in your grammar +DSL through the `RustRegex` class. Simply pass your regex pattern as a string: + + ```js + new RustRegex('(?i)[a-z_][a-z0-9_]*') // matches a simple identifier + ``` + + Unlike JavaScript's builtin `RegExp` class, which takes a pattern and flags as separate arguments, `RustRegex` only + accepts a single pattern string. While it doesn't support separate flags, you can use inline flags within the pattern + itself. For more details about Rust's regex syntax and capabilities, check out the [Rust regex documentation][rust regex]. + + ```admonish note + Only a subset of the Regex engine is actually supported. This is due to certain features like lookahead and lookaround + assertions not feasible to use in an LR(1) grammar, as well as certain flags being unnecessary for tree-sitter. However, + plenty of features are supported by default: + + - Character classes + - Character ranges + - Character sets + - Quantifiers + - Alternation + - Grouping + - Unicode character escapes + - Unicode property escapes + ``` + +- **Sequences : `seq(rule1, rule2, ...)`** — This function creates a rule that matches any number of other rules, one after +another. It is analogous to simply writing multiple symbols next to each other in [EBNF notation][ebnf]. + +- **Alternatives : `choice(rule1, rule2, ...)`** — This function creates a rule that matches *one* of a set of possible +rules. The order of the arguments does not matter. This is analogous to the `|` (pipe) operator in EBNF notation. + +- **Repetitions : `repeat(rule)`** — This function creates a rule that matches *zero-or-more* occurrences of a given rule. +It is analogous to the `{x}` (curly brace) syntax in EBNF notation. + +- **Repetitions : `repeat1(rule)`** — This function creates a rule that matches *one-or-more* occurrences of a given rule. +The previous `repeat` rule is implemented in `repeat1` but is included because it is very commonly used. + +- **Options : `optional(rule)`** — This function creates a rule that matches *zero or one* occurrence of a given rule. +It is analogous to the `[x]` (square bracket) syntax in EBNF notation. + +- **Precedence : `prec(number, rule)`** — This function marks the given rule with a numerical precedence, which will be +used to resolve [*LR(1) Conflicts*][lr-conflict] at parser-generation time. When two rules overlap in a way that represents +either a true ambiguity or a *local* ambiguity given one token of lookahead, Tree-sitter will try to resolve the conflict +by matching the rule with the higher precedence. The default precedence of all rules is zero. This works similarly to the +[precedence directives][yacc-prec] in Yacc grammars. + + This function can also be used to assign lexical precedence to a given + token, but it must be wrapped in a `token` call, such as `token(prec(1, 'foo'))`. This reads as "the token `foo` has a + lexical precedence of 1". The purpose of lexical precedence is to solve the issue where multiple tokens can match the same + set of characters, but one token should be preferred over the other. See [Lexical Precedence vs Parse Precedence][lexical vs parse] + for a more detailed explanation. + +- **Left Associativity : `prec.left([number], rule)`** — This function marks the given rule as left-associative (and optionally +applies a numerical precedence). When an LR(1) conflict arises in which all the rules have the same numerical precedence, +Tree-sitter will consult the rules' associativity. If there is a left-associative rule, Tree-sitter will prefer matching +a rule that ends *earlier*. This works similarly to [associativity directives][yacc-prec] in Yacc grammars. + +- **Right Associativity : `prec.right([number], rule)`** — This function is like `prec.left`, but it instructs Tree-sitter +to prefer matching a rule that ends *later*. + +- **Dynamic Precedence : `prec.dynamic(number, rule)`** — This function is similar to `prec`, but the given numerical precedence +is applied at *runtime* instead of at parser generation time. This is only necessary when handling a conflict dynamically +using the `conflicts` field in the grammar, and when there is a genuine *ambiguity*: multiple rules correctly match a given +piece of code. In that event, Tree-sitter compares the total dynamic precedence associated with each rule, and selects the +one with the highest total. This is similar to [dynamic precedence directives][bison-dprec] in Bison grammars. + +- **Tokens : `token(rule)`** — This function marks the given rule as producing only +a single token. Tree-sitter's default is to treat each String or RegExp literal +in the grammar as a separate token. Each token is matched separately by the lexer +and returned as its own leaf node in the tree. The `token` function allows you to +express a complex rule using the functions described above (rather than as a single +regular expression) but still have Tree-sitter treat it as a single token. +The token function will only accept terminal rules, so `token($.foo)` will not work. +You can think of it as a shortcut for squashing complex rules of strings or regexes +down to a single token. + +- **Immediate Tokens : `token.immediate(rule)`** — Usually, whitespace (and any other extras, such as comments) is optional +before each token. This function means that the token will only match if there is no whitespace. + +- **Aliases : `alias(rule, name)`** — This function causes the given rule to *appear* with an alternative name in the syntax +tree. If `name` is a *symbol*, as in `alias($.foo, $.bar)`, then the aliased rule will *appear* as a [named node][named-vs-anonymous-nodes] +called `bar`. And if `name` is a *string literal*, as in `alias($.foo, 'bar')`, then the aliased rule will appear as an +[anonymous node][named-vs-anonymous-nodes], as if the rule had been written as the simple string. + +- **Field Names : `field(name, rule)`** — This function assigns a *field name* to the child node(s) matched by the given +rule. In the resulting syntax tree, you can then use that field name to access specific children. + +- **Reserved Keywords : `reserved(wordset, rule)`** — This function will override the global reserved word set with the +one passed into the `wordset` parameter. This is useful for contextual keywords, such as `if` in JavaScript, which cannot +be used as a variable name in most contexts, but can be used as a property name. + +In addition to the `name` and `rules` fields, grammars have a few other optional public fields that influence the behavior +of the parser. Each of these fields is a function that accepts the grammar object (`$`) as its only parameter, like the +grammar rules themselves. These fields are: + +- **`extras`** — an array of tokens that may appear *anywhere* in the language. This is often used for whitespace and +comments. The default value of `extras` is to accept whitespace. To control whitespace explicitly, specify +`extras: $ => []` in your grammar. See the section on [using extras][extras] for more details. + +- **`inline`** — an array of rule names that should be automatically *removed* from the grammar by replacing all of their +usages with a copy of their definition. This is useful for rules that are used in multiple places but for which you *don't* +want to create syntax tree nodes at runtime. + +- **`conflicts`** — an array of arrays of rule names. Each inner array represents a set of rules that's involved in an +*LR(1) conflict* that is *intended to exist* in the grammar. When these conflicts occur at runtime, Tree-sitter will use +the GLR algorithm to explore all the possible interpretations. If *multiple* parses end up succeeding, Tree-sitter will +pick the subtree whose corresponding rule has the highest total *dynamic precedence*. + +- **`externals`** — an array of token names which can be returned by an +[*external scanner*][external-scanners]. External scanners allow you to write custom C code which runs during the lexing +process to handle lexical rules (e.g. Python's indentation tokens) that cannot be described by regular expressions. + +- **`precedences`** — an array of arrays of strings, where each array of strings defines named precedence levels in descending +order. These names can be used in the `prec` functions to define precedence relative only to other names in the array, rather +than globally. Can only be used with parse precedence, not lexical precedence. + +- **`word`** — the name of a token that will match keywords to the +[keyword extraction][keyword-extraction] optimization. + +- **`supertypes`** — an array of rule names which should be considered to be 'supertypes' in the generated +[*node types* file][static-node-types-supertypes]. Supertype rules are automatically hidden from the parse tree, regardless +of whether their names start with an underscore. The main use case for supertypes is to group together multiple different +kinds of nodes under a single abstract category, such as "expression" or "declaration". See the section on [`using supertypes`][supertypes] +for more details. + +- **`reserved`** — similar in structure to the main `rules` property, an object of reserved word sets associated with an +array of reserved rules. The reserved rule in the array must be a terminal token meaning it must be a string, regex, token, +or terminal rule. The reserved rule must also exist and be used in the grammar, specifying arbitrary tokens will not work. +The *first* reserved word set in the object is the global word set, meaning it applies to every rule in every parse state. +However, certain keywords are contextual, depending on the rule. For example, in JavaScript, keywords are typically not +allowed as ordinary variables, however, they *can* be used as a property name. In this situation, the `reserved` function +would be used, and the word set to pass in would be the name of the word set that is declared in the `reserved` object that +corresponds to an empty array, signifying *no* keywords are reserved. + +[bison-dprec]: https://www.gnu.org/software/bison/manual/html_node/Generalized-LR-Parsing.html +[ebnf]: https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form +[external-scanners]: ./4-external-scanners.md +[extras]: ./3-writing-the-grammar.md#using-extras +[keyword-extraction]: ./3-writing-the-grammar.md#keyword-extraction +[lexical vs parse]: ./3-writing-the-grammar.md#lexical-precedence-vs-parse-precedence +[lr-conflict]: https://en.wikipedia.org/wiki/LR_parser#Conflicts_in_the_constructed_tables +[named-vs-anonymous-nodes]: ../using-parsers/2-basic-parsing.md#named-vs-anonymous-nodes +[rust regex]: https://docs.rs/regex/1.1.8/regex/#grouping-and-flags +[static-node-types]: ../using-parsers/6-static-node-types.md +[static-node-types-supertypes]: ../using-parsers/6-static-node-types.md#supertype-nodes +[supertypes]: ./3-writing-the-grammar.md#using-supertypes +[yacc-prec]: https://docs.oracle.com/cd/E19504-01/802-5880/6i9k05dh3/index.html diff --git a/docs/src/creating-parsers/3-writing-the-grammar.md b/docs/src/creating-parsers/3-writing-the-grammar.md new file mode 100644 index 00000000..5048ff5b --- /dev/null +++ b/docs/src/creating-parsers/3-writing-the-grammar.md @@ -0,0 +1,657 @@ +# Writing the Grammar + +Writing a grammar requires creativity. There are an infinite number of CFGs (context-free grammars) that can be used to +describe any given language. To produce a good Tree-sitter parser, you need to create a grammar with two important properties: + +1. **An intuitive structure** — Tree-sitter's output is a [concrete syntax tree][cst]; each node in the tree corresponds +directly to a [terminal or non-terminal symbol][non-terminal] in the grammar. So to produce an easy-to-analyze tree, there +should be a direct correspondence between the symbols in your grammar and the recognizable constructs in the language. +This might seem obvious, but it is very different from the way that context-free grammars are often written in contexts +like [language specifications][language-spec] or [Yacc][yacc]/[Bison][bison] parsers. + +2. **A close adherence to LR(1)** — Tree-sitter is based on the [GLR parsing][glr-parsing] algorithm. This means that while +it can handle any context-free grammar, it works most efficiently with a class of context-free grammars called [LR(1) Grammars][lr-grammars]. +In this respect, Tree-sitter's grammars are similar to (but less restrictive than) [Yacc][yacc] and [Bison][bison] grammars, +but _different_ from [ANTLR grammars][antlr], [Parsing Expression Grammars][peg], or the [ambiguous grammars][ambiguous-grammar] +commonly used in language specifications. + +It's unlikely that you'll be able to satisfy these two properties just by translating an existing context-free grammar directly +into Tree-sitter's grammar format. There are a few kinds of adjustments that are often required. +The following sections will explain these adjustments in more depth. + +## The First Few Rules + +It's usually a good idea to find a formal specification for the language you're trying to parse. This specification will +most likely contain a context-free grammar. As you read through the rules of this CFG, you will probably discover a complex +and cyclic graph of relationships. It might be unclear how you should navigate this graph as you define your grammar. + +Although languages have very different constructs, their constructs can often be categorized in to similar groups like +_Declarations_, _Definitions_, _Statements_, _Expressions_, _Types_ and _Patterns_. In writing your grammar, a good first +step is to create just enough structure to include all of these basic _groups_ of symbols. For a language like Go, +you might start with something like this: + +```js +{ + // ... + + rules: { + source_file: $ => repeat($._definition), + + _definition: $ => choice( + $.function_definition + // TODO: other kinds of definitions + ), + + function_definition: $ => seq( + 'func', + $.identifier, + $.parameter_list, + $._type, + $.block + ), + + parameter_list: $ => seq( + '(', + // TODO: parameters + ')' + ), + + _type: $ => choice( + 'bool' + // TODO: other kinds of types + ), + + block: $ => seq( + '{', + repeat($._statement), + '}' + ), + + _statement: $ => choice( + $.return_statement + // TODO: other kinds of statements + ), + + return_statement: $ => seq( + 'return', + $.expression, + ';' + ), + + expression: $ => choice( + $.identifier, + $.number + // TODO: other kinds of expressions + ), + + identifier: $ => /[a-z]+/, + + number: $ => /\d+/ + } +} +``` + +One important fact to know up front is that the start rule for the grammar is the first property in the `rules` object. +In the example above, that would correspond to `source_file`, but it can be named anything. + +Some details of this grammar will be explained in more depth later on, but if you focus on the `TODO` comments, you can +see that the overall strategy is _breadth-first_. Notably, this initial skeleton does not need to directly match an exact +subset of the context-free grammar in the language specification. It just needs to touch on the major groupings of rules +in as simple and obvious a way as possible. + +With this structure in place, you can now freely decide what part of the grammar to flesh out next. For example, you might +decide to start with _types_. One-by-one, you could define the rules for writing basic types and composing them into more +complex types: + +```js +{ + // ... + + _type: $ => choice( + $.primitive_type, + $.array_type, + $.pointer_type + ), + + primitive_type: $ => choice( + 'bool', + 'int' + ), + + array_type: $ => seq( + '[', + ']', + $._type + ), + + pointer_type: $ => seq( + '*', + $._type + ) +} +``` + +After developing the _type_ sublanguage a bit further, you might decide to switch to working on _statements_ or _expressions_ +instead. It's often useful to check your progress by trying to parse some real code using `tree-sitter parse`. + +**And remember to add tests for each rule in your `test/corpus` folder!** + +## Structuring Rules Well + +Imagine that you were just starting work on the [Tree-sitter JavaScript parser][tree-sitter-javascript]. Naively, you might +try to directly mirror the structure of the [ECMAScript Language Spec][ecmascript-spec]. To illustrate the problem with +this approach, consider the following line of code: + +```js +return x + y; +``` + +According to the specification, this line is a `ReturnStatement`, the fragment `x + y` is an `AdditiveExpression`, +and `x` and `y` are both `IdentifierReferences`. The relationship between these constructs is captured by a complex series +of production rules: + +```text +ReturnStatement -> 'return' Expression +Expression -> AssignmentExpression +AssignmentExpression -> ConditionalExpression +ConditionalExpression -> LogicalORExpression +LogicalORExpression -> LogicalANDExpression +LogicalANDExpression -> BitwiseORExpression +BitwiseORExpression -> BitwiseXORExpression +BitwiseXORExpression -> BitwiseANDExpression +BitwiseANDExpression -> EqualityExpression +EqualityExpression -> RelationalExpression +RelationalExpression -> ShiftExpression +ShiftExpression -> AdditiveExpression +AdditiveExpression -> MultiplicativeExpression +MultiplicativeExpression -> ExponentiationExpression +ExponentiationExpression -> UnaryExpression +UnaryExpression -> UpdateExpression +UpdateExpression -> LeftHandSideExpression +LeftHandSideExpression -> NewExpression +NewExpression -> MemberExpression +MemberExpression -> PrimaryExpression +PrimaryExpression -> IdentifierReference +``` + +The language spec encodes the twenty different precedence levels of JavaScript expressions using twenty levels of indirection +between `IdentifierReference` and `Expression`. If we were to create a concrete syntax tree representing this statement +according to the language spec, it would have twenty levels of nesting, and it would contain nodes with names like `BitwiseXORExpression`, +which are unrelated to the actual code. + +## Standard Rule Names + +Tree-sitter places no restrictions on how to name the rules of your grammar. It can be helpful, however, to follow certain +conventions used by many other established grammars in the ecosystem. Some of these well-established patterns are listed +below: + +- `source_file`: Represents an entire source file, this rule is commonly used as the root node for a grammar, +- `expression`/`statement`: Used to represent statements and expressions for a given language. Commonly defined as a choice +between several more specific sub-expression/sub-statement rules. +- `block`: Used as the parent node for block scopes, with its children representing the block's contents. +- `type`: Represents the types of a language such as `int`, `char`, and `void`. +- `identifier`: Used for constructs like variable names, function arguments, and object fields; this rule is commonly used +as the `word` token in grammars. +- `string`: Used to represent `"string literals"`. +- `comment`: Used to represent comments, this rule is commonly used as an `extra`. + +## Using Precedence + +To produce a readable syntax tree, we'd like to model JavaScript expressions using a much flatter structure like this: + +```js +{ + // ... + + expression: $ => choice( + $.identifier, + $.unary_expression, + $.binary_expression, + // ... + ), + + unary_expression: $ => choice( + seq('-', $.expression), + seq('!', $.expression), + // ... + ), + + binary_expression: $ => choice( + seq($.expression, '*', $.expression), + seq($.expression, '+', $.expression), + // ... + ), +} +``` + +Of course, this flat structure is highly ambiguous. If we try to generate a parser, Tree-sitter gives us an error message: + +```text +Error: Unresolved conflict for symbol sequence: + + '-' _expression • '*' … + +Possible interpretations: + + 1: '-' (binary_expression _expression • '*' _expression) + 2: (unary_expression '-' _expression) • '*' … + +Possible resolutions: + + 1: Specify a higher precedence in `binary_expression` than in the other rules. + 2: Specify a higher precedence in `unary_expression` than in the other rules. + 3: Specify a left or right associativity in `unary_expression` + 4: Add a conflict for these rules: `binary_expression` `unary_expression` +``` + +```admonish hint +The • character in the error message indicates where exactly during +parsing the conflict occurs, or in other words, where the parser is encountering +ambiguity. +``` + +For an expression like `-a * b`, it's not clear whether the `-` operator applies to the `a * b` or just to the `a`. This +is where the `prec` function [described in the previous page][grammar dsl] comes into play. By wrapping a rule with `prec`, +we can indicate that certain sequence of symbols should _bind to each other more tightly_ than others. For example, the +`'-', $.expression` sequence in `unary_expression` should bind more tightly than the `$.expression, '+', $.expression` +sequence in `binary_expression`: + +```js +{ + // ... + + unary_expression: $ => + prec( + 2, + choice( + seq("-", $.expression), + seq("!", $.expression), + // ... + ), + ); +} +``` + +## Using Associativity + +Applying a higher precedence in `unary_expression` fixes that conflict, but there is still another conflict: + +```text +Error: Unresolved conflict for symbol sequence: + + _expression '*' _expression • '*' … + +Possible interpretations: + + 1: _expression '*' (binary_expression _expression • '*' _expression) + 2: (binary_expression _expression '*' _expression) • '*' … + +Possible resolutions: + + 1: Specify a left or right associativity in `binary_expression` + 2: Add a conflict for these rules: `binary_expression` +``` + +For an expression like `a * b * c`, it's not clear whether we mean `a * (b * c)` or `(a * b) * c`. +This is where `prec.left` and `prec.right` come into use. We want to select the second interpretation, so we use `prec.left`. + +```js +{ + // ... + + binary_expression: $ => choice( + prec.left(2, seq($.expression, '*', $.expression)), + prec.left(1, seq($.expression, '+', $.expression)), + // ... + ), +} +``` + +## Using Conflicts + +Sometimes, conflicts are actually desirable. In our JavaScript grammar, expressions and patterns can create intentional +ambiguity. A construct like `[x, y]` could be legitimately parsed as both an array literal (like in `let a = [x, y]`) or +as a destructuring pattern (like in `let [x, y] = arr`). + +```js +export default grammar({ + name: "javascript", + + rules: { + expression: $ => choice( + $.identifier, + $.array, + $.pattern, + ), + + array: $ => seq( + "[", + optional(seq( + $.expression, repeat(seq(",", $.expression)) + )), + "]" + ), + + array_pattern: $ => seq( + "[", + optional(seq( + $.pattern, repeat(seq(",", $.pattern)) + )), + "]" + ), + + pattern: $ => choice( + $.identifier, + $.array_pattern, + ), + }, +}) +``` + +In such cases, we want the parser to explore both possibilities by explicitly declaring this ambiguity: + +```js +{ + name: "javascript", + + conflicts: $ => [ + [$.array, $.array_pattern], + ], + + rules: { + // ... + }, +} +``` + +```admonish note +The example is a bit contrived for the purpose of illustrating the usage of conflicts. The actual JavaScript grammar isn't +structured like that, but this conflict is actually present in the +[Tree-sitter JavaScript grammar](https://github.com/tree-sitter/tree-sitter-javascript/blob/108b2d4d17a04356a340aea809e4dd5b801eb40d/grammar.js#L100). +``` + +## Hiding Rules + +You may have noticed in the above examples that some grammar rule name like `_expression` and `_type` began with an underscore. +Starting a rule's name with an underscore causes the rule to be _hidden_ in the syntax tree. This is useful for rules like +`_expression` in the grammars above, which always just wrap a single child node. If these nodes were not hidden, they would +add substantial depth and noise to the syntax tree without making it any easier to understand. + +## Using Fields + +Often, it's easier to analyze a syntax node if you can refer to its children by _name_ instead of by their position in an +ordered list. Tree-sitter grammars support this using the `field` function. This function allows you to assign unique names +to some or all of a node's children: + +```js +function_definition: $ => + seq( + "func", + field("name", $.identifier), + field("parameters", $.parameter_list), + field("return_type", $._type), + field("body", $.block), + ); +``` + +Adding fields like this allows you to retrieve nodes using the [field APIs][field-names-section]. + +## Using Extras + +Extras are tokens that can appear anywhere in the grammar, without being explicitly mentioned in a rule. This is useful +for things like whitespace and comments, which can appear between any two tokens in most programming languages. To define +an extra, you can use the `extras` function: + +```js +module.exports = grammar({ + name: "my_language", + + extras: ($) => [ + /\s/, // whitespace + $.comment, + ], + + rules: { + comment: ($) => + token( + choice(seq("//", /.*/), seq("/*", /[^*]*\*+([^/*][^*]*\*+)*/, "/")), + ), + }, +}); +``` + +```admonish warning +When adding more complicated tokens to `extras`, it's preferable to associate the pattern +with a rule. This way, you avoid the lexer inlining this pattern in a bunch of spots, +which can dramatically reduce the parser size. +``` + +For example, instead of defining the `comment` token inline in `extras`: + +```js +// ❌ Less preferable + +const comment = token( + choice(seq("//", /.*/), seq("/*", /[^*]*\*+([^/*][^*]*\*+)*/, "/")), +); + +module.exports = grammar({ + name: "my_language", + extras: ($) => [ + /\s/, // whitespace + comment, + ], + rules: { + // ... + }, +}); +``` + +We can define it as a rule and then reference it in `extras`: + +```js +// ✅ More preferable + +module.exports = grammar({ + name: "my_language", + + extras: ($) => [ + /\s/, // whitespace + $.comment, + ], + + rules: { + // ... + + comment: ($) => + token( + choice(seq("//", /.*/), seq("/*", /[^*]*\*+([^/*][^*]*\*+)*/, "/")), + ), + }, +}); +``` + +```admonish note +Tree-sitter intentionally simplifies the whitespace character class, `\s`, to `[ \t\n\r]` as a performance +optimization. This is because typically users do not require the full Unicode definition of whitespace. +``` + +## Using Supertypes + +Some rules in your grammar will represent abstract categories of syntax nodes, such as "expression", "type", or "declaration". +These rules are often defined as simple choices between several other rules. For example, in the JavaScript grammar, the +`_expression` rule is defined as a choice between many different kinds of expressions: + +```js +expression: $ => choice( + $.identifier, + $.unary_expression, + $.binary_expression, + $.call_expression, + $.member_expression, + // ... +), +``` + +By default, Tree-sitter will generate a visible node type for each of these abstract category rules, which can lead to +unnecessarily deep and complex syntax trees. To avoid this, you can add these abstract category rules to the grammar's `supertypes` +definition. Tree-sitter will then treat these rules as _supertypes_, and will not generate visible node types for them in +the syntax tree. + +```js +module.exports = grammar({ + name: "javascript", + + supertypes: $ => [ + $.expression, + ], + + rules: { + expression: $ => choice( + $.identifier, + // ... + ), + + // ... + }, +}); +_ +``` + +Although supertype rules are hidden from the syntax tree, they can still be used in queries. See the chapter on +[Query Syntax][query syntax] for more information. + +# Lexical Analysis + +Tree-sitter's parsing process is divided into two phases: parsing (which is described above) and [lexing][lexing] — the +process of grouping individual characters into the language's fundamental _tokens_. There are a few important things to +know about how Tree-sitter's lexing works. + +## Conflicting Tokens + +Grammars often contain multiple tokens that can match the same characters. For example, a grammar might contain the tokens +(`"if"` and `/[a-z]+/`). Tree-sitter differentiates between these conflicting tokens in a few ways. + +1. **Context-aware Lexing** — Tree-sitter performs lexing on-demand, during the parsing process. At any given position +in a source document, the lexer only tries to recognize tokens that are _valid_ at that position in the document. + +2. **Lexical Precedence** — When the precedence functions described [in the previous page][grammar dsl] are used _within_ +the `token` function, the given explicit precedence values serve as instructions to the lexer. If there are two valid tokens +that match the characters at a given position in the document, Tree-sitter will select the one with the higher precedence. + +3. **Match Length** — If multiple valid tokens with the same precedence match the characters at a given position in a document, +Tree-sitter will select the token that matches the [longest sequence of characters][longest-match]. + +4. **Match Specificity** — If there are two valid tokens with the same precedence, and they both match the same number +of characters, Tree-sitter will prefer a token that is specified in the grammar as a `String` over a token specified as +a `RegExp`. + +5. **Rule Order** — If none of the above criteria can be used to select one token over another, Tree-sitter will prefer +the token that appears earlier in the grammar. + +If there is an external scanner it may have [an additional impact][external scanner] over regular tokens +defined in the grammar. + +## Lexical Precedence vs. Parse Precedence + +One common mistake involves not distinguishing _lexical precedence_ from _parse precedence_. Parse precedence determines +which rule is chosen to interpret a given sequence of tokens. _Lexical precedence_ determines which token is chosen to interpret +at a given position of text, and it is a lower-level operation that is done first. The above list fully captures Tree-sitter's +lexical precedence rules, and you will probably refer back to this section of the documentation more often than any other. +Most of the time when you really get stuck, you're dealing with a lexical precedence problem. Pay particular attention to +the difference in meaning between using `prec` inside the `token` function versus outside it. The _lexical precedence_ syntax, +as mentioned in the previous page, is `token(prec(N, ...))`. + +## Keywords + +Many languages have a set of _keyword_ tokens (e.g. `if`, `for`, `return`), as well as a more general token (e.g. `identifier`) +that matches any word, including many of the keyword strings. For example, JavaScript has a keyword `instanceof`, which +is used as a binary operator, like this: + +```js +if (a instanceof Something) b(); +``` + +The following, however, is not valid JavaScript: + +```js +if (a instanceofSomething) b(); +``` + +A keyword like `instanceof` cannot be followed immediately by another letter, because then it would be tokenized as an `identifier`, +**even though an identifier is not valid at that position**. Because Tree-sitter uses context-aware lexing, as described +[above](#conflicting-tokens), it would not normally impose this restriction. By default, Tree-sitter would recognize `instanceofSomething` +as two separate tokens: the `instanceof` keyword followed by an `identifier`. + +## Keyword Extraction + +Fortunately, Tree-sitter has a feature that allows you to fix this, so that you can match the behavior of other standard +parsers: the `word` token. If you specify a `word` token in your grammar, Tree-sitter will find the set of _keyword_ tokens +that match strings also matched by the `word` token. Then, during lexing, instead of matching each of these keywords individually, +Tree-sitter will match the keywords via a two-step process where it _first_ matches the `word` token. + +For example, suppose we added `identifier` as the `word` token in our JavaScript grammar: + +```js +grammar({ + name: "javascript", + + word: $ => $.identifier, + + rules: { + expression: $ => + choice( + $.identifier, + $.unary_expression, + $.binary_expression, + // ... + ), + + binary_expression: $ => + choice( + prec.left(1, seq($.expression, "instanceof", $.expression)), + // ... + ), + + unary_expression: $ => + choice( + prec.left(2, seq("typeof", $.expression)), + // ... + ), + + identifier: $ => /[a-z_]+/, + }, +}); +``` + +Tree-sitter would identify `typeof` and `instanceof` as keywords. Then, when parsing the invalid code above, rather than +scanning for the `instanceof` token individually, it would scan for an `identifier` first, and find `instanceofSomething`. +It would then correctly recognize the code as invalid. + +Aside from improving error detection, keyword extraction also has performance benefits. It allows Tree-sitter to generate +a smaller, simpler lexing function, which means that **the parser will compile much more quickly**. + +```admonish note +The word token must be a unique token that is not reused by another rule. If you want to have a word token used in a +rule that's called something else, you should just alias the word token instead, like how the Rust grammar does it +here +``` + +[ambiguous-grammar]: https://en.wikipedia.org/wiki/Ambiguous_grammar +[antlr]: https://www.antlr.org +[bison]: https://en.wikipedia.org/wiki/GNU_bison +[cst]: https://en.wikipedia.org/wiki/Parse_tree +[ecmascript-spec]: https://262.ecma-international.org/6.0/ +[external scanner]: ./4-external-scanners.md#other-external-scanner-details +[glr-parsing]: https://en.wikipedia.org/wiki/GLR_parser +[grammar dsl]: ./2-the-grammar-dsl.md +[language-spec]: https://en.wikipedia.org/wiki/Programming_language_specification +[lexing]: https://en.wikipedia.org/wiki/Lexical_analysis +[longest-match]: https://en.wikipedia.org/wiki/Maximal_munch +[lr-grammars]: https://en.wikipedia.org/wiki/LR_parser +[field-names-section]: ../using-parsers/2-basic-parsing.md#node-field-names +[non-terminal]: https://en.wikipedia.org/wiki/Terminal_and_nonterminal_symbols +[peg]: https://en.wikipedia.org/wiki/Parsing_expression_grammar +[query syntax]: ../using-parsers/queries/1-syntax.md#supertype-nodes +[tree-sitter-javascript]: https://github.com/tree-sitter/tree-sitter-javascript +[yacc]: https://en.wikipedia.org/wiki/Yacc diff --git a/docs/src/creating-parsers/4-external-scanners.md b/docs/src/creating-parsers/4-external-scanners.md new file mode 100644 index 00000000..6c89e221 --- /dev/null +++ b/docs/src/creating-parsers/4-external-scanners.md @@ -0,0 +1,384 @@ +# External Scanners + +Many languages have some tokens whose structure is impossible or inconvenient to describe with a regular expression. +Some examples: + +- [Indent and dedent][indent-tokens] tokens in Python +- [Heredocs][heredoc] in Bash and Ruby +- [Percent strings][percent-string] in Ruby + +Tree-sitter allows you to handle these kinds of tokens using _external scanners_. An external scanner is a set of C functions +that you, the grammar author, can write by hand to add custom logic for recognizing certain tokens. + +To use an external scanner, there are a few steps. First, add an `externals` section to your grammar. This section should +list the names of all of your external tokens. These names can then be used elsewhere in your grammar. + +```js +grammar({ + name: "my_language", + + externals: $ => [$.indent, $.dedent, $.newline], + + // ... +}); +``` + +Then, add another C source file to your project. Its path must be src/scanner.c for the CLI to recognize it. + +In this new source file, define an [`enum`][enum] type containing the names of all of your external tokens. The ordering +of this enum must match the order in your grammar's `externals` array; the actual names do not matter. + +```c +#include "tree_sitter/parser.h" +#include "tree_sitter/alloc.h" +#include "tree_sitter/array.h" + +enum TokenType { + INDENT, + DEDENT, + NEWLINE +} +``` + +Finally, you must define five functions with specific names, based on your language's name and five actions: +_create_, _destroy_, _serialize_, _deserialize_, and _scan_. + +## Create + +```c +void * tree_sitter_my_language_external_scanner_create() { + // ... +} +``` + +This function should create your scanner object. It will only be called once anytime your language is set on a parser. +Often, you will want to allocate memory on the heap and return a pointer to it. If your external scanner doesn't need to +maintain any state, it's ok to return `NULL`. + +## Destroy + +```c +void tree_sitter_my_language_external_scanner_destroy(void *payload) { + // ... +} +``` + +This function should free any memory used by your scanner. It is called once when a parser is deleted or assigned a different +language. It receives as an argument the same pointer that was returned from the _create_ function. If your _create_ function +didn't allocate any memory, this function can be a no-op. + +## Serialize + +```c +unsigned tree_sitter_my_language_external_scanner_serialize( + void *payload, + char *buffer +) { + // ... +} +``` + +This function should copy the complete state of your scanner into a given byte buffer, and return the number of bytes written. +The function is called every time the external scanner successfully recognizes a token. It receives a pointer to your scanner +and a pointer to a buffer. The maximum number of bytes that you can write is given by the `TREE_SITTER_SERIALIZATION_BUFFER_SIZE` +constant, defined in the `tree_sitter/parser.h` header file. + +The data that this function writes will ultimately be stored in the syntax tree so that the scanner can be restored to the +right state when handling edits or ambiguities. For your parser to work correctly, the `serialize` function must store its +entire state, and `deserialize` must restore the entire state. For good performance, you should design your scanner so that +its state can be serialized as quickly and compactly as possible. + +## Deserialize + +```c +void tree_sitter_my_language_external_scanner_deserialize( + void *payload, + const char *buffer, + unsigned length +) { + // ... +} +``` + +This function should _restore_ the state of your scanner based the bytes that were previously written by the `serialize` +function. It is called with a pointer to your scanner, a pointer to the buffer of bytes, and the number of bytes that should +be read. It is good practice to explicitly erase your scanner state variables at the start of this function, before restoring +their values from the byte buffer. + +## Scan + +Typically, one will + +- Call `lexer->advance` several times, if the characters are valid for the token being lexed. + +- Optionally, call `lexer->mark_end` to mark the end of the token, and "peek ahead" +to check if the next character (or set of characters) invalidates the token. + +- Set `lexer->result_symbol` to the token type. + +- Return `true` from the scanning function, indicating that a token was successfully lexed. + +Tree-sitter will then push resulting node to the parse stack, and the input position will remain where it reached at the +point `lexer->mark_end` was called. + +```c +bool tree_sitter_my_language_external_scanner_scan( + void *payload, + TSLexer *lexer, + const bool *valid_symbols +) { + // ... +} +``` + +The second parameter to this function is the lexer, of type `TSLexer`. The `TSLexer` struct has the following fields: + +- **`int32_t lookahead`** — The current next character in the input stream, represented as a 32-bit unicode code point. + +- **`TSSymbol result_symbol`** — The symbol that was recognized. Your scan function should _assign_ to this field one of +the values from the `TokenType` enum, described above. + +- **`void (*advance)(TSLexer *, bool skip)`** — A function for advancing to the next character. If you pass `true` for +the second argument, the current character will be treated as whitespace; whitespace won't be included in the text range +associated with tokens emitted by the external scanner. + +- **`void (*mark_end)(TSLexer *)`** — A function for marking the end of the recognized token. This allows matching tokens +that require multiple characters of lookahead. By default, (if you don't call `mark_end`), any character that you moved +past using the `advance` function will be included in the size of the token. But once you call `mark_end`, then any later +calls to `advance` will _not_ increase the size of the returned token. You can call `mark_end` multiple times to increase +the size of the token. + +- **`uint32_t (*get_column)(TSLexer *)`** — A function for querying the current column position of the lexer. It returns +the number of codepoints since the start of the current line. The codepoint position is recalculated on every call to this +function by reading from the start of the line. + +- **`bool (*is_at_included_range_start)(const TSLexer *)`** — A function for checking whether the parser has just skipped +some characters in the document. When parsing an embedded document using the `ts_parser_set_included_ranges` function +(described in the [multi-language document section][multi-language-section]), the scanner may want to apply some special +behavior when moving to a disjoint part of the document. For example, in [EJS documents][ejs], the JavaScript parser uses +this function to enable inserting automatic semicolon tokens in between the code directives, delimited by `<%` and `%>`. + +- **`bool (*eof)(const TSLexer *)`** — A function for determining whether the lexer is at the end of the file. The value +of `lookahead` will be `0` at the end of a file, but this function should be used instead of checking for that value because +the `0` or "NUL" value is also a valid character that could be present in the file being parsed. + +The third argument to the `scan` function is an array of booleans that indicates which of external tokens are expected by +the parser. You should only look for a given token if it is valid according to this array. At the same time, you cannot +backtrack, so you may need to combine certain pieces of logic. + +```c +if (valid_symbols[INDENT] || valid_symbols[DEDENT]) { + + // ... logic that is common to both `INDENT` and `DEDENT` + + if (valid_symbols[INDENT]) { + + // ... logic that is specific to `INDENT` + + lexer->result_symbol = INDENT; + return true; + } +} +``` + +## External Scanner Helpers + +### Allocator + +Instead of using libc's `malloc`, `calloc`, `realloc`, and `free`, you should use the versions prefixed with `ts_` from +`tree_sitter/alloc.h`. These macros can allow a potential consumer to override the default allocator with their own implementation, +but by default will use the libc functions. + +As a consumer of the tree-sitter core library as well as any parser libraries that might use allocations, you can enable +overriding the default allocator and have it use the same one as the library allocator, of which you can set with `ts_set_allocator`. +To enable this overriding in scanners, you must compile them with the `TREE_SITTER_REUSE_ALLOCATOR` macro defined, and tree-sitter +the library must be linked into your final app dynamically, since it needs to resolve the internal functions at runtime. +If you are compiling an executable binary that uses the core library, but want to load parsers dynamically at runtime, then +you will have to use a special linker flag on Unix. For non-Darwin systems, that would be `--dynamic-list` and for Darwin +systems, that would be `-exported_symbols_list`. The CLI does exactly this, so you can use it as a reference (check out +`cli/build.rs`). + +For example, assuming you wanted to allocate 100 bytes for your scanner, you'd do so like the following example: + +```c +#include "tree_sitter/parser.h" +#include "tree_sitter/alloc.h" + +// ... + +void* tree_sitter_my_language_external_scanner_create() { + return ts_calloc(100, 1); // or ts_malloc(100) +} + +// ... + +``` + +### Arrays + +If you need to use array-like types in your scanner, such as tracking a stack of indentations or tags, you should use the +array macros from `tree_sitter/array.h`. + +There are quite a few of them provided for you, but here's how you could get started tracking some . Check out the header +itself for more detailed documentation. + +```admonish attention +Do not use any of the array functions or macros that are prefixed with an underscore and have comments saying +that it is not what you are looking for. These are internal functions used as helpers by other macros that are public. +They are not meant to be used directly, nor are they what you want. +``` + +```c +#include "tree_sitter/parser.h" +#include "tree_sitter/array.h" + +enum TokenType { + INDENT, + DEDENT, + NEWLINE, + STRING, +} + +// Create the array in your create function + +void* tree_sitter_my_language_external_scanner_create() { + return ts_calloc(1, sizeof(Array(int))); + + // or if you want to zero out the memory yourself + + Array(int) *stack = ts_malloc(sizeof(Array(int))); + array_init(&stack); + return stack; +} + +bool tree_sitter_my_language_external_scanner_scan( + void *payload, + TSLexer *lexer, + const bool *valid_symbols +) { + Array(int) *stack = payload; + if (valid_symbols[INDENT]) { + array_push(stack, lexer->get_column(lexer)); + lexer->result_symbol = INDENT; + return true; + } + if (valid_symbols[DEDENT]) { + array_pop(stack); // this returns the popped element by value, but we don't need it + lexer->result_symbol = DEDENT; + return true; + } + + // we can also use an array on the stack to keep track of a string + + Array(char) next_string = array_new(); + + if (valid_symbols[STRING] && lexer->lookahead == '"') { + lexer->advance(lexer, false); + while (lexer->lookahead != '"' && lexer->lookahead != '\n' && !lexer->eof(lexer)) { + array_push(&next_string, lexer->lookahead); + lexer->advance(lexer, false); + } + + // assume we have some arbitrary constraint of not having more than 100 characters in a string + if (lexer->lookahead == '"' && next_string.size <= 100) { + lexer->advance(lexer, false); + lexer->result_symbol = STRING; + return true; + } + } + + return false; +} + +``` + +## Other External Scanner Details + +External scanners have priority over Tree-sitter's normal lexing process. When a token listed in the externals array is +valid at a given position, the external scanner is called first. This makes external scanners a powerful way to override +Tree-sitter's default lexing behavior, especially for cases that can't be handled with regular lexical rules, parsing, or +dynamic precedence. + +During error recovery, Tree-sitter's first step is to call the external scanner's scan function with all tokens marked as +valid. Your scanner should detect and handle this case appropriately. One simple approach is to add an unused "sentinel" +token at the end of your externals array: + +```js +{ + name: "my_language", + + externals: $ => [$.token1, $.token2, $.error_sentinel] + + // ... +} +``` + +You can then check if this sentinel token is marked valid to determine if Tree-sitter is in error recovery mode. + +If you would rather not handle the error recovery case explicitly, the easiest way to "opt-out" and let tree-sitter's internal +lexer handle it is to return `false` from your scan function when `valid_symbols` contains the error sentinel. + +```c +bool tree_sitter_my_language_external_scanner_scan( + void *payload, + TSLexer *lexer, + const bool *valid_symbols +) { + if (valid_symbols[ERROR_SENTINEL]) { + return false; + } + // ... +} +``` + +When you include literal keywords in the externals array, for example: + +```js +externals: $ => ['if', 'then', 'else'] +``` + +_those_ keywords will +be tokenized by the external scanner whenever they appear in the grammar. + +This is equivalent to declaring named tokens and aliasing them: + +```js +{ + name: "my_language", + + externals: $ => [$.if_keyword, $.then_keyword, $.else_keyword], + + rules: { + + // then using it in a rule like so: + if_statement: $ => seq(alias($.if_keyword, 'if'), ...), + + // ... + } +} +``` + +The tokenization process for external keywords works in two stages: + +1. The external scanner attempts to recognize the token first +2. If the scanner returns true and sets a token, that token is used +3. If the scanner returns false, Tree-sitter falls back to its internal lexer + +However, when you use rule references (like `$.if_keyword`) in the externals array without defining the corresponding rules +in the grammar, Tree-sitter cannot fall back to its internal lexer. In this case, the external scanner is solely responsible +for recognizing these tokens. + +```admonish danger +- External scanners can easily create infinite loops + +- Be extremely careful when emitting zero-width tokens + +- Always use the `eof` function when looping through characters +``` + +[ejs]: https://ejs.co +[enum]: https://en.wikipedia.org/wiki/Enumerated_type#C +[heredoc]: https://en.wikipedia.org/wiki/Here_document +[indent-tokens]: https://en.wikipedia.org/wiki/Off-side_rule +[multi-language-section]: ../using-parsers/3-advanced-parsing.md#multi-language-documents +[percent-string]: https://docs.ruby-lang.org/en/2.5.0/doc/syntax/literals_rdoc.html#label-Percent+Strings diff --git a/docs/src/creating-parsers/5-writing-tests.md b/docs/src/creating-parsers/5-writing-tests.md new file mode 100644 index 00000000..33155cca --- /dev/null +++ b/docs/src/creating-parsers/5-writing-tests.md @@ -0,0 +1,175 @@ +# Writing Tests + +For each rule that you add to the grammar, you should first create a *test* that describes how the syntax trees should look +when parsing that rule. These tests are written using specially-formatted text files in the `test/corpus/` directory within +your parser's root folder. + +For example, you might have a file called `test/corpus/statements.txt` that contains a series of entries like this: + +```text +================== +Return statements +================== + +func x() int { + return 1; +} + +--- + +(source_file + (function_definition + (identifier) + (parameter_list) + (primitive_type) + (block + (return_statement (number))))) +``` + +* The **name** of each test is written between two lines containing only `=` (equal sign) characters. + +* Then the **input source code** is written, followed by a line containing three or more `-` (dash) characters. + +* Then, the **expected output syntax tree** is written as an [S-expression][s-exp]. The exact placement of whitespace in +the S-expression doesn't matter, but ideally the syntax tree should be legible. + +```admonish tip +The S-expression does not show syntax nodes like `func`, `(` and `;`, which are expressed as strings and regexes in the grammar. +It only shows the *named* nodes, as described in [this section][named-vs-anonymous-nodes] of the page on parser usage. +``` + + The expected output section can also *optionally* show the [*field names*][node-field-names] associated with each child + node. To include field names in your tests, you write a node's field name followed by a colon, before the node itself + in the S-expression: + +```query +(source_file + (function_definition + name: (identifier) + parameters: (parameter_list) + result: (primitive_type) + body: (block + (return_statement (number))))) +``` + +* If your language's syntax conflicts with the `===` and `---` test separators, you can optionally add an arbitrary identical +suffix (in the below example, `|||`) to disambiguate them: + +```text +==================||| +Basic module +==================||| + +---- MODULE Test ---- +increment(n) == n + 1 +==== + +---||| + +(source_file + (module (identifier) + (operator (identifier) + (parameter_list (identifier)) + (plus (identifier_ref) (number))))) +``` + +These tests are important. They serve as the parser's API documentation, and they can be run every time you change the grammar +to verify that everything still parses correctly. + +By default, the `tree-sitter test` command runs all the tests in your `test/corpus/` folder. To run a particular test, you +can use the `-i` flag: + +```sh +tree-sitter test -i 'Return statements' +``` + +The recommendation is to be comprehensive in adding tests. If it's a visible node, add it to a test file in your `test/corpus` +directory. It's typically a good idea to test all the permutations of each language construct. This increases test coverage, +but doubly acquaints readers with a way to examine expected outputs and understand the "edges" of a language. + +```admonish tip +After modifying the grammar, you can run `tree-sitter test -u` +to update all syntax trees in corpus files with current parser output. +``` + +## Attributes + +Tests can be annotated with a few `attributes`. Attributes must be put in the header, below the test name, and start with +a `:`. A couple of attributes also take in a parameter, which require the use of parenthesis. + +```admonish tip +If you'd like to supply in multiple parameters, e.g. to run tests on multiple platforms or to test multiple languages, +you can repeat the attribute on a new line. +``` + +The following attributes are available: + +* `:cst` - This attribute specifies that the expected output should be in the form of a CST instead of the normal S-expression. +This CST matches the format given by `parse --cst`. +* `:error` — This attribute will assert that the parse tree contains an error. It's useful to just validate that a certain +input is invalid without displaying the whole parse tree, as such you should omit the parse tree below the `---` line. +* `:fail-fast` — This attribute will stop the testing of additional cases if the test marked with this attribute fails. +* `:language(LANG)` — This attribute will run the tests using the parser for the specified language. This is useful for +multi-parser repos, such as XML and DTD, or Typescript and TSX. The default parser used will always be the first entry in +the `grammars` field in the `tree-sitter.json` config file, so having a way to pick a second or even third parser is useful. +* `:platform(PLATFORM)` — This attribute specifies the platform on which the test should run. It is useful to test platform-specific +behavior (e.g. Windows newlines are different from Unix). This attribute must match up with Rust's [`std::env::consts::OS`][constants]. +* `:skip` — This attribute will skip the test when running `tree-sitter test`. +This is useful when you want to temporarily disable running a test without deleting it. + +Examples using attributes: + +```text +========================= +Test that will be skipped +:skip +========================= + +int main() {} + +------------------------- + +==================================== +Test that will run on Linux or macOS + +:platform(linux) +:platform(macos) +==================================== + +int main() {} + +------------------------------------ + +======================================================================== +Test that expects an error, and will fail fast if there's no parse error +:fail-fast +:error +======================================================================== + +int main ( {} + +------------------------------------------------------------------------ + +================================================= +Test that will parse with both Typescript and TSX +:language(typescript) +:language(tsx) +================================================= + +console.log('Hello, world!'); + +------------------------------------------------- +``` + +### Automatic Compilation + +You might notice that the first time you run `tree-sitter test` after regenerating your parser, it takes some extra time. +This is because Tree-sitter automatically compiles your C code into a dynamically-loadable library. It recompiles your parser +as-needed whenever you update it by re-running `tree-sitter generate`, or whenever the [external scanner][external-scanners] +file is changed. + +[constants]: https://doc.rust-lang.org/std/env/consts/constant.OS.html +[external-scanners]: ./4-external-scanners.md +[node-field-names]: ../using-parsers/2-basic-parsing.md#node-field-names +[s-exp]: https://en.wikipedia.org/wiki/S-expression +[named-vs-anonymous-nodes]: ../using-parsers/2-basic-parsing.md#named-vs-anonymous-nodes diff --git a/docs/src/creating-parsers/6-publishing.md b/docs/src/creating-parsers/6-publishing.md new file mode 100644 index 00000000..af961422 --- /dev/null +++ b/docs/src/creating-parsers/6-publishing.md @@ -0,0 +1,49 @@ +# Publishing your grammar + +Once you feel that your parser is in a stable working state for consumers to use, you can publish it to various registries. +It's strongly recommended to publish grammars to GitHub, [crates.io][crates.io] (Rust), [npm][npm] (JavaScript), and [PyPI][pypi] +(Python) to make it easier for others to find and use your grammar. + +If your grammar is hosted on GitHub, you can make use of our [reusable workflows][workflows] to handle the publishing process +for you. This action will automatically handle regenerating and publishing your grammar in CI, so long as you have the required +tokens setup for the various registries. For an example of this workflow in action, see the [Python grammar's GitHub][python-gh] + +## From start to finish + +To release a new grammar (or publish your first version), these are the steps you should follow: + +1. Bump your version to the desired version with `tree-sitter version`. For example, if you're releasing version `1.0.0` +of your grammar, you'd run `tree-sitter version 1.0.0`. +2. Commit the changes with `git commit -am "Release 1.0.0" (or however you like)` (ensure that your working directory is +clean). +3. Tag the commit with `git tag -- v1.0.0`. +4. Push the commit and tag with `git push --tags origin main` (assuming you're on the `main` branch, and `origin` is your +remote). +5. (optional) If you've set up the GitHub workflows for your grammar, the release will be automatically published to GitHub, +crates.io, npm, and PyPI. + +### Adhering to Semantic Versioning + +When releasing new versions of your grammar, it's important to adhere to [Semantic Versioning][semver]. This ensures that +consumers can predictably update their dependencies and that their existing tree-sitter integrations (queries, tree traversal +code, node type checks) will continue to work as expected when upgrading. + +1. Increment the major version when you make incompatible changes to the grammar's node types or structure +2. Increment the minor version when you add new node types or patterns while maintaining backward compatibility +3. Increment the patch version when you fix bugs without changing the grammar's structure + +For grammars in version 0.y.z (zero version), the usual semantic versioning rules are technically relaxed. However, if your +grammar already has users, it's recommended to treat version changes more conservatively: + +- Treat patch version (`z`) changes as if they were minor version changes +- Treat minor version (`y`) changes as if they were major version changes + +This helps maintain stability for existing users during the pre-1.0 phase. By following these versioning guidelines, you +ensure that downstream users can safely upgrade without their existing queries breaking. + +[crates.io]: https://crates.io +[npm]: https://www.npmjs.com +[pypi]: https://pypi.org +[python-gh]: https://github.com/tree-sitter/tree-sitter-python/blob/master/.github/workflows/publish.yml +[semver]: https://semver.org/ +[workflows]: https://github.com/tree-sitter/workflows diff --git a/docs/src/creating-parsers/index.md b/docs/src/creating-parsers/index.md new file mode 100644 index 00000000..4fb2c112 --- /dev/null +++ b/docs/src/creating-parsers/index.md @@ -0,0 +1,4 @@ +# Creating parsers + +Developing Tree-sitter grammars can have a difficult learning curve, but once you get the hang of it, it can be fun and +even zen-like. This document will help you to get started and to develop a useful mental model. diff --git a/docs/src/index.md b/docs/src/index.md new file mode 100644 index 00000000..ee92966a --- /dev/null +++ b/docs/src/index.md @@ -0,0 +1,106 @@ +
+ Tree-sitter logo +
+ +# Introduction + +Tree-sitter is a parser generator tool and an incremental parsing library. It can build a concrete syntax tree for a source +file and efficiently update the syntax tree as the source file is edited. Tree-sitter aims to be: + +- **General** enough to parse any programming language +- **Fast** enough to parse on every keystroke in a text editor +- **Robust** enough to provide useful results even in the presence of syntax errors +- **Dependency-free** so that the runtime library (which is written in pure [C11](https://github.com/tree-sitter/tree-sitter/tree/master/lib)) +can be embedded in any application + +## Language Bindings + +There are bindings that allow Tree-sitter to be used from the following languages: + +### Official + +- [C#](https://github.com/tree-sitter/csharp-tree-sitter) +- [Go](https://github.com/tree-sitter/go-tree-sitter) +- [Haskell](https://github.com/tree-sitter/haskell-tree-sitter) +- [Java (JDK 22+)](https://github.com/tree-sitter/java-tree-sitter) +- [JavaScript (Node.js)](https://github.com/tree-sitter/node-tree-sitter) +- [JavaScript (Wasm)](https://github.com/tree-sitter/tree-sitter/tree/master/lib/binding_web) +- [Kotlin](https://github.com/tree-sitter/kotlin-tree-sitter) +- [Python](https://github.com/tree-sitter/py-tree-sitter) +- [Rust](https://github.com/tree-sitter/tree-sitter/tree/master/lib/binding_rust) +- [Swift](https://github.com/tree-sitter/swift-tree-sitter) +- [Zig](https://github.com/tree-sitter/zig-tree-sitter) + +### Third-party + +- [C# (.NET)](https://github.com/zabbius/dotnet-tree-sitter) +- [C++](https://github.com/nsumner/cpp-tree-sitter) +- [Crystal](https://github.com/crystal-lang-tools/crystal-tree-sitter) +- [D](https://github.com/aminya/d-tree-sitter) +- [Delphi](https://github.com/modersohn/delphi-tree-sitter) +- [ELisp](https://www.gnu.org/software/emacs/manual/html_node/elisp/Parsing-Program-Source.html) +- [Go](https://github.com/alexaandru/go-tree-sitter-bare) +- [Guile](https://github.com/Z572/guile-ts) +- [Janet](https://github.com/sogaiu/janet-tree-sitter) +- [Java (JDK 8+)](https://github.com/bonede/tree-sitter-ng) +- [Java (JDK 11+)](https://github.com/seart-group/java-tree-sitter) +- [Julia](https://github.com/MichaelHatherly/TreeSitter.jl) +- [Lua](https://github.com/euclidianAce/ltreesitter) +- [Lua](https://github.com/xcb-xwii/lua-tree-sitter) +- [OCaml](https://github.com/semgrep/ocaml-tree-sitter-core) +- [Odin](https://github.com/laytan/odin-tree-sitter) +- [Perl](https://metacpan.org/pod/Text::Treesitter) +- [Pharo](https://github.com/Evref-BL/Pharo-Tree-Sitter) +- [PHP](https://github.com/soulseekah/ext-treesitter) +- [R](https://github.com/DavisVaughan/r-tree-sitter) +- [Ruby](https://github.com/Faveod/ruby-tree-sitter) + +_Keep in mind that some of the bindings may be incomplete or out of date._ + +## Parsers + +The following parsers can be found in the upstream organization: + +- [Agda](https://github.com/tree-sitter/tree-sitter-agda) +- [Bash](https://github.com/tree-sitter/tree-sitter-bash) +- [C](https://github.com/tree-sitter/tree-sitter-c) +- [C++](https://github.com/tree-sitter/tree-sitter-cpp) +- [C#](https://github.com/tree-sitter/tree-sitter-c-sharp) +- [CSS](https://github.com/tree-sitter/tree-sitter-css) +- [ERB / EJS](https://github.com/tree-sitter/tree-sitter-embedded-template) +- [Go](https://github.com/tree-sitter/tree-sitter-go) +- [Haskell](https://github.com/tree-sitter/tree-sitter-haskell) +- [HTML](https://github.com/tree-sitter/tree-sitter-html) +- [Java](https://github.com/tree-sitter/tree-sitter-java) +- [JavaScript](https://github.com/tree-sitter/tree-sitter-javascript) +- [JSDoc](https://github.com/tree-sitter/tree-sitter-jsdoc) +- [JSON](https://github.com/tree-sitter/tree-sitter-json) +- [Julia](https://github.com/tree-sitter/tree-sitter-julia) +- [OCaml](https://github.com/tree-sitter/tree-sitter-ocaml) +- [PHP](https://github.com/tree-sitter/tree-sitter-php) +- [Python](https://github.com/tree-sitter/tree-sitter-python) +- [Regex](https://github.com/tree-sitter/tree-sitter-regex) +- [Ruby](https://github.com/tree-sitter/tree-sitter-ruby) +- [Rust](https://github.com/tree-sitter/tree-sitter-rust) +- [Scala](https://github.com/tree-sitter/tree-sitter-scala) +- [TypeScript](https://github.com/tree-sitter/tree-sitter-typescript) +- [Verilog](https://github.com/tree-sitter/tree-sitter-verilog) + +A list of known parsers can be found in the [wiki](https://github.com/tree-sitter/tree-sitter/wiki/List-of-parsers). + +## Talks on Tree-sitter + +- [Strange Loop 2018](https://www.thestrangeloop.com/2018/tree-sitter---a-new-parsing-system-for-programming-tools.html) +- [FOSDEM 2018](https://www.youtube.com/watch?v=0CGzC_iss-8) +- [GitHub Universe 2017](https://www.youtube.com/watch?v=a1rC79DHpmY) + +## Underlying Research + +The design of Tree-sitter was greatly influenced by the following research papers: + +- [Practical Algorithms for Incremental Software Development Environments](https://www2.eecs.berkeley.edu/Pubs/TechRpts/1997/CSD-97-946.pdf) +- [Context Aware Scanning for Parsing Extensible Languages](https://www-users.cse.umn.edu/~evw/pubs/vanwyk07gpce/vanwyk07gpce.pdf) +- [Efficient and Flexible Incremental Parsing](https://harmonia.cs.berkeley.edu/papers/twagner-parsing.pdf) +- [Incremental Analysis of Real Programming Languages](https://harmonia.cs.berkeley.edu/papers/twagner-glr.pdf) +- [Error Detection and Recovery in LR Parsers](https://web.archive.org/web/20240302031213/https://what-when-how.com/compiler-writing/bottom-up-parsing-compiler-writing-part-13) +- [Error Recovery for LR Parsers](https://apps.dtic.mil/sti/pdfs/ADA043470.pdf) diff --git a/docs/src/using-parsers/1-getting-started.md b/docs/src/using-parsers/1-getting-started.md new file mode 100644 index 00000000..803cc3e8 --- /dev/null +++ b/docs/src/using-parsers/1-getting-started.md @@ -0,0 +1,134 @@ +# Getting Started + +## Building the Library + +To build the library on a POSIX system, just run `make` in the Tree-sitter directory. This will create a static library +called `libtree-sitter.a` as well as dynamic libraries. + +Alternatively, you can incorporate the library in a larger project's build system by adding one source file to the build. +This source file needs two directories to be in the include path when compiled: + +**source file:** + +- `tree-sitter/lib/src/lib.c` + +**include directories:** + +- `tree-sitter/lib/src` +- `tree-sitter/lib/include` + +## The Basic Objects + +There are four main types of objects involved when using Tree-sitter: languages, parsers, syntax trees, and syntax nodes. +In C, these are called `TSLanguage`, `TSParser`, `TSTree`, and `TSNode`. + +- A `TSLanguage` is an opaque object that defines how to parse a particular programming language. The code for each `TSLanguage` +is generated by Tree-sitter. Many languages are already available in separate git repositories within the +[Tree-sitter GitHub organization][ts org] and the [Tree-sitter grammars GitHub organization][tsg org]. +See [the next section][creating parsers] for how to create new languages. + +- A `TSParser` is a stateful object that can be assigned a `TSLanguage` and used to produce a `TSTree` based on some +source code. + +- A `TSTree` represents the syntax tree of an entire source code file. It contains `TSNode` instances that indicate the +structure of the source code. It can also be edited and used to produce a new `TSTree` in the event that the +source code changes. + +- A `TSNode` represents a single node in the syntax tree. It tracks its start and end positions in the source code, as +well as its relation to other nodes like its parent, siblings and children. + +## An Example Program + +Here's an example of a simple C program that uses the Tree-sitter [JSON parser][json]. + +```c +// Filename - test-json-parser.c + +#include +#include +#include +#include + +// Declare the `tree_sitter_json` function, which is +// implemented by the `tree-sitter-json` library. +const TSLanguage *tree_sitter_json(void); + +int main() { + // Create a parser. + TSParser *parser = ts_parser_new(); + + // Set the parser's language (JSON in this case). + ts_parser_set_language(parser, tree_sitter_json()); + + // Build a syntax tree based on source code stored in a string. + const char *source_code = "[1, null]"; + TSTree *tree = ts_parser_parse_string( + parser, + NULL, + source_code, + strlen(source_code) + ); + + // Get the root node of the syntax tree. + TSNode root_node = ts_tree_root_node(tree); + + // Get some child nodes. + TSNode array_node = ts_node_named_child(root_node, 0); + TSNode number_node = ts_node_named_child(array_node, 0); + + // Check that the nodes have the expected types. + assert(strcmp(ts_node_type(root_node), "document") == 0); + assert(strcmp(ts_node_type(array_node), "array") == 0); + assert(strcmp(ts_node_type(number_node), "number") == 0); + + // Check that the nodes have the expected child counts. + assert(ts_node_child_count(root_node) == 1); + assert(ts_node_child_count(array_node) == 5); + assert(ts_node_named_child_count(array_node) == 2); + assert(ts_node_child_count(number_node) == 0); + + // Print the syntax tree as an S-expression. + char *string = ts_node_string(root_node); + printf("Syntax tree: %s\n", string); + + // Free all of the heap-allocated memory. + free(string); + ts_tree_delete(tree); + ts_parser_delete(parser); + return 0; +} +``` + +This program requires three components to build: + +1. The Tree-sitter C API from `tree-sitter/api.h` (requiring `tree-sitter/lib/include` in our include path) +2. The Tree-sitter library (`libtree-sitter.a`) +3. The JSON grammar's source code, which we compile directly into the binary + +```sh +clang \ + -I tree-sitter/lib/include \ + test-json-parser.c \ + tree-sitter-json/src/parser.c \ + tree-sitter/libtree-sitter.a \ + -o test-json-parser +./test-json-parser +``` + +When using dynamic linking, you'll need to ensure the shared library is discoverable through `LD_LIBRARY_PATH` or your system's +equivalent environment variable. Here's how to compile with dynamic linking: + +```sh +clang \ + -I tree-sitter/lib/include \ + test-json-parser.c \ + tree-sitter-json/src/parser.c \ + -ltree-sitter \ + -o test-json-parser +./test-json-parser +``` + +[creating parsers]: ../creating-parsers/index.md +[json]: https://github.com/tree-sitter/tree-sitter-json +[ts org]: https://github.com/tree-sitter +[tsg org]: https://github.com/tree-sitter-grammars diff --git a/docs/src/using-parsers/2-basic-parsing.md b/docs/src/using-parsers/2-basic-parsing.md new file mode 100644 index 00000000..8c425d6b --- /dev/null +++ b/docs/src/using-parsers/2-basic-parsing.md @@ -0,0 +1,197 @@ +# Basic Parsing + +## Providing the Code + +In the example on the previous page, we parsed source code stored in a simple string using the `ts_parser_parse_string` +function: + +```c +TSTree *ts_parser_parse_string( + TSParser *self, + const TSTree *old_tree, + const char *string, + uint32_t length +); +``` + +You may want to parse source code that's stored in a custom data structure, like a [piece table][piece table] or a [rope][rope]. +In this case, you can use the more general `ts_parser_parse` function: + +```c +TSTree *ts_parser_parse( + TSParser *self, + const TSTree *old_tree, + TSInput input +); +``` + +The `TSInput` structure lets you provide your own function for reading a chunk of text at a given byte offset and row/column +position. The function can return text encoded in either UTF-8 or UTF-16. This interface allows you to efficiently parse +text that is stored in your own data structure. + +```c +typedef struct { + void *payload; + const char *(*read)( + void *payload, + uint32_t byte_offset, + TSPoint position, + uint32_t *bytes_read + ); + TSInputEncoding encoding; + TSDecodeFunction decode; +} TSInput; +``` + +If you want to decode text that is not encoded in UTF-8 or UTF-16, you can set the `decode` field of the input to your function +that will decode text. The signature of the `TSDecodeFunction` is as follows: + +```c +typedef uint32_t (*TSDecodeFunction)( + const uint8_t *string, + uint32_t length, + int32_t *code_point +); +``` + +```admonish attention +The `TSInputEncoding` must be set to `TSInputEncodingCustom` for the `decode` function to be called. +``` + +The `string` argument is a pointer to the text to decode, which comes from the `read` function, and the `length` argument +is the length of the `string`. The `code_point` argument is a pointer to an integer that represents the decoded code point, +and should be written to in your `decode` callback. The function should return the number of bytes decoded. + +## Syntax Nodes + +Tree-sitter provides a [DOM][dom]-style interface for inspecting syntax trees. +A syntax node's _type_ is a string that indicates which grammar rule the node represents. + +```c +const char *ts_node_type(TSNode); +``` + +Syntax nodes store their position in the source code both in raw bytes and row/column +coordinates. In a point, rows and columns are zero-based. The `row` field represents +the number of newlines before a given position, while `column` represents the number +of bytes between the position and beginning of the line. + +```c +uint32_t ts_node_start_byte(TSNode); +uint32_t ts_node_end_byte(TSNode); +typedef struct { + uint32_t row; + uint32_t column; +} TSPoint; +TSPoint ts_node_start_point(TSNode); +TSPoint ts_node_end_point(TSNode); +``` + +```admonish note +A *newline* is considered to be a single line feed (`\n`) character. +``` + +## Retrieving Nodes + +Every tree has a _root node_: + +```c +TSNode ts_tree_root_node(const TSTree *); +``` + +Once you have a node, you can access the node's children: + +```c +uint32_t ts_node_child_count(TSNode); +TSNode ts_node_child(TSNode, uint32_t); +``` + +You can also access its siblings and parent: + +```c +TSNode ts_node_next_sibling(TSNode); +TSNode ts_node_prev_sibling(TSNode); +TSNode ts_node_parent(TSNode); +``` + +These methods may all return a _null node_ to indicate, for example, that a node does not _have_ a next sibling. +You can check if a node is null: + +```c +bool ts_node_is_null(TSNode); +``` + +## Named vs Anonymous Nodes + +Tree-sitter produces [_concrete_ syntax trees][cst] — trees that contain nodes for +every individual token in the source code, including things like commas and parentheses. This is important for use-cases +that deal with individual tokens, like [syntax highlighting][syntax highlighting]. But some +types of code analysis are easier to perform using an [_abstract_ syntax tree][ast] — a tree in which the less important +details have been removed. Tree-sitter's trees support these use cases by making a distinction between +_named_ and _anonymous_ nodes. + +Consider a grammar rule like this: + +```js +if_statement: $ => seq("if", "(", $._expression, ")", $._statement); +``` + +A syntax node representing an `if_statement` in this language would have 5 children: the condition expression, the body +statement, as well as the `if`, `(`, and `)` tokens. The expression and the statement would be marked as _named_ nodes, +because they have been given explicit names in the grammar. But the `if`, `(`, and `)` nodes would _not_ be named nodes, +because they are represented in the grammar as simple strings. + +You can check whether any given node is named: + +```c +bool ts_node_is_named(TSNode); +``` + +When traversing the tree, you can also choose to skip over anonymous nodes by using the `_named_` variants of all of the +methods described above: + +```c +TSNode ts_node_named_child(TSNode, uint32_t); +uint32_t ts_node_named_child_count(TSNode); +TSNode ts_node_next_named_sibling(TSNode); +TSNode ts_node_prev_named_sibling(TSNode); +``` + +If you use this group of methods, the syntax tree functions much like an abstract syntax tree. + +## Node Field Names + +To make syntax nodes easier to analyze, many grammars assign unique _field names_ to particular child nodes. +In the [creating parsers][using fields] section, it's explained how to do this in your own grammars. If a syntax node has +fields, you can access its children using their field name: + +```c +TSNode ts_node_child_by_field_name( + TSNode self, + const char *field_name, + uint32_t field_name_length +); +``` + +Fields also have numeric ids that you can use, if you want to avoid repeated string comparisons. You can convert between +strings and ids using the `TSLanguage`: + +```c +uint32_t ts_language_field_count(const TSLanguage *); +const char *ts_language_field_name_for_id(const TSLanguage *, TSFieldId); +TSFieldId ts_language_field_id_for_name(const TSLanguage *, const char *, uint32_t); +``` + +The field ids can be used in place of the name: + +```c +TSNode ts_node_child_by_field_id(TSNode, TSFieldId); +``` + +[ast]: https://en.wikipedia.org/wiki/Abstract_syntax_tree +[cst]: https://en.wikipedia.org/wiki/Parse_tree +[dom]: https://en.wikipedia.org/wiki/Document_Object_Model +[piece table]: +[rope]: +[syntax highlighting]: https://en.wikipedia.org/wiki/Syntax_highlighting +[using fields]: ../creating-parsers/3-writing-the-grammar.md#using-fields diff --git a/docs/src/using-parsers/3-advanced-parsing.md b/docs/src/using-parsers/3-advanced-parsing.md new file mode 100644 index 00000000..c1c92e24 --- /dev/null +++ b/docs/src/using-parsers/3-advanced-parsing.md @@ -0,0 +1,164 @@ +# Advanced Parsing + +## Editing + +In applications like text editors, you often need to re-parse a file after its source code has changed. Tree-sitter is designed +to support this use case efficiently. There are two steps required. First, you must _edit_ the syntax tree, which adjusts +the ranges of its nodes so that they stay in sync with the code. + +```c +typedef struct { + uint32_t start_byte; + uint32_t old_end_byte; + uint32_t new_end_byte; + TSPoint start_point; + TSPoint old_end_point; + TSPoint new_end_point; +} TSInputEdit; + +void ts_tree_edit(TSTree *, const TSInputEdit *); +``` + +Then, you can call `ts_parser_parse` again, passing in the old tree. This will create a new tree that internally shares +structure with the old tree. + +When you edit a syntax tree, the positions of its nodes will change. If you have stored any `TSNode` instances outside of +the `TSTree`, you must update their positions separately, using the same `TSInputEdit` value, in order to update their +cached positions. + +```c +void ts_node_edit(TSNode *, const TSInputEdit *); +``` + +This `ts_node_edit` function is _only_ needed in the case where you have retrieved `TSNode` instances _before_ editing the +tree, and then _after_ editing the tree, you want to continue to use those specific node instances. Often, you'll just want +to re-fetch nodes from the edited tree, in which case `ts_node_edit` is not needed. + +## Multi-language Documents + +Sometimes, different parts of a file may be written in different languages. For example, templating languages like [EJS][ejs] +and [ERB][erb] allow you to generate HTML by writing a mixture of HTML and another language like JavaScript or Ruby. + +Tree-sitter handles these types of documents by allowing you to create a syntax tree based on the text in certain +_ranges_ of a file. + +```c +typedef struct { + TSPoint start_point; + TSPoint end_point; + uint32_t start_byte; + uint32_t end_byte; +} TSRange; + +void ts_parser_set_included_ranges( + TSParser *self, + const TSRange *ranges, + uint32_t range_count +); +``` + +For example, consider this ERB document: + +```erb +
    + <% people.each do |person| %> +
  • <%= person.name %>
  • + <% end %> +
+``` + +Conceptually, it can be represented by three syntax trees with overlapping ranges: an ERB syntax tree, a Ruby syntax tree, +and an HTML syntax tree. You could generate these syntax trees with the following code: + +```c +#include +#include + +// These functions are each implemented in their own repo. +const TSLanguage *tree_sitter_embedded_template(void); +const TSLanguage *tree_sitter_html(void); +const TSLanguage *tree_sitter_ruby(void); + +int main(int argc, const char **argv) { + const char *text = argv[1]; + unsigned len = strlen(text); + + // Parse the entire text as ERB. + TSParser *parser = ts_parser_new(); + ts_parser_set_language(parser, tree_sitter_embedded_template()); + TSTree *erb_tree = ts_parser_parse_string(parser, NULL, text, len); + TSNode erb_root_node = ts_tree_root_node(erb_tree); + + // In the ERB syntax tree, find the ranges of the `content` nodes, + // which represent the underlying HTML, and the `code` nodes, which + // represent the interpolated Ruby. + TSRange html_ranges[10]; + TSRange ruby_ranges[10]; + unsigned html_range_count = 0; + unsigned ruby_range_count = 0; + unsigned child_count = ts_node_child_count(erb_root_node); + + for (unsigned i = 0; i < child_count; i++) { + TSNode node = ts_node_child(erb_root_node, i); + if (strcmp(ts_node_type(node), "content") == 0) { + html_ranges[html_range_count++] = (TSRange) { + ts_node_start_point(node), + ts_node_end_point(node), + ts_node_start_byte(node), + ts_node_end_byte(node), + }; + } else { + TSNode code_node = ts_node_named_child(node, 0); + ruby_ranges[ruby_range_count++] = (TSRange) { + ts_node_start_point(code_node), + ts_node_end_point(code_node), + ts_node_start_byte(code_node), + ts_node_end_byte(code_node), + }; + } + } + + // Use the HTML ranges to parse the HTML. + ts_parser_set_language(parser, tree_sitter_html()); + ts_parser_set_included_ranges(parser, html_ranges, html_range_count); + TSTree *html_tree = ts_parser_parse_string(parser, NULL, text, len); + TSNode html_root_node = ts_tree_root_node(html_tree); + + // Use the Ruby ranges to parse the Ruby. + ts_parser_set_language(parser, tree_sitter_ruby()); + ts_parser_set_included_ranges(parser, ruby_ranges, ruby_range_count); + TSTree *ruby_tree = ts_parser_parse_string(parser, NULL, text, len); + TSNode ruby_root_node = ts_tree_root_node(ruby_tree); + + // Print all three trees. + char *erb_sexp = ts_node_string(erb_root_node); + char *html_sexp = ts_node_string(html_root_node); + char *ruby_sexp = ts_node_string(ruby_root_node); + printf("ERB: %s\n", erb_sexp); + printf("HTML: %s\n", html_sexp); + printf("Ruby: %s\n", ruby_sexp); + return 0; +} +``` + +This API allows for great flexibility in how languages can be composed. Tree-sitter is not responsible for mediating the +interactions between languages. Instead, you are free to do that using arbitrary application-specific logic. + +## Concurrency + +Tree-sitter supports multi-threaded use cases by making syntax trees very cheap to copy. + +```c +TSTree *ts_tree_copy(const TSTree *); +``` + +Internally, copying a syntax tree just entails incrementing an atomic reference count. Conceptually, it provides you a new +tree which you can freely query, edit, reparse, or delete on a new thread while continuing to use the original tree on a +different thread. + +```admonish danger +Individual `TSTree` instances are _not_ thread safe; you must copy a tree if you want to use it on multiple threads simultaneously. +``` + +[ejs]: https://ejs.co +[erb]: https://ruby-doc.org/stdlib-2.5.1/libdoc/erb/rdoc/ERB.html diff --git a/docs/src/using-parsers/4-walking-trees.md b/docs/src/using-parsers/4-walking-trees.md new file mode 100644 index 00000000..94ac4eb4 --- /dev/null +++ b/docs/src/using-parsers/4-walking-trees.md @@ -0,0 +1,41 @@ +# Walking Trees with Tree Cursors + +You can access every node in a syntax tree using the `TSNode` APIs [described earlier][retrieving nodes], but if you need +to access a large number of nodes, the fastest way to do so is with a _tree cursor_. A cursor is a stateful object that +allows you to walk a syntax tree with maximum efficiency. + +```admonish note +The given input node is considered the root of the cursor, and the cursor cannot walk outside this node. +Going to the parent or any sibling of the root node will always return `false`. + +This has no unexpected effects if the given input node is the actual `root` node of the tree, but is something to keep in +mind when using cursors constructed with a node that is not the `root` node. +``` + +You can initialize a cursor from any node: + +```c +TSTreeCursor ts_tree_cursor_new(TSNode); +``` + +You can move the cursor around the tree: + +```c +bool ts_tree_cursor_goto_first_child(TSTreeCursor *); +bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *); +bool ts_tree_cursor_goto_parent(TSTreeCursor *); +``` + +These methods return `true` if the cursor successfully moved and `false` if there was no node to move to. + +You can always retrieve the cursor's current node, as well as the [field name][node-field-names] that is associated with +the current node. + +```c +TSNode ts_tree_cursor_current_node(const TSTreeCursor *); +const char *ts_tree_cursor_current_field_name(const TSTreeCursor *); +TSFieldId ts_tree_cursor_current_field_id(const TSTreeCursor *); +``` + +[retrieving nodes]: ./2-basic-parsing.md#retrieving-nodes +[node-field-names]: ./2-basic-parsing.md#node-field-names diff --git a/docs/src/using-parsers/6-static-node-types.md b/docs/src/using-parsers/6-static-node-types.md new file mode 100644 index 00000000..171f4314 --- /dev/null +++ b/docs/src/using-parsers/6-static-node-types.md @@ -0,0 +1,162 @@ +# Static Node Types + +In languages with static typing, it can be helpful for syntax trees to provide specific type information about individual +syntax nodes. Tree-sitter makes this information available via a generated file called `node-types.json`. This _node types_ +file provides structured data about every possible syntax node in a grammar. + +You can use this data to generate type declarations in statically-typed programming languages. + +The node types file contains an array of objects, each of which describes a particular type of syntax node using the +following entries: + +## Basic Info + +Every object in this array has these two entries: + +- `"type"` — A string that indicates, which grammar rule the node represents. This corresponds to the `ts_node_type` function +described [here][syntax nodes]. +- `"named"` — A boolean that indicates whether this kind of node corresponds to a rule name in the grammar or just a string +literal. See [here][named-vs-anonymous-nodes] for more info. + +Examples: + +```json +{ + "type": "string_literal", + "named": true +} +{ + "type": "+", + "named": false +} +``` + +Together, these two fields constitute a unique identifier for a node type; no two top-level objects in the `node-types.json` +should have the same values for both `"type"` and `"named"`. + +## Internal Nodes + +Many syntax nodes can have _children_. The node type object describes the possible children that a node can have using the +following entries: + +- `"fields"` — An object that describes the possible [fields][node-field-names] that the node can have. The keys of this +object are field names, and the values are _child type_ objects, described below. +- `"children"` — Another _child type_ object that describes all the node's possible _named_ children _without_ fields. + +A _child type_ object describes a set of child nodes using the following entries: + +- `"required"` — A boolean indicating whether there is always _at least one_ node in this set. +- `"multiple"` — A boolean indicating whether there can be _multiple_ nodes in this set. +- `"types"`- An array of objects that represent the possible types of nodes in this set. Each object has two keys: `"type"` +and `"named"`, whose meanings are described above. + +Example with fields: + +```json +{ + "type": "method_definition", + "named": true, + "fields": { + "body": { + "multiple": false, + "required": true, + "types": [{ "type": "statement_block", "named": true }] + }, + "decorator": { + "multiple": true, + "required": false, + "types": [{ "type": "decorator", "named": true }] + }, + "name": { + "multiple": false, + "required": true, + "types": [ + { "type": "computed_property_name", "named": true }, + { "type": "property_identifier", "named": true } + ] + }, + "parameters": { + "multiple": false, + "required": true, + "types": [{ "type": "formal_parameters", "named": true }] + } + } +} +``` + +Example with children: + +```json +{ + "type": "array", + "named": true, + "fields": {}, + "children": { + "multiple": true, + "required": false, + "types": [ + { "type": "_expression", "named": true }, + { "type": "spread_element", "named": true } + ] + } +} +``` + +## Supertype Nodes + +In Tree-sitter grammars, there are usually certain rules that represent abstract _categories_ of syntax nodes (e.g. "expression", +"type", "declaration"). In the `grammar.js` file, these are often written as [hidden rules][hidden rules] +whose definition is a simple [`choice`][grammar dsl] where each member is just a single symbol. + +Normally, hidden rules are not mentioned in the node types file, since they don't appear in the syntax tree. But if you +add a hidden rule to the grammar's [`supertypes` list][grammar dsl], then it _will_ show up in the node types file, with +the following special entry: + +- `"subtypes"` — An array of objects that specify the _types_ of nodes that this 'supertype' node can wrap. + +Example: + +```json +{ + "type": "_declaration", + "named": true, + "subtypes": [ + { "type": "class_declaration", "named": true }, + { "type": "function_declaration", "named": true }, + { "type": "generator_function_declaration", "named": true }, + { "type": "lexical_declaration", "named": true }, + { "type": "variable_declaration", "named": true } + ] +} +``` + +Supertype nodes will also appear elsewhere in the node types file, as children of other node types, in a way that corresponds +with how the supertype rule was used in the grammar. This can make the node types much shorter and easier to read, because +a single supertype will take the place of multiple subtypes. + +Example: + +```json +{ + "type": "export_statement", + "named": true, + "fields": { + "declaration": { + "multiple": false, + "required": false, + "types": [{ "type": "_declaration", "named": true }] + }, + "source": { + "multiple": false, + "required": false, + "types": [{ "type": "string", "named": true }] + } + } +} +``` + +[grammar dsl]: ../creating-parsers/2-the-grammar-dsl.md +[hidden rules]: ../creating-parsers/3-writing-the-grammar.md#hiding-rules +[named-vs-anonymous-nodes]: ./2-basic-parsing.md#named-vs-anonymous-nodes +[node-field-names]: ./2-basic-parsing.md#node-field-names +[syntax nodes]: ./2-basic-parsing.md#syntax-nodes diff --git a/docs/src/using-parsers/7-abi-versions.md b/docs/src/using-parsers/7-abi-versions.md new file mode 100644 index 00000000..1db42a5d --- /dev/null +++ b/docs/src/using-parsers/7-abi-versions.md @@ -0,0 +1,25 @@ +# ABI versions + +Parsers generated with tree-sitter have an associated ABI version, which establishes hard compatibility boundaries +between the generated parser and the tree-sitter library. + +A given version of the tree-sitter library is only able to load parsers generated with supported ABI versions: + +| tree-sitter version | Min parser ABI version | Max parser ABI version | +|---------------------|------------------------|------------------------| +| 0.14 | 9 | 9 | +| >=0.15.0, <=0.15.7 | 9 | 10 | +| >=0.15.8, <=0.16 | 9 | 11 | +| 0.17, 0.18 | 9 | 12 | +| >=0.19, <=0.20.2 | 13 | 13 | +| >=0.20.3, <=0.24 | 13 | 14 | +| >=0.25 | 13 | 15 | + +By default, the tree-sitter CLI will generate parsers using the latest available ABI for that version, but an older ABI +(supported by the CLI) can be selected by passing the [`--abi` option][abi_option] to the `generate` command. + +Note that the ABI version range supported by the CLI can be smaller than for the library: When a new ABI version is released, +older versions will be phased out over a deprecation period, which starts with no longer being able to generate parsers +with the oldest ABI version. + +[abi_option]: ../cli/generate.md#--abi-version diff --git a/docs/src/using-parsers/index.md b/docs/src/using-parsers/index.md new file mode 100644 index 00000000..5b2c146d --- /dev/null +++ b/docs/src/using-parsers/index.md @@ -0,0 +1,27 @@ +# Using Parsers + +This guide covers the fundamental concepts of using Tree-sitter, which is applicable across all programming languages. +Although we'll explore some C-specific details that are valuable for direct C API usage or creating new language bindings, +the core concepts remain the same. + +Tree-sitter's parsing functionality is implemented through its C API, with all functions documented in the [tree_sitter/api.h][api.h] +header file, but if you're working in another language, you can use one of the following bindings found [here](../index.md#language-bindings), +each providing idiomatic access to Tree-sitter's functionality. Of these bindings, the official ones have their own API +doc hosted online at the following pages: + +- [Go][go] +- [Java] +- [JavaScript (Node.js)][javascript] +- [Kotlin][kotlin] +- [Python][python] +- [Rust][rust] +- [Zig][zig] + +[api.h]: https://github.com/tree-sitter/tree-sitter/blob/master/lib/include/tree_sitter/api.h +[go]: https://pkg.go.dev/github.com/tree-sitter/go-tree-sitter +[java]: https://tree-sitter.github.io/java-tree-sitter +[javascript]: https://tree-sitter.github.io/node-tree-sitter +[kotlin]: https://tree-sitter.github.io/kotlin-tree-sitter +[python]: https://tree-sitter.github.io/py-tree-sitter +[rust]: https://docs.rs/tree-sitter +[zig]: https://tree-sitter.github.io/zig-tree-sitter diff --git a/docs/src/using-parsers/queries/1-syntax.md b/docs/src/using-parsers/queries/1-syntax.md new file mode 100644 index 00000000..2e0a8853 --- /dev/null +++ b/docs/src/using-parsers/queries/1-syntax.md @@ -0,0 +1,127 @@ +# Query Syntax + +A _query_ consists of one or more _patterns_, where each pattern is an [S-expression][s-exp] that matches a certain set +of nodes in a syntax tree. The expression to match a given node consists of a pair of parentheses containing two things: +the node's type, and optionally, a series of other S-expressions that match the node's children. For example, this pattern +would match any `binary_expression` node whose children are both `number_literal` nodes: + +```query +(binary_expression (number_literal) (number_literal)) +``` + +Children can also be omitted. For example, this would match any `binary_expression` where at least _one_ of child is a +`string_literal` node: + +```query +(binary_expression (string_literal)) +``` + +## Fields + +In general, it's a good idea to make patterns more specific by specifying [field names][node-field-names] associated with +child nodes. You do this by prefixing a child pattern with a field name followed by a colon. For example, this pattern would +match an `assignment_expression` node where the `left` child is a `member_expression` whose `object` is a `call_expression`. + +```query +(assignment_expression + left: (member_expression + object: (call_expression))) +``` + +## Negated Fields + +You can also constrain a pattern so that it only matches nodes that _lack_ a certain field. To do this, add a field name +prefixed by a `!` within the parent pattern. For example, this pattern would match a class declaration with no type parameters: + +```query +(class_declaration + name: (identifier) @class_name + !type_parameters) +``` + +## Anonymous Nodes + +The parenthesized syntax for writing nodes only applies to [named nodes][named-vs-anonymous-nodes]. To match specific anonymous +nodes, you write their name between double quotes. For example, this pattern would match any `binary_expression` where the +operator is `!=` and the right side is `null`: + +```query +(binary_expression + operator: "!=" + right: (null)) +``` + +## Special Nodes + +### The Wildcard Node + +A wildcard node is represented with an underscore (`_`), it matches any node. +This is similar to `.` in regular expressions. +There are two types, `(_)` will match any named node, +and `_` will match any named or anonymous node. + +For example, this pattern would match any node inside a call: + +```query +(call (_) @call.inner) +``` + +### The `ERROR` Node + +When the parser encounters text it does not recognize, it represents this node +as `(ERROR)` in the syntax tree. These error nodes can be queried just like +normal nodes: + +```scheme +(ERROR) @error-node +``` + +### The `MISSING` Node + +If the parser is able to recover from erroneous text by inserting a missing token and then reducing, it will insert that +missing node in the final tree so long as that tree has the lowest error cost. These missing nodes appear as seemingly normal +nodes in the tree, but they are zero tokens wide, and are internally represented as a property of the actual terminal node +that was inserted, instead of being its own kind of node, like the `ERROR` node. These special missing nodes can be queried +using `(MISSING)`: + +```scheme +(MISSING) @missing-node +``` + +This is useful when attempting to detect all syntax errors in a given parse tree, since these missing node are not captured +by `(ERROR)` queries. Specific missing node types can also be queried: + +```scheme +(MISSING identifier) @missing-identifier +(MISSING ";") @missing-semicolon +``` + +### Supertype Nodes + +Some node types are marked as _supertypes_ in a grammar. A supertype is a node type that contains multiple +subtypes. For example, in the [JavaScript grammar example][grammar], `expression` is a supertype that can represent any +kind of expression, such as a `binary_expression`, `call_expression`, or `identifier`. You can use supertypes in queries +to match any of their subtypes, rather than having to list out each subtype individually. For example, this pattern would +match any kind of expression, even though it's not a visible node in the syntax tree: + +```query +(expression) @any-expression +``` + +To query specific subtypes of a supertype, you can use the syntax `supertype/subtype`. For example, this pattern would +match a `binary_expression` only if it is a child of `expression`: + +```query +(expression/binary_expression) @binary-expression +``` + +This also applies to anonymous nodes. For example, this pattern would match `"()"` only if it is a child of `expression`: + +```query +(expression/"()") @empty-expression +``` + +[grammar]: ../../creating-parsers/3-writing-the-grammar.md#structuring-rules-well +[node-field-names]: ../2-basic-parsing.md#node-field-names +[named-vs-anonymous-nodes]: ../2-basic-parsing.md#named-vs-anonymous-nodes +[s-exp]: https://en.wikipedia.org/wiki/S-expression diff --git a/docs/src/using-parsers/queries/2-operators.md b/docs/src/using-parsers/queries/2-operators.md new file mode 100644 index 00000000..6f9a8ca4 --- /dev/null +++ b/docs/src/using-parsers/queries/2-operators.md @@ -0,0 +1,151 @@ +# Operators + +## Capturing Nodes + +When matching patterns, you may want to process specific nodes within the pattern. Captures allow you to associate names +with specific nodes in a pattern, so that you can later refer to those nodes by those names. Capture names are written _after_ +the nodes that they refer to, and start with an `@` character. + +For example, this pattern would match any assignment of a `function` to an `identifier`, and it would associate the name +`the-function-name` with the identifier: + +```query +(assignment_expression + left: (identifier) @the-function-name + right: (function)) +``` + +And this pattern would match all method definitions, associating the name `the-method-name` with the method name, `the-class-name` +with the containing class name: + +```query +(class_declaration + name: (identifier) @the-class-name + body: (class_body + (method_definition + name: (property_identifier) @the-method-name))) +``` + +## Quantification Operators + +You can match a repeating sequence of sibling nodes using the postfix `+` and `*` _repetition_ operators, which work analogously +to the `+` and `*` operators [in regular expressions][regex]. The `+` operator matches _one or more_ repetitions of a pattern, +and the `*` operator matches _zero or more_. + +For example, this pattern would match a sequence of one or more comments: + +```query +(comment)+ +``` + +This pattern would match a class declaration, capturing all of the decorators if any were present: + +```query +(class_declaration + (decorator)* @the-decorator + name: (identifier) @the-name) +``` + +You can also mark a node as optional using the `?` operator. For example, this pattern would match all function calls, capturing +a string argument if one was present: + +```query +(call_expression + function: (identifier) @the-function + arguments: (arguments (string)? @the-string-arg)) +``` + +## Grouping Sibling Nodes + +You can also use parentheses for grouping a sequence of _sibling_ nodes. For example, this pattern would match a comment +followed by a function declaration: + +```query +( + (comment) + (function_declaration) +) +``` + +Any of the quantification operators mentioned above (`+`, `*`, and `?`) can also be applied to groups. For example, this +pattern would match a comma-separated series of numbers: + +```query +( + (number) + ("," (number))* +) +``` + +## Alternations + +An alternation is written as a pair of square brackets (`[]`) containing a list of alternative patterns. +This is similar to _character classes_ from regular expressions (`[abc]` matches either a, b, or c). + +For example, this pattern would match a call to either a variable or an object property. +In the case of a variable, capture it as `@function`, and in the case of a property, capture it as `@method`: + +```query +(call_expression + function: [ + (identifier) @function + (member_expression + property: (property_identifier) @method) + ]) +``` + +This pattern would match a set of possible keyword tokens, capturing them as `@keyword`: + +```query +[ + "break" + "delete" + "else" + "for" + "function" + "if" + "return" + "try" + "while" +] @keyword +``` + +## Anchors + +The anchor operator, `.`, is used to constrain the ways in which child patterns are matched. It has different behaviors +depending on where it's placed inside a query. + +When `.` is placed before the _first_ child within a parent pattern, the child will only match when it is the first named +node in the parent. For example, the below pattern matches a given `array` node at most once, assigning the `@the-element` +capture to the first `identifier` node in the parent `array`: + +```query +(array . (identifier) @the-element) +``` + +Without this anchor, the pattern would match once for every identifier in the array, with `@the-element` bound +to each matched identifier. + +Similarly, an anchor placed after a pattern's _last_ child will cause that child pattern to only match nodes that are the +last named child of their parent. The below pattern matches only nodes that are the last named child within a `block`. + +```query +(block (_) @last-expression .) +``` + +Finally, an anchor _between_ two child patterns will cause the patterns to only match nodes that are immediate siblings. +The pattern below, given a long dotted name like `a.b.c.d`, will only match pairs of consecutive identifiers: +`a, b`, `b, c`, and `c, d`. + +```query +(dotted_name + (identifier) @prev-id + . + (identifier) @next-id) +``` + +Without the anchor, non-consecutive pairs like `a, c` and `b, d` would also be matched. + +The restrictions placed on a pattern by an anchor operator ignore anonymous nodes. + +[regex]: https://en.wikipedia.org/wiki/Regular_expression#Basic_concepts diff --git a/docs/src/using-parsers/queries/3-predicates-and-directives.md b/docs/src/using-parsers/queries/3-predicates-and-directives.md new file mode 100644 index 00000000..1f9aeada --- /dev/null +++ b/docs/src/using-parsers/queries/3-predicates-and-directives.md @@ -0,0 +1,201 @@ +# Predicates + +You can also specify arbitrary metadata and conditions associated with a pattern +by adding _predicate_ S-expressions anywhere within your pattern. Predicate S-expressions +start with a _predicate name_ beginning with a `#` character, and ending with a `?` character. After that, they can +contain an arbitrary number of `@`-prefixed capture names or strings. + +Tree-sitter's CLI supports the following predicates by default: + +## The `eq?` predicate + +This family of predicates allows you to match against a single capture or string +value. + +The first argument to this predicate must be a capture, but the second can be either a capture to +compare the two captures' text, or a string to compare first capture's text +against. + +The base predicate is `#eq?`, but its complement, `#not-eq?`, can be used to _not_ +match a value. Additionally, you can prefix either of these with `any-` to match +if _any_ of the nodes match the predicate. This is only useful when dealing with +quantified captures, as by default a quantified capture will only match if _all_ the captured nodes match the predicate. + +Thus, there are four predicates in total: + +- `#eq?` +- `#not-eq?` +- `#any-eq?` +- `#any-not-eq?` + +Consider the following example targeting C: + +```query +((identifier) @variable.builtin + (#eq? @variable.builtin "self")) +``` + +This pattern would match any identifier that is `self`. + +Now consider the following example: + +```query +( + (pair + key: (property_identifier) @key-name + value: (identifier) @value-name) + (#eq? @key-name @value-name) +) +``` + +This pattern would match key-value pairs where the `value` is an identifier +with the same text as the key (meaning they are the same): + +As mentioned earlier, the `any-` prefix is meant for use with quantified captures. Here's +an example finding an empty comment within a group of comments: + +```query +((comment)+ @comment.empty + (#any-eq? @comment.empty "//")) +``` + +## The `match?` predicate + +These predicates are similar to the `eq?` predicates, but they use regular expressions +to match against the capture's text instead of string comparisons. + +The first argument must be a capture, and the second must be a string containing +a regular expression. + +Like the `eq?` predicate family, we can tack on `not-` to the beginning of the predicate +to negate the match, and `any-` to match if _any_ of the nodes in a quantified capture match the predicate. + +This pattern matches identifiers written in `SCREAMING_SNAKE_CASE`. + +```query +((identifier) @constant + (#match? @constant "^[A-Z][A-Z_]+")) +``` + +This query identifies documentation comments in C that begin with three forward slashes (`///`). + +```query +((comment)+ @comment.documentation + (#match? @comment.documentation "^///\\s+.*")) +``` + +This query finds C code embedded in Go comments that appear just before a "C" import statement. +These are known as [`Cgo`][cgo] comments and are used to inject C code into Go programs. + +```query +((comment)+ @injection.content + . + (import_declaration + (import_spec path: (interpreted_string_literal) @_import_c)) + (#eq? @_import_c "\"C\"") + (#match? @injection.content "^//")) +``` + +## The `any-of?` predicate + +The `any-of?` predicate allows you to match a capture against multiple strings, +and will match if the capture's text is equal to any of the strings. + +The query below will match any of the builtin variables in JavaScript. + +```query +((identifier) @variable.builtin + (#any-of? @variable.builtin + "arguments" + "module" + "console" + "window" + "document")) +``` + +## The `is?` predicate + +The `is?` predicate allows you to assert that a capture has a given property. This isn't widely used, but the CLI uses it +to determine whether a given node is a local variable or not, for example: + +```query +((identifier) @variable.builtin + (#match? @variable.builtin "^(arguments|module|console|window|document)$") + (#is-not? local)) +``` + +This pattern would match any builtin variable that is not a local variable, because the `#is-not? local` predicate is used. + +# Directives + +Similar to predicates, directives are a way to associate arbitrary metadata with a pattern. The only difference between +predicates and directives is that directives end in a `!` character instead of `?` character. + +Tree-sitter's CLI supports the following directives by default: + +## The `set!` directive + +This directive allows you to associate key-value pairs with a pattern. The key and value can be any arbitrary text that +you see fit. + +```query +((comment) @injection.content + (#match? @injection.content "/[*\/][!*\/] for the CLI," + echo " and .#lib- for the library (e.g. nix build .#cli-aarch64-linux)." + echo "" + echo " Available targets:" + echo " aarch64-linux - ARM64 Linux" + echo " armv7l-linux - ARMv7 Linux" + echo " x86_64-linux - x86_64 Linux" + echo " i686-linux - i686 Linux" + echo " loongarch64 - LoongArch64 Linux" + echo " mips - MIPS Linux" + echo " mips64 - MIPS64 Linux" + echo " musl64 - x86_64 MUSL Linux" + echo " powerpc64-linux - PowerPC64 Linux" + echo " riscv32 - RISC-V 32-bit Linux" + echo " riscv64 - RISC-V 64-bit Linux" + echo " s390x - s390x Linux" + echo " x86_64-windows - x86_64 Windows" + echo " x86_64-darwin - x86_64 macOS (Darwin only)" + echo " aarch64-darwin - ARM64 macOS (Darwin only)" + echo "" + echo "Apps:" + echo " nix run .#cli - Run tree-sitter CLI" + echo " nix run .#docs - Serve docs locally" + echo " nix run .#format - Format all code" + echo " nix run .#lint - Run all linting checks" + echo "" + echo "Tests & Checks:" + echo " nix flake check - Run all tests and checks" + echo "" + echo "Version: ${version}" + ''; + + env = { + RUST_BACKTRACE = 1; + LIBCLANG_PATH = "${pkgs.libclang.lib}/lib"; + LLVM_COV = "${pkgs.llvm}/bin/llvm-cov"; + LLVM_PROFDATA = "${pkgs.llvm}/bin/llvm-profdata"; + TREE_SITTER_WASI_SDK_PATH = "${pkgs.pkgsCross.wasi32.stdenv.cc}"; + }; + }; + } + ); + }; +} diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 08455a91..64a9e8eb 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -4,10 +4,11 @@ version.workspace = true description = "Rust bindings to the Tree-sitter parsing library" authors.workspace = true edition.workspace = true -rust-version = "1.65" +rust-version = "1.77" readme = "binding_rust/README.md" homepage.workspace = true repository.workspace = true +documentation = "https://docs.rs/tree-sitter" license.workspace = true keywords.workspace = true categories = [ @@ -29,6 +30,7 @@ include = [ "/src/unicode/*", "/src/wasm/*", "/include/tree_sitter/api.h", + "/LICENSE", ] [package.metadata.docs.rs] @@ -45,21 +47,22 @@ std = ["regex/std", "regex/perf", "regex-syntax/unicode"] wasm = ["std", "wasmtime-c-api"] [dependencies] -regex = { version = "1.10.6", default-features = false, features = ["unicode"] } -regex-syntax = { version = "0.8.4", default-features = false } -tree-sitter-language = { version = "0.1", path = "language" } +regex = { version = "1.11.3", default-features = false, features = ["unicode"] } +regex-syntax = { version = "0.8.6", default-features = false } +tree-sitter-language.workspace = true streaming-iterator = "0.1.9" [dependencies.wasmtime-c-api] -version = "26.0.1" +version = "33.0.2" optional = true package = "wasmtime-c-api-impl" default-features = false -features = ["cranelift"] +features = ["cranelift", "gc-drc"] [build-dependencies] -bindgen = { version = "0.70.1", optional = true } +bindgen = { version = "0.72.0", optional = true } cc.workspace = true +serde_json.workspace = true [lib] path = "binding_rust/lib.rs" diff --git a/lib/LICENSE b/lib/LICENSE new file mode 100644 index 00000000..971b81f9 --- /dev/null +++ b/lib/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2018 Max Brunsfeld + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/lib/binding_rust/README.md b/lib/binding_rust/README.md index 602f647b..39c13b7c 100644 --- a/lib/binding_rust/README.md +++ b/lib/binding_rust/README.md @@ -17,13 +17,6 @@ use tree_sitter::{InputEdit, Language, Parser, Point}; let mut parser = Parser::new(); ``` -Add the `cc` crate to your `Cargo.toml` under `[build-dependencies]`: - -```toml -[build-dependencies] -cc="*" -``` - Then, add a language as a dependency: ```toml @@ -32,7 +25,7 @@ tree-sitter = "0.24" tree-sitter-rust = "0.23" ``` -To then use a language, you assign them to the parser. +To use a language, you assign them to the parser. ```rust parser.set_language(&tree_sitter_rust::LANGUAGE.into()).expect("Error loading Rust grammar"); @@ -105,6 +98,49 @@ assert_eq!( ); ``` +## Using Wasm Grammar Files + +> Requires the feature **wasm** to be enabled. + +First, create a parser with a Wasm store: + +```rust +use tree_sitter::{wasmtime::Engine, Parser, WasmStore}; + +let engine = Engine::default(); +let store = WasmStore::new(&engine).unwrap(); + +let mut parser = Parser::new(); +parser.set_wasm_store(store).unwrap(); +``` + +Then, load the language from a Wasm file: + +```rust +const JAVASCRIPT_GRAMMAR: &[u8] = include_bytes!("path/to/tree-sitter-javascript.wasm"); + +let mut store = WasmStore::new(&engine).unwrap(); +let javascript = store + .load_language("javascript", JAVASCRIPT_GRAMMAR) + .unwrap(); + +// The language may be loaded from a different WasmStore than the one set on +// the parser but it must use the same underlying WasmEngine. +parser.set_language(&javascript).unwrap(); +``` + +Now you can parse source code: + +```rust +let source_code = "let x = 1;"; +let tree = parser.parse(source_code, None).unwrap(); + +assert_eq!( + tree.root_node().to_sexp(), + "(program (lexical_declaration (variable_declarator name: (identifier) value: (number))))" +); +``` + [tree-sitter]: https://github.com/tree-sitter/tree-sitter ## Features diff --git a/lib/binding_rust/bindings.rs b/lib/binding_rust/bindings.rs index 807bf019..ec00a7c6 100644 --- a/lib/binding_rust/bindings.rs +++ b/lib/binding_rust/bindings.rs @@ -1,4 +1,4 @@ -/* automatically generated by rust-bindgen 0.70.1 */ +/* automatically generated by rust-bindgen 0.72.1 */ pub const TREE_SITTER_LANGUAGE_VERSION: u32 = 15; pub const TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION: u32 = 13; @@ -35,7 +35,7 @@ pub struct TSQueryCursor { pub struct TSLookaheadIterator { _unused: [u8; 0], } -pub type DecodeFunction = ::core::option::Option< +pub type TSDecodeFunction = ::core::option::Option< unsafe extern "C" fn(string: *const u8, length: u32, code_point: *mut i32) -> u32, >; pub const TSInputEncodingUTF8: TSInputEncoding = 0; @@ -75,13 +75,14 @@ pub struct TSInput { ) -> *const ::core::ffi::c_char, >, pub encoding: TSInputEncoding, - pub decode: DecodeFunction, + pub decode: TSDecodeFunction, } #[repr(C)] #[derive(Debug, Copy, Clone)] pub struct TSParseState { pub payload: *mut ::core::ffi::c_void, pub current_byte_offset: u32, + pub has_error: bool, } #[repr(C)] #[derive(Debug, Copy, Clone)] @@ -180,6 +181,14 @@ pub struct TSQueryCursorOptions { pub progress_callback: ::core::option::Option bool>, } +#[doc = " The metadata associated with a language.\n\n Currently, this metadata can be used to check the [Semantic Version](https://semver.org/)\n of the language. This version information should be used to signal if a given parser might\n be incompatible with existing queries when upgrading between major versions, or minor versions\n if it's in zerover."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct TSLanguageMetadata { + pub major_version: u8, + pub minor_version: u8, + pub patch_version: u8, +} extern "C" { #[doc = " Create a new parser."] pub fn ts_parser_new() -> *mut TSParser; @@ -193,7 +202,7 @@ extern "C" { pub fn ts_parser_language(self_: *const TSParser) -> *const TSLanguage; } extern "C" { - #[doc = " Set the language that the parser should use for parsing.\n\n Returns a boolean indicating whether or not the language was successfully\n assigned. True means assignment succeeded. False means there was a version\n mismatch: the language was generated with an incompatible version of the\n Tree-sitter CLI. Check the language's version using [`ts_language_version`]\n and compare it to this library's [`TREE_SITTER_LANGUAGE_VERSION`] and\n [`TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION`] constants."] + #[doc = " Set the language that the parser should use for parsing.\n\n Returns a boolean indicating whether or not the language was successfully\n assigned. True means assignment succeeded. False means there was a version\n mismatch: the language was generated with an incompatible version of the\n Tree-sitter CLI. Check the language's ABI version using [`ts_language_abi_version`]\n and compare it to this library's [`TREE_SITTER_LANGUAGE_VERSION`] and\n [`TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION`] constants."] pub fn ts_parser_set_language(self_: *mut TSParser, language: *const TSLanguage) -> bool; } extern "C" { @@ -209,7 +218,7 @@ extern "C" { pub fn ts_parser_included_ranges(self_: *const TSParser, count: *mut u32) -> *const TSRange; } extern "C" { - #[doc = " Use the parser to parse some source code and create a syntax tree.\n\n If you are parsing this document for the first time, pass `NULL` for the\n `old_tree` parameter. Otherwise, if you have already parsed an earlier\n version of this document and the document has since been edited, pass the\n previous syntax tree so that the unchanged parts of it can be reused.\n This will save time and memory. For this to work correctly, you must have\n already edited the old syntax tree using the [`ts_tree_edit`] function in a\n way that exactly matches the source code changes.\n\n The [`TSInput`] parameter lets you specify how to read the text. It has the\n following three fields:\n 1. [`read`]: A function to retrieve a chunk of text at a given byte offset\n and (row, column) position. The function should return a pointer to the\n text and write its length to the [`bytes_read`] pointer. The parser does\n not take ownership of this buffer; it just borrows it until it has\n finished reading it. The function should write a zero value to the\n [`bytes_read`] pointer to indicate the end of the document.\n 2. [`payload`]: An arbitrary pointer that will be passed to each invocation\n of the [`read`] function.\n 3. [`encoding`]: An indication of how the text is encoded. Either\n `TSInputEncodingUTF8` or `TSInputEncodingUTF16`.\n\n This function returns a syntax tree on success, and `NULL` on failure. There\n are four possible reasons for failure:\n 1. The parser does not have a language assigned. Check for this using the\n[`ts_parser_language`] function.\n 2. Parsing was cancelled due to a timeout that was set by an earlier call to\n the [`ts_parser_set_timeout_micros`] function. You can resume parsing from\n where the parser left out by calling [`ts_parser_parse`] again with the\n same arguments. Or you can start parsing from scratch by first calling\n [`ts_parser_reset`].\n 3. Parsing was cancelled using a cancellation flag that was set by an\n earlier call to [`ts_parser_set_cancellation_flag`]. You can resume parsing\n from where the parser left out by calling [`ts_parser_parse`] again with\n the same arguments.\n 4. Parsing was cancelled due to the progress callback returning true. This callback\n is passed in [`ts_parser_parse_with_options`] inside the [`TSParseOptions`] struct.\n\n [`read`]: TSInput::read\n [`payload`]: TSInput::payload\n [`encoding`]: TSInput::encoding\n [`bytes_read`]: TSInput::read"] + #[doc = " Use the parser to parse some source code and create a syntax tree.\n\n If you are parsing this document for the first time, pass `NULL` for the\n `old_tree` parameter. Otherwise, if you have already parsed an earlier\n version of this document and the document has since been edited, pass the\n previous syntax tree so that the unchanged parts of it can be reused.\n This will save time and memory. For this to work correctly, you must have\n already edited the old syntax tree using the [`ts_tree_edit`] function in a\n way that exactly matches the source code changes.\n\n The [`TSInput`] parameter lets you specify how to read the text. It has the\n following three fields:\n 1. [`read`]: A function to retrieve a chunk of text at a given byte offset\n and (row, column) position. The function should return a pointer to the\n text and write its length to the [`bytes_read`] pointer. The parser does\n not take ownership of this buffer; it just borrows it until it has\n finished reading it. The function should write a zero value to the\n [`bytes_read`] pointer to indicate the end of the document.\n 2. [`payload`]: An arbitrary pointer that will be passed to each invocation\n of the [`read`] function.\n 3. [`encoding`]: An indication of how the text is encoded. Either\n `TSInputEncodingUTF8` or `TSInputEncodingUTF16`.\n\n This function returns a syntax tree on success, and `NULL` on failure. There\n are four possible reasons for failure:\n 1. The parser does not have a language assigned. Check for this using the\n[`ts_parser_language`] function.\n 2. Parsing was cancelled due to the progress callback returning true. This callback\n is passed in [`ts_parser_parse_with_options`] inside the [`TSParseOptions`] struct.\n\n [`read`]: TSInput::read\n [`payload`]: TSInput::payload\n [`encoding`]: TSInput::encoding\n [`bytes_read`]: TSInput::read"] pub fn ts_parser_parse( self_: *mut TSParser, old_tree: *const TSTree, @@ -245,25 +254,9 @@ extern "C" { ) -> *mut TSTree; } extern "C" { - #[doc = " Instruct the parser to start the next parse from the beginning.\n\n If the parser previously failed because of a timeout or a cancellation, then\n by default, it will resume where it left off on the next call to\n [`ts_parser_parse`] or other parsing functions. If you don't want to resume,\n and instead intend to use this parser to parse some other document, you must\n call [`ts_parser_reset`] first."] + #[doc = " Instruct the parser to start the next parse from the beginning.\n\n If the parser previously failed because of the progress callback, then\n by default, it will resume where it left off on the next call to\n [`ts_parser_parse`] or other parsing functions. If you don't want to resume,\n and instead intend to use this parser to parse some other document, you must\n call [`ts_parser_reset`] first."] pub fn ts_parser_reset(self_: *mut TSParser); } -extern "C" { - #[doc = " @deprecated use [`ts_parser_parse_with_options`] and pass in a callback instead, this will be removed in 0.26.\n\n Set the maximum duration in microseconds that parsing should be allowed to\n take before halting.\n\n If parsing takes longer than this, it will halt early, returning NULL.\n See [`ts_parser_parse`] for more information."] - pub fn ts_parser_set_timeout_micros(self_: *mut TSParser, timeout_micros: u64); -} -extern "C" { - #[doc = " @deprecated use [`ts_parser_parse_with_options`] and pass in a callback instead, this will be removed in 0.26.\n\n Get the duration in microseconds that parsing is allowed to take."] - pub fn ts_parser_timeout_micros(self_: *const TSParser) -> u64; -} -extern "C" { - #[doc = " @deprecated use [`ts_parser_parse_with_options`] and pass in a callback instead, this will be removed in 0.26.\n\n Set the parser's current cancellation flag pointer.\n\n If a non-null pointer is assigned, then the parser will periodically read\n from this pointer during parsing. If it reads a non-zero value, it will\n halt early, returning NULL. See [`ts_parser_parse`] for more information."] - pub fn ts_parser_set_cancellation_flag(self_: *mut TSParser, flag: *const usize); -} -extern "C" { - #[doc = " @deprecated use [`ts_parser_parse_with_options`] and pass in a callback instead, this will be removed in 0.26.\n\n Get the parser's current cancellation flag pointer."] - pub fn ts_parser_cancellation_flag(self_: *const TSParser) -> *const usize; -} extern "C" { #[doc = " Set the logger that a parser should use during parsing.\n\n The parser does not take ownership over the logger payload. If a logger was\n previously assigned, the caller is responsible for releasing any memory\n owned by the previous logger."] pub fn ts_parser_set_logger(self_: *mut TSParser, logger: TSLogger); @@ -309,7 +302,7 @@ extern "C" { pub fn ts_tree_edit(self_: *mut TSTree, edit: *const TSInputEdit); } extern "C" { - #[doc = " Compare an old edited syntax tree to a new syntax tree representing the same\n document, returning an array of ranges whose syntactic structure has changed.\n\n For this to work correctly, the old syntax tree must have been edited such\n that its ranges match up to the new tree. Generally, you'll want to call\n this function right after calling one of the [`ts_parser_parse`] functions.\n You need to pass the old tree that was passed to parse, as well as the new\n tree that was returned from that function.\n\n The returned array is allocated using `malloc` and the caller is responsible\n for freeing it using `free`. The length of the array will be written to the\n given `length` pointer."] + #[doc = " Compare an old edited syntax tree to a new syntax tree representing the same\n document, returning an array of ranges whose syntactic structure has changed.\n\n For this to work correctly, the old syntax tree must have been edited such\n that its ranges match up to the new tree. Generally, you'll want to call\n this function right after calling one of the [`ts_parser_parse`] functions.\n You need to pass the old tree that was passed to parse, as well as the new\n tree that was returned from that function.\n\n The returned ranges indicate areas where the hierarchical structure of syntax\n nodes (from root to leaf) has changed between the old and new trees. Characters\n outside these ranges have identical ancestor nodes in both trees.\n\n Note that the returned ranges may be slightly larger than the exact changed areas,\n but Tree-sitter attempts to make them as small as possible.\n\n The returned array is allocated using `malloc` and the caller is responsible\n for freeing it using `free`. The length of the array will be written to the\n given `length` pointer."] pub fn ts_tree_get_changed_ranges( old_tree: *const TSTree, new_tree: *const TSTree, @@ -397,15 +390,11 @@ extern "C" { pub fn ts_node_next_parse_state(self_: TSNode) -> TSStateId; } extern "C" { - #[doc = " Get the node's immediate parent.\n Prefer [`ts_node_child_containing_descendant`] for\n iterating over the node's ancestors."] + #[doc = " Get the node's immediate parent.\n Prefer [`ts_node_child_with_descendant`] for\n iterating over the node's ancestors."] pub fn ts_node_parent(self_: TSNode) -> TSNode; } extern "C" { - #[doc = " @deprecated use [`ts_node_contains_descendant`] instead, this will be removed in 0.25\n\n Get the node's child containing `descendant`. This will not return\n the descendant if it is a direct child of `self`, for that use\n `ts_node_contains_descendant`."] - pub fn ts_node_child_containing_descendant(self_: TSNode, descendant: TSNode) -> TSNode; -} -extern "C" { - #[doc = " Get the node that contains `descendant`.\n\n Note that this can return `descendant` itself, unlike the deprecated function\n [`ts_node_child_containing_descendant`]."] + #[doc = " Get the node that contains `descendant`.\n\n Note that this can return `descendant` itself."] pub fn ts_node_child_with_descendant(self_: TSNode, descendant: TSNode) -> TSNode; } extern "C" { @@ -465,11 +454,11 @@ extern "C" { pub fn ts_node_prev_named_sibling(self_: TSNode) -> TSNode; } extern "C" { - #[doc = " Get the node's first child that extends beyond the given byte offset."] + #[doc = " Get the node's first child that contains or starts after the given byte offset."] pub fn ts_node_first_child_for_byte(self_: TSNode, byte: u32) -> TSNode; } extern "C" { - #[doc = " Get the node's first named child that extends beyond the given byte offset."] + #[doc = " Get the node's first named child that contains or starts after the given byte offset."] pub fn ts_node_first_named_child_for_byte(self_: TSNode, byte: u32) -> TSNode; } extern "C" { @@ -507,7 +496,15 @@ extern "C" { pub fn ts_node_eq(self_: TSNode, other: TSNode) -> bool; } extern "C" { - #[doc = " Create a new tree cursor starting from the given node.\n\n A tree cursor allows you to walk a syntax tree more efficiently than is\n possible using the [`TSNode`] functions. It is a mutable object that is always\n on a certain syntax node, and can be moved imperatively to different nodes."] + #[doc = " Edit a point to keep it in-sync with source code that has been edited.\n\n This function updates a single point's byte offset and row/column position\n based on an edit operation. This is useful for editing points without\n requiring a tree or node instance."] + pub fn ts_point_edit(point: *mut TSPoint, point_byte: *mut u32, edit: *const TSInputEdit); +} +extern "C" { + #[doc = " Edit a range to keep it in-sync with source code that has been edited.\n\n This function updates a range's start and end positions based on an edit\n operation. This is useful for editing ranges without requiring a tree\n or node instance."] + pub fn ts_range_edit(range: *mut TSRange, edit: *const TSInputEdit); +} +extern "C" { + #[doc = " Create a new tree cursor starting from the given node.\n\n A tree cursor allows you to walk a syntax tree more efficiently than is\n possible using the [`TSNode`] functions. It is a mutable object that is always\n on a certain syntax node, and can be moved imperatively to different nodes.\n\n Note that the given node is considered the root of the cursor,\n and the cursor cannot walk outside this node."] pub fn ts_tree_cursor_new(node: TSNode) -> TSTreeCursor; } extern "C" { @@ -537,15 +534,15 @@ extern "C" { pub fn ts_tree_cursor_current_field_id(self_: *const TSTreeCursor) -> TSFieldId; } extern "C" { - #[doc = " Move the cursor to the parent of its current node.\n\n This returns `true` if the cursor successfully moved, and returns `false`\n if there was no parent node (the cursor was already on the root node)."] + #[doc = " Move the cursor to the parent of its current node.\n\n This returns `true` if the cursor successfully moved, and returns `false`\n if there was no parent node (the cursor was already on the root node).\n\n Note that the node the cursor was constructed with is considered the root\n of the cursor, and the cursor cannot walk outside this node."] pub fn ts_tree_cursor_goto_parent(self_: *mut TSTreeCursor) -> bool; } extern "C" { - #[doc = " Move the cursor to the next sibling of its current node.\n\n This returns `true` if the cursor successfully moved, and returns `false`\n if there was no next sibling node."] + #[doc = " Move the cursor to the next sibling of its current node.\n\n This returns `true` if the cursor successfully moved, and returns `false`\n if there was no next sibling node.\n\n Note that the node the cursor was constructed with is considered the root\n of the cursor, and the cursor cannot walk outside this node."] pub fn ts_tree_cursor_goto_next_sibling(self_: *mut TSTreeCursor) -> bool; } extern "C" { - #[doc = " Move the cursor to the previous sibling of its current node.\n\n This returns `true` if the cursor successfully moved, and returns `false` if\n there was no previous sibling node.\n\n Note, that this function may be slower than\n [`ts_tree_cursor_goto_next_sibling`] due to how node positions are stored. In\n the worst case, this will need to iterate through all the children upto the\n previous sibling node to recalculate its position."] + #[doc = " Move the cursor to the previous sibling of its current node.\n\n This returns `true` if the cursor successfully moved, and returns `false` if\n there was no previous sibling node.\n\n Note, that this function may be slower than\n [`ts_tree_cursor_goto_next_sibling`] due to how node positions are stored. In\n the worst case, this will need to iterate through all the children up to the\n previous sibling node to recalculate its position. Also note that the node the cursor\n was constructed with is considered the root of the cursor, and the cursor cannot\n walk outside this node."] pub fn ts_tree_cursor_goto_previous_sibling(self_: *mut TSTreeCursor) -> bool; } extern "C" { @@ -569,7 +566,7 @@ extern "C" { pub fn ts_tree_cursor_current_depth(self_: *const TSTreeCursor) -> u32; } extern "C" { - #[doc = " Move the cursor to the first child of its current node that extends beyond\n the given byte offset or point.\n\n This returns the index of the child node if one was found, and returns -1\n if no such child was found."] + #[doc = " Move the cursor to the first child of its current node that contains or starts after\n the given byte offset or point.\n\n This returns the index of the child node if one was found, and returns -1\n if no such child was found."] pub fn ts_tree_cursor_goto_first_child_for_byte( self_: *mut TSTreeCursor, goal_byte: u32, @@ -681,7 +678,7 @@ extern "C" { pub fn ts_query_cursor_exec(self_: *mut TSQueryCursor, query: *const TSQuery, node: TSNode); } extern "C" { - #[doc = " Start running a gievn query on a given node, with some options."] + #[doc = " Start running a given query on a given node, with some options."] pub fn ts_query_cursor_exec_with_options( self_: *mut TSQueryCursor, query: *const TSQuery, @@ -700,15 +697,7 @@ extern "C" { pub fn ts_query_cursor_set_match_limit(self_: *mut TSQueryCursor, limit: u32); } extern "C" { - #[doc = " @deprecated use [`ts_query_cursor_exec_with_options`] and pass in a callback instead, this will be removed in 0.26.\n\n Set the maximum duration in microseconds that query execution should be allowed to\n take before halting.\n\n If query execution takes longer than this, it will halt early, returning NULL.\n See [`ts_query_cursor_next_match`] or [`ts_query_cursor_next_capture`] for more information."] - pub fn ts_query_cursor_set_timeout_micros(self_: *mut TSQueryCursor, timeout_micros: u64); -} -extern "C" { - #[doc = " @deprecated use [`ts_query_cursor_exec_with_options`] and pass in a callback instead, this will be removed in 0.26.\n\n Get the duration in microseconds that query execution is allowed to take.\n\n This is set via [`ts_query_cursor_set_timeout_micros`]."] - pub fn ts_query_cursor_timeout_micros(self_: *const TSQueryCursor) -> u64; -} -extern "C" { - #[doc = " Set the range of bytes in which the query will be executed.\n\n This will return `false` if the start byte is greater than the end byte, otherwise\n it will return `true`."] + #[doc = " Set the range of bytes in which the query will be executed.\n\n The query cursor will return matches that intersect with the given point range.\n This means that a match may be returned even if some of its captures fall\n outside the specified range, as long as at least part of the match\n overlaps with the range.\n\n For example, if a query pattern matches a node that spans a larger area\n than the specified range, but part of that node intersects with the range,\n the entire match will be returned.\n\n This will return `false` if the start byte is greater than the end byte, otherwise\n it will return `true`."] pub fn ts_query_cursor_set_byte_range( self_: *mut TSQueryCursor, start_byte: u32, @@ -716,13 +705,29 @@ extern "C" { ) -> bool; } extern "C" { - #[doc = " Set the range of (row, column) positions in which the query will be executed.\n\n This will return `false` if the start point is greater than the end point, otherwise\n it will return `true`."] + #[doc = " Set the range of (row, column) positions in which the query will be executed.\n\n The query cursor will return matches that intersect with the given point range.\n This means that a match may be returned even if some of its captures fall\n outside the specified range, as long as at least part of the match\n overlaps with the range.\n\n For example, if a query pattern matches a node that spans a larger area\n than the specified range, but part of that node intersects with the range,\n the entire match will be returned.\n\n This will return `false` if the start point is greater than the end point, otherwise\n it will return `true`."] pub fn ts_query_cursor_set_point_range( self_: *mut TSQueryCursor, start_point: TSPoint, end_point: TSPoint, ) -> bool; } +extern "C" { + #[doc = " Set the byte range within which all matches must be fully contained.\n\n Set the range of bytes in which matches will be searched for. In contrast to\n `ts_query_cursor_set_byte_range`, this will restrict the query cursor to only return\n matches where _all_ nodes are _fully_ contained within the given range. Both functions\n can be used together, e.g. to search for any matches that intersect line 5000, as\n long as they are fully contained within lines 4500-5500"] + pub fn ts_query_cursor_set_containing_byte_range( + self_: *mut TSQueryCursor, + start_byte: u32, + end_byte: u32, + ) -> bool; +} +extern "C" { + #[doc = " Set the point range within which all matches must be fully contained.\n\n Set the range of bytes in which matches will be searched for. In contrast to\n `ts_query_cursor_set_point_range`, this will restrict the query cursor to only return\n matches where _all_ nodes are _fully_ contained within the given range. Both functions\n can be used together, e.g. to search for any matches that intersect line 5000, as\n long as they are fully contained within lines 4500-5500"] + pub fn ts_query_cursor_set_containing_point_range( + self_: *mut TSQueryCursor, + start_point: TSPoint, + end_point: TSPoint, + ) -> bool; +} extern "C" { #[doc = " Advance to the next match of the currently running query.\n\n If there is a match, write it to `*match` and return `true`.\n Otherwise, return `false`."] pub fn ts_query_cursor_next_match(self_: *mut TSQueryCursor, match_: *mut TSQueryMatch) @@ -759,13 +764,6 @@ extern "C" { #[doc = " Get the number of valid states in this language."] pub fn ts_language_state_count(self_: *const TSLanguage) -> u32; } -extern "C" { - #[doc = " Get a node type string for the given numerical id."] - pub fn ts_language_symbol_name( - self_: *const TSLanguage, - symbol: TSSymbol, - ) -> *const ::core::ffi::c_char; -} extern "C" { #[doc = " Get the numerical id for the given node type string."] pub fn ts_language_symbol_for_name( @@ -794,13 +792,36 @@ extern "C" { name_length: u32, ) -> TSFieldId; } +extern "C" { + #[doc = " Get a list of all supertype symbols for the language."] + pub fn ts_language_supertypes(self_: *const TSLanguage, length: *mut u32) -> *const TSSymbol; +} +extern "C" { + #[doc = " Get a list of all subtype symbol ids for a given supertype symbol.\n\n See [`ts_language_supertypes`] for fetching all supertype symbols."] + pub fn ts_language_subtypes( + self_: *const TSLanguage, + supertype: TSSymbol, + length: *mut u32, + ) -> *const TSSymbol; +} +extern "C" { + #[doc = " Get a node type string for the given numerical id."] + pub fn ts_language_symbol_name( + self_: *const TSLanguage, + symbol: TSSymbol, + ) -> *const ::core::ffi::c_char; +} extern "C" { #[doc = " Check whether the given node type id belongs to named nodes, anonymous nodes,\n or a hidden nodes.\n\n See also [`ts_node_is_named`]. Hidden nodes are never returned from the API."] pub fn ts_language_symbol_type(self_: *const TSLanguage, symbol: TSSymbol) -> TSSymbolType; } extern "C" { #[doc = " Get the ABI version number for this language. This version number is used\n to ensure that languages were generated by a compatible version of\n Tree-sitter.\n\n See also [`ts_parser_set_language`]."] - pub fn ts_language_version(self_: *const TSLanguage) -> u32; + pub fn ts_language_abi_version(self_: *const TSLanguage) -> u32; +} +extern "C" { + #[doc = " Get the metadata for this language. This information is generated by the\n CLI, and relies on the language author providing the correct metadata in\n the language's `tree-sitter.json` file.\n\n See also [`TSMetadata`]."] + pub fn ts_language_metadata(self_: *const TSLanguage) -> *const TSLanguageMetadata; } extern "C" { #[doc = " Get the next parse state. Combine this with lookahead iterators to generate\n completion suggestions or valid symbols in error nodes. Use\n [`ts_node_grammar_symbol`] for valid symbols."] @@ -903,7 +924,7 @@ extern "C" { ) -> *const TSLanguage; } extern "C" { - #[doc = " Get the number of languages instantiated in the given wasm store."] + #[doc = " Get the number of languages instantiated in the given Wasm store."] pub fn ts_wasm_store_language_count(arg1: *const TSWasmStore) -> usize; } extern "C" { diff --git a/lib/binding_rust/build.rs b/lib/binding_rust/build.rs index a9e553de..57c5bc94 100644 --- a/lib/binding_rust/build.rs +++ b/lib/binding_rust/build.rs @@ -2,6 +2,7 @@ use std::{env, fs, path::PathBuf}; fn main() { let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + let target = env::var("TARGET").unwrap(); #[cfg(feature = "bindgen")] generate_bindings(&out_dir); @@ -26,6 +27,11 @@ fn main() { let include_path = manifest_path.join("include"); let src_path = manifest_path.join("src"); let wasm_path = src_path.join("wasm"); + + if target.starts_with("wasm32-unknown") { + configure_wasm_build(&mut config); + } + for entry in fs::read_dir(&src_path).unwrap() { let entry = entry.unwrap(); let path = src_path.join(entry.file_name()); @@ -43,6 +49,8 @@ fn main() { .include(&include_path) .define("_POSIX_C_SOURCE", "200112L") .define("_DEFAULT_SOURCE", None) + .define("_BSD_SOURCE", None) + .define("_DARWIN_C_SOURCE", None) .warnings(false) .file(src_path.join("lib.c")) .compile("tree-sitter"); @@ -50,8 +58,28 @@ fn main() { println!("cargo:include={}", include_path.display()); } +fn configure_wasm_build(config: &mut cc::Build) { + let Ok(wasm_headers) = env::var("DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS") else { + panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS must be set by the language crate"); + }; + let Ok(wasm_src) = env::var("DEP_TREE_SITTER_LANGUAGE_WASM_SRC").map(PathBuf::from) else { + panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_SRC must be set by the language crate"); + }; + + config.include(&wasm_headers); + config.files([ + wasm_src.join("stdio.c"), + wasm_src.join("stdlib.c"), + wasm_src.join("string.c"), + ]); +} + #[cfg(feature = "bindgen")] fn generate_bindings(out_dir: &std::path::Path) { + use std::str::FromStr; + + use bindgen::RustTarget; + const HEADER_PATH: &str = "include/tree_sitter/api.h"; println!("cargo:rerun-if-changed={HEADER_PATH}"); @@ -70,6 +98,8 @@ fn generate_bindings(out_dir: &std::path::Path) { "TSQueryPredicateStep", ]; + let rust_version = env!("CARGO_PKG_RUST_VERSION"); + let bindings = bindgen::Builder::default() .header(HEADER_PATH) .layout_tests(false) @@ -79,11 +109,16 @@ fn generate_bindings(out_dir: &std::path::Path) { .no_copy(no_copy.join("|")) .prepend_enum_name(false) .use_core() + .clang_arg("-D TREE_SITTER_FEATURE_WASM") + .rust_target(RustTarget::from_str(rust_version).unwrap()) .generate() .expect("Failed to generate bindings"); let bindings_rs = out_dir.join("bindings.rs"); - bindings - .write_to_file(&bindings_rs) - .unwrap_or_else(|_| panic!("Failed to write bindings into path: {bindings_rs:?}")); + bindings.write_to_file(&bindings_rs).unwrap_or_else(|_| { + panic!( + "Failed to write bindings into path: {}", + bindings_rs.display() + ) + }); } diff --git a/lib/binding_rust/ffi.rs b/lib/binding_rust/ffi.rs index 2ea2cbff..4c68a633 100644 --- a/lib/binding_rust/ffi.rs +++ b/lib/binding_rust/ffi.rs @@ -1,6 +1,7 @@ #![allow(dead_code)] #![allow(non_upper_case_globals)] #![allow(non_camel_case_types)] +#![allow(clippy::missing_const_for_fn)] #[cfg(feature = "bindgen")] include!(concat!(env!("OUT_DIR"), "/bindings.rs")); @@ -15,6 +16,7 @@ extern "C" { } #[cfg(windows)] +#[cfg(feature = "std")] extern "C" { pub(crate) fn _ts_dup(handle: *mut std::os::raw::c_void) -> std::os::raw::c_int; } diff --git a/lib/binding_rust/lib.rs b/lib/binding_rust/lib.rs index 6950c63b..51fd7f25 100644 --- a/lib/binding_rust/lib.rs +++ b/lib/binding_rust/lib.rs @@ -1,4 +1,4 @@ -#![doc = include_str!("./README.md")] +#![cfg_attr(not(any(test, doctest)), doc = include_str!("./README.md"))] #![cfg_attr(not(feature = "std"), no_std)] #![cfg_attr(docsrs, feature(doc_cfg))] @@ -16,10 +16,9 @@ use core::{ marker::PhantomData, mem::MaybeUninit, num::NonZeroU16, - ops::{self, Deref}, + ops::{self, ControlFlow, Deref}, ptr::{self, NonNull}, slice, str, - sync::atomic::AtomicUsize, }; #[cfg(feature = "std")] use std::error; @@ -28,7 +27,7 @@ use std::os::fd::AsRawFd; #[cfg(all(windows, feature = "std"))] use std::os::windows::io::AsRawHandle; -use streaming_iterator::{StreamingIterator, StreamingIteratorMut}; +pub use streaming_iterator::{StreamingIterator, StreamingIteratorMut}; use tree_sitter_language::LanguageFn; #[cfg(feature = "wasm")] @@ -64,6 +63,29 @@ pub struct Language(*const ffi::TSLanguage); pub struct LanguageRef<'a>(*const ffi::TSLanguage, PhantomData<&'a ()>); +/// The metadata associated with a language. +/// +/// Currently, this metadata can be used to check the [Semantic Version](https://semver.org/) +/// of the language. This version information should be used to signal if a given parser might +/// be incompatible with existing queries when upgrading between major versions, or minor versions +/// if it's in zerover. +#[doc(alias = "TSLanguageMetadata")] +pub struct LanguageMetadata { + pub major_version: u8, + pub minor_version: u8, + pub patch_version: u8, +} + +impl From for LanguageMetadata { + fn from(val: ffi::TSLanguageMetadata) -> Self { + Self { + major_version: val.major_version, + minor_version: val.minor_version, + patch_version: val.patch_version, + } + } +} + /// A tree that represents the syntactic structure of a source code file. #[doc(alias = "TSTree")] pub struct Tree(NonNull); @@ -98,6 +120,48 @@ pub struct InputEdit { pub new_end_position: Point, } +impl InputEdit { + /// Edit a point to keep it in-sync with source code that has been edited. + /// + /// This function updates a single point's byte offset and row/column position + /// based on this edit operation. This is useful for editing points without + /// requiring a tree or node instance. + #[doc(alias = "ts_point_edit")] + pub fn edit_point(&self, point: &mut Point, byte: &mut usize) { + let edit = self.into(); + let mut ts_point = (*point).into(); + let mut ts_byte = *byte as u32; + + unsafe { + ffi::ts_point_edit( + core::ptr::addr_of_mut!(ts_point), + core::ptr::addr_of_mut!(ts_byte), + &edit, + ); + } + + *point = ts_point.into(); + *byte = ts_byte as usize; + } + + /// Edit a range to keep it in-sync with source code that has been edited. + /// + /// This function updates a range's start and end positions based on this edit + /// operation. This is useful for editing ranges without requiring a tree + /// or node instance. + #[doc(alias = "ts_range_edit")] + pub fn edit_range(&self, range: &mut Range) { + let edit = self.into(); + let mut ts_range = (*range).into(); + + unsafe { + ffi::ts_range_edit(core::ptr::addr_of_mut!(ts_range), &edit); + } + + *range = ts_range.into(); + } +} + /// A single node within a syntax [`Tree`]. #[doc(alias = "TSNode")] #[derive(Clone, Copy)] @@ -121,9 +185,14 @@ pub struct ParseState(NonNull); impl ParseState { #[must_use] - pub fn current_byte_offset(&self) -> usize { + pub const fn current_byte_offset(&self) -> usize { unsafe { self.0.as_ref() }.current_byte_offset as usize } + + #[must_use] + pub const fn has_error(&self) -> bool { + unsafe { self.0.as_ref() }.has_error + } } /// A stateful object that is passed into a [`QueryProgressCallback`] @@ -132,7 +201,7 @@ pub struct QueryCursorState(NonNull); impl QueryCursorState { #[must_use] - pub fn current_byte_offset(&self) -> usize { + pub const fn current_byte_offset(&self) -> usize { unsafe { self.0.as_ref() }.current_byte_offset as usize } } @@ -149,10 +218,27 @@ impl<'a> ParseOptions<'a> { } #[must_use] - pub fn progress_callback bool>(mut self, callback: &'a mut F) -> Self { + pub fn progress_callback ControlFlow<()>>( + mut self, + callback: &'a mut F, + ) -> Self { self.progress_callback = Some(callback); self } + + /// Create a new `ParseOptions` with a shorter lifetime, borrowing from this one. + /// + /// This is useful when you need to reuse parse options multiple times, e.g., calling + /// [`Parser::parse_with_options`] multiple times with the same options. + #[must_use] + pub fn reborrow(&mut self) -> ParseOptions { + ParseOptions { + progress_callback: match &mut self.progress_callback { + Some(cb) => Some(*cb), + None => None, + }, + } + } } #[derive(Default)] @@ -167,13 +253,27 @@ impl<'a> QueryCursorOptions<'a> { } #[must_use] - pub fn progress_callback bool>( + pub fn progress_callback ControlFlow<()>>( mut self, callback: &'a mut F, ) -> Self { self.progress_callback = Some(callback); self } + + /// Create a new `QueryCursorOptions` with a shorter lifetime, borrowing from this one. + /// + /// This is useful when you need to reuse query cursor options multiple times, e.g., calling + /// [`QueryCursor::matches`] multiple times with the same options. + #[must_use] + pub fn reborrow(&mut self) -> QueryCursorOptions { + QueryCursorOptions { + progress_callback: match &mut self.progress_callback { + Some(cb) => Some(*cb), + None => None, + }, + } + } } struct QueryCursorOptionsDrop(*mut ffi::TSQueryCursorOptions); @@ -204,10 +304,10 @@ type FieldId = NonZeroU16; type Logger<'a> = Box; /// A callback that receives the parse state during parsing. -type ParseProgressCallback<'a> = &'a mut dyn FnMut(&ParseState) -> bool; +type ParseProgressCallback<'a> = &'a mut dyn FnMut(&ParseState) -> ControlFlow<()>; /// A callback that receives the query state during query execution. -type QueryProgressCallback<'a> = &'a mut dyn FnMut(&QueryCursorState) -> bool; +type QueryProgressCallback<'a> = &'a mut dyn FnMut(&QueryCursorState) -> ControlFlow<()>; pub trait Decode { /// A callback that decodes the next code point from the input slice. It should return the code @@ -217,7 +317,7 @@ pub trait Decode { /// A stateful object for walking a syntax [`Tree`] efficiently. #[doc(alias = "TSTreeCursor")] -pub struct TreeCursor<'cursor>(ffi::TSTreeCursor, PhantomData<&'cursor ()>); +pub struct TreeCursor<'tree>(ffi::TSTreeCursor, PhantomData<&'tree ()>); /// A set of patterns that match nodes in a syntax tree. #[doc(alias = "TSQuery")] @@ -251,7 +351,7 @@ impl From for CaptureQuantifier { ffi::TSQuantifierZeroOrMore => Self::ZeroOrMore, ffi::TSQuantifierOne => Self::One, ffi::TSQuantifierOneOrMore => Self::OneOrMore, - _ => panic!("Unrecognized quantifier: {value}"), + _ => unreachable!(), } } } @@ -292,7 +392,7 @@ pub struct QueryMatch<'cursor, 'tree> { } /// A sequence of [`QueryMatch`]es associated with a given [`QueryCursor`]. -pub struct QueryMatches<'query, 'tree: 'query, T: TextProvider, I: AsRef<[u8]>> { +pub struct QueryMatches<'query, 'tree, T: TextProvider, I: AsRef<[u8]>> { ptr: *mut ffi::TSQueryCursor, query: &'query Query, text_provider: T, @@ -304,7 +404,10 @@ pub struct QueryMatches<'query, 'tree: 'query, T: TextProvider, I: AsRef<[u8] } /// A sequence of [`QueryCapture`]s associated with a given [`QueryCursor`]. -pub struct QueryCaptures<'query, 'tree: 'query, T: TextProvider, I: AsRef<[u8]>> { +/// +/// During iteration, each element contains a [`QueryMatch`] and index. The index can +/// be used to access the new capture inside of the [`QueryMatch::captures`]'s [`captures`]. +pub struct QueryCaptures<'query, 'tree, T: TextProvider, I: AsRef<[u8]>> { ptr: *mut ffi::TSQueryCursor, query: &'query Query, text_provider: T, @@ -333,10 +436,13 @@ pub struct QueryCapture<'tree> { } /// An error that occurred when trying to assign an incompatible [`Language`] to -/// a [`Parser`]. +/// a [`Parser`]. If the `wasm` feature is enabled, this can also indicate a failure +/// to load the Wasm store. #[derive(Debug, PartialEq, Eq)] -pub struct LanguageError { - version: usize, +pub enum LanguageError { + Version(usize), + #[cfg(feature = "wasm")] + Wasm, } /// An error that occurred in [`Parser::set_included_ranges`]. @@ -377,7 +483,7 @@ enum TextPredicateCapture { AnyString(u32, Box<[Box]>, bool), } -// TODO: Remove this struct at at some point. If `core::str::lossy::Utf8Lossy` +// TODO: Remove this struct at some point. If `core::str::lossy::Utf8Lossy` // is ever stabilized. pub struct LossyUtf8<'a> { bytes: &'a [u8], @@ -391,7 +497,7 @@ impl Language { } /// Get the name of this language. This returns `None` in older parsers. - #[doc(alias = "ts_language_version")] + #[doc(alias = "ts_language_name")] #[must_use] pub fn name(&self) -> Option<&'static str> { let ptr = unsafe { ffi::ts_language_name(self.0) }; @@ -400,10 +506,24 @@ impl Language { /// Get the ABI version number that indicates which version of the /// Tree-sitter CLI that was used to generate this [`Language`]. - #[doc(alias = "ts_language_version")] + #[doc(alias = "ts_language_abi_version")] #[must_use] - pub fn version(&self) -> usize { - unsafe { ffi::ts_language_version(self.0) as usize } + pub fn abi_version(&self) -> usize { + unsafe { ffi::ts_language_abi_version(self.0) as usize } + } + + /// Get the metadata for this language. This information is generated by the + /// CLI, and relies on the language author providing the correct metadata in + /// the language's `tree-sitter.json` file. + /// + /// See also [`LanguageMetadata`]. + #[doc(alias = "ts_language_metadata")] + #[must_use] + pub fn metadata(&self) -> Option { + unsafe { + let ptr = ffi::ts_language_metadata(self.0); + (!ptr.is_null()).then(|| (*ptr).into()) + } } /// Get the number of distinct node types in this language. @@ -420,6 +540,36 @@ impl Language { unsafe { ffi::ts_language_state_count(self.0) as usize } } + /// Get a list of all supertype symbols for the language. + #[doc(alias = "ts_language_supertypes")] + #[must_use] + pub fn supertypes(&self) -> &[u16] { + let mut length = 0u32; + unsafe { + let ptr = ffi::ts_language_supertypes(self.0, core::ptr::addr_of_mut!(length)); + if length == 0 { + &[] + } else { + slice::from_raw_parts(ptr.cast_mut(), length as usize) + } + } + } + + /// Get a list of all subtype symbols for a given supertype symbol. + #[doc(alias = "ts_language_supertype_map")] + #[must_use] + pub fn subtypes_for_supertype(&self, supertype: u16) -> &[u16] { + unsafe { + let mut length = 0u32; + let ptr = ffi::ts_language_subtypes(self.0, supertype, core::ptr::addr_of_mut!(length)); + if length == 0 { + &[] + } else { + slice::from_raw_parts(ptr.cast_mut(), length as usize) + } + } + } + /// Get the name of the node kind for the given numerical id. #[doc(alias = "ts_language_symbol_name")] #[must_use] @@ -469,7 +619,7 @@ impl Language { unsafe { ffi::ts_language_field_count(self.0) as usize } } - /// Get the field names for the given numerical id. + /// Get the field name for the given numerical id. #[doc(alias = "ts_language_field_name_for_id")] #[must_use] pub fn field_name_for_id(&self, field_id: u16) -> Option<&'static str> { @@ -497,7 +647,7 @@ impl Language { /// generate completion suggestions or valid symbols in error nodes. /// /// Example: - /// ``` + /// ```ignore /// let state = language.next_state(node.parse_state(), node.grammar_id()); /// ``` #[doc(alias = "ts_language_next_state")] @@ -515,7 +665,7 @@ impl Language { /// symbol from [`LookaheadIterator::current_symbol`]. /// /// Lookahead iterators can be useful to generate suggestions and improve - /// syntax error diagnostics. To get symbols valid in an ERROR node, use the + /// syntax error diagnostics. To get symbols valid in an `ERROR` node, use the /// lookahead iterator on its first leaf node state. For `MISSING` nodes, a /// lookahead iterator created on the previous non-extra leaf node may be /// appropriate. @@ -580,14 +730,17 @@ impl Parser { /// [`LANGUAGE_VERSION`] and [`MIN_COMPATIBLE_LANGUAGE_VERSION`] constants. #[doc(alias = "ts_parser_set_language")] pub fn set_language(&mut self, language: &Language) -> Result<(), LanguageError> { - let version = language.version(); + let version = language.abi_version(); if (MIN_COMPATIBLE_LANGUAGE_VERSION..=LANGUAGE_VERSION).contains(&version) { - unsafe { - ffi::ts_parser_set_language(self.0.as_ptr(), language.0); + #[allow(unused_variables)] + let success = unsafe { ffi::ts_parser_set_language(self.0.as_ptr(), language.0) }; + #[cfg(feature = "wasm")] + if !success { + return Err(LanguageError::Wasm); } Ok(()) } else { - Err(LanguageError { version }) + Err(LanguageError::Version(version)) } } @@ -607,7 +760,7 @@ impl Parser { unsafe { logger.payload.cast::().as_ref() } } - /// Set the logging callback that a parser should use during parsing. + /// Set the logging callback that the parser should use during parsing. #[doc(alias = "ts_parser_set_logger")] pub fn set_logger(&mut self, logger: Option) { let prev_logger = unsafe { ffi::ts_parser_logger(self.0.as_ptr()) }; @@ -615,8 +768,7 @@ impl Parser { drop(unsafe { Box::from_raw(prev_logger.payload.cast::()) }); } - let c_logger; - if let Some(logger) = logger { + let c_logger = if let Some(logger) = logger { let container = Box::new(logger); unsafe extern "C" fn log( @@ -637,16 +789,16 @@ impl Parser { let raw_container = Box::into_raw(container); - c_logger = ffi::TSLogger { + ffi::TSLogger { payload: raw_container.cast::(), log: Some(log), - }; + } } else { - c_logger = ffi::TSLogger { + ffi::TSLogger { payload: ptr::null_mut(), log: None, - }; - } + } + }; unsafe { ffi::ts_parser_set_logger(self.0.as_ptr(), c_logger) }; } @@ -700,8 +852,6 @@ impl Parser { /// /// Returns a [`Tree`] if parsing succeeded, or `None` if: /// * The parser has not yet had a language assigned with [`Parser::set_language`] - /// * The timeout set with [`Parser::set_timeout_micros`] expired (deprecated) - /// * The cancellation flag set with [`Parser::set_cancellation_flag`] was flipped (deprecated) #[doc(alias = "ts_parser_parse")] pub fn parse(&mut self, text: impl AsRef<[u8]>, old_tree: Option<&Tree>) -> Option { let bytes = text.as_ref(); @@ -713,47 +863,6 @@ impl Parser { ) } - /// Parse a slice of UTF16 text. - /// - /// # Arguments: - /// * `text` The UTF16-encoded text to parse. - /// * `old_tree` A previous syntax tree parsed from the same document. If the text of the - /// document has changed since `old_tree` was created, then you must edit `old_tree` to match - /// the new text using [`Tree::edit`]. - #[deprecated(since = "0.25.0", note = "Prefer parse_utf16_le instead")] - pub fn parse_utf16( - &mut self, - input: impl AsRef<[u16]>, - old_tree: Option<&Tree>, - ) -> Option { - let code_points = input.as_ref(); - let len = code_points.len(); - self.parse_utf16_le_with_options( - &mut |i, _| (i < len).then(|| &code_points[i..]).unwrap_or_default(), - old_tree, - None, - ) - } - - /// Parse UTF8 text provided in chunks by a callback. - /// - /// # Arguments: - /// * `callback` A function that takes a byte offset and position and returns a slice of - /// UTF8-encoded text starting at that byte offset and position. The slices can be of any - /// length. If the given position is at the end of the text, the callback should return an - /// empty slice. - /// * `old_tree` A previous syntax tree parsed from the same document. If the text of the - /// document has changed since `old_tree` was created, then you must edit `old_tree` to match - /// the new text using [`Tree::edit`]. - #[deprecated(since = "0.25.0", note = "Prefer `parse_with_options` instead")] - pub fn parse_with, F: FnMut(usize, Point) -> T>( - &mut self, - callback: &mut F, - old_tree: Option<&Tree>, - ) -> Option { - self.parse_with_options(callback, old_tree, None) - } - /// Parse text provided in chunks by a callback. /// /// # Arguments: @@ -780,7 +889,10 @@ impl Parser { .cast::() .as_mut() .unwrap(); - callback(&ParseState::from_raw(state)) + match callback(&ParseState::from_raw(state)) { + ControlFlow::Continue(()) => false, + ControlFlow::Break(()) => true, + } } // This C function is passed to Tree-sitter as the input callback. @@ -844,28 +956,6 @@ impl Parser { } } - /// Parse UTF16 text provided in chunks by a callback. - /// - /// # Arguments: - /// * `callback` A function that takes a code point offset and position and returns a slice of - /// UTF16-encoded text starting at that byte offset and position. The slices can be of any - /// length. If the given position is at the end of the text, the callback should return an - /// empty slice. - /// * `old_tree` A previous syntax tree parsed from the same document. If the text of the - /// document has changed since `old_tree` was created, then you must edit `old_tree` to match - /// the new text using [`Tree::edit`]. - #[deprecated( - since = "0.25.0", - note = "Prefer `parse_utf16_le_with_options` instead" - )] - pub fn parse_utf16_with, F: FnMut(usize, Point) -> T>( - &mut self, - callback: &mut F, - old_tree: Option<&Tree>, - ) -> Option { - self.parse_utf16_le_with_options(callback, old_tree, None) - } - /// Parse a slice of UTF16 little-endian text. /// /// # Arguments: @@ -912,7 +1002,10 @@ impl Parser { .cast::() .as_mut() .unwrap(); - callback(&ParseState::from_raw(state)) + match callback(&ParseState::from_raw(state)) { + ControlFlow::Continue(()) => false, + ControlFlow::Break(()) => true, + } } // This C function is passed to Tree-sitter as the input callback. @@ -1029,7 +1122,10 @@ impl Parser { .cast::() .as_mut() .unwrap(); - callback(&ParseState::from_raw(state)) + match callback(&ParseState::from_raw(state)) { + ControlFlow::Continue(()) => false, + ControlFlow::Break(()) => true, + } } // This C function is passed to Tree-sitter as the input callback. @@ -1129,7 +1225,10 @@ impl Parser { .cast::() .as_mut() .unwrap(); - callback(&ParseState::from_raw(state)) + match callback(&ParseState::from_raw(state)) { + ControlFlow::Continue(()) => false, + ControlFlow::Break(()) => true, + } } // At compile time, create a C-compatible callback that calls the custom `decode` method. @@ -1138,7 +1237,7 @@ impl Parser { len: u32, code_point: *mut i32, ) -> u32 { - let (c, len) = D::decode(std::slice::from_raw_parts(data, len as usize)); + let (c, len) = D::decode(core::slice::from_raw_parts(data, len as usize)); if let Some(code_point) = code_point.as_mut() { *code_point = c; } @@ -1209,43 +1308,15 @@ impl Parser { /// Instruct the parser to start the next parse from the beginning. /// - /// If the parser previously failed because of a timeout, cancellation, - /// or callback, then by default, it will resume where it left off on the - /// next call to [`parse`](Parser::parse) or other parsing functions. - /// If you don't want to resume, and instead intend to use this parser to - /// parse some other document, you must call `reset` first. + /// If the parser previously failed because of a callback, then by default, + /// it will resume where it left off on the next call to [`parse`](Parser::parse) + /// or other parsing functions. If you don't want to resume, and instead intend to use + /// this parser to parse some other document, you must call `reset` first. #[doc(alias = "ts_parser_reset")] pub fn reset(&mut self) { unsafe { ffi::ts_parser_reset(self.0.as_ptr()) } } - /// Get the duration in microseconds that parsing is allowed to take. - /// - /// This is set via [`set_timeout_micros`](Parser::set_timeout_micros). - #[doc(alias = "ts_parser_timeout_micros")] - #[deprecated( - since = "0.25.0", - note = "Prefer using `parse_with_options` and using a callback" - )] - #[must_use] - pub fn timeout_micros(&self) -> u64 { - unsafe { ffi::ts_parser_timeout_micros(self.0.as_ptr()) } - } - - /// Set the maximum duration in microseconds that parsing should be allowed - /// to take before halting. - /// - /// If parsing takes longer than this, it will halt early, returning `None`. - /// See [`parse`](Parser::parse) for more information. - #[doc(alias = "ts_parser_set_timeout_micros")] - #[deprecated( - since = "0.25.0", - note = "Prefer using `parse_with_options` and using a callback" - )] - pub fn set_timeout_micros(&mut self, timeout_micros: u64) { - unsafe { ffi::ts_parser_set_timeout_micros(self.0.as_ptr(), timeout_micros) } - } - /// Set the ranges of text that the parser should include when parsing. /// /// By default, the parser will always include entire documents. This @@ -1301,59 +1372,25 @@ impl Parser { result } } - - /// Get the parser's current cancellation flag pointer. - /// - /// # Safety - /// - /// It uses FFI - #[doc(alias = "ts_parser_cancellation_flag")] - #[deprecated( - since = "0.25.0", - note = "Prefer using `parse_with_options` and using a callback" - )] - #[must_use] - pub unsafe fn cancellation_flag(&self) -> Option<&AtomicUsize> { - ffi::ts_parser_cancellation_flag(self.0.as_ptr()) - .cast::() - .as_ref() - } - - /// Set the parser's current cancellation flag pointer. - /// - /// If a pointer is assigned, then the parser will periodically read from - /// this pointer during parsing. If it reads a non-zero value, it will halt - /// early, returning `None`. See [`parse`](Parser::parse) for more - /// information. - /// - /// # Safety - /// - /// It uses FFI - #[doc(alias = "ts_parser_set_cancellation_flag")] - #[deprecated( - since = "0.25.0", - note = "Prefer using `parse_with_options` and using a callback" - )] - pub unsafe fn set_cancellation_flag(&mut self, flag: Option<&AtomicUsize>) { - if let Some(flag) = flag { - ffi::ts_parser_set_cancellation_flag( - self.0.as_ptr(), - (flag as *const AtomicUsize).cast::(), - ); - } else { - ffi::ts_parser_set_cancellation_flag(self.0.as_ptr(), ptr::null()); - } - } } impl Drop for Parser { fn drop(&mut self) { - self.stop_printing_dot_graphs(); + #[cfg(feature = "std")] + #[cfg(not(target_os = "wasi"))] + { + self.stop_printing_dot_graphs(); + } self.set_logger(None); unsafe { ffi::ts_parser_delete(self.0.as_ptr()) } } } +#[cfg(windows)] +extern "C" { + fn _open_osfhandle(osfhandle: isize, flags: core::ffi::c_int) -> core::ffi::c_int; +} + impl Tree { /// Get the root node of the syntax tree. #[doc(alias = "ts_tree_root_node")] @@ -1463,7 +1500,8 @@ impl Tree { #[cfg(windows)] { let handle = file.as_raw_handle(); - unsafe { ffi::ts_tree_print_dot_graph(self.0.as_ptr(), handle as i32) } + let fd = unsafe { _open_osfhandle(handle as isize, 0) }; + unsafe { ffi::ts_tree_print_dot_graph(self.0.as_ptr(), fd) } } } } @@ -1493,10 +1531,14 @@ impl<'tree> Node<'tree> { /// Get a numeric id for this node that is unique. /// - /// Within a given syntax tree, no two nodes have the same id. However, if - /// a new tree is created based on an older tree, and a node from the old - /// tree is reused in the process, then that node will have the same id in - /// both trees. + /// Within a given syntax tree, no two nodes have the same id. However: + /// + /// - If a new tree is created based on an older tree, and a node from the old tree is reused in + /// the process, then that node will have the same id in both trees. + /// + /// - A node not marked as having changes does not guarantee it was reused. + /// + /// - If a node is marked as having changed in the old tree, it will not be reused. #[must_use] pub fn id(&self) -> usize { self.0.id as usize @@ -1539,7 +1581,7 @@ impl<'tree> Node<'tree> { /// Get the [`Language`] that was used to parse this node's syntax tree. #[doc(alias = "ts_node_language")] #[must_use] - pub fn language(&self) -> LanguageRef { + pub fn language(&self) -> LanguageRef<'tree> { LanguageRef(unsafe { ffi::ts_node_language(self.0) }, PhantomData) } @@ -1555,7 +1597,7 @@ impl<'tree> Node<'tree> { /// Check if this node is *extra*. /// - /// Extra nodes represent things like comments, which are not required the + /// Extra nodes represent things like comments, which are not required by the /// grammar, but can appear anywhere. #[doc(alias = "ts_node_is_extra")] #[must_use] @@ -1612,14 +1654,14 @@ impl<'tree> Node<'tree> { unsafe { ffi::ts_node_is_missing(self.0) } } - /// Get the byte offsets where this node starts. + /// Get the byte offset where this node starts. #[doc(alias = "ts_node_start_byte")] #[must_use] pub fn start_byte(&self) -> usize { unsafe { ffi::ts_node_start_byte(self.0) as usize } } - /// Get the byte offsets where this node end. + /// Get the byte offset where this node ends. #[doc(alias = "ts_node_end_byte")] #[must_use] pub fn end_byte(&self) -> usize { @@ -1668,8 +1710,8 @@ impl<'tree> Node<'tree> { /// [`Node::children`] instead. #[doc(alias = "ts_node_child")] #[must_use] - pub fn child(&self, i: usize) -> Option { - Self::new(unsafe { ffi::ts_node_child(self.0, i as u32) }) + pub fn child(&self, i: u32) -> Option { + Self::new(unsafe { ffi::ts_node_child(self.0, i) }) } /// Get this node's number of children. @@ -1687,8 +1729,8 @@ impl<'tree> Node<'tree> { /// [`Node::named_children`] instead. #[doc(alias = "ts_node_named_child")] #[must_use] - pub fn named_child(&self, i: usize) -> Option { - Self::new(unsafe { ffi::ts_node_named_child(self.0, i as u32) }) + pub fn named_child(&self, i: u32) -> Option { + Self::new(unsafe { ffi::ts_node_named_child(self.0, i) }) } /// Get this node's number of *named* children. @@ -1849,7 +1891,7 @@ impl<'tree> Node<'tree> { } /// Get this node's immediate parent. - /// Prefer [`child_containing_descendant`](Node::child_containing_descendant) + /// Prefer [`child_with_descendant`](Node::child_with_descendant) /// for iterating over this node's ancestors. #[doc(alias = "ts_node_parent")] #[must_use] @@ -1857,20 +1899,9 @@ impl<'tree> Node<'tree> { Self::new(unsafe { ffi::ts_node_parent(self.0) }) } - /// Get this node's child containing `descendant`. This will not return - /// the descendant if it is a direct child of `self`, for that use - /// [`Node::child_with_descendant`]. - #[doc(alias = "ts_node_child_containing_descendant")] - #[must_use] - #[deprecated(since = "0.24.0", note = "Prefer child_with_descendant instead")] - pub fn child_containing_descendant(&self, descendant: Self) -> Option { - Self::new(unsafe { ffi::ts_node_child_containing_descendant(self.0, descendant.0) }) - } - /// Get the node that contains `descendant`. /// - /// Note that this can return `descendant` itself, unlike the deprecated function - /// [`Node::child_containing_descendant`]. + /// Note that this can return `descendant` itself. #[doc(alias = "ts_node_child_with_descendant")] #[must_use] pub fn child_with_descendant(&self, descendant: Self) -> Option { @@ -1905,14 +1936,14 @@ impl<'tree> Node<'tree> { Self::new(unsafe { ffi::ts_node_prev_named_sibling(self.0) }) } - /// Get the node's first child that extends beyond the given byte offset. + /// Get this node's first child that contains or starts after the given byte offset. #[doc(alias = "ts_node_first_child_for_byte")] #[must_use] pub fn first_child_for_byte(&self, byte: usize) -> Option { Self::new(unsafe { ffi::ts_node_first_child_for_byte(self.0, byte as u32) }) } - /// Get the node's first named child that extends beyond the given byte offset. + /// Get this node's first named child that contains or starts after the given byte offset. #[doc(alias = "ts_node_first_named_child_for_point")] #[must_use] pub fn first_named_child_for_byte(&self, byte: usize) -> Option { @@ -1926,7 +1957,7 @@ impl<'tree> Node<'tree> { unsafe { ffi::ts_node_descendant_count(self.0) as usize } } - /// Get the smallest node within this node that spans the given range. + /// Get the smallest node within this node that spans the given byte range. #[doc(alias = "ts_node_descendant_for_byte_range")] #[must_use] pub fn descendant_for_byte_range(&self, start: usize, end: usize) -> Option { @@ -1935,7 +1966,7 @@ impl<'tree> Node<'tree> { }) } - /// Get the smallest named node within this node that spans the given range. + /// Get the smallest named node within this node that spans the given byte range. #[doc(alias = "ts_node_named_descendant_for_byte_range")] #[must_use] pub fn named_descendant_for_byte_range(&self, start: usize, end: usize) -> Option { @@ -1944,7 +1975,7 @@ impl<'tree> Node<'tree> { }) } - /// Get the smallest node within this node that spans the given range. + /// Get the smallest node within this node that spans the given point range. #[doc(alias = "ts_node_descendant_for_point_range")] #[must_use] pub fn descendant_for_point_range(&self, start: Point, end: Point) -> Option { @@ -1953,7 +1984,7 @@ impl<'tree> Node<'tree> { }) } - /// Get the smallest named node within this node that spans the given range. + /// Get the smallest named node within this node that spans the given point range. #[doc(alias = "ts_node_named_descendant_for_point_range")] #[must_use] pub fn named_descendant_for_point_range(&self, start: Point, end: Point) -> Option { @@ -1962,6 +1993,7 @@ impl<'tree> Node<'tree> { }) } + /// Get an S-expression representing the node. #[doc(alias = "ts_node_string")] #[must_use] pub fn to_sexp(&self) -> String { @@ -1980,10 +2012,13 @@ impl<'tree> Node<'tree> { #[must_use] pub fn utf16_text<'a>(&self, source: &'a [u16]) -> &'a [u16] { - &source[self.start_byte()..self.end_byte()] + &source[self.start_byte() / 2..self.end_byte() / 2] } /// Create a new [`TreeCursor`] starting from this node. + /// + /// Note that the given node is considered the root of the cursor, + /// and the cursor cannot walk outside this node. #[doc(alias = "ts_tree_cursor_new")] #[must_use] pub fn walk(&self) -> TreeCursor<'tree> { @@ -2006,7 +2041,7 @@ impl<'tree> Node<'tree> { impl PartialEq for Node<'_> { fn eq(&self, other: &Self) -> bool { - self.0.id == other.0.id + core::ptr::eq(self.0.id, other.0.id) } } @@ -2047,11 +2082,11 @@ impl fmt::Display for Node<'_> { } } -impl<'cursor> TreeCursor<'cursor> { +impl<'tree> TreeCursor<'tree> { /// Get the tree cursor's current [`Node`]. #[doc(alias = "ts_tree_cursor_current_node")] #[must_use] - pub fn node(&self) -> Node<'cursor> { + pub fn node(&self) -> Node<'tree> { Node( unsafe { ffi::ts_tree_cursor_current_node(&self.0) }, PhantomData, @@ -2121,6 +2156,9 @@ impl<'cursor> TreeCursor<'cursor> { /// This returns `true` if the cursor successfully moved, and returns /// `false` if there was no parent node (the cursor was already on the /// root node). + /// + /// Note that the node the cursor was constructed with is considered the root + /// of the cursor, and the cursor cannot walk outside this node. #[doc(alias = "ts_tree_cursor_goto_parent")] pub fn goto_parent(&mut self) -> bool { unsafe { ffi::ts_tree_cursor_goto_parent(&mut self.0) } @@ -2130,6 +2168,9 @@ impl<'cursor> TreeCursor<'cursor> { /// /// This returns `true` if the cursor successfully moved, and returns /// `false` if there was no next sibling node. + /// + /// Note that the node the cursor was constructed with is considered the root + /// of the cursor, and the cursor cannot walk outside this node. #[doc(alias = "ts_tree_cursor_goto_next_sibling")] pub fn goto_next_sibling(&mut self) -> bool { unsafe { ffi::ts_tree_cursor_goto_next_sibling(&mut self.0) } @@ -2151,15 +2192,16 @@ impl<'cursor> TreeCursor<'cursor> { /// Note, that this function may be slower than /// [`goto_next_sibling`](TreeCursor::goto_next_sibling) due to how node /// positions are stored. In the worst case, this will need to iterate - /// through all the children upto the previous sibling node to recalculate - /// its position. + /// through all the children up to the previous sibling node to recalculate + /// its position. Also note that the node the cursor was constructed with is + /// considered the root of the cursor, and the cursor cannot walk outside this node. #[doc(alias = "ts_tree_cursor_goto_previous_sibling")] pub fn goto_previous_sibling(&mut self) -> bool { unsafe { ffi::ts_tree_cursor_goto_previous_sibling(&mut self.0) } } - /// Move this cursor to the first child of its current node that extends - /// beyond the given byte offset. + /// Move this cursor to the first child of its current node that contains or + /// starts after the given byte offset. /// /// This returns the index of the child node if one was found, and returns /// `None` if no such child was found. @@ -2167,11 +2209,11 @@ impl<'cursor> TreeCursor<'cursor> { pub fn goto_first_child_for_byte(&mut self, index: usize) -> Option { let result = unsafe { ffi::ts_tree_cursor_goto_first_child_for_byte(&mut self.0, index as u32) }; - (result >= 0).then_some(result as usize) + result.try_into().ok() } - /// Move this cursor to the first child of its current node that extends - /// beyond the given byte offset. + /// Move this cursor to the first child of its current node that contains or + /// starts after the given byte offset. /// /// This returns the index of the child node if one was found, and returns /// `None` if no such child was found. @@ -2179,13 +2221,13 @@ impl<'cursor> TreeCursor<'cursor> { pub fn goto_first_child_for_point(&mut self, point: Point) -> Option { let result = unsafe { ffi::ts_tree_cursor_goto_first_child_for_point(&mut self.0, point.into()) }; - (result >= 0).then_some(result as usize) + result.try_into().ok() } /// Re-initialize this tree cursor to start at the original node that the /// cursor was constructed with. #[doc(alias = "ts_tree_cursor_reset")] - pub fn reset(&mut self, node: Node<'cursor>) { + pub fn reset(&mut self, node: Node<'tree>) { unsafe { ffi::ts_tree_cursor_reset(&mut self.0, node.0) }; } @@ -2301,6 +2343,16 @@ impl Query { /// on syntax nodes parsed with that language. References to Queries can be /// shared between multiple threads. pub fn new(language: &Language, source: &str) -> Result { + let ptr = Self::new_raw(language, source)?; + unsafe { Self::from_raw_parts(ptr, source) } + } + + /// Constructs a raw [`TSQuery`](ffi::TSQuery) pointer without performing extra checks specific to the rust + /// bindings, such as predicate validation. A [`Query`] object can be constructed from the + /// returned pointer using [`from_raw_parts`](Query::from_raw_parts). The caller is + /// responsible for ensuring that the returned pointer is eventually freed by calling + /// [`ts_query_delete`](ffi::ts_query_delete). + pub fn new_raw(language: &Language, source: &str) -> Result<*mut ffi::TSQuery, QueryError> { let mut error_offset = 0u32; let mut error_type: ffi::TSQueryError = 0; let bytes = source.as_bytes(); @@ -2316,93 +2368,90 @@ impl Query { ) }; + if !ptr.is_null() { + return Ok(ptr); + } + // On failure, build an error based on the error code and offset. - if ptr.is_null() { - if error_type == ffi::TSQueryErrorLanguage { - return Err(QueryError { - row: 0, - column: 0, - offset: 0, - message: LanguageError { - version: language.version(), - } - .to_string(), - kind: QueryErrorKind::Language, - }); - } - - let offset = error_offset as usize; - let mut line_start = 0; - let mut row = 0; - let mut line_containing_error = None; - for line in source.lines() { - let line_end = line_start + line.len() + 1; - if line_end > offset { - line_containing_error = Some(line); - break; - } - line_start = line_end; - row += 1; - } - let column = offset - line_start; - - let kind; - let message; - match error_type { - // Error types that report names - ffi::TSQueryErrorNodeType | ffi::TSQueryErrorField | ffi::TSQueryErrorCapture => { - let suffix = source.split_at(offset).1; - let in_quotes = source.as_bytes()[offset - 1] == b'"'; - let mut backslashes = 0; - let end_offset = suffix - .find(|c| { - if in_quotes { - if c == '"' && backslashes % 2 == 0 { - true - } else if c == '\\' { - backslashes += 1; - false - } else { - backslashes = 0; - false - } - } else { - !char::is_alphanumeric(c) && c != '_' && c != '-' - } - }) - .unwrap_or(suffix.len()); - message = suffix.split_at(end_offset).0.to_string(); - kind = match error_type { - ffi::TSQueryErrorNodeType => QueryErrorKind::NodeType, - ffi::TSQueryErrorField => QueryErrorKind::Field, - ffi::TSQueryErrorCapture => QueryErrorKind::Capture, - _ => unreachable!(), - }; - } - - // Error types that report positions - _ => { - message = line_containing_error.map_or_else( - || "Unexpected EOF".to_string(), - |line| line.to_string() + "\n" + &" ".repeat(offset - line_start) + "^", - ); - kind = match error_type { - ffi::TSQueryErrorStructure => QueryErrorKind::Structure, - _ => QueryErrorKind::Syntax, - }; - } - }; - + if error_type == ffi::TSQueryErrorLanguage { return Err(QueryError { - row, - column, - offset, - message, - kind, + row: 0, + column: 0, + offset: 0, + message: LanguageError::Version(language.abi_version()).to_string(), + kind: QueryErrorKind::Language, }); } - unsafe { Self::from_raw_parts(ptr, source) } + let offset = error_offset as usize; + let mut line_start = 0; + let mut row = 0; + let mut line_containing_error = None; + for line in source.lines() { + let line_end = line_start + line.len() + 1; + if line_end > offset { + line_containing_error = Some(line); + break; + } + line_start = line_end; + row += 1; + } + let column = offset - line_start; + + let kind; + let message; + match error_type { + // Error types that report names + ffi::TSQueryErrorNodeType | ffi::TSQueryErrorField | ffi::TSQueryErrorCapture => { + let suffix = source.split_at(offset).1; + let in_quotes = offset > 0 && source.as_bytes()[offset - 1] == b'"'; + let mut backslashes = 0; + let end_offset = suffix + .find(|c| { + if in_quotes { + if c == '"' && backslashes % 2 == 0 { + true + } else if c == '\\' { + backslashes += 1; + false + } else { + backslashes = 0; + false + } + } else { + !char::is_alphanumeric(c) && c != '_' && c != '-' + } + }) + .unwrap_or(suffix.len()); + message = format!("\"{}\"", suffix.split_at(end_offset).0); + kind = match error_type { + ffi::TSQueryErrorNodeType => QueryErrorKind::NodeType, + ffi::TSQueryErrorField => QueryErrorKind::Field, + ffi::TSQueryErrorCapture => QueryErrorKind::Capture, + _ => unreachable!(), + }; + } + + // Error types that report positions + _ => { + message = line_containing_error.map_or_else( + || "Unexpected EOF".to_string(), + |line| line.to_string() + "\n" + &" ".repeat(offset - line_start) + "^", + ); + kind = match error_type { + ffi::TSQueryErrorStructure => QueryErrorKind::Structure, + _ => QueryErrorKind::Syntax, + }; + } + } + + Err(QueryError { + row, + column, + offset, + message, + kind, + }) } #[doc(hidden)] @@ -2441,7 +2490,7 @@ impl Query { } } - // Build a vector to store capture qunatifiers. + // Build a vector to store capture quantifiers. for i in 0..pattern_count { let mut capture_quantifiers = Vec::with_capacity(capture_count as usize); for j in 0..capture_count { @@ -2916,34 +2965,6 @@ impl QueryCursor { } } - /// Set the maximum duration in microseconds that query execution should be allowed to - /// take before halting. - /// - /// If query execution takes longer than this, it will halt early, returning None. - #[doc(alias = "ts_query_cursor_set_timeout_micros")] - #[deprecated( - since = "0.25.0", - note = "Prefer using `matches_with_options` or `captures_with_options` and using a callback" - )] - pub fn set_timeout_micros(&mut self, timeout: u64) { - unsafe { - ffi::ts_query_cursor_set_timeout_micros(self.ptr.as_ptr(), timeout); - } - } - - /// Get the duration in microseconds that query execution is allowed to take. - /// - /// This is set via [`set_timeout_micros`](QueryCursor::set_timeout_micros). - #[doc(alias = "ts_query_cursor_timeout_micros")] - #[deprecated( - since = "0.25.0", - note = "Prefer using `matches_with_options` or `captures_with_options` and using a callback" - )] - #[must_use] - pub fn timeout_micros(&self) -> u64 { - unsafe { ffi::ts_query_cursor_timeout_micros(self.ptr.as_ptr()) } - } - /// Check if, on its last execution, this cursor exceeded its maximum number /// of in-progress matches. #[doc(alias = "ts_query_cursor_did_exceed_match_limit")] @@ -2958,6 +2979,10 @@ impl QueryCursor { /// captures. Because multiple patterns can match the same set of nodes, /// one match may contain captures that appear *before* some of the /// captures from a previous match. + /// + /// Iterating over a `QueryMatches` object requires the `StreamingIterator` + /// or `StreamingIteratorMut` trait to be in scope. This can be done via + /// `use tree_sitter::StreamingIterator` or `use tree_sitter::StreamingIteratorMut` #[doc(alias = "ts_query_cursor_exec")] pub fn matches<'query, 'cursor: 'query, 'tree, T: TextProvider, I: AsRef<[u8]>>( &'cursor mut self, @@ -3005,7 +3030,10 @@ impl QueryCursor { .cast::() .as_mut() .unwrap(); - (callback)(&QueryCursorState::from_raw(state)) + match callback(&QueryCursorState::from_raw(state)) { + ControlFlow::Continue(()) => false, + ControlFlow::Break(()) => true, + } } let query_options = options.progress_callback.map(|cb| { @@ -3041,6 +3069,10 @@ impl QueryCursor { /// /// This is useful if you don't care about which pattern matched, and just /// want a single, ordered sequence of captures. + /// + /// Iterating over a `QueryCaptures` object requires the `StreamingIterator` + /// or `StreamingIteratorMut` trait to be in scope. This can be done via + /// `use tree_sitter::StreamingIterator` or `use tree_sitter::StreamingIteratorMut` #[doc(alias = "ts_query_cursor_exec")] pub fn captures<'query, 'cursor: 'query, 'tree, T: TextProvider, I: AsRef<[u8]>>( &'cursor mut self, @@ -3087,7 +3119,10 @@ impl QueryCursor { .cast::() .as_mut() .unwrap(); - (callback)(&QueryCursorState::from_raw(state)) + match callback(&QueryCursorState::from_raw(state)) { + ControlFlow::Continue(()) => false, + ControlFlow::Break(()) => true, + } } let query_options = options.progress_callback.map(|cb| { @@ -3146,6 +3181,44 @@ impl QueryCursor { self } + /// Set the byte range within which all matches must be fully contained. + /// + /// Set the range of bytes in which matches will be searched for. In contrast to + /// `ts_query_cursor_set_byte_range`, this will restrict the query cursor to only return + /// matches where _all_ nodes are _fully_ contained within the given range. Both functions + /// can be used together, e.g. to search for any matches that intersect line 5000, as + /// long as they are fully contained within lines 4500-5500 + #[doc(alias = "ts_query_cursor_set_containing_byte_range")] + pub fn set_containing_byte_range(&mut self, range: ops::Range) -> &mut Self { + unsafe { + ffi::ts_query_cursor_set_containing_byte_range( + self.ptr.as_ptr(), + range.start as u32, + range.end as u32, + ); + } + self + } + + /// Set the point range within which all matches must be fully contained. + /// + /// Set the range of bytes in which matches will be searched for. In contrast to + /// `ts_query_cursor_set_point_range`, this will restrict the query cursor to only return + /// matches where _all_ nodes are _fully_ contained within the given range. Both functions + /// can be used together, e.g. to search for any matches that intersect line 5000, as + /// long as they are fully contained within lines 4500-5500 + #[doc(alias = "ts_query_cursor_set_containing_point_range")] + pub fn set_containing_point_range(&mut self, range: ops::Range) -> &mut Self { + unsafe { + ffi::ts_query_cursor_set_containing_point_range( + self.ptr.as_ptr(), + range.start.into(), + range.end.into(), + ); + } + self + } + /// Set the maximum start depth for a query cursor. /// /// This prevents cursors from exploring children nodes at a certain depth. @@ -3155,9 +3228,9 @@ impl QueryCursor { /// The zero max start depth value can be used as a special behavior and /// it helps to destructure a subtree by staying on a node and using /// captures for interested parts. Note that the zero max start depth - /// only limit a search depth for a pattern's root node but other nodes - /// that are parts of the pattern may be searched at any depth what - /// defined by the pattern structure. + /// only limits a search depth for a pattern's root node but other nodes + /// that are parts of the pattern may be searched at any depth depending on + /// what is defined by the pattern structure. /// /// Set to `None` to remove the maximum start depth. #[doc(alias = "ts_query_cursor_set_max_start_depth")] @@ -3253,9 +3326,11 @@ impl<'tree> QueryMatch<'_, 'tree> { .iter() .all(|predicate| match predicate { TextPredicateCapture::EqCapture(i, j, is_positive, match_all_nodes) => { - let mut nodes_1 = self.nodes_for_capture_index(*i); - let mut nodes_2 = self.nodes_for_capture_index(*j); - while let (Some(node1), Some(node2)) = (nodes_1.next(), nodes_2.next()) { + let mut nodes_1 = self.nodes_for_capture_index(*i).peekable(); + let mut nodes_2 = self.nodes_for_capture_index(*j).peekable(); + while nodes_1.peek().is_some() && nodes_2.peek().is_some() { + let node1 = nodes_1.next().unwrap(); + let node2 = nodes_2.next().unwrap(); let mut text1 = text_provider.text(node1); let mut text2 = text_provider.text(node2); let text1 = node_text1.get_text(&mut text1); @@ -3329,7 +3404,7 @@ impl QueryProperty { /// Provide a `StreamingIterator` instead of the traditional `Iterator`, as the /// underlying object in the C library gets updated on each iteration. Copies would /// have their internal state overwritten, leading to Undefined Behavior -impl<'query, 'tree: 'query, T: TextProvider, I: AsRef<[u8]>> StreamingIterator +impl<'query, 'tree, T: TextProvider, I: AsRef<[u8]>> StreamingIterator for QueryMatches<'query, 'tree, T, I> { type Item = QueryMatch<'query, 'tree>; @@ -3360,15 +3435,13 @@ impl<'query, 'tree: 'query, T: TextProvider, I: AsRef<[u8]>> StreamingIterato } } -impl<'query, 'tree: 'query, T: TextProvider, I: AsRef<[u8]>> StreamingIteratorMut - for QueryMatches<'query, 'tree, T, I> -{ +impl, I: AsRef<[u8]>> StreamingIteratorMut for QueryMatches<'_, '_, T, I> { fn get_mut(&mut self) -> Option<&mut Self::Item> { self.current_match.as_mut() } } -impl<'query, 'tree: 'query, T: TextProvider, I: AsRef<[u8]>> StreamingIterator +impl<'query, 'tree, T: TextProvider, I: AsRef<[u8]>> StreamingIterator for QueryCaptures<'query, 'tree, T, I> { type Item = (QueryMatch<'query, 'tree>, usize); @@ -3405,9 +3478,7 @@ impl<'query, 'tree: 'query, T: TextProvider, I: AsRef<[u8]>> StreamingIterato } } -impl<'query, 'tree: 'query, T: TextProvider, I: AsRef<[u8]>> StreamingIteratorMut - for QueryCaptures<'query, 'tree, T, I> -{ +impl, I: AsRef<[u8]>> StreamingIteratorMut for QueryCaptures<'_, '_, T, I> { fn get_mut(&mut self) -> Option<&mut Self::Item> { self.current_match.as_mut() } @@ -3547,8 +3618,8 @@ impl From for Range { } } -impl From<&'_ InputEdit> for ffi::TSInputEdit { - fn from(val: &'_ InputEdit) -> Self { +impl From<&InputEdit> for ffi::TSInputEdit { + fn from(val: &InputEdit) -> Self { Self { start_byte: val.start_byte as u32, old_end_byte: val.old_end_byte as u32, @@ -3626,11 +3697,18 @@ impl fmt::Display for IncludedRangesError { impl fmt::Display for LanguageError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!( - f, - "Incompatible language version {}. Expected minimum {}, maximum {}", - self.version, MIN_COMPATIBLE_LANGUAGE_VERSION, LANGUAGE_VERSION, - ) + match self { + Self::Version(version) => { + write!( + f, + "Incompatible language version {version}. Expected minimum {MIN_COMPATIBLE_LANGUAGE_VERSION}, maximum {LANGUAGE_VERSION}", + ) + } + #[cfg(feature = "wasm")] + Self::Wasm => { + write!(f, "Failed to load the Wasm store.") + } + } } } diff --git a/lib/binding_rust/wasm_language.rs b/lib/binding_rust/wasm_language.rs index c7cc0793..66df377a 100644 --- a/lib/binding_rust/wasm_language.rs +++ b/lib/binding_rust/wasm_language.rs @@ -26,6 +26,9 @@ pub struct wasm_engine_t { pub struct WasmStore(*mut ffi::TSWasmStore); +unsafe impl Send for WasmStore {} +unsafe impl Sync for WasmStore {} + #[derive(Debug, PartialEq, Eq)] pub struct WasmError { pub kind: WasmErrorKind, @@ -45,7 +48,9 @@ impl WasmStore { unsafe { let mut error = MaybeUninit::::uninit(); let store = ffi::ts_wasm_store_new( - (engine as *const wasmtime::Engine).cast_mut().cast(), + std::ptr::from_ref::(engine) + .cast_mut() + .cast(), error.as_mut_ptr(), ); if store.is_null() { @@ -130,9 +135,9 @@ impl Drop for WasmStore { impl fmt::Display for WasmError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let kind = match self.kind { - WasmErrorKind::Parse => "Failed to parse wasm", - WasmErrorKind::Compile => "Failed to compile wasm", - WasmErrorKind::Instantiate => "Failed to instantiate wasm module", + WasmErrorKind::Parse => "Failed to parse Wasm", + WasmErrorKind::Compile => "Failed to compile Wasm", + WasmErrorKind::Instantiate => "Failed to instantiate Wasm module", WasmErrorKind::Other => "Unknown error", }; write!(f, "{kind}: {}", self.message) diff --git a/lib/binding_web/.eslintrc.js b/lib/binding_web/.eslintrc.js deleted file mode 100644 index 38709eb8..00000000 --- a/lib/binding_web/.eslintrc.js +++ /dev/null @@ -1,22 +0,0 @@ -module.exports = { - 'env': { - 'commonjs': true, - 'es2021': true, - }, - 'extends': 'google', - 'overrides': [ - ], - 'parserOptions': { - 'ecmaVersion': 'latest', - 'sourceType': 'module', - }, - 'rules': { - 'indent': ['error', 2, {'SwitchCase': 1}], - 'max-len': [ - 'error', - {'code': 120, 'ignoreComments': true, 'ignoreUrls': true, 'ignoreStrings': true, 'ignoreTemplateLiterals': true}, - ], - 'require-jsdoc': 0, - 'new-cap': 0, - }, -}; diff --git a/lib/binding_web/.gitattributes b/lib/binding_web/.gitattributes new file mode 100644 index 00000000..ab19847d --- /dev/null +++ b/lib/binding_web/.gitattributes @@ -0,0 +1 @@ +lib/web-tree-sitter.d.ts linguist-generated diff --git a/lib/binding_web/.gitignore b/lib/binding_web/.gitignore index eec0cfe6..8f45e5f9 100644 --- a/lib/binding_web/.gitignore +++ b/lib/binding_web/.gitignore @@ -1,6 +1,10 @@ -/tree-sitter.js -/tree-sitter.wasm -package-lock.json +debug/ +dist/ +web-tree-sitter* +lib/*.c +lib/*.h +!lib/tree-sitter.c +!lib/web-tree-sitter.d.ts node_modules *.tgz LICENSE diff --git a/lib/binding_web/.npmignore b/lib/binding_web/.npmignore deleted file mode 100644 index f110063a..00000000 --- a/lib/binding_web/.npmignore +++ /dev/null @@ -1,5 +0,0 @@ -* -!README.md -!tree-sitter.js -!tree-sitter.wasm -!tree-sitter-web.d.ts diff --git a/lib/binding_web/CONTRIBUTING.md b/lib/binding_web/CONTRIBUTING.md new file mode 100644 index 00000000..eb4e5fc3 --- /dev/null +++ b/lib/binding_web/CONTRIBUTING.md @@ -0,0 +1,142 @@ +# Contributing + +## Code of Conduct + +Contributors to Tree-sitter should abide by the [Contributor Covenant][covenant]. + +## Developing Web-tree-sitter + +### Prerequisites + +To make changes to Web-tree-sitter, you should have: + +1. A [Rust toolchain][rust], for running the xtasks necessary to build the library. +2. Node.js and NPM (or an equivalent package manager). +3. Either [Emscripten][emscripten], [Docker][docker], or [podman][podman] for +compiling the library to Wasm. + +### Building + +Clone the repository: + +```sh +git clone https://github.com/tree-sitter/tree-sitter +cd tree-sitter/lib/binding_web +``` + +Install the necessary dependencies: + +```sh +npm install +``` + +Build the library: + +```sh +npm run build +``` + +Note that the build process requires a Rust toolchain to be installed. If you don't have one installed, you can install it +by visiting the [Rust website][rust] and following the instructions there. + +> [!NOTE] +> By default, the build process will emit an ES6 module. If you need a CommonJS module, export `CJS` to `true`, or just +> run `CJS=true npm run build` (or the equivalent command for Windows). + +> [!TIP] +> To build the library with debug information, you can run `npm run build:debug`. The `CJS` environment variable is still +> taken into account. + +### Putting it together + +#### The C side + +There are several components that come together to build the final JS and Wasm files. First, we use `emscripten` in our +xtask located at `xtask/src/build_wasm.rs` from the root directory to compile the Wasm files. This Wasm module is output into the +local `lib` folder, and is used only in [`src/bindings.ts`][bindings.ts] to handle loading the Wasm module. The C code that +is compiled into the Wasm module is located in at [`lib/tree-sitter.c`][tree-sitter.c], and contains all the necessary +glue code to interact with the JS environment. If you need to update the imported functions from the tree-sitter library, +or anywhere else, you must update [`lib/exports.txt`][exports.txt]. Lastly, the type information for the Wasm module is +located at [`lib/tree-sitter.d.ts`][tree-sitter.d.ts], and can be updated by running `cargo xtask build-wasm --emit-tsd` +from the root directory. + +#### The TypeScript side + +The TypeScript library is a higher level abstraction over the Wasm module, and is located in `src`. This is where the +public API is defined, and where the Wasm module is loaded and initialized. The TypeScript library is built into a single +ES6 (or CommonJS) module, and is output into the same directory as `package.json`. If you need to update the public API, +you can do so by editing the files in `src`. + +If you make changes to the library that require updating the type definitions, such as adding a new public API method, +you should run: + +```sh +npm run build:dts +``` + +This uses [`dts-buddy`][dts-buddy] to generate `web-tree-sitter.d.ts` from the public types in `src`. Additionally, a sourcemap +is generated for the `.d.ts` file, which enables `go-to definition` and other editor integrations to take you straight +to the TypeScript source code. + +This TypeScript code is then compiled into a single JavaScript file with `esbuild`. The build configuration for this can +be found in [`script/build.js`][build.js], but this shouldn't need to be updated. This step is responsible for emitting +the final JS and Wasm files that are shipped with the library, as well as their sourcemaps. + +### Testing + +Before you can run the tests, you need to fetch and build some upstream grammars that are used for testing. +Run this in the root of the repository: + +```sh +cargo xtask fetch-fixtures +``` + +Optionally, to update the generated parser.c files: + +```sh +cargo xtask generate-fixtures +``` + +Then you can build the Wasm modules: + +```sh +cargo xtask generate-fixtures --wasm +``` + +Now, you can run the tests. In the `lib/binding_web` directory, run: + +```sh +npm test +``` + +> [!NOTE] +> We use `vitest` to run the tests. If you want to run a specific test, you can use the `-t` flag to pass in a pattern. +> If you want to run a specific file, you can just pass the name of the file as is. For example, to run the `parser` tests +> in `test/parser.test.ts`, you can run `npm test parser`. To run tests that have the name `descendant` somewhere, run +> `npm test -- -t descendant`. +> +> For coverage information, you can run `npm test -- --coverage`. + +### Debugging + +You might have noticed that when you ran `npm build`, the build process generated a couple of [sourcemaps][sourcemap]: +`web-tree-sitter.js.map` and `web-tree-sitter.wasm.map`. These sourcemaps can be used to debug the library in the browser, and are +shipped with the library on both NPM and the GitHub releases. + +#### Tweaking the Emscripten build + +If you're trying to tweak the Emscripten build, or are trying to debug an issue, the code for this lies in `xtask/src/build_wasm.rs` +file mentioned earlier, namely in the `run_wasm` function. + +[bindings.ts]: src/bindings.ts +[build.js]: script/build.js +[covenant]: https://www.contributor-covenant.org/version/1/4/code-of-conduct +[docker]: https://www.docker.com +[dts-buddy]: https://github.com/Rich-Harris/dts-buddy +[emscripten]: https://emscripten.org +[exports.txt]: lib/exports.txt +[podman]: https://podman.io +[rust]: https://www.rust-lang.org/tools/install +[sourcemap]: https://developer.mozilla.org/en-US/docs/Glossary/Source_map +[tree-sitter.c]: lib/tree-sitter.c +[tree-sitter.d.ts]: lib/tree-sitter.d.ts diff --git a/lib/binding_web/README.md b/lib/binding_web/README.md index ce5def96..08a939dc 100644 --- a/lib/binding_web/README.md +++ b/lib/binding_web/README.md @@ -7,46 +7,77 @@ WebAssembly bindings to the [Tree-sitter](https://github.com/tree-sitter/tree-sitter) parsing library. -### Setup +## Setup -You can download the `tree-sitter.js` and `tree-sitter.wasm` files from [the latest GitHub release](https://github.com/tree-sitter/tree-sitter/releases/latest) and load them using a standalone script: +You can download the `web-tree-sitter.js` and `web-tree-sitter.wasm` files from [the latest GitHub release][gh release] and load +them using a standalone script: ```html - + ``` -You can also install [the `web-tree-sitter` module](https://www.npmjs.com/package/web-tree-sitter) from NPM and load it using a system like Webpack: +You can also install [the `web-tree-sitter` module][npm module] from NPM and load it using a system like Webpack: ```js -const Parser = require('web-tree-sitter'); +const { Parser } = require('web-tree-sitter'); Parser.init().then(() => { /* the library is ready */ }); ``` -You can use this module with [deno](https://deno.land/): +or Vite: ```js -import Parser from "npm:web-tree-sitter"; +import { Parser } from 'web-tree-sitter'; +Parser.init().then(() => { /* the library is ready */ }); +``` + +With Vite, you also need to make sure your server provides the `tree-sitter.wasm` +file to your `public` directory. You can do this automatically with a `postinstall` +[script](https://docs.npmjs.com/cli/v10/using-npm/scripts) in your `package.json`: + +```js +"postinstall": "cp node_modules/web-tree-sitter/tree-sitter.wasm public" +``` + +You can also use this module with [deno](https://deno.land/): + +```js +import { Parser } from "npm:web-tree-sitter"; await Parser.init(); // the library is ready ``` +To use the debug version of the library, replace your import of `web-tree-sitter` with `web-tree-sitter/debug`: + +```js +import { Parser } from 'web-tree-sitter/debug'; // or require('web-tree-sitter/debug') + +Parser.init().then(() => { /* the library is ready */ }); +``` + +This will load the debug version of the `.js` and `.wasm` file, which includes debug symbols and assertions. + +> [!NOTE] +> The `web-tree-sitter.js` file on GH releases is an ES6 module. If you are interested in using a pure CommonJS library, such +> as for Electron, you should use the `web-tree-sitter.cjs` file instead. + ### Basic Usage First, create a parser: ```js -const parser = new Parser; +const parser = new Parser(); ``` Then assign a language to the parser. Tree-sitter languages are packaged as individual `.wasm` files (more on this below): ```js -const JavaScript = await Parser.Language.load('/path/to/tree-sitter-javascript.wasm'); +const { Language } = require('web-tree-sitter'); +const JavaScript = await Language.load('/path/to/tree-sitter-javascript.wasm'); parser.setLanguage(JavaScript); ``` @@ -102,7 +133,8 @@ const newTree = parser.parse(newSourceCode, tree); ### Parsing Text From a Custom Data Structure -If your text is stored in a data structure other than a single string, you can parse it by supplying a callback to `parse` instead of a string: +If your text is stored in a data structure other than a single string, you can parse it by supplying a callback to `parse` +instead of a string: ```javascript const sourceLines = [ @@ -116,13 +148,36 @@ const tree = parser.parse((index, position) => { }); ``` -### Generate .wasm language files +### Getting the `.wasm` language files -The following example shows how to generate `.wasm` file for tree-sitter JavaScript grammar. +There are several options on how to get the `.wasm` files for the languages you want to parse. -**IMPORTANT**: [emscripten](https://emscripten.org/docs/getting_started/downloads.html), [docker](https://www.docker.com/), or [podman](https://podman.io) need to be installed. +#### From npmjs.com -First install `tree-sitter-cli` and the tree-sitter language for which to generate `.wasm` (`tree-sitter-javascript` in this example): +The recommended way is to just install the package from npm. For example, to parse JavaScript, you can install the `tree-sitter-javascript` +package: + +```sh +npm install tree-sitter-javascript +``` + +Then you can find the `.wasm` file in the `node_modules/tree-sitter-javascript` directory. + +#### From GitHub + +You can also download the `.wasm` files from GitHub releases, so long as the repository uses our reusable workflow to publish +them. +For example, you can download the JavaScript `.wasm` file from the tree-sitter-javascript [releases page][gh release js]. + +#### Generating `.wasm` files + +You can also generate the `.wasm` file for your desired grammar. Shown below is an example of how to generate the `.wasm` +file for the JavaScript grammar. + +**IMPORTANT**: [Emscripten][emscripten], [Docker][docker], or [Podman][podman] need to be installed. + +First install `tree-sitter-cli`, and the tree-sitter language for which to generate `.wasm` +(`tree-sitter-javascript` in this example): ```sh npm install --save-dev tree-sitter-cli tree-sitter-javascript @@ -136,9 +191,10 @@ npx tree-sitter build --wasm node_modules/tree-sitter-javascript If everything is fine, file `tree-sitter-javascript.wasm` should be generated in current directory. -#### Running .wasm in Node.js +### Running .wasm in Node.js -Notice that executing `.wasm` files in node.js is considerably slower than running [node.js bindings](https://github.com/tree-sitter/node-tree-sitter). However could be useful for testing purposes: +Notice that executing `.wasm` files in Node.js is considerably slower than running [Node.js bindings][node bindings]. +However, this could be useful for testing purposes: ```javascript const Parser = require('web-tree-sitter'); @@ -153,18 +209,18 @@ const Parser = require('web-tree-sitter'); })(); ``` -#### Running .wasm in browser +### Running .wasm in browser `web-tree-sitter` can run in the browser, but there are some common pitfalls. -##### Loading the .wasm file +#### Loading the .wasm file `web-tree-sitter` needs to load the `tree-sitter.wasm` file. By default, it assumes that this file is available in the same path as the JavaScript code. Therefore, if the code is being served from `http://localhost:3000/bundle.js`, then -the wasm file should be at `http://localhost:3000/tree-sitter.wasm`. +the Wasm file should be at `http://localhost:3000/tree-sitter.wasm`. For server side frameworks like NextJS, this can be tricky as pages are often served from a path such as -`http://localhost:3000/_next/static/chunks/pages/index.js`. The loader will therefore look for the wasm file at +`http://localhost:3000/_next/static/chunks/pages/index.js`. The loader will therefore look for the Wasm file at `http://localhost:3000/_next/static/chunks/pages/tree-sitter.wasm`. The solution is to pass a `locateFile` function in the `moduleOptions` argument to `Parser.init()`: @@ -176,15 +232,17 @@ await Parser.init({ }); ``` -`locateFile` takes in two parameters, `scriptName`, i.e. the wasm file name, and `scriptDirectory`, i.e. the directory -where the loader expects the script to be. It returns the path where the loader will look for the wasm file. In the NextJS +`locateFile` takes in two parameters, `scriptName`, i.e. the Wasm file name, and `scriptDirectory`, i.e. the directory +where the loader expects the script to be. It returns the path where the loader will look for the Wasm file. In the NextJS case, we want to return just the `scriptName` so that the loader will look at `http://localhost:3000/tree-sitter.wasm` and not `http://localhost:3000/_next/static/chunks/pages/tree-sitter.wasm`. -##### `Can't resolve 'fs' in 'node_modules/web-tree-sitter'` +For more information on the module options you can pass in, see the [emscripten documentation][emscripten-module-options]. -Most bundlers will notice that the `tree-sitter.js` file is attempting to import `fs`, i.e. node's file system library. -Since this doesn't exist in the browser, the bundlers will get confused. For webpack you can fix this by adding the +#### "Can't resolve 'fs' in 'node_modules/web-tree-sitter" + +Most bundlers will notice that the `web-tree-sitter.js` file is attempting to import `fs`, i.e. node's file system library. +Since this doesn't exist in the browser, the bundlers will get confused. For Webpack, you can fix this by adding the following to your webpack config: ```javascript @@ -196,3 +254,12 @@ following to your webpack config: } } ``` + +[docker]: https://www.docker.com +[emscripten]: https://emscripten.org +[emscripten-module-options]: https://emscripten.org/docs/api_reference/module.html#affecting-execution +[gh release]: https://github.com/tree-sitter/tree-sitter/releases/latest +[gh release js]: https://github.com/tree-sitter/tree-sitter-javascript/releases/latest +[node bindings]: https://github.com/tree-sitter/node-tree-sitter +[npm module]: https://www.npmjs.com/package/web-tree-sitter +[podman]: https://podman.io diff --git a/lib/binding_web/binding.js b/lib/binding_web/binding.js deleted file mode 100644 index 8d54af2d..00000000 --- a/lib/binding_web/binding.js +++ /dev/null @@ -1,1559 +0,0 @@ -/* eslint-disable-next-line spaced-comment */ -/// -/* eslint-disable-next-line spaced-comment */ -/// - -const C = Module; -const INTERNAL = {}; -const SIZE_OF_INT = 4; -const SIZE_OF_CURSOR = 4 * SIZE_OF_INT; -const SIZE_OF_NODE = 5 * SIZE_OF_INT; -const SIZE_OF_POINT = 2 * SIZE_OF_INT; -const SIZE_OF_RANGE = 2 * SIZE_OF_INT + 2 * SIZE_OF_POINT; -const ZERO_POINT = {row: 0, column: 0}; -const QUERY_WORD_REGEX = /[\w-.]*/g; - -const PREDICATE_STEP_TYPE_CAPTURE = 1; -const PREDICATE_STEP_TYPE_STRING = 2; - -const LANGUAGE_FUNCTION_REGEX = /^_?tree_sitter_\w+/; - -let VERSION; -let MIN_COMPATIBLE_VERSION; -let TRANSFER_BUFFER; -let currentParseCallback; -// eslint-disable-next-line no-unused-vars -let currentLogCallback; - -// eslint-disable-next-line no-unused-vars -class ParserImpl { - static init() { - TRANSFER_BUFFER = C._ts_init(); - VERSION = getValue(TRANSFER_BUFFER, 'i32'); - MIN_COMPATIBLE_VERSION = getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); - } - - initialize() { - C._ts_parser_new_wasm(); - this[0] = getValue(TRANSFER_BUFFER, 'i32'); - this[1] = getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); - } - - delete() { - C._ts_parser_delete(this[0]); - C._free(this[1]); - this[0] = 0; - this[1] = 0; - } - - setLanguage(language) { - let address; - if (!language) { - address = 0; - language = null; - } else if (language.constructor === Language) { - address = language[0]; - const version = C._ts_language_version(address); - if (version < MIN_COMPATIBLE_VERSION || VERSION < version) { - throw new Error( - `Incompatible language version ${version}. ` + - `Compatibility range ${MIN_COMPATIBLE_VERSION} through ${VERSION}.`, - ); - } - } else { - throw new Error('Argument must be a Language'); - } - this.language = language; - C._ts_parser_set_language(this[0], address); - return this; - } - - getLanguage() { - return this.language; - } - - parse(callback, oldTree, options) { - if (typeof callback === 'string') { - currentParseCallback = (index, _) => callback.slice(index); - } else if (typeof callback === 'function') { - currentParseCallback = callback; - } else { - throw new Error('Argument must be a string or a function'); - } - - if (this.logCallback) { - currentLogCallback = this.logCallback; - C._ts_parser_enable_logger_wasm(this[0], 1); - } else { - currentLogCallback = null; - C._ts_parser_enable_logger_wasm(this[0], 0); - } - - let rangeCount = 0; - let rangeAddress = 0; - if (options?.includedRanges) { - rangeCount = options.includedRanges.length; - rangeAddress = C._calloc(rangeCount, SIZE_OF_RANGE); - let address = rangeAddress; - for (let i = 0; i < rangeCount; i++) { - marshalRange(address, options.includedRanges[i]); - address += SIZE_OF_RANGE; - } - } - - const treeAddress = C._ts_parser_parse_wasm( - this[0], - this[1], - oldTree ? oldTree[0] : 0, - rangeAddress, - rangeCount, - ); - - if (!treeAddress) { - currentParseCallback = null; - currentLogCallback = null; - throw new Error('Parsing failed'); - } - - const result = new Tree(INTERNAL, treeAddress, this.language, currentParseCallback); - currentParseCallback = null; - currentLogCallback = null; - return result; - } - - reset() { - C._ts_parser_reset(this[0]); - } - - getIncludedRanges() { - C._ts_parser_included_ranges_wasm(this[0]); - const count = getValue(TRANSFER_BUFFER, 'i32'); - const buffer = getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); - const result = new Array(count); - if (count > 0) { - let address = buffer; - for (let i = 0; i < count; i++) { - result[i] = unmarshalRange(address); - address += SIZE_OF_RANGE; - } - C._free(buffer); - } - return result; - } - - getTimeoutMicros() { - return C._ts_parser_timeout_micros(this[0]); - } - - setTimeoutMicros(timeout) { - C._ts_parser_set_timeout_micros(this[0], timeout); - } - - setLogger(callback) { - if (!callback) { - callback = null; - } else if (typeof callback !== 'function') { - throw new Error('Logger callback must be a function'); - } - this.logCallback = callback; - return this; - } - - getLogger() { - return this.logCallback; - } -} - -class Tree { - constructor(internal, address, language, textCallback) { - assertInternal(internal); - this[0] = address; - this.language = language; - this.textCallback = textCallback; - } - - copy() { - const address = C._ts_tree_copy(this[0]); - return new Tree(INTERNAL, address, this.language, this.textCallback); - } - - delete() { - C._ts_tree_delete(this[0]); - this[0] = 0; - } - - edit(edit) { - marshalEdit(edit); - C._ts_tree_edit_wasm(this[0]); - } - - get rootNode() { - C._ts_tree_root_node_wasm(this[0]); - return unmarshalNode(this); - } - - rootNodeWithOffset(offsetBytes, offsetExtent) { - const address = TRANSFER_BUFFER + SIZE_OF_NODE; - setValue(address, offsetBytes, 'i32'); - marshalPoint(address + SIZE_OF_INT, offsetExtent); - C._ts_tree_root_node_with_offset_wasm(this[0]); - return unmarshalNode(this); - } - - getLanguage() { - return this.language; - } - - walk() { - return this.rootNode.walk(); - } - - getChangedRanges(other) { - if (other.constructor !== Tree) { - throw new TypeError('Argument must be a Tree'); - } - - C._ts_tree_get_changed_ranges_wasm(this[0], other[0]); - const count = getValue(TRANSFER_BUFFER, 'i32'); - const buffer = getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); - const result = new Array(count); - if (count > 0) { - let address = buffer; - for (let i = 0; i < count; i++) { - result[i] = unmarshalRange(address); - address += SIZE_OF_RANGE; - } - C._free(buffer); - } - return result; - } - - getIncludedRanges() { - C._ts_tree_included_ranges_wasm(this[0]); - const count = getValue(TRANSFER_BUFFER, 'i32'); - const buffer = getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); - const result = new Array(count); - if (count > 0) { - let address = buffer; - for (let i = 0; i < count; i++) { - result[i] = unmarshalRange(address); - address += SIZE_OF_RANGE; - } - C._free(buffer); - } - return result; - } -} - -class Node { - constructor(internal, tree) { - assertInternal(internal); - this.tree = tree; - } - - get typeId() { - marshalNode(this); - return C._ts_node_symbol_wasm(this.tree[0]); - } - - get grammarId() { - marshalNode(this); - return C._ts_node_grammar_symbol_wasm(this.tree[0]); - } - - get type() { - return this.tree.language.types[this.typeId] || 'ERROR'; - } - - get grammarType() { - return this.tree.language.types[this.grammarId] || 'ERROR'; - } - - get endPosition() { - marshalNode(this); - C._ts_node_end_point_wasm(this.tree[0]); - return unmarshalPoint(TRANSFER_BUFFER); - } - - get endIndex() { - marshalNode(this); - return C._ts_node_end_index_wasm(this.tree[0]); - } - - get text() { - return getText(this.tree, this.startIndex, this.endIndex); - } - - get parseState() { - marshalNode(this); - return C._ts_node_parse_state_wasm(this.tree[0]); - } - - get nextParseState() { - marshalNode(this); - return C._ts_node_next_parse_state_wasm(this.tree[0]); - } - - get isNamed() { - marshalNode(this); - return C._ts_node_is_named_wasm(this.tree[0]) === 1; - } - - get hasError() { - marshalNode(this); - return C._ts_node_has_error_wasm(this.tree[0]) === 1; - } - - get hasChanges() { - marshalNode(this); - return C._ts_node_has_changes_wasm(this.tree[0]) === 1; - } - - get isError() { - marshalNode(this); - return C._ts_node_is_error_wasm(this.tree[0]) === 1; - } - - get isMissing() { - marshalNode(this); - return C._ts_node_is_missing_wasm(this.tree[0]) === 1; - } - - get isExtra() { - marshalNode(this); - return C._ts_node_is_extra_wasm(this.tree[0]) === 1; - } - - equals(other) { - return this.id === other.id; - } - - child(index) { - marshalNode(this); - C._ts_node_child_wasm(this.tree[0], index); - return unmarshalNode(this.tree); - } - - namedChild(index) { - marshalNode(this); - C._ts_node_named_child_wasm(this.tree[0], index); - return unmarshalNode(this.tree); - } - - childForFieldId(fieldId) { - marshalNode(this); - C._ts_node_child_by_field_id_wasm(this.tree[0], fieldId); - return unmarshalNode(this.tree); - } - - childForFieldName(fieldName) { - const fieldId = this.tree.language.fields.indexOf(fieldName); - if (fieldId !== -1) return this.childForFieldId(fieldId); - return null; - } - - fieldNameForChild(index) { - marshalNode(this); - const address = C._ts_node_field_name_for_child_wasm(this.tree[0], index); - if (!address) { - return null; - } - const result = AsciiToString(address); - // must not free, the string memory is owned by the language - return result; - } - - childrenForFieldName(fieldName) { - const fieldId = this.tree.language.fields.indexOf(fieldName); - if (fieldId !== -1 && fieldId !== 0) return this.childrenForFieldId(fieldId); - return []; - } - - childrenForFieldId(fieldId) { - marshalNode(this); - C._ts_node_children_by_field_id_wasm(this.tree[0], fieldId); - const count = getValue(TRANSFER_BUFFER, 'i32'); - const buffer = getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); - const result = new Array(count); - if (count > 0) { - let address = buffer; - for (let i = 0; i < count; i++) { - result[i] = unmarshalNode(this.tree, address); - address += SIZE_OF_NODE; - } - C._free(buffer); - } - return result; - } - - firstChildForIndex(index) { - marshalNode(this); - const address = TRANSFER_BUFFER + SIZE_OF_NODE; - setValue(address, index, 'i32'); - C._ts_node_first_child_for_byte_wasm(this.tree[0]); - return unmarshalNode(this.tree); - } - - firstNamedChildForIndex(index) { - marshalNode(this); - const address = TRANSFER_BUFFER + SIZE_OF_NODE; - setValue(address, index, 'i32'); - C._ts_node_first_named_child_for_byte_wasm(this.tree[0]); - return unmarshalNode(this.tree); - } - - get childCount() { - marshalNode(this); - return C._ts_node_child_count_wasm(this.tree[0]); - } - - get namedChildCount() { - marshalNode(this); - return C._ts_node_named_child_count_wasm(this.tree[0]); - } - - get firstChild() { - return this.child(0); - } - - get firstNamedChild() { - return this.namedChild(0); - } - - get lastChild() { - return this.child(this.childCount - 1); - } - - get lastNamedChild() { - return this.namedChild(this.namedChildCount - 1); - } - - get children() { - if (!this._children) { - marshalNode(this); - C._ts_node_children_wasm(this.tree[0]); - const count = getValue(TRANSFER_BUFFER, 'i32'); - const buffer = getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); - this._children = new Array(count); - if (count > 0) { - let address = buffer; - for (let i = 0; i < count; i++) { - this._children[i] = unmarshalNode(this.tree, address); - address += SIZE_OF_NODE; - } - C._free(buffer); - } - } - return this._children; - } - - get namedChildren() { - if (!this._namedChildren) { - marshalNode(this); - C._ts_node_named_children_wasm(this.tree[0]); - const count = getValue(TRANSFER_BUFFER, 'i32'); - const buffer = getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); - this._namedChildren = new Array(count); - if (count > 0) { - let address = buffer; - for (let i = 0; i < count; i++) { - this._namedChildren[i] = unmarshalNode(this.tree, address); - address += SIZE_OF_NODE; - } - C._free(buffer); - } - } - return this._namedChildren; - } - - descendantsOfType(types, startPosition, endPosition) { - if (!Array.isArray(types)) types = [types]; - if (!startPosition) startPosition = ZERO_POINT; - if (!endPosition) endPosition = ZERO_POINT; - - // Convert the type strings to numeric type symbols. - const symbols = []; - const typesBySymbol = this.tree.language.types; - for (let i = 0, n = typesBySymbol.length; i < n; i++) { - if (types.includes(typesBySymbol[i])) { - symbols.push(i); - } - } - - // Copy the array of symbols to the WASM heap. - const symbolsAddress = C._malloc(SIZE_OF_INT * symbols.length); - for (let i = 0, n = symbols.length; i < n; i++) { - setValue(symbolsAddress + i * SIZE_OF_INT, symbols[i], 'i32'); - } - - // Call the C API to compute the descendants. - marshalNode(this); - C._ts_node_descendants_of_type_wasm( - this.tree[0], - symbolsAddress, - symbols.length, - startPosition.row, - startPosition.column, - endPosition.row, - endPosition.column, - ); - - // Instantiate the nodes based on the data returned. - const descendantCount = getValue(TRANSFER_BUFFER, 'i32'); - const descendantAddress = getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); - const result = new Array(descendantCount); - if (descendantCount > 0) { - let address = descendantAddress; - for (let i = 0; i < descendantCount; i++) { - result[i] = unmarshalNode(this.tree, address); - address += SIZE_OF_NODE; - } - } - - // Free the intermediate buffers - C._free(descendantAddress); - C._free(symbolsAddress); - return result; - } - - get nextSibling() { - marshalNode(this); - C._ts_node_next_sibling_wasm(this.tree[0]); - return unmarshalNode(this.tree); - } - - get previousSibling() { - marshalNode(this); - C._ts_node_prev_sibling_wasm(this.tree[0]); - return unmarshalNode(this.tree); - } - - get nextNamedSibling() { - marshalNode(this); - C._ts_node_next_named_sibling_wasm(this.tree[0]); - return unmarshalNode(this.tree); - } - - get previousNamedSibling() { - marshalNode(this); - C._ts_node_prev_named_sibling_wasm(this.tree[0]); - return unmarshalNode(this.tree); - } - - get descendantCount() { - marshalNode(this); - return C._ts_node_descendant_count_wasm(this.tree[0]); - } - - get parent() { - marshalNode(this); - C._ts_node_parent_wasm(this.tree[0]); - return unmarshalNode(this.tree); - } - - descendantForIndex(start, end = start) { - if (typeof start !== 'number' || typeof end !== 'number') { - throw new Error('Arguments must be numbers'); - } - - marshalNode(this); - const address = TRANSFER_BUFFER + SIZE_OF_NODE; - setValue(address, start, 'i32'); - setValue(address + SIZE_OF_INT, end, 'i32'); - C._ts_node_descendant_for_index_wasm(this.tree[0]); - return unmarshalNode(this.tree); - } - - namedDescendantForIndex(start, end = start) { - if (typeof start !== 'number' || typeof end !== 'number') { - throw new Error('Arguments must be numbers'); - } - - marshalNode(this); - const address = TRANSFER_BUFFER + SIZE_OF_NODE; - setValue(address, start, 'i32'); - setValue(address + SIZE_OF_INT, end, 'i32'); - C._ts_node_named_descendant_for_index_wasm(this.tree[0]); - return unmarshalNode(this.tree); - } - - descendantForPosition(start, end = start) { - if (!isPoint(start) || !isPoint(end)) { - throw new Error('Arguments must be {row, column} objects'); - } - - marshalNode(this); - const address = TRANSFER_BUFFER + SIZE_OF_NODE; - marshalPoint(address, start); - marshalPoint(address + SIZE_OF_POINT, end); - C._ts_node_descendant_for_position_wasm(this.tree[0]); - return unmarshalNode(this.tree); - } - - namedDescendantForPosition(start, end = start) { - if (!isPoint(start) || !isPoint(end)) { - throw new Error('Arguments must be {row, column} objects'); - } - - marshalNode(this); - const address = TRANSFER_BUFFER + SIZE_OF_NODE; - marshalPoint(address, start); - marshalPoint(address + SIZE_OF_POINT, end); - C._ts_node_named_descendant_for_position_wasm(this.tree[0]); - return unmarshalNode(this.tree); - } - - walk() { - marshalNode(this); - C._ts_tree_cursor_new_wasm(this.tree[0]); - return new TreeCursor(INTERNAL, this.tree); - } - - toString() { - marshalNode(this); - const address = C._ts_node_to_string_wasm(this.tree[0]); - const result = AsciiToString(address); - C._free(address); - return result; - } -} - -class TreeCursor { - constructor(internal, tree) { - assertInternal(internal); - this.tree = tree; - unmarshalTreeCursor(this); - } - - delete() { - marshalTreeCursor(this); - C._ts_tree_cursor_delete_wasm(this.tree[0]); - this[0] = this[1] = this[2] = 0; - } - - reset(node) { - marshalNode(node); - marshalTreeCursor(this, TRANSFER_BUFFER + SIZE_OF_NODE); - C._ts_tree_cursor_reset_wasm(this.tree[0]); - unmarshalTreeCursor(this); - } - - resetTo(cursor) { - marshalTreeCursor(this, TRANSFER_BUFFER); - marshalTreeCursor(cursor, TRANSFER_BUFFER + SIZE_OF_CURSOR); - C._ts_tree_cursor_reset_to_wasm(this.tree[0], cursor.tree[0]); - unmarshalTreeCursor(this); - } - - get nodeType() { - return this.tree.language.types[this.nodeTypeId] || 'ERROR'; - } - - get nodeTypeId() { - marshalTreeCursor(this); - return C._ts_tree_cursor_current_node_type_id_wasm(this.tree[0]); - } - - get nodeStateId() { - marshalTreeCursor(this); - return C._ts_tree_cursor_current_node_state_id_wasm(this.tree[0]); - } - - get nodeId() { - marshalTreeCursor(this); - return C._ts_tree_cursor_current_node_id_wasm(this.tree[0]); - } - - get nodeIsNamed() { - marshalTreeCursor(this); - return C._ts_tree_cursor_current_node_is_named_wasm(this.tree[0]) === 1; - } - - get nodeIsMissing() { - marshalTreeCursor(this); - return C._ts_tree_cursor_current_node_is_missing_wasm(this.tree[0]) === 1; - } - - get nodeText() { - marshalTreeCursor(this); - const startIndex = C._ts_tree_cursor_start_index_wasm(this.tree[0]); - const endIndex = C._ts_tree_cursor_end_index_wasm(this.tree[0]); - return getText(this.tree, startIndex, endIndex); - } - - get startPosition() { - marshalTreeCursor(this); - C._ts_tree_cursor_start_position_wasm(this.tree[0]); - return unmarshalPoint(TRANSFER_BUFFER); - } - - get endPosition() { - marshalTreeCursor(this); - C._ts_tree_cursor_end_position_wasm(this.tree[0]); - return unmarshalPoint(TRANSFER_BUFFER); - } - - get startIndex() { - marshalTreeCursor(this); - return C._ts_tree_cursor_start_index_wasm(this.tree[0]); - } - - get endIndex() { - marshalTreeCursor(this); - return C._ts_tree_cursor_end_index_wasm(this.tree[0]); - } - - get currentNode() { - marshalTreeCursor(this); - C._ts_tree_cursor_current_node_wasm(this.tree[0]); - return unmarshalNode(this.tree); - } - - get currentFieldId() { - marshalTreeCursor(this); - return C._ts_tree_cursor_current_field_id_wasm(this.tree[0]); - } - - get currentFieldName() { - return this.tree.language.fields[this.currentFieldId]; - } - - get currentDepth() { - marshalTreeCursor(this); - return C._ts_tree_cursor_current_depth_wasm(this.tree[0]); - } - - get currentDescendantIndex() { - marshalTreeCursor(this); - return C._ts_tree_cursor_current_descendant_index_wasm(this.tree[0]); - } - - gotoFirstChild() { - marshalTreeCursor(this); - const result = C._ts_tree_cursor_goto_first_child_wasm(this.tree[0]); - unmarshalTreeCursor(this); - return result === 1; - } - - gotoLastChild() { - marshalTreeCursor(this); - const result = C._ts_tree_cursor_goto_last_child_wasm(this.tree[0]); - unmarshalTreeCursor(this); - return result === 1; - } - - gotoFirstChildForIndex(goalIndex) { - marshalTreeCursor(this); - setValue(TRANSFER_BUFFER + SIZE_OF_CURSOR, goalIndex, 'i32'); - const result = C._ts_tree_cursor_goto_first_child_for_index_wasm(this.tree[0]); - unmarshalTreeCursor(this); - return result === 1; - } - - gotoFirstChildForPosition(goalPosition) { - marshalTreeCursor(this); - marshalPoint(TRANSFER_BUFFER + SIZE_OF_CURSOR, goalPosition); - const result = C._ts_tree_cursor_goto_first_child_for_position_wasm(this.tree[0]); - unmarshalTreeCursor(this); - return result === 1; - } - - gotoNextSibling() { - marshalTreeCursor(this); - const result = C._ts_tree_cursor_goto_next_sibling_wasm(this.tree[0]); - unmarshalTreeCursor(this); - return result === 1; - } - - gotoPreviousSibling() { - marshalTreeCursor(this); - const result = C._ts_tree_cursor_goto_previous_sibling_wasm(this.tree[0]); - unmarshalTreeCursor(this); - return result === 1; - } - - gotoDescendant(goalDescendantindex) { - marshalTreeCursor(this); - C._ts_tree_cursor_goto_descendant_wasm(this.tree[0], goalDescendantindex); - unmarshalTreeCursor(this); - } - - gotoParent() { - marshalTreeCursor(this); - const result = C._ts_tree_cursor_goto_parent_wasm(this.tree[0]); - unmarshalTreeCursor(this); - return result === 1; - } -} - -class Language { - constructor(internal, address) { - assertInternal(internal); - this[0] = address; - this.types = new Array(C._ts_language_symbol_count(this[0])); - for (let i = 0, n = this.types.length; i < n; i++) { - if (C._ts_language_symbol_type(this[0], i) < 2) { - this.types[i] = UTF8ToString(C._ts_language_symbol_name(this[0], i)); - } - } - this.fields = new Array(C._ts_language_field_count(this[0]) + 1); - for (let i = 0, n = this.fields.length; i < n; i++) { - const fieldName = C._ts_language_field_name_for_id(this[0], i); - if (fieldName !== 0) { - this.fields[i] = UTF8ToString(fieldName); - } else { - this.fields[i] = null; - } - } - } - - get version() { - return C._ts_language_version(this[0]); - } - - get fieldCount() { - return this.fields.length - 1; - } - - get stateCount() { - return C._ts_language_state_count(this[0]); - } - - fieldIdForName(fieldName) { - const result = this.fields.indexOf(fieldName); - if (result !== -1) { - return result; - } else { - return null; - } - } - - fieldNameForId(fieldId) { - return this.fields[fieldId] || null; - } - - idForNodeType(type, named) { - const typeLength = lengthBytesUTF8(type); - const typeAddress = C._malloc(typeLength + 1); - stringToUTF8(type, typeAddress, typeLength + 1); - const result = C._ts_language_symbol_for_name(this[0], typeAddress, typeLength, named); - C._free(typeAddress); - return result || null; - } - - get nodeTypeCount() { - return C._ts_language_symbol_count(this[0]); - } - - nodeTypeForId(typeId) { - const name = C._ts_language_symbol_name(this[0], typeId); - return name ? UTF8ToString(name) : null; - } - - nodeTypeIsNamed(typeId) { - return C._ts_language_type_is_named_wasm(this[0], typeId) ? true : false; - } - - nodeTypeIsVisible(typeId) { - return C._ts_language_type_is_visible_wasm(this[0], typeId) ? true : false; - } - - nextState(stateId, typeId) { - return C._ts_language_next_state(this[0], stateId, typeId); - } - - lookaheadIterator(stateId) { - const address = C._ts_lookahead_iterator_new(this[0], stateId); - if (address) return new LookaheadIterable(INTERNAL, address, this); - return null; - } - - query(source) { - const sourceLength = lengthBytesUTF8(source); - const sourceAddress = C._malloc(sourceLength + 1); - stringToUTF8(source, sourceAddress, sourceLength + 1); - const address = C._ts_query_new( - this[0], - sourceAddress, - sourceLength, - TRANSFER_BUFFER, - TRANSFER_BUFFER + SIZE_OF_INT, - ); - - if (!address) { - const errorId = getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); - const errorByte = getValue(TRANSFER_BUFFER, 'i32'); - const errorIndex = UTF8ToString(sourceAddress, errorByte).length; - const suffix = source.substr(errorIndex, 100).split('\n')[0]; - let word = suffix.match(QUERY_WORD_REGEX)[0]; - let error; - switch (errorId) { - case 2: - error = new RangeError(`Bad node name '${word}'`); - break; - case 3: - error = new RangeError(`Bad field name '${word}'`); - break; - case 4: - error = new RangeError(`Bad capture name @${word}`); - break; - case 5: - error = new TypeError(`Bad pattern structure at offset ${errorIndex}: '${suffix}'...`); - word = ''; - break; - default: - error = new SyntaxError(`Bad syntax at offset ${errorIndex}: '${suffix}'...`); - word = ''; - break; - } - error.index = errorIndex; - error.length = word.length; - C._free(sourceAddress); - throw error; - } - - const stringCount = C._ts_query_string_count(address); - const captureCount = C._ts_query_capture_count(address); - const patternCount = C._ts_query_pattern_count(address); - const captureNames = new Array(captureCount); - const stringValues = new Array(stringCount); - - for (let i = 0; i < captureCount; i++) { - const nameAddress = C._ts_query_capture_name_for_id( - address, - i, - TRANSFER_BUFFER, - ); - const nameLength = getValue(TRANSFER_BUFFER, 'i32'); - captureNames[i] = UTF8ToString(nameAddress, nameLength); - } - - for (let i = 0; i < stringCount; i++) { - const valueAddress = C._ts_query_string_value_for_id( - address, - i, - TRANSFER_BUFFER, - ); - const nameLength = getValue(TRANSFER_BUFFER, 'i32'); - stringValues[i] = UTF8ToString(valueAddress, nameLength); - } - - const setProperties = new Array(patternCount); - const assertedProperties = new Array(patternCount); - const refutedProperties = new Array(patternCount); - const predicates = new Array(patternCount); - const textPredicates = new Array(patternCount); - - for (let i = 0; i < patternCount; i++) { - const predicatesAddress = C._ts_query_predicates_for_pattern( - address, - i, - TRANSFER_BUFFER, - ); - const stepCount = getValue(TRANSFER_BUFFER, 'i32'); - - predicates[i] = []; - textPredicates[i] = []; - - const steps = []; - let stepAddress = predicatesAddress; - for (let j = 0; j < stepCount; j++) { - const stepType = getValue(stepAddress, 'i32'); - stepAddress += SIZE_OF_INT; - const stepValueId = getValue(stepAddress, 'i32'); - stepAddress += SIZE_OF_INT; - if (stepType === PREDICATE_STEP_TYPE_CAPTURE) { - steps.push({type: 'capture', name: captureNames[stepValueId]}); - } else if (stepType === PREDICATE_STEP_TYPE_STRING) { - steps.push({type: 'string', value: stringValues[stepValueId]}); - } else if (steps.length > 0) { - if (steps[0].type !== 'string') { - throw new Error('Predicates must begin with a literal value'); - } - const operator = steps[0].value; - let isPositive = true; - let matchAll = true; - let captureName; - switch (operator) { - case 'any-not-eq?': - case 'not-eq?': - isPositive = false; - case 'any-eq?': - case 'eq?': - if (steps.length !== 3) { - throw new Error( - `Wrong number of arguments to \`#${operator}\` predicate. Expected 2, got ${steps.length - 1}`, - ); - } - if (steps[1].type !== 'capture') { - throw new Error( - `First argument of \`#${operator}\` predicate must be a capture. Got "${steps[1].value}"`, - ); - } - matchAll = !operator.startsWith('any-'); - if (steps[2].type === 'capture') { - const captureName1 = steps[1].name; - const captureName2 = steps[2].name; - textPredicates[i].push((captures) => { - const nodes1 = []; - const nodes2 = []; - for (const c of captures) { - if (c.name === captureName1) nodes1.push(c.node); - if (c.name === captureName2) nodes2.push(c.node); - } - const compare = (n1, n2, positive) => { - return positive ? - n1.text === n2.text : - n1.text !== n2.text; - }; - return matchAll ? - nodes1.every((n1) => nodes2.some((n2) => compare(n1, n2, isPositive))) : - nodes1.some((n1) => nodes2.some((n2) => compare(n1, n2, isPositive))); - }); - } else { - captureName = steps[1].name; - const stringValue = steps[2].value; - const matches = (n) => n.text === stringValue; - const doesNotMatch = (n) => n.text !== stringValue; - textPredicates[i].push((captures) => { - const nodes = []; - for (const c of captures) { - if (c.name === captureName) nodes.push(c.node); - } - const test = isPositive ? matches : doesNotMatch; - return matchAll ? - nodes.every(test) : - nodes.some(test); - }); - } - break; - - case 'any-not-match?': - case 'not-match?': - isPositive = false; - case 'any-match?': - case 'match?': - if (steps.length !== 3) { - throw new Error( - `Wrong number of arguments to \`#${operator}\` predicate. Expected 2, got ${steps.length - 1}.`, - ); - } - if (steps[1].type !== 'capture') { - throw new Error( - `First argument of \`#${operator}\` predicate must be a capture. Got "${steps[1].value}".`, - ); - } - if (steps[2].type !== 'string') { - throw new Error( - `Second argument of \`#${operator}\` predicate must be a string. Got @${steps[2].value}.`, - ); - } - captureName = steps[1].name; - const regex = new RegExp(steps[2].value); - matchAll = !operator.startsWith('any-'); - textPredicates[i].push((captures) => { - const nodes = []; - for (const c of captures) { - if (c.name === captureName) nodes.push(c.node.text); - } - const test = (text, positive) => { - return positive ? - regex.test(text) : - !regex.test(text); - }; - if (nodes.length === 0) return !isPositive; - return matchAll ? - nodes.every((text) => test(text, isPositive)) : - nodes.some((text) => test(text, isPositive)); - }); - break; - - case 'set!': - if (steps.length < 2 || steps.length > 3) { - throw new Error( - `Wrong number of arguments to \`#set!\` predicate. Expected 1 or 2. Got ${steps.length - 1}.`, - ); - } - if (steps.some((s) => s.type !== 'string')) { - throw new Error( - `Arguments to \`#set!\` predicate must be a strings.".`, - ); - } - if (!setProperties[i]) setProperties[i] = {}; - setProperties[i][steps[1].value] = steps[2] ? steps[2].value : null; - break; - - case 'is?': - case 'is-not?': - if (steps.length < 2 || steps.length > 3) { - throw new Error( - `Wrong number of arguments to \`#${operator}\` predicate. Expected 1 or 2. Got ${steps.length - 1}.`, - ); - } - if (steps.some((s) => s.type !== 'string')) { - throw new Error( - `Arguments to \`#${operator}\` predicate must be a strings.".`, - ); - } - const properties = operator === 'is?' ? assertedProperties : refutedProperties; - if (!properties[i]) properties[i] = {}; - properties[i][steps[1].value] = steps[2] ? steps[2].value : null; - break; - - case 'not-any-of?': - isPositive = false; - case 'any-of?': - if (steps.length < 2) { - throw new Error( - `Wrong number of arguments to \`#${operator}\` predicate. Expected at least 1. Got ${steps.length - 1}.`, - ); - } - if (steps[1].type !== 'capture') { - throw new Error( - `First argument of \`#${operator}\` predicate must be a capture. Got "${steps[1].value}".`, - ); - } - for (let i = 2; i < steps.length; i++) { - if (steps[i].type !== 'string') { - throw new Error( - `Arguments to \`#${operator}\` predicate must be a strings.".`, - ); - } - } - captureName = steps[1].name; - const values = steps.slice(2).map((s) => s.value); - textPredicates[i].push((captures) => { - const nodes = []; - for (const c of captures) { - if (c.name === captureName) nodes.push(c.node.text); - } - if (nodes.length === 0) return !isPositive; - return nodes.every((text) => values.includes(text)) === isPositive; - }); - break; - - default: - predicates[i].push({operator, operands: steps.slice(1)}); - } - - steps.length = 0; - } - } - - Object.freeze(setProperties[i]); - Object.freeze(assertedProperties[i]); - Object.freeze(refutedProperties[i]); - } - - C._free(sourceAddress); - return new Query( - INTERNAL, - address, - captureNames, - textPredicates, - predicates, - Object.freeze(setProperties), - Object.freeze(assertedProperties), - Object.freeze(refutedProperties), - ); - } - - static load(input) { - let bytes; - if (input instanceof Uint8Array) { - bytes = Promise.resolve(input); - } else { - if (globalThis.process?.versions?.node) { - const fs = require('fs/promises'); - bytes = fs.readFile(input); - } else { - bytes = fetch(input) - .then((response) => response.arrayBuffer() - .then((buffer) => { - if (response.ok) { - return new Uint8Array(buffer); - } else { - const body = new TextDecoder('utf-8').decode(buffer); - throw new Error(`Language.load failed with status ${response.status}.\n\n${body}`); - } - })); - } - } - - return bytes - .then((bytes) => loadWebAssemblyModule(bytes, {loadAsync: true})) - .then((mod) => { - const symbolNames = Object.keys(mod); - const functionName = symbolNames.find((key) => - LANGUAGE_FUNCTION_REGEX.test(key) && - !key.includes('external_scanner_'), - ); - if (!functionName) { - console.log(`Couldn't find language function in WASM file. Symbols:\n${JSON.stringify(symbolNames, null, 2)}`); - } - const languageAddress = mod[functionName](); - return new Language(INTERNAL, languageAddress); - }); - } -} - -class LookaheadIterable { - constructor(internal, address, language) { - assertInternal(internal); - this[0] = address; - this.language = language; - } - - get currentTypeId() { - return C._ts_lookahead_iterator_current_symbol(this[0]); - } - - get currentType() { - return this.language.types[this.currentTypeId] || 'ERROR'; - } - - delete() { - C._ts_lookahead_iterator_delete(this[0]); - this[0] = 0; - } - - resetState(stateId) { - return C._ts_lookahead_iterator_reset_state(this[0], stateId); - } - - reset(language, stateId) { - if (C._ts_lookahead_iterator_reset(this[0], language[0], stateId)) { - this.language = language; - return true; - } - - return false; - } - - [Symbol.iterator]() { - const self = this; - return { - next() { - if (C._ts_lookahead_iterator_next(self[0])) { - return {done: false, value: self.currentType}; - } - - return {done: true, value: ''}; - }, - }; - } -} - -class Query { - constructor( - internal, address, captureNames, textPredicates, predicates, - setProperties, assertedProperties, refutedProperties, - ) { - assertInternal(internal); - this[0] = address; - this.captureNames = captureNames; - this.textPredicates = textPredicates; - this.predicates = predicates; - this.setProperties = setProperties; - this.assertedProperties = assertedProperties; - this.refutedProperties = refutedProperties; - this.exceededMatchLimit = false; - } - - delete() { - C._ts_query_delete(this[0]); - this[0] = 0; - } - - matches( - node, - { - startPosition = ZERO_POINT, - endPosition = ZERO_POINT, - startIndex = 0, - endIndex = 0, - matchLimit = 0xFFFFFFFF, - maxStartDepth = 0xFFFFFFFF, - timeoutMicros = 0, - } = {}, - ) { - if (typeof matchLimit !== 'number') { - throw new Error('Arguments must be numbers'); - } - if (endIndex != 0 && startIndex > endIndex) { - throw new Error('`startIndex` cannot be greater than `endIndex`'); - } - if (endPosition != ZERO_POINT && (startPosition.row > endPosition.row || - (startPosition.row == endPosition.row && startPosition.column > endPosition.row))) { - throw new Error('`startPosition` cannot be greater than `endPosition`'); - } - - marshalNode(node); - - C._ts_query_matches_wasm( - this[0], - node.tree[0], - startPosition.row, - startPosition.column, - endPosition.row, - endPosition.column, - startIndex, - endIndex, - matchLimit, - maxStartDepth, - timeoutMicros, - ); - - const rawCount = getValue(TRANSFER_BUFFER, 'i32'); - const startAddress = getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); - const didExceedMatchLimit = getValue(TRANSFER_BUFFER + 2 * SIZE_OF_INT, 'i32'); - const result = new Array(rawCount); - this.exceededMatchLimit = Boolean(didExceedMatchLimit); - - let filteredCount = 0; - let address = startAddress; - for (let i = 0; i < rawCount; i++) { - const pattern = getValue(address, 'i32'); - address += SIZE_OF_INT; - const captureCount = getValue(address, 'i32'); - address += SIZE_OF_INT; - - const captures = new Array(captureCount); - address = unmarshalCaptures(this, node.tree, address, captures); - if (this.textPredicates[pattern].every((p) => p(captures))) { - result[filteredCount] = {pattern, captures}; - const setProperties = this.setProperties[pattern]; - if (setProperties) result[filteredCount].setProperties = setProperties; - const assertedProperties = this.assertedProperties[pattern]; - if (assertedProperties) result[filteredCount].assertedProperties = assertedProperties; - const refutedProperties = this.refutedProperties[pattern]; - if (refutedProperties) result[filteredCount].refutedProperties = refutedProperties; - filteredCount++; - } - } - result.length = filteredCount; - - C._free(startAddress); - return result; - } - - captures( - node, - { - startPosition = ZERO_POINT, - endPosition = ZERO_POINT, - startIndex = 0, - endIndex = 0, - matchLimit = 0xFFFFFFFF, - maxStartDepth = 0xFFFFFFFF, - timeoutMicros = 0, - } = {}, - ) { - if (typeof matchLimit !== 'number') { - throw new Error('Arguments must be numbers'); - } - if (endIndex != 0 && startIndex > endIndex) { - throw new Error('`startIndex` cannot be greater than `endIndex`'); - } - if (endPosition != ZERO_POINT && (startPosition.row > endPosition.row || - (startPosition.row == endPosition.row && startPosition.column > endPosition.row))) { - throw new Error('`startPosition` cannot be greater than `endPosition`'); - } - - marshalNode(node); - - C._ts_query_captures_wasm( - this[0], - node.tree[0], - startPosition.row, - startPosition.column, - endPosition.row, - endPosition.column, - startIndex, - endIndex, - matchLimit, - maxStartDepth, - timeoutMicros, - ); - - const count = getValue(TRANSFER_BUFFER, 'i32'); - const startAddress = getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); - const didExceedMatchLimit = getValue(TRANSFER_BUFFER + 2 * SIZE_OF_INT, 'i32'); - const result = []; - this.exceededMatchLimit = Boolean(didExceedMatchLimit); - - const captures = []; - let address = startAddress; - for (let i = 0; i < count; i++) { - const pattern = getValue(address, 'i32'); - address += SIZE_OF_INT; - const captureCount = getValue(address, 'i32'); - address += SIZE_OF_INT; - const captureIndex = getValue(address, 'i32'); - address += SIZE_OF_INT; - - captures.length = captureCount; - address = unmarshalCaptures(this, node.tree, address, captures); - - if (this.textPredicates[pattern].every((p) => p(captures))) { - const capture = captures[captureIndex]; - const setProperties = this.setProperties[pattern]; - if (setProperties) capture.setProperties = setProperties; - const assertedProperties = this.assertedProperties[pattern]; - if (assertedProperties) capture.assertedProperties = assertedProperties; - const refutedProperties = this.refutedProperties[pattern]; - if (refutedProperties) capture.refutedProperties = refutedProperties; - result.push(capture); - } - } - - C._free(startAddress); - return result; - } - - predicatesForPattern(patternIndex) { - return this.predicates[patternIndex]; - } - - disableCapture(captureName) { - const captureNameLength = lengthBytesUTF8(captureName); - const captureNameAddress = C._malloc(captureNameLength + 1); - stringToUTF8(captureName, captureNameAddress, captureNameLength + 1); - C._ts_query_disable_capture(this[0], captureNameAddress, captureNameLength); - C._free(captureNameAddress); - } - - didExceedMatchLimit() { - return this.exceededMatchLimit; - } -} - -function getText(tree, startIndex, endIndex) { - const length = endIndex - startIndex; - let result = tree.textCallback(startIndex, null, endIndex); - startIndex += result.length; - while (startIndex < endIndex) { - const string = tree.textCallback(startIndex, null, endIndex); - if (string && string.length > 0) { - startIndex += string.length; - result += string; - } else { - break; - } - } - if (startIndex > endIndex) { - result = result.slice(0, length); - } - return result; -} - -function unmarshalCaptures(query, tree, address, result) { - for (let i = 0, n = result.length; i < n; i++) { - const captureIndex = getValue(address, 'i32'); - address += SIZE_OF_INT; - const node = unmarshalNode(tree, address); - address += SIZE_OF_NODE; - result[i] = {name: query.captureNames[captureIndex], node}; - } - return address; -} - -function assertInternal(x) { - if (x !== INTERNAL) throw new Error('Illegal constructor'); -} - -function isPoint(point) { - return ( - point && - typeof point.row === 'number' && - typeof point.column === 'number' - ); -} - -function marshalNode(node) { - let address = TRANSFER_BUFFER; - setValue(address, node.id, 'i32'); - address += SIZE_OF_INT; - setValue(address, node.startIndex, 'i32'); - address += SIZE_OF_INT; - setValue(address, node.startPosition.row, 'i32'); - address += SIZE_OF_INT; - setValue(address, node.startPosition.column, 'i32'); - address += SIZE_OF_INT; - setValue(address, node[0], 'i32'); -} - -function unmarshalNode(tree, address = TRANSFER_BUFFER) { - const id = getValue(address, 'i32'); - address += SIZE_OF_INT; - if (id === 0) return null; - - const index = getValue(address, 'i32'); - address += SIZE_OF_INT; - const row = getValue(address, 'i32'); - address += SIZE_OF_INT; - const column = getValue(address, 'i32'); - address += SIZE_OF_INT; - const other = getValue(address, 'i32'); - - const result = new Node(INTERNAL, tree); - result.id = id; - result.startIndex = index; - result.startPosition = {row, column}; - result[0] = other; - - return result; -} - -function marshalTreeCursor(cursor, address = TRANSFER_BUFFER) { - setValue(address + 0 * SIZE_OF_INT, cursor[0], 'i32'); - setValue(address + 1 * SIZE_OF_INT, cursor[1], 'i32'); - setValue(address + 2 * SIZE_OF_INT, cursor[2], 'i32'); - setValue(address + 3 * SIZE_OF_INT, cursor[3], 'i32'); -} - -function unmarshalTreeCursor(cursor) { - cursor[0] = getValue(TRANSFER_BUFFER + 0 * SIZE_OF_INT, 'i32'); - cursor[1] = getValue(TRANSFER_BUFFER + 1 * SIZE_OF_INT, 'i32'); - cursor[2] = getValue(TRANSFER_BUFFER + 2 * SIZE_OF_INT, 'i32'); - cursor[3] = getValue(TRANSFER_BUFFER + 3 * SIZE_OF_INT, 'i32'); -} - -function marshalPoint(address, point) { - setValue(address, point.row, 'i32'); - setValue(address + SIZE_OF_INT, point.column, 'i32'); -} - -function unmarshalPoint(address) { - const result = { - row: getValue(address, 'i32') >>> 0, - column: getValue(address + SIZE_OF_INT, 'i32') >>> 0, - }; - return result; -} - -function marshalRange(address, range) { - marshalPoint(address, range.startPosition); address += SIZE_OF_POINT; - marshalPoint(address, range.endPosition); address += SIZE_OF_POINT; - setValue(address, range.startIndex, 'i32'); address += SIZE_OF_INT; - setValue(address, range.endIndex, 'i32'); address += SIZE_OF_INT; -} - -function unmarshalRange(address) { - const result = {}; - result.startPosition = unmarshalPoint(address); address += SIZE_OF_POINT; - result.endPosition = unmarshalPoint(address); address += SIZE_OF_POINT; - result.startIndex = getValue(address, 'i32') >>> 0; address += SIZE_OF_INT; - result.endIndex = getValue(address, 'i32') >>> 0; - return result; -} - -function marshalEdit(edit) { - let address = TRANSFER_BUFFER; - marshalPoint(address, edit.startPosition); address += SIZE_OF_POINT; - marshalPoint(address, edit.oldEndPosition); address += SIZE_OF_POINT; - marshalPoint(address, edit.newEndPosition); address += SIZE_OF_POINT; - setValue(address, edit.startIndex, 'i32'); address += SIZE_OF_INT; - setValue(address, edit.oldEndIndex, 'i32'); address += SIZE_OF_INT; - setValue(address, edit.newEndIndex, 'i32'); address += SIZE_OF_INT; -} diff --git a/lib/binding_web/check-artifacts-fresh.js b/lib/binding_web/check-artifacts-fresh.js deleted file mode 100755 index a0c24933..00000000 --- a/lib/binding_web/check-artifacts-fresh.js +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env node - -const fs = require('fs'); -const path = require('path'); - -const inputFiles = [ - 'binding.c', - 'binding.js', - 'exports.txt', - 'imports.js', - 'prefix.js', - ...list('../include/tree_sitter'), - ...list('../src'), -]; - -const outputFiles = ['tree-sitter.js', 'tree-sitter.wasm']; - -const outputMtime = Math.min(...outputFiles.map(mtime)); - -for (const inputFile of inputFiles) { - if (mtime(inputFile) > outputMtime) { - console.log(`File '${inputFile}' has changed. Re-run 'script/build-wasm'.`); - process.exit(1); - } -} - -function list(dir) { - return fs - .readdirSync(path.join(__dirname, dir), 'utf8') - .filter((p) => !p.startsWith('.')) - .map((p) => path.join(dir, p)); -} - -function mtime(p) { - return fs.statSync(path.join(__dirname, p)).mtime; -} diff --git a/lib/binding_web/eslint.config.mjs b/lib/binding_web/eslint.config.mjs new file mode 100644 index 00000000..a8bb9739 --- /dev/null +++ b/lib/binding_web/eslint.config.mjs @@ -0,0 +1,27 @@ +import eslint from '@eslint/js'; +import tseslint from 'typescript-eslint'; + +export default tseslint.config( + eslint.configs.recommended, + tseslint.configs.recommendedTypeChecked, + tseslint.configs.strictTypeChecked, + tseslint.configs.stylisticTypeChecked, + { + languageOptions: { + parserOptions: { + projectService: true, + tsconfigRootDir: import.meta.dirname, + }, + }, + rules: { + 'no-fallthrough': 'off', + '@typescript-eslint/no-non-null-assertion': 'off', + '@typescript-eslint/no-unnecessary-condition': ['error', { + allowConstantLoopConditions: true + }], + '@typescript-eslint/restrict-template-expressions': ['error', { + allowNumber: true + }], + } + }, +); diff --git a/lib/binding_web/imports.js b/lib/binding_web/imports.js deleted file mode 100644 index dfb65bbe..00000000 --- a/lib/binding_web/imports.js +++ /dev/null @@ -1,25 +0,0 @@ -mergeInto(LibraryManager.library, { - tree_sitter_parse_callback( - inputBufferAddress, - index, - row, - column, - lengthAddress, - ) { - const INPUT_BUFFER_SIZE = 10 * 1024; - const string = currentParseCallback(index, {row, column}); - if (typeof string === 'string') { - setValue(lengthAddress, string.length, 'i32'); - stringToUTF16(string, inputBufferAddress, INPUT_BUFFER_SIZE); - } else { - setValue(lengthAddress, 0, 'i32'); - } - }, - - tree_sitter_log_callback(isLexMessage, messageAddress) { - if (currentLogCallback) { - const message = UTF8ToString(messageAddress); - currentLogCallback(message, isLexMessage !== 0); - } - }, -}); diff --git a/lib/binding_web/exports.txt b/lib/binding_web/lib/exports.txt similarity index 86% rename from lib/binding_web/exports.txt rename to lib/binding_web/lib/exports.txt index 7507c51f..86104496 100644 --- a/lib/binding_web/exports.txt +++ b/lib/binding_web/lib/exports.txt @@ -5,12 +5,17 @@ "ts_language_type_is_visible_wasm", "ts_language_symbol_count", "ts_language_state_count", +"ts_language_supertypes_wasm", +"ts_language_subtypes_wasm", "ts_language_symbol_for_name", "ts_language_symbol_name", "ts_language_symbol_type", -"ts_language_version", +"ts_language_name", +"ts_language_abi_version", +"ts_language_metadata_wasm", "ts_language_next_state", "ts_node_field_name_for_child_wasm", +"ts_node_field_name_for_named_child_wasm", "ts_node_children_by_field_id_wasm", "ts_node_first_child_for_byte_wasm", "ts_node_first_named_child_for_byte_wasm", @@ -39,6 +44,7 @@ "ts_node_next_named_sibling_wasm", "ts_node_next_sibling_wasm", "ts_node_parent_wasm", +"ts_node_child_with_descendant_wasm", "ts_node_prev_named_sibling_wasm", "ts_node_prev_sibling_wasm", "ts_node_descendant_count_wasm", @@ -55,8 +61,7 @@ "ts_parser_set_language", "ts_parser_set_included_ranges", "ts_parser_included_ranges_wasm", -"ts_parser_set_timeout_micros", -"ts_parser_timeout_micros", +"ts_point_edit", "ts_query_capture_count", "ts_query_capture_name_for_id", "ts_query_captures_wasm", @@ -66,8 +71,16 @@ "ts_query_pattern_count", "ts_query_predicates_for_pattern", "ts_query_disable_capture", +"ts_query_start_byte_for_pattern", +"ts_query_end_byte_for_pattern", "ts_query_string_count", "ts_query_string_value_for_id", +"ts_query_disable_pattern", +"ts_query_capture_quantifier_for_id", +"ts_query_is_pattern_non_local", +"ts_query_is_pattern_rooted", +"ts_query_is_pattern_guaranteed_at_step", +"ts_range_edit", "ts_tree_copy", "ts_tree_cursor_current_field_id_wasm", "ts_tree_cursor_current_depth_wasm", @@ -94,6 +107,7 @@ "ts_tree_cursor_reset_to_wasm", "ts_tree_cursor_start_index_wasm", "ts_tree_cursor_start_position_wasm", +"ts_tree_cursor_copy_wasm", "ts_tree_delete", "ts_tree_included_ranges_wasm", "ts_tree_edit_wasm", diff --git a/lib/binding_web/lib/imports.js b/lib/binding_web/lib/imports.js new file mode 100644 index 00000000..01e789ae --- /dev/null +++ b/lib/binding_web/lib/imports.js @@ -0,0 +1,39 @@ +mergeInto(LibraryManager.library, { + tree_sitter_parse_callback( + inputBufferAddress, + index, + row, + column, + lengthAddress, + ) { + const INPUT_BUFFER_SIZE = 10 * 1024; + const string = Module.currentParseCallback(index, { row, column }); + if (typeof string === 'string') { + setValue(lengthAddress, string.length, 'i32'); + stringToUTF16(string, inputBufferAddress, INPUT_BUFFER_SIZE); + } else { + setValue(lengthAddress, 0, 'i32'); + } + }, + + tree_sitter_log_callback(isLexMessage, messageAddress) { + if (Module.currentLogCallback) { + const message = UTF8ToString(messageAddress); + Module.currentLogCallback(message, isLexMessage !== 0); + } + }, + + tree_sitter_progress_callback(currentOffset, hasError) { + if (Module.currentProgressCallback) { + return Module.currentProgressCallback({ currentOffset, hasError }); + } + return false; + }, + + tree_sitter_query_progress_callback(currentOffset) { + if (Module.currentQueryProgressCallback) { + return Module.currentQueryProgressCallback({ currentOffset }); + } + return false; + }, +}); diff --git a/lib/binding_web/lib/prefix.js b/lib/binding_web/lib/prefix.js new file mode 100644 index 00000000..bd953d5d --- /dev/null +++ b/lib/binding_web/lib/prefix.js @@ -0,0 +1,4 @@ +Module.currentQueryProgressCallback = null; +Module.currentProgressCallback = null; +Module.currentLogCallback = null; +Module.currentParseCallback = null; diff --git a/lib/binding_web/binding.c b/lib/binding_web/lib/tree-sitter.c similarity index 80% rename from lib/binding_web/binding.c rename to lib/binding_web/lib/tree-sitter.c index 23faeafe..828132ac 100644 --- a/lib/binding_web/binding.c +++ b/lib/binding_web/lib/tree-sitter.c @@ -16,10 +16,16 @@ const void *TRANSFER_BUFFER[12] = { NULL, NULL, NULL, NULL, }; +static const int SIZE_OF_CURSOR = 4; +static const int SIZE_OF_NODE = 5; +static const int SIZE_OF_POINT = 2; +static const int SIZE_OF_RANGE = 2 + (2 * SIZE_OF_POINT); +static const int SIZE_OF_CAPTURE = 1 + SIZE_OF_NODE; + void *ts_init() { TRANSFER_BUFFER[0] = (const void *)TREE_SITTER_LANGUAGE_VERSION; TRANSFER_BUFFER[1] = (const void *)TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION; - return TRANSFER_BUFFER; + return (void*)TRANSFER_BUFFER; } static uint32_t code_unit_to_byte(uint32_t unit) { @@ -38,17 +44,22 @@ static inline void marshal_node(const void **buffer, TSNode node) { buffer[4] = (const void *)node.context[3]; } -static inline TSNode unmarshal_node(const TSTree *tree) { +static inline TSNode unmarshal_node_at(const TSTree *tree, uint32_t index) { TSNode node; - node.id = TRANSFER_BUFFER[0]; - node.context[0] = code_unit_to_byte((uint32_t)TRANSFER_BUFFER[1]); - node.context[1] = (uint32_t)TRANSFER_BUFFER[2]; - node.context[2] = code_unit_to_byte((uint32_t)TRANSFER_BUFFER[3]); - node.context[3] = (uint32_t)TRANSFER_BUFFER[4]; + const void **buffer = TRANSFER_BUFFER + index * SIZE_OF_NODE; + node.id = buffer[0]; + node.context[0] = code_unit_to_byte((uint32_t)buffer[1]); + node.context[1] = (uint32_t)buffer[2]; + node.context[2] = code_unit_to_byte((uint32_t)buffer[3]); + node.context[3] = (uint32_t)buffer[4]; node.tree = tree; return node; } +static inline TSNode unmarshal_node(const TSTree *tree) { + return unmarshal_node_at(tree, 0); +} + static inline void marshal_cursor(const TSTreeCursor *cursor) { TRANSFER_BUFFER[0] = cursor->id; TRANSFER_BUFFER[1] = (const void *)cursor->context[0]; @@ -95,15 +106,26 @@ static void unmarshal_range(TSRange *range) { static TSInputEdit unmarshal_edit() { TSInputEdit edit; const void **address = TRANSFER_BUFFER; - edit.start_point = unmarshal_point(address); address += 2; - edit.old_end_point = unmarshal_point(address); address += 2; - edit.new_end_point = unmarshal_point(address); address += 2; + edit.start_point = unmarshal_point(address); address += SIZE_OF_POINT; + edit.old_end_point = unmarshal_point(address); address += SIZE_OF_POINT; + edit.new_end_point = unmarshal_point(address); address += SIZE_OF_POINT; edit.start_byte = code_unit_to_byte((uint32_t)*address); address += 1; edit.old_end_byte = code_unit_to_byte((uint32_t)*address); address += 1; edit.new_end_byte = code_unit_to_byte((uint32_t)*address); address += 1; return edit; } +static void marshal_language_metadata(const TSLanguageMetadata *metadata) { + if (metadata == NULL) { + TRANSFER_BUFFER[0] = 0; + return; + } + TRANSFER_BUFFER[0] = (const void*)3; + TRANSFER_BUFFER[1] = (const void*)(uint32_t)metadata->major_version; + TRANSFER_BUFFER[2] = (const void*)(uint32_t)metadata->minor_version; + TRANSFER_BUFFER[3] = (const void*)(uint32_t)metadata->patch_version; +} + /********************/ /* Section - Parser */ /********************/ @@ -121,6 +143,15 @@ extern void tree_sitter_log_callback( const char *message ); +extern bool tree_sitter_progress_callback( + uint32_t current_offset, + bool has_error +); + +extern bool tree_sitter_query_progress_callback( + uint32_t current_offset +); + static const char *call_parse_callback( void *payload, uint32_t byte, @@ -150,6 +181,18 @@ static void call_log_callback( tree_sitter_log_callback(log_type == TSLogTypeLex, message); } +static bool progress_callback( + TSParseState *state +) { + return tree_sitter_progress_callback(state->current_byte_offset, state->has_error); +} + +static bool query_progress_callback( + TSQueryCursorState *state +) { + return tree_sitter_query_progress_callback(state->current_byte_offset); +} + void ts_parser_new_wasm() { TSParser *parser = ts_parser_new(); char *input_buffer = calloc(INPUT_BUFFER_SIZE, sizeof(char)); @@ -172,7 +215,8 @@ TSTree *ts_parser_parse_wasm( TSInput input = { input_buffer, call_parse_callback, - TSInputEncodingUTF16LE + TSInputEncodingUTF16LE, + NULL, }; if (range_count) { for (unsigned i = 0; i < range_count; i++) { @@ -183,7 +227,10 @@ TSTree *ts_parser_parse_wasm( } else { ts_parser_set_included_ranges(self, NULL, 0); } - return ts_parser_parse(self, old_tree, input); + + TSParseOptions options = {.payload = NULL, .progress_callback = progress_callback}; + + return ts_parser_parse_with_options(self, old_tree, input, options); } void ts_parser_included_ranges_wasm(TSParser *self) { @@ -212,6 +259,25 @@ int ts_language_type_is_visible_wasm(const TSLanguage *self, TSSymbol typeId) { return symbolType <= TSSymbolTypeAnonymous; } +void ts_language_metadata_wasm(const TSLanguage *self) { + const TSLanguageMetadata *metadata = ts_language_metadata(self); + marshal_language_metadata(metadata); +} + +void ts_language_supertypes_wasm(const TSLanguage *self) { + uint32_t length; + const TSSymbol *supertypes = ts_language_supertypes(self, &length); + TRANSFER_BUFFER[0] = (const void *)length; + TRANSFER_BUFFER[1] = supertypes; +} + +void ts_language_subtypes_wasm(const TSLanguage *self, TSSymbol supertype) { + uint32_t length; + const TSSymbol *subtypes = ts_language_subtypes(self, supertype, &length); + TRANSFER_BUFFER[0] = (const void *)length; + TRANSFER_BUFFER[1] = subtypes; +} + /******************/ /* Section - Tree */ /******************/ @@ -222,7 +288,7 @@ void ts_tree_root_node_wasm(const TSTree *tree) { void ts_tree_root_node_with_offset_wasm(const TSTree *tree) { // read int and point from transfer buffer - const void **address = TRANSFER_BUFFER + 5; + const void **address = TRANSFER_BUFFER + SIZE_OF_NODE; uint32_t offset = code_unit_to_byte((uint32_t)address[0]); TSPoint extent = unmarshal_point(address + 1); TSNode node = ts_tree_root_node_with_offset(tree, offset, extent); @@ -264,6 +330,12 @@ void ts_tree_cursor_new_wasm(const TSTree *tree) { marshal_cursor(&cursor); } +void ts_tree_cursor_copy_wasm(const TSTree *tree) { + TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); + TSTreeCursor copy = ts_tree_cursor_copy(&cursor); + marshal_cursor(©); +} + void ts_tree_cursor_delete_wasm(const TSTree *tree) { TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); ts_tree_cursor_delete(&cursor); @@ -271,14 +343,14 @@ void ts_tree_cursor_delete_wasm(const TSTree *tree) { void ts_tree_cursor_reset_wasm(const TSTree *tree) { TSNode node = unmarshal_node(tree); - TSTreeCursor cursor = unmarshal_cursor(&TRANSFER_BUFFER[5], tree); + TSTreeCursor cursor = unmarshal_cursor(&TRANSFER_BUFFER[SIZE_OF_NODE], tree); ts_tree_cursor_reset(&cursor, node); marshal_cursor(&cursor); } void ts_tree_cursor_reset_to_wasm(const TSTree *_dst, const TSTree *_src) { TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, _dst); - TSTreeCursor src = unmarshal_cursor(&TRANSFER_BUFFER[4], _src); + TSTreeCursor src = unmarshal_cursor(&TRANSFER_BUFFER[SIZE_OF_CURSOR], _src); ts_tree_cursor_reset_to(&cursor, &src); marshal_cursor(&cursor); } @@ -433,6 +505,11 @@ const char *ts_node_field_name_for_child_wasm(const TSTree *tree, uint32_t index return ts_node_field_name_for_child(node, index); } +const char *ts_node_field_name_for_named_child_wasm(const TSTree *tree, uint32_t index) { + TSNode node = unmarshal_node(tree); + return ts_node_field_name_for_named_child(node, index); +} + void ts_node_children_by_field_id_wasm(const TSTree *tree, uint32_t field_id) { TSNode node = unmarshal_node(tree); TSTreeCursor cursor = ts_tree_cursor_new(node); @@ -459,25 +536,25 @@ void ts_node_children_by_field_id_wasm(const TSTree *tree, uint32_t field_id) { if (!ts_tree_cursor_goto_next_sibling(&cursor)) { done = true; } - array_grow_by(&result, 5); - marshal_node(result.contents + result.size - 5, result_node); + array_grow_by(&result, SIZE_OF_NODE); + marshal_node(result.contents + result.size - SIZE_OF_NODE, result_node); } ts_tree_cursor_delete(&cursor); - TRANSFER_BUFFER[0] = (const void*)(result.size / 5); - TRANSFER_BUFFER[1] = result.contents; + TRANSFER_BUFFER[0] = (const void*)(result.size / SIZE_OF_NODE); + TRANSFER_BUFFER[1] = (const void*)result.contents; } void ts_node_first_child_for_byte_wasm(const TSTree *tree) { TSNode node = unmarshal_node(tree); - const void** address = TRANSFER_BUFFER + 5; + const void** address = TRANSFER_BUFFER + SIZE_OF_NODE; uint32_t byte = code_unit_to_byte((uint32_t)address[0]); marshal_node(TRANSFER_BUFFER, ts_node_first_child_for_byte(node, byte)); } void ts_node_first_named_child_for_byte_wasm(const TSTree *tree) { TSNode node = unmarshal_node(tree); - const void** address = TRANSFER_BUFFER + 5; + const void** address = TRANSFER_BUFFER + SIZE_OF_NODE; uint32_t byte = code_unit_to_byte((uint32_t)address[0]); marshal_node(TRANSFER_BUFFER, ts_node_first_named_child_for_byte(node, byte)); } @@ -542,9 +619,15 @@ void ts_node_parent_wasm(const TSTree *tree) { marshal_node(TRANSFER_BUFFER, ts_node_parent(node)); } +void ts_node_child_with_descendant_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + TSNode descendant = unmarshal_node_at(tree, 1); + marshal_node(TRANSFER_BUFFER, ts_node_child_with_descendant(node, descendant)); +} + void ts_node_descendant_for_index_wasm(const TSTree *tree) { TSNode node = unmarshal_node(tree); - const void **address = TRANSFER_BUFFER + 5; + const void **address = TRANSFER_BUFFER + SIZE_OF_NODE; uint32_t start = code_unit_to_byte((uint32_t)address[0]); uint32_t end = code_unit_to_byte((uint32_t)address[1]); marshal_node(TRANSFER_BUFFER, ts_node_descendant_for_byte_range(node, start, end)); @@ -552,7 +635,7 @@ void ts_node_descendant_for_index_wasm(const TSTree *tree) { void ts_node_named_descendant_for_index_wasm(const TSTree *tree) { TSNode node = unmarshal_node(tree); - const void **address = TRANSFER_BUFFER + 5; + const void **address = TRANSFER_BUFFER + SIZE_OF_NODE; uint32_t start = code_unit_to_byte((uint32_t)address[0]); uint32_t end = code_unit_to_byte((uint32_t)address[1]); marshal_node(TRANSFER_BUFFER, ts_node_named_descendant_for_byte_range(node, start, end)); @@ -560,16 +643,16 @@ void ts_node_named_descendant_for_index_wasm(const TSTree *tree) { void ts_node_descendant_for_position_wasm(const TSTree *tree) { TSNode node = unmarshal_node(tree); - const void **address = TRANSFER_BUFFER + 5; - TSPoint start = unmarshal_point(address); address += 2; + const void **address = TRANSFER_BUFFER + SIZE_OF_NODE; + TSPoint start = unmarshal_point(address); address += SIZE_OF_POINT; TSPoint end = unmarshal_point(address); marshal_node(TRANSFER_BUFFER, ts_node_descendant_for_point_range(node, start, end)); } void ts_node_named_descendant_for_position_wasm(const TSTree *tree) { TSNode node = unmarshal_node(tree); - const void **address = TRANSFER_BUFFER + 5; - TSPoint start = unmarshal_point(address); address += 2; + const void **address = TRANSFER_BUFFER + SIZE_OF_NODE; + TSPoint start = unmarshal_point(address); address += SIZE_OF_POINT; TSPoint end = unmarshal_point(address); marshal_node(TRANSFER_BUFFER, ts_node_named_descendant_for_point_range(node, start, end)); } @@ -604,20 +687,20 @@ void ts_node_children_wasm(const TSTree *tree) { uint32_t count = ts_node_child_count(node); const void **result = NULL; if (count > 0) { - result = calloc(sizeof(void *), 5 * count); + result = (const void**)calloc(sizeof(void *), SIZE_OF_NODE * count); const void **address = result; ts_tree_cursor_reset(&scratch_cursor, node); ts_tree_cursor_goto_first_child(&scratch_cursor); marshal_node(address, ts_tree_cursor_current_node(&scratch_cursor)); for (uint32_t i = 1; i < count; i++) { - address += 5; + address += SIZE_OF_NODE; ts_tree_cursor_goto_next_sibling(&scratch_cursor); TSNode child = ts_tree_cursor_current_node(&scratch_cursor); marshal_node(address, child); } } TRANSFER_BUFFER[0] = (const void *)count; - TRANSFER_BUFFER[1] = result; + TRANSFER_BUFFER[1] = (const void *)result; } void ts_node_named_children_wasm(const TSTree *tree) { @@ -625,7 +708,7 @@ void ts_node_named_children_wasm(const TSTree *tree) { uint32_t count = ts_node_named_child_count(node); const void **result = NULL; if (count > 0) { - result = calloc(sizeof(void *), 5 * count); + result = (const void**)calloc(sizeof(void *), SIZE_OF_NODE * count); const void **address = result; ts_tree_cursor_reset(&scratch_cursor, node); ts_tree_cursor_goto_first_child(&scratch_cursor); @@ -634,7 +717,7 @@ void ts_node_named_children_wasm(const TSTree *tree) { TSNode child = ts_tree_cursor_current_node(&scratch_cursor); if (ts_node_is_named(child)) { marshal_node(address, child); - address += 5; + address += SIZE_OF_NODE; i++; if (i == count) { break; @@ -646,7 +729,7 @@ void ts_node_named_children_wasm(const TSTree *tree) { } } TRANSFER_BUFFER[0] = (const void *)count; - TRANSFER_BUFFER[1] = result; + TRANSFER_BUFFER[1] = (const void *)result; } bool symbols_contain(const uint32_t *set, uint32_t length, uint32_t value) { @@ -708,8 +791,8 @@ void ts_node_descendants_of_type_wasm( // Add the node to the result if its type matches one of the given // node types. if (symbols_contain(symbols, symbol_count, ts_node_symbol(descendant))) { - array_grow_by(&result, 5); - marshal_node(result.contents + result.size - 5, descendant); + array_grow_by(&result, SIZE_OF_NODE); + marshal_node(result.contents + result.size - SIZE_OF_NODE, descendant); } // Continue walking. @@ -734,8 +817,8 @@ void ts_node_descendants_of_type_wasm( } } - TRANSFER_BUFFER[0] = (const void *)(result.size / 5); - TRANSFER_BUFFER[1] = result.contents; + TRANSFER_BUFFER[0] = (const void *)(result.size / SIZE_OF_NODE); + TRANSFER_BUFFER[1] = (const void *)result.contents; } int ts_node_is_named_wasm(const TSTree *tree) { @@ -791,9 +874,14 @@ void ts_query_matches_wasm( uint32_t end_column, uint32_t start_index, uint32_t end_index, + uint32_t start_containing_row, + uint32_t start_containing_column, + uint32_t end_containing_row, + uint32_t end_containing_column, + uint32_t start_containing_index, + uint32_t end_containing_index, uint32_t match_limit, - uint32_t max_start_depth, - uint32_t timeout_micros + uint32_t max_start_depth ) { if (!scratch_query_cursor) { scratch_query_cursor = ts_query_cursor_new(); @@ -807,12 +895,26 @@ void ts_query_matches_wasm( TSNode node = unmarshal_node(tree); TSPoint start_point = {start_row, code_unit_to_byte(start_column)}; TSPoint end_point = {end_row, code_unit_to_byte(end_column)}; + TSPoint start_containing_point = {start_containing_row, code_unit_to_byte(start_containing_column)}; + TSPoint end_containing_point = {end_containing_row, code_unit_to_byte(end_containing_column)}; ts_query_cursor_set_point_range(scratch_query_cursor, start_point, end_point); ts_query_cursor_set_byte_range(scratch_query_cursor, start_index, end_index); + ts_query_cursor_set_containing_point_range( + scratch_query_cursor, + start_containing_point, + end_containing_point + ); + ts_query_cursor_set_containing_byte_range( + scratch_query_cursor, + start_containing_index, + end_containing_index + ); ts_query_cursor_set_match_limit(scratch_query_cursor, match_limit); ts_query_cursor_set_max_start_depth(scratch_query_cursor, max_start_depth); - ts_query_cursor_set_timeout_micros(scratch_query_cursor, timeout_micros); - ts_query_cursor_exec(scratch_query_cursor, self, node); + + TSQueryCursorOptions options = {.payload = NULL, .progress_callback = query_progress_callback}; + + ts_query_cursor_exec_with_options(scratch_query_cursor, self, node, &options); uint32_t index = 0; uint32_t match_count = 0; @@ -821,21 +923,21 @@ void ts_query_matches_wasm( TSQueryMatch match; while (ts_query_cursor_next_match(scratch_query_cursor, &match)) { match_count++; - array_grow_by(&result, 2 + 6 * match.capture_count); + array_grow_by(&result, 2 + (SIZE_OF_CAPTURE * match.capture_count)); result.contents[index++] = (const void *)(uint32_t)match.pattern_index; result.contents[index++] = (const void *)(uint32_t)match.capture_count; for (unsigned i = 0; i < match.capture_count; i++) { const TSQueryCapture *capture = &match.captures[i]; result.contents[index++] = (const void *)capture->index; marshal_node(result.contents + index, capture->node); - index += 5; + index += SIZE_OF_NODE; } } bool did_exceed_match_limit = ts_query_cursor_did_exceed_match_limit(scratch_query_cursor); TRANSFER_BUFFER[0] = (const void *)(match_count); - TRANSFER_BUFFER[1] = result.contents; + TRANSFER_BUFFER[1] = (const void *)result.contents; TRANSFER_BUFFER[2] = (const void *)(did_exceed_match_limit); } @@ -848,9 +950,14 @@ void ts_query_captures_wasm( uint32_t end_column, uint32_t start_index, uint32_t end_index, + uint32_t start_containing_row, + uint32_t start_containing_column, + uint32_t end_containing_row, + uint32_t end_containing_column, + uint32_t start_containing_index, + uint32_t end_containing_index, uint32_t match_limit, - uint32_t max_start_depth, - uint32_t timeout_micros + uint32_t max_start_depth ) { if (!scratch_query_cursor) { scratch_query_cursor = ts_query_cursor_new(); @@ -861,11 +968,22 @@ void ts_query_captures_wasm( TSNode node = unmarshal_node(tree); TSPoint start_point = {start_row, code_unit_to_byte(start_column)}; TSPoint end_point = {end_row, code_unit_to_byte(end_column)}; + TSPoint start_containing_point = {start_containing_row, code_unit_to_byte(start_containing_column)}; + TSPoint end_containing_point = {end_containing_row, code_unit_to_byte(end_containing_column)}; ts_query_cursor_set_point_range(scratch_query_cursor, start_point, end_point); ts_query_cursor_set_byte_range(scratch_query_cursor, start_index, end_index); + ts_query_cursor_set_containing_point_range( + scratch_query_cursor, + start_containing_point, + end_containing_point + ); + ts_query_cursor_set_containing_byte_range( + scratch_query_cursor, + start_containing_index, + end_containing_index + ); ts_query_cursor_set_match_limit(scratch_query_cursor, match_limit); ts_query_cursor_set_max_start_depth(scratch_query_cursor, max_start_depth); - ts_query_cursor_set_timeout_micros(scratch_query_cursor, timeout_micros); ts_query_cursor_exec(scratch_query_cursor, self, node); unsigned index = 0; @@ -881,7 +999,7 @@ void ts_query_captures_wasm( )) { capture_count++; - array_grow_by(&result, 3 + 6 * match.capture_count); + array_grow_by(&result, 3 + (SIZE_OF_CAPTURE * match.capture_count)); result.contents[index++] = (const void *)(uint32_t)match.pattern_index; result.contents[index++] = (const void *)(uint32_t)match.capture_count; result.contents[index++] = (const void *)capture_index; @@ -889,13 +1007,13 @@ void ts_query_captures_wasm( const TSQueryCapture *capture = &match.captures[i]; result.contents[index++] = (const void *)capture->index; marshal_node(result.contents + index, capture->node); - index += 5; + index += SIZE_OF_NODE; } } bool did_exceed_match_limit = ts_query_cursor_did_exceed_match_limit(scratch_query_cursor); TRANSFER_BUFFER[0] = (const void *)(capture_count); - TRANSFER_BUFFER[1] = result.contents; + TRANSFER_BUFFER[1] = (const void *)result.contents; TRANSFER_BUFFER[2] = (const void *)(did_exceed_match_limit); } diff --git a/lib/binding_web/lib/web-tree-sitter.d.ts b/lib/binding_web/lib/web-tree-sitter.d.ts new file mode 100644 index 00000000..c1e0e0dd --- /dev/null +++ b/lib/binding_web/lib/web-tree-sitter.d.ts @@ -0,0 +1,205 @@ +// TypeScript bindings for emscripten-generated code. Automatically @generated at compile time. +declare namespace RuntimeExports { + function AsciiToString(ptr: number): string; + function stringToUTF8(str: string, outPtr: number, maxBytesToWrite: number): number; + /** + * Given a pointer 'ptr' to a null-terminated UTF8-encoded string in the + * emscripten HEAP, returns a copy of that string as a Javascript String object. + * + * @param {number} ptr + * @param {number=} maxBytesToRead - An optional length that specifies the + * maximum number of bytes to read. You can omit this parameter to scan the + * string until the first 0 byte. If maxBytesToRead is passed, and the string + * at [ptr, ptr+maxBytesToReadr[ contains a null byte in the middle, then the + * string will cut short at that byte index. + * @param {boolean=} ignoreNul - If true, the function will not stop on a NUL character. + * @return {string} + */ + function UTF8ToString(ptr: number, maxBytesToRead?: number | undefined, ignoreNul?: boolean | undefined): string; + function lengthBytesUTF8(str: string): number; + function stringToUTF16(str: string, outPtr: number, maxBytesToWrite: number): number; + /** + * @param {string=} libName + * @param {Object=} localScope + * @param {number=} handle + */ + function loadWebAssemblyModule(binary: Uint8Array | WebAssembly.Module, flags: Record, libName?: string, localScope?: Record, handle?: number): Promise number>>; + /** + * @param {number} ptr + * @param {string} type + */ + function getValue(ptr: number, type?: string): number; + /** + * @param {number} ptr + * @param {number} value + * @param {string} type + */ + function setValue(ptr: number, value: number, type?: string): void; + let HEAPF32: Float32Array; + let HEAPF64: Float64Array; + let HEAP_DATA_VIEW: DataView; + let HEAP8: Int8Array; + let HEAPU8: Uint8Array; + let HEAP16: Int16Array; + let HEAPU16: Uint16Array; + let HEAP32: Int32Array; + let HEAPU32: Uint32Array; + let HEAP64: BigInt64Array; + let HEAPU64: BigUint64Array; + function LE_HEAP_STORE_I64(byteOffset: any, value: any): any; +} +interface WasmModule { + _malloc(_0: number): number; + _calloc(_0: number, _1: number): number; + _realloc(_0: number, _1: number): number; + _free(_0: number): void; + _memcmp(_0: number, _1: number, _2: number): number; + _ts_language_symbol_count(_0: number): number; + _ts_language_state_count(_0: number): number; + _ts_language_abi_version(_0: number): number; + _ts_language_name(_0: number): number; + _ts_language_field_count(_0: number): number; + _ts_language_next_state(_0: number, _1: number, _2: number): number; + _ts_language_symbol_name(_0: number, _1: number): number; + _ts_language_symbol_for_name(_0: number, _1: number, _2: number, _3: number): number; + _strncmp(_0: number, _1: number, _2: number): number; + _ts_language_symbol_type(_0: number, _1: number): number; + _ts_language_field_name_for_id(_0: number, _1: number): number; + _ts_lookahead_iterator_new(_0: number, _1: number): number; + _ts_lookahead_iterator_delete(_0: number): void; + _ts_lookahead_iterator_reset_state(_0: number, _1: number): number; + _ts_lookahead_iterator_reset(_0: number, _1: number, _2: number): number; + _ts_lookahead_iterator_next(_0: number): number; + _ts_lookahead_iterator_current_symbol(_0: number): number; + _ts_parser_delete(_0: number): void; + _ts_parser_reset(_0: number): void; + _ts_parser_set_language(_0: number, _1: number): number; + _ts_parser_set_included_ranges(_0: number, _1: number, _2: number): number; + _ts_query_new(_0: number, _1: number, _2: number, _3: number, _4: number): number; + _ts_query_delete(_0: number): void; + _iswspace(_0: number): number; + _iswalnum(_0: number): number; + _ts_query_pattern_count(_0: number): number; + _ts_query_capture_count(_0: number): number; + _ts_query_string_count(_0: number): number; + _ts_query_capture_name_for_id(_0: number, _1: number, _2: number): number; + _ts_query_capture_quantifier_for_id(_0: number, _1: number, _2: number): number; + _ts_query_string_value_for_id(_0: number, _1: number, _2: number): number; + _ts_query_predicates_for_pattern(_0: number, _1: number, _2: number): number; + _ts_query_start_byte_for_pattern(_0: number, _1: number): number; + _ts_query_end_byte_for_pattern(_0: number, _1: number): number; + _ts_query_is_pattern_rooted(_0: number, _1: number): number; + _ts_query_is_pattern_non_local(_0: number, _1: number): number; + _ts_query_is_pattern_guaranteed_at_step(_0: number, _1: number): number; + _ts_query_disable_capture(_0: number, _1: number, _2: number): void; + _ts_query_disable_pattern(_0: number, _1: number): void; + _ts_tree_copy(_0: number): number; + _ts_tree_delete(_0: number): void; + _ts_init(): number; + _ts_parser_new_wasm(): void; + _ts_parser_enable_logger_wasm(_0: number, _1: number): void; + _ts_parser_parse_wasm(_0: number, _1: number, _2: number, _3: number, _4: number): number; + _ts_parser_included_ranges_wasm(_0: number): void; + _ts_language_type_is_named_wasm(_0: number, _1: number): number; + _ts_language_type_is_visible_wasm(_0: number, _1: number): number; + _ts_language_metadata_wasm(_0: number): void; + _ts_language_supertypes_wasm(_0: number): void; + _ts_language_subtypes_wasm(_0: number, _1: number): void; + _ts_tree_root_node_wasm(_0: number): void; + _ts_tree_root_node_with_offset_wasm(_0: number): void; + _ts_tree_edit_wasm(_0: number): void; + _ts_tree_included_ranges_wasm(_0: number): void; + _ts_tree_get_changed_ranges_wasm(_0: number, _1: number): void; + _ts_tree_cursor_new_wasm(_0: number): void; + _ts_tree_cursor_copy_wasm(_0: number): void; + _ts_tree_cursor_delete_wasm(_0: number): void; + _ts_tree_cursor_reset_wasm(_0: number): void; + _ts_tree_cursor_reset_to_wasm(_0: number, _1: number): void; + _ts_tree_cursor_goto_first_child_wasm(_0: number): number; + _ts_tree_cursor_goto_last_child_wasm(_0: number): number; + _ts_tree_cursor_goto_first_child_for_index_wasm(_0: number): number; + _ts_tree_cursor_goto_first_child_for_position_wasm(_0: number): number; + _ts_tree_cursor_goto_next_sibling_wasm(_0: number): number; + _ts_tree_cursor_goto_previous_sibling_wasm(_0: number): number; + _ts_tree_cursor_goto_descendant_wasm(_0: number, _1: number): void; + _ts_tree_cursor_goto_parent_wasm(_0: number): number; + _ts_tree_cursor_current_node_type_id_wasm(_0: number): number; + _ts_tree_cursor_current_node_state_id_wasm(_0: number): number; + _ts_tree_cursor_current_node_is_named_wasm(_0: number): number; + _ts_tree_cursor_current_node_is_missing_wasm(_0: number): number; + _ts_tree_cursor_current_node_id_wasm(_0: number): number; + _ts_tree_cursor_start_position_wasm(_0: number): void; + _ts_tree_cursor_end_position_wasm(_0: number): void; + _ts_tree_cursor_start_index_wasm(_0: number): number; + _ts_tree_cursor_end_index_wasm(_0: number): number; + _ts_tree_cursor_current_field_id_wasm(_0: number): number; + _ts_tree_cursor_current_depth_wasm(_0: number): number; + _ts_tree_cursor_current_descendant_index_wasm(_0: number): number; + _ts_tree_cursor_current_node_wasm(_0: number): void; + _ts_node_symbol_wasm(_0: number): number; + _ts_node_field_name_for_child_wasm(_0: number, _1: number): number; + _ts_node_field_name_for_named_child_wasm(_0: number, _1: number): number; + _ts_node_children_by_field_id_wasm(_0: number, _1: number): void; + _ts_node_first_child_for_byte_wasm(_0: number): void; + _ts_node_first_named_child_for_byte_wasm(_0: number): void; + _ts_node_grammar_symbol_wasm(_0: number): number; + _ts_node_child_count_wasm(_0: number): number; + _ts_node_named_child_count_wasm(_0: number): number; + _ts_node_child_wasm(_0: number, _1: number): void; + _ts_node_named_child_wasm(_0: number, _1: number): void; + _ts_node_child_by_field_id_wasm(_0: number, _1: number): void; + _ts_node_next_sibling_wasm(_0: number): void; + _ts_node_prev_sibling_wasm(_0: number): void; + _ts_node_next_named_sibling_wasm(_0: number): void; + _ts_node_prev_named_sibling_wasm(_0: number): void; + _ts_node_descendant_count_wasm(_0: number): number; + _ts_node_parent_wasm(_0: number): void; + _ts_node_child_with_descendant_wasm(_0: number): void; + _ts_node_descendant_for_index_wasm(_0: number): void; + _ts_node_named_descendant_for_index_wasm(_0: number): void; + _ts_node_descendant_for_position_wasm(_0: number): void; + _ts_node_named_descendant_for_position_wasm(_0: number): void; + _ts_node_start_point_wasm(_0: number): void; + _ts_node_end_point_wasm(_0: number): void; + _ts_node_start_index_wasm(_0: number): number; + _ts_node_end_index_wasm(_0: number): number; + _ts_node_to_string_wasm(_0: number): number; + _ts_node_children_wasm(_0: number): void; + _ts_node_named_children_wasm(_0: number): void; + _ts_node_descendants_of_type_wasm(_0: number, _1: number, _2: number, _3: number, _4: number, _5: number, _6: number): void; + _ts_node_is_named_wasm(_0: number): number; + _ts_node_has_changes_wasm(_0: number): number; + _ts_node_has_error_wasm(_0: number): number; + _ts_node_is_error_wasm(_0: number): number; + _ts_node_is_missing_wasm(_0: number): number; + _ts_node_is_extra_wasm(_0: number): number; + _ts_node_parse_state_wasm(_0: number): number; + _ts_node_next_parse_state_wasm(_0: number): number; + _ts_query_matches_wasm(_0: number, _1: number, _2: number, _3: number, _4: number, _5: number, _6: number, _7: number, _8: number, _9: number, _10: number, _11: number, _12: number, _13: number, _14: number, _15: number): void; + _ts_query_captures_wasm(_0: number, _1: number, _2: number, _3: number, _4: number, _5: number, _6: number, _7: number, _8: number, _9: number, _10: number, _11: number, _12: number, _13: number, _14: number, _15: number): void; + _memset(_0: number, _1: number, _2: number): number; + _memcpy(_0: number, _1: number, _2: number): number; + _memmove(_0: number, _1: number, _2: number): number; + _iswalpha(_0: number): number; + _iswblank(_0: number): number; + _iswdigit(_0: number): number; + _iswlower(_0: number): number; + _iswupper(_0: number): number; + _iswxdigit(_0: number): number; + _memchr(_0: number, _1: number, _2: number): number; + _strlen(_0: number): number; + _strcmp(_0: number, _1: number): number; + _strncat(_0: number, _1: number, _2: number): number; + _strncpy(_0: number, _1: number, _2: number): number; + _towlower(_0: number): number; + _towupper(_0: number): number; +} + +export type MainModule = WasmModule & typeof RuntimeExports & { + currentParseCallback: ((index: number, position: {row: number, column: number}) => string | undefined) | null; + currentLogCallback: ((message: string, isLex: boolean) => void) | null; + currentProgressCallback: ((state: {currentOffset: number, hasError: boolean}) => void) | null; + currentQueryProgressCallback: ((state: {currentOffset: number}) => void) | null; +}; + +export default function MainModuleFactory(options?: Partial): Promise; diff --git a/lib/binding_web/package-lock.json b/lib/binding_web/package-lock.json new file mode 100644 index 00000000..4ec07f7b --- /dev/null +++ b/lib/binding_web/package-lock.json @@ -0,0 +1,4297 @@ +{ + "name": "web-tree-sitter", + "version": "0.27.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "web-tree-sitter", + "version": "0.27.0", + "license": "MIT", + "devDependencies": { + "@eslint/js": "^9.39.1", + "@types/emscripten": "^1.41.5", + "@types/node": "^24.10.1", + "@vitest/coverage-v8": "^3.0.5", + "dts-buddy": "^0.6.2", + "esbuild": "^0.27.1", + "eslint": "^9.39.1", + "source-map": "^0.7.4", + "tsx": "^4.21.0", + "typescript": "^5.7.3", + "typescript-eslint": "^8.48.1", + "vitest": "^3.0.5" + } + }, + "node_modules/@ampproject/remapping": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", + "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.25.9.tgz", + "integrity": "sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.9.tgz", + "integrity": "sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.26.5", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.26.5.tgz", + "integrity": "sha512-SRJ4jYmXRqV1/Xc+TIVG84WjHBXKlxO9sHQnA2Pf12QQEAp1LOh6kDzNHXcUnbH1QI0FDoPPVOt+vyUDucxpaw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.26.5" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/types": { + "version": "7.26.5", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.5.tgz", + "integrity": "sha512-L6mZmwFDK6Cjh1nRCLXpa6no13ZIioJDz7mdkzHv399pThrTa/k0nUlNaenOeh2kWu/iaOQYElEpKPUswUa9Vg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.25.9", + "@babel/helper-validator-identifier": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@bcoe/v8-coverage": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-1.0.2.tgz", + "integrity": "sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.1.tgz", + "integrity": "sha512-HHB50pdsBX6k47S4u5g/CaLjqS3qwaOVE5ILsq64jyzgMhLuCuZ8rGzM9yhsAjfjkbgUPMzZEPa7DAp7yz6vuA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.1.tgz", + "integrity": "sha512-kFqa6/UcaTbGm/NncN9kzVOODjhZW8e+FRdSeypWe6j33gzclHtwlANs26JrupOntlcWmB0u8+8HZo8s7thHvg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.1.tgz", + "integrity": "sha512-45fuKmAJpxnQWixOGCrS+ro4Uvb4Re9+UTieUY2f8AEc+t7d4AaZ6eUJ3Hva7dtrxAAWHtlEFsXFMAgNnGU9uQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.1.tgz", + "integrity": "sha512-LBEpOz0BsgMEeHgenf5aqmn/lLNTFXVfoWMUox8CtWWYK9X4jmQzWjoGoNb8lmAYml/tQ/Ysvm8q7szu7BoxRQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.1.tgz", + "integrity": "sha512-veg7fL8eMSCVKL7IW4pxb54QERtedFDfY/ASrumK/SbFsXnRazxY4YykN/THYqFnFwJ0aVjiUrVG2PwcdAEqQQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.1.tgz", + "integrity": "sha512-+3ELd+nTzhfWb07Vol7EZ+5PTbJ/u74nC6iv4/lwIU99Ip5uuY6QoIf0Hn4m2HoV0qcnRivN3KSqc+FyCHjoVQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.1.tgz", + "integrity": "sha512-/8Rfgns4XD9XOSXlzUDepG8PX+AVWHliYlUkFI3K3GB6tqbdjYqdhcb4BKRd7C0BhZSoaCxhv8kTcBrcZWP+xg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.1.tgz", + "integrity": "sha512-GITpD8dK9C+r+5yRT/UKVT36h/DQLOHdwGVwwoHidlnA168oD3uxA878XloXebK4Ul3gDBBIvEdL7go9gCUFzQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.1.tgz", + "integrity": "sha512-ieMID0JRZY/ZeCrsFQ3Y3NlHNCqIhTprJfDgSB3/lv5jJZ8FX3hqPyXWhe+gvS5ARMBJ242PM+VNz/ctNj//eA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.1.tgz", + "integrity": "sha512-W9//kCrh/6in9rWIBdKaMtuTTzNj6jSeG/haWBADqLLa9P8O5YSRDzgD5y9QBok4AYlzS6ARHifAb75V6G670Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.1.tgz", + "integrity": "sha512-VIUV4z8GD8rtSVMfAj1aXFahsi/+tcoXXNYmXgzISL+KB381vbSTNdeZHHHIYqFyXcoEhu9n5cT+05tRv13rlw==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.1.tgz", + "integrity": "sha512-l4rfiiJRN7sTNI//ff65zJ9z8U+k6zcCg0LALU5iEWzY+a1mVZ8iWC1k5EsNKThZ7XCQ6YWtsZ8EWYm7r1UEsg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.1.tgz", + "integrity": "sha512-U0bEuAOLvO/DWFdygTHWY8C067FXz+UbzKgxYhXC0fDieFa0kDIra1FAhsAARRJbvEyso8aAqvPdNxzWuStBnA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.1.tgz", + "integrity": "sha512-NzdQ/Xwu6vPSf/GkdmRNsOfIeSGnh7muundsWItmBsVpMoNPVpM61qNzAVY3pZ1glzzAxLR40UyYM23eaDDbYQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.1.tgz", + "integrity": "sha512-7zlw8p3IApcsN7mFw0O1Z1PyEk6PlKMu18roImfl3iQHTnr/yAfYv6s4hXPidbDoI2Q0pW+5xeoM4eTCC0UdrQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.1.tgz", + "integrity": "sha512-cGj5wli+G+nkVQdZo3+7FDKC25Uh4ZVwOAK6A06Hsvgr8WqBBuOy/1s+PUEd/6Je+vjfm6stX0kmib5b/O2Ykw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.1.tgz", + "integrity": "sha512-z3H/HYI9MM0HTv3hQZ81f+AKb+yEoCRlUby1F80vbQ5XdzEMyY/9iNlAmhqiBKw4MJXwfgsh7ERGEOhrM1niMA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.1.tgz", + "integrity": "sha512-wzC24DxAvk8Em01YmVXyjl96Mr+ecTPyOuADAvjGg+fyBpGmxmcr2E5ttf7Im8D0sXZihpxzO1isus8MdjMCXQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.1.tgz", + "integrity": "sha512-1YQ8ybGi2yIXswu6eNzJsrYIGFpnlzEWRl6iR5gMgmsrR0FcNoV1m9k9sc3PuP5rUBLshOZylc9nqSgymI+TYg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.1.tgz", + "integrity": "sha512-5Z+DzLCrq5wmU7RDaMDe2DVXMRm2tTDvX2KU14JJVBN2CT/qov7XVix85QoJqHltpvAOZUAc3ndU56HSMWrv8g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.1.tgz", + "integrity": "sha512-Q73ENzIdPF5jap4wqLtsfh8YbYSZ8Q0wnxplOlZUOyZy7B4ZKW8DXGWgTCZmF8VWD7Tciwv5F4NsRf6vYlZtqg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.1.tgz", + "integrity": "sha512-ajbHrGM/XiK+sXM0JzEbJAen+0E+JMQZ2l4RR4VFwvV9JEERx+oxtgkpoKv1SevhjavK2z2ReHk32pjzktWbGg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.1.tgz", + "integrity": "sha512-IPUW+y4VIjuDVn+OMzHc5FV4GubIwPnsz6ubkvN8cuhEqH81NovB53IUlrlBkPMEPxvNnf79MGBoz8rZ2iW8HA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.1.tgz", + "integrity": "sha512-RIVRWiljWA6CdVu8zkWcRmGP7iRRIIwvhDKem8UMBjPql2TXM5PkDVvvrzMtj1V+WFPB4K7zkIGM7VzRtFkjdg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.1.tgz", + "integrity": "sha512-2BR5M8CPbptC1AK5JbJT1fWrHLvejwZidKx3UMSF0ecHMa+smhi16drIrCEggkgviBwLYd5nwrFLSl5Kho96RQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.1.tgz", + "integrity": "sha512-d5X6RMYv6taIymSk8JBP+nxv8DQAMY6A51GPgusqLdK9wBz5wWIXy1KjTck6HnjE9hqJzJRdk+1p/t5soSbCtw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.0.tgz", + "integrity": "sha512-ayVFHdtZ+hsq1t2Dy24wCmGXGe4q9Gu3smhLYALJrr473ZH27MsnSL+LKUlimp4BWJqMDMLmPpx/Q9R3OAlL4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "eslint-visitor-keys": "^3.4.3" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.12.1", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.1.tgz", + "integrity": "sha512-CCZCDJuduB9OUkFkY2IgppNZMi2lBQgD2qzwXkEia16cge2pijY/aXi96CJMquDMn3nJdlPV1A5KrJEXwfLNzQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/config-array": { + "version": "0.21.1", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.1.tgz", + "integrity": "sha512-aw1gNayWpdI/jSYVgzN5pL0cfzU02GT3NBpeT/DXbx1/1x7ZKxFPd9bwrzygx/qiwIQiJ1sw/zD8qY/kRvlGHA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/object-schema": "^2.1.7", + "debug": "^4.3.1", + "minimatch": "^3.1.2" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/config-helpers": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.4.2.tgz", + "integrity": "sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^0.17.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/core": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.17.0.tgz", + "integrity": "sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@types/json-schema": "^7.0.15" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/eslintrc": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.1.tgz", + "integrity": "sha512-gtF186CXhIl1p4pJNGZw8Yc6RlshoePRvE0X91oPGb3vZ8pM3qOS9W9NGPat9LziaBV7XrJWGylNQXkGcnM3IQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^6.12.4", + "debug": "^4.3.2", + "espree": "^10.0.1", + "globals": "^14.0.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.1.0", + "minimatch": "^3.1.2", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint/js": { + "version": "9.39.1", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.1.tgz", + "integrity": "sha512-S26Stp4zCy88tH94QbBv3XCuzRQiZ9yXofEILmglYTh/Ug/a9/umqvgFtYBAo3Lp0nsI/5/qH1CCrbdK3AP1Tw==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://eslint.org/donate" + } + }, + "node_modules/@eslint/object-schema": { + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-2.1.7.tgz", + "integrity": "sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/plugin-kit": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.4.1.tgz", + "integrity": "sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^0.17.0", + "levn": "^0.4.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@humanfs/core": { + "version": "0.19.1", + "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", + "integrity": "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/node": { + "version": "0.16.6", + "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.6.tgz", + "integrity": "sha512-YuI2ZHQL78Q5HbhDiBA1X4LmYdXCKCMQIfw0pw7piHJwyREFebJUvrQN4cMssyES6x+vfUbx1CIpaQUKYdQZOw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanfs/core": "^0.19.1", + "@humanwhocodes/retry": "^0.3.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/node/node_modules/@humanwhocodes/retry": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.3.1.tgz", + "integrity": "sha512-JBxkERygn7Bv/GbN5Rv8Ul6LVknS+5Bp6RgDC/O8gEBU/yeH5Ui5C/OlWrTb6qct7LjjfT6Re2NxB0ln0yYybA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/retry": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", + "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@istanbuljs/schema": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", + "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.8", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.8.tgz", + "integrity": "sha512-imAbBGkb+ebQyxKgzv5Hu2nmROxoDOXHh80evxdoXNOrvAnVx7zimzc1Oo5h9RlfV4vPXaE2iM5pOFbvOCClWA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/set-array": "^1.2.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/set-array": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.2.1.tgz", + "integrity": "sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/source-map": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.6.tgz", + "integrity": "sha512-1ZJTZebgqllO79ue2bm3rIGud/bOe0pP5BjSRCRxxYkEZS8STV7zN84UBbiYu7jy+eCKSnVIUgoWWE/tt+shMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.25" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", + "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.30", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.30.tgz", + "integrity": "sha512-GQ7Nw5G2lTu/BtHTKfXhKHok2WGetd4XYcVKGx00SjAk8GMwgJM3zr6zORiPGuOE+/vkc90KtTosSSvaCjKb2Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "dev": true, + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.49.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.49.0.tgz", + "integrity": "sha512-rlKIeL854Ed0e09QGYFlmDNbka6I3EQFw7iZuugQjMb11KMpJCLPFL4ZPbMfaEhLADEL1yx0oujGkBQ7+qW3eA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.49.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.49.0.tgz", + "integrity": "sha512-cqPpZdKUSQYRtLLr6R4X3sD4jCBO1zUmeo3qrWBCqYIeH8Q3KRL4F3V7XJ2Rm8/RJOQBZuqzQGWPjjvFUcYa/w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.49.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.49.0.tgz", + "integrity": "sha512-99kMMSMQT7got6iYX3yyIiJfFndpojBmkHfTc1rIje8VbjhmqBXE+nb7ZZP3A5skLyujvT0eIUCUsxAe6NjWbw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.49.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.49.0.tgz", + "integrity": "sha512-y8cXoD3wdWUDpjOLMKLx6l+NFz3NlkWKcBCBfttUn+VGSfgsQ5o/yDUGtzE9HvsodkP0+16N0P4Ty1VuhtRUGg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.49.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.49.0.tgz", + "integrity": "sha512-3mY5Pr7qv4GS4ZvWoSP8zha8YoiqrU+e0ViPvB549jvliBbdNLrg2ywPGkgLC3cmvN8ya3za+Q2xVyT6z+vZqA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.49.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.49.0.tgz", + "integrity": "sha512-C9KzzOAQU5gU4kG8DTk+tjdKjpWhVWd5uVkinCwwFub2m7cDYLOdtXoMrExfeBmeRy9kBQMkiyJ+HULyF1yj9w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.49.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.49.0.tgz", + "integrity": "sha512-OVSQgEZDVLnTbMq5NBs6xkmz3AADByCWI4RdKSFNlDsYXdFtlxS59J+w+LippJe8KcmeSSM3ba+GlsM9+WwC1w==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.49.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.49.0.tgz", + "integrity": "sha512-ZnfSFA7fDUHNa4P3VwAcfaBLakCbYaxCk0jUnS3dTou9P95kwoOLAMlT3WmEJDBCSrOEFFV0Y1HXiwfLYJuLlA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.49.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.49.0.tgz", + "integrity": "sha512-Z81u+gfrobVK2iV7GqZCBfEB1y6+I61AH466lNK+xy1jfqFLiQ9Qv716WUM5fxFrYxwC7ziVdZRU9qvGHkYIJg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.49.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.49.0.tgz", + "integrity": "sha512-zoAwS0KCXSnTp9NH/h9aamBAIve0DXeYpll85shf9NJ0URjSTzzS+Z9evmolN+ICfD3v8skKUPyk2PO0uGdFqg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loongarch64-gnu": { + "version": "4.49.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loongarch64-gnu/-/rollup-linux-loongarch64-gnu-4.49.0.tgz", + "integrity": "sha512-2QyUyQQ1ZtwZGiq0nvODL+vLJBtciItC3/5cYN8ncDQcv5avrt2MbKt1XU/vFAJlLta5KujqyHdYtdag4YEjYQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.49.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.49.0.tgz", + "integrity": "sha512-k9aEmOWt+mrMuD3skjVJSSxHckJp+SiFzFG+v8JLXbc/xi9hv2icSkR3U7uQzqy+/QbbYY7iNB9eDTwrELo14g==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.49.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.49.0.tgz", + "integrity": "sha512-rDKRFFIWJ/zJn6uk2IdYLc09Z7zkE5IFIOWqpuU0o6ZpHcdniAyWkwSUWE/Z25N/wNDmFHHMzin84qW7Wzkjsw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.49.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.49.0.tgz", + "integrity": "sha512-FkkhIY/hYFVnOzz1WeV3S9Bd1h0hda/gRqvZCMpHWDHdiIHn6pqsY3b5eSbvGccWHMQ1uUzgZTKS4oGpykf8Tw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.49.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.49.0.tgz", + "integrity": "sha512-gRf5c+A7QiOG3UwLyOOtyJMD31JJhMjBvpfhAitPAoqZFcOeK3Kc1Veg1z/trmt+2P6F/biT02fU19GGTS529A==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.49.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.49.0.tgz", + "integrity": "sha512-BR7+blScdLW1h/2hB/2oXM+dhTmpW3rQt1DeSiCP9mc2NMMkqVgjIN3DDsNpKmezffGC9R8XKVOLmBkRUcK/sA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.49.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.49.0.tgz", + "integrity": "sha512-hDMOAe+6nX3V5ei1I7Au3wcr9h3ktKzDvF2ne5ovX8RZiAHEtX1A5SNNk4zt1Qt77CmnbqT+upb/umzoPMWiPg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.49.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.49.0.tgz", + "integrity": "sha512-wkNRzfiIGaElC9kXUT+HLx17z7D0jl+9tGYRKwd8r7cUqTL7GYAvgUY++U2hK6Ar7z5Z6IRRoWC8kQxpmM7TDA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.49.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.49.0.tgz", + "integrity": "sha512-gq5aW/SyNpjp71AAzroH37DtINDcX1Qw2iv9Chyz49ZgdOP3NV8QCyKZUrGsYX9Yyggj5soFiRCgsL3HwD8TdA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.49.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.49.0.tgz", + "integrity": "sha512-gEtqFbzmZLFk2xKh7g0Rlo8xzho8KrEFEkzvHbfUGkrgXOpZ4XagQ6n+wIZFNh1nTb8UD16J4nFSFKXYgnbdBg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@types/chai": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.2.tgz", + "integrity": "sha512-8kB30R7Hwqf40JPiKhVzodJs2Qc1ZJ5zuT3uzw5Hq/dhNCl3G3l83jfpdI1e20BP348+fV7VIL/+FxaXkqBmWg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/emscripten": { + "version": "1.41.5", + "resolved": "https://registry.npmjs.org/@types/emscripten/-/emscripten-1.41.5.tgz", + "integrity": "sha512-cMQm7pxu6BxtHyqJ7mQZ2kXWV5SLmugybFdHCBbJ5eHzOo6VhBckEgAT3//rP5FwPHNPeEiq4SmQ5ucBwsOo4Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", + "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "24.10.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz", + "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "undici-types": "~7.16.0" + } + }, + "node_modules/@typescript-eslint/eslint-plugin": { + "version": "8.48.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.48.1.tgz", + "integrity": "sha512-X63hI1bxl5ohelzr0LY5coufyl0LJNthld+abwxpCoo6Gq+hSqhKwci7MUWkXo67mzgUK6YFByhmaHmUcuBJmA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/regexpp": "^4.10.0", + "@typescript-eslint/scope-manager": "8.48.1", + "@typescript-eslint/type-utils": "8.48.1", + "@typescript-eslint/utils": "8.48.1", + "@typescript-eslint/visitor-keys": "8.48.1", + "graphemer": "^1.4.0", + "ignore": "^7.0.0", + "natural-compare": "^1.4.0", + "ts-api-utils": "^2.1.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "@typescript-eslint/parser": "^8.48.1", + "eslint": "^8.57.0 || ^9.0.0", + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", + "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/@typescript-eslint/parser": { + "version": "8.48.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.48.1.tgz", + "integrity": "sha512-PC0PDZfJg8sP7cmKe6L3QIL8GZwU5aRvUFedqSIpw3B+QjRSUZeeITC2M5XKeMXEzL6wccN196iy3JLwKNvDVA==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@typescript-eslint/scope-manager": "8.48.1", + "@typescript-eslint/types": "8.48.1", + "@typescript-eslint/typescript-estree": "8.48.1", + "@typescript-eslint/visitor-keys": "8.48.1", + "debug": "^4.3.4" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0", + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/project-service": { + "version": "8.48.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.48.1.tgz", + "integrity": "sha512-HQWSicah4s9z2/HifRPQ6b6R7G+SBx64JlFQpgSSHWPKdvCZX57XCbszg/bapbRsOEv42q5tayTYcEFpACcX1w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/tsconfig-utils": "^8.48.1", + "@typescript-eslint/types": "^8.48.1", + "debug": "^4.3.4" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/scope-manager": { + "version": "8.48.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.48.1.tgz", + "integrity": "sha512-rj4vWQsytQbLxC5Bf4XwZ0/CKd362DkWMUkviT7DCS057SK64D5lH74sSGzhI6PDD2HCEq02xAP9cX68dYyg1w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.48.1", + "@typescript-eslint/visitor-keys": "8.48.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/tsconfig-utils": { + "version": "8.48.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.48.1.tgz", + "integrity": "sha512-k0Jhs4CpEffIBm6wPaCXBAD7jxBtrHjrSgtfCjUvPp9AZ78lXKdTR8fxyZO5y4vWNlOvYXRtngSZNSn+H53Jkw==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/type-utils": { + "version": "8.48.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.48.1.tgz", + "integrity": "sha512-1jEop81a3LrJQLTf/1VfPQdhIY4PlGDBc/i67EVWObrtvcziysbLN3oReexHOM6N3jyXgCrkBsZpqwH0hiDOQg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.48.1", + "@typescript-eslint/typescript-estree": "8.48.1", + "@typescript-eslint/utils": "8.48.1", + "debug": "^4.3.4", + "ts-api-utils": "^2.1.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0", + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/types": { + "version": "8.48.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.48.1.tgz", + "integrity": "sha512-+fZ3LZNeiELGmimrujsDCT4CRIbq5oXdHe7chLiW8qzqyPMnn1puNstCrMNVAqwcl2FdIxkuJ4tOs/RFDBVc/Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/typescript-estree": { + "version": "8.48.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.48.1.tgz", + "integrity": "sha512-/9wQ4PqaefTK6POVTjJaYS0bynCgzh6ClJHGSBj06XEHjkfylzB+A3qvyaXnErEZSaxhIo4YdyBgq6j4RysxDg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/project-service": "8.48.1", + "@typescript-eslint/tsconfig-utils": "8.48.1", + "@typescript-eslint/types": "8.48.1", + "@typescript-eslint/visitor-keys": "8.48.1", + "debug": "^4.3.4", + "minimatch": "^9.0.4", + "semver": "^7.6.0", + "tinyglobby": "^0.2.15", + "ts-api-utils": "^2.1.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@typescript-eslint/utils": { + "version": "8.48.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.48.1.tgz", + "integrity": "sha512-fAnhLrDjiVfey5wwFRwrweyRlCmdz5ZxXz2G/4cLn0YDLjTapmN4gcCsTBR1N2rWnZSDeWpYtgLDsJt+FpmcwA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.7.0", + "@typescript-eslint/scope-manager": "8.48.1", + "@typescript-eslint/types": "8.48.1", + "@typescript-eslint/typescript-estree": "8.48.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0", + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/visitor-keys": { + "version": "8.48.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.48.1.tgz", + "integrity": "sha512-BmxxndzEWhE4TIEEMBs8lP3MBWN3jFPs/p6gPm/wkv02o41hI6cq9AuSmGAaTTHPtA1FTi2jBre4A9rm5ZmX+Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.48.1", + "eslint-visitor-keys": "^4.2.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@vitest/coverage-v8": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-3.2.4.tgz", + "integrity": "sha512-EyF9SXU6kS5Ku/U82E259WSnvg6c8KTjppUncuNdm5QHpe17mwREHnjDzozC8x9MZ0xfBUFSaLkRv4TMA75ALQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@ampproject/remapping": "^2.3.0", + "@bcoe/v8-coverage": "^1.0.2", + "ast-v8-to-istanbul": "^0.3.3", + "debug": "^4.4.1", + "istanbul-lib-coverage": "^3.2.2", + "istanbul-lib-report": "^3.0.1", + "istanbul-lib-source-maps": "^5.0.6", + "istanbul-reports": "^3.1.7", + "magic-string": "^0.30.17", + "magicast": "^0.3.5", + "std-env": "^3.9.0", + "test-exclude": "^7.0.1", + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@vitest/browser": "3.2.4", + "vitest": "3.2.4" + }, + "peerDependenciesMeta": { + "@vitest/browser": { + "optional": true + } + } + }, + "node_modules/@vitest/expect": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-3.2.4.tgz", + "integrity": "sha512-Io0yyORnB6sikFlt8QW5K7slY4OjqNX9jmJQ02QDda8lyM6B5oNgVWoSoKPac8/kgnCUzuHQKrSLtu/uOqqrig==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/spy": "3.2.4", + "@vitest/utils": "3.2.4", + "chai": "^5.2.0", + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-3.2.4.tgz", + "integrity": "sha512-46ryTE9RZO/rfDd7pEqFl7etuyzekzEhUbTW3BvmeO/BcCMEgq59BKhek3dXDWgAj4oMK6OZi+vRr1wPW6qjEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "3.2.4", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.17" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-3.2.4.tgz", + "integrity": "sha512-IVNZik8IVRJRTr9fxlitMKeJeXFFFN0JaB9PHPGQ8NKQbGpfjlTx9zO4RefN8gp7eqjNy8nyK3NZmBzOPeIxtA==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-3.2.4.tgz", + "integrity": "sha512-oukfKT9Mk41LreEW09vt45f8wx7DordoWUZMYdY/cyAk7w5TWkTRCNZYF7sX7n2wB7jyGAl74OxgwhPgKaqDMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "3.2.4", + "pathe": "^2.0.3", + "strip-literal": "^3.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-3.2.4.tgz", + "integrity": "sha512-dEYtS7qQP2CjU27QBC5oUOxLE/v5eLkGqPE0ZKEIDGMs4vKWe7IjgLOeauHsR0D5YuuycGRO5oSRXnwnmA78fQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "3.2.4", + "magic-string": "^0.30.17", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-3.2.4.tgz", + "integrity": "sha512-vAfasCOe6AIK70iP5UD11Ac4siNUNJ9i/9PZ3NKx07sG6sUxeag1LWdNrMWeKKYBLlzuK+Gn65Yd5nyL6ds+nw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyspy": "^4.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-3.2.4.tgz", + "integrity": "sha512-fB2V0JFrQSMsCo9HiSq3Ezpdv4iYaXRG1Sx8edX3MwxfyNn83mKiGzOcH+Fkxt4MHxr3y42fQi1oeAInqgX2QA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "3.2.4", + "loupe": "^3.1.4", + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/acorn": { + "version": "8.15.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", + "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "dev": true, + "license": "MIT", + "peer": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ansi-regex": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", + "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true, + "license": "Python-2.0" + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/ast-v8-to-istanbul": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/ast-v8-to-istanbul/-/ast-v8-to-istanbul-0.3.5.tgz", + "integrity": "sha512-9SdXjNheSiE8bALAQCQQuT6fgQaoxJh7IRYrRGZ8/9nv8WhJeC1aXAwN8TbaOssGOukUvyvnkgD9+Yuykvl1aA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.30", + "estree-walker": "^3.0.3", + "js-tokens": "^9.0.1" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/cac": { + "version": "6.7.14", + "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", + "integrity": "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/chai": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/chai/-/chai-5.3.3.tgz", + "integrity": "sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "assertion-error": "^2.0.1", + "check-error": "^2.1.1", + "deep-eql": "^5.0.1", + "loupe": "^3.1.0", + "pathval": "^2.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/check-error": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.1.tgz", + "integrity": "sha512-OAlb+T7V4Op9OwdkjmguYRqncdlx5JiofwOAUkmTF+jNdHwzTaTs4sRAGpzLF3oOz5xAyDGrPgeIDFQmDOTiJw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 16" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true, + "license": "MIT" + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", + "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deep-eql": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-5.0.2.tgz", + "integrity": "sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/dts-buddy": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/dts-buddy/-/dts-buddy-0.6.2.tgz", + "integrity": "sha512-KUmYrRKpVpjmnqM/JY93p1PWezMXodKCiMd5CFvfLtRCRc5i2GZiojrkVQXO2Dd8HeaU1C3sgTKDWxCEK5cyXA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/source-map": "^0.3.5", + "@jridgewell/sourcemap-codec": "^1.4.15", + "kleur": "^4.1.5", + "locate-character": "^3.0.0", + "magic-string": "^0.30.4", + "sade": "^1.8.1", + "tinyglobby": "^0.2.10", + "ts-api-utils": "^1.0.3" + }, + "bin": { + "dts-buddy": "src/cli.js" + }, + "peerDependencies": { + "typescript": ">=5.0.4 <5.9" + } + }, + "node_modules/dts-buddy/node_modules/ts-api-utils": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.4.3.tgz", + "integrity": "sha512-i3eMG77UTMD0hZhgRS562pv83RC6ukSAC2GMNWc+9dieh/+jDM5u5YG+NHX6VNDRHQcHwmsTHctP9LhbC3WxVw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16" + }, + "peerDependencies": { + "typescript": ">=4.2.0" + } + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "dev": true, + "license": "MIT" + }, + "node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true, + "license": "MIT" + }, + "node_modules/es-module-lexer": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", + "integrity": "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==", + "dev": true, + "license": "MIT" + }, + "node_modules/esbuild": { + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.1.tgz", + "integrity": "sha512-yY35KZckJJuVVPXpvjgxiCuVEJT67F6zDeVTv4rizyPrfGBUpZQsvmxnN+C371c2esD/hNMjj4tpBhuueLN7aA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.1", + "@esbuild/android-arm": "0.27.1", + "@esbuild/android-arm64": "0.27.1", + "@esbuild/android-x64": "0.27.1", + "@esbuild/darwin-arm64": "0.27.1", + "@esbuild/darwin-x64": "0.27.1", + "@esbuild/freebsd-arm64": "0.27.1", + "@esbuild/freebsd-x64": "0.27.1", + "@esbuild/linux-arm": "0.27.1", + "@esbuild/linux-arm64": "0.27.1", + "@esbuild/linux-ia32": "0.27.1", + "@esbuild/linux-loong64": "0.27.1", + "@esbuild/linux-mips64el": "0.27.1", + "@esbuild/linux-ppc64": "0.27.1", + "@esbuild/linux-riscv64": "0.27.1", + "@esbuild/linux-s390x": "0.27.1", + "@esbuild/linux-x64": "0.27.1", + "@esbuild/netbsd-arm64": "0.27.1", + "@esbuild/netbsd-x64": "0.27.1", + "@esbuild/openbsd-arm64": "0.27.1", + "@esbuild/openbsd-x64": "0.27.1", + "@esbuild/openharmony-arm64": "0.27.1", + "@esbuild/sunos-x64": "0.27.1", + "@esbuild/win32-arm64": "0.27.1", + "@esbuild/win32-ia32": "0.27.1", + "@esbuild/win32-x64": "0.27.1" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "9.39.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.1.tgz", + "integrity": "sha512-BhHmn2yNOFA9H9JmmIVKJmd288g9hrVRDkdoIgRCRuSySRUHH7r/DI6aAXW9T1WwUuY3DFgrcaqB+deURBLR5g==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@eslint-community/eslint-utils": "^4.8.0", + "@eslint-community/regexpp": "^4.12.1", + "@eslint/config-array": "^0.21.1", + "@eslint/config-helpers": "^0.4.2", + "@eslint/core": "^0.17.0", + "@eslint/eslintrc": "^3.3.1", + "@eslint/js": "9.39.1", + "@eslint/plugin-kit": "^0.4.1", + "@humanfs/node": "^0.16.6", + "@humanwhocodes/module-importer": "^1.0.1", + "@humanwhocodes/retry": "^0.4.2", + "@types/estree": "^1.0.6", + "ajv": "^6.12.4", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.6", + "debug": "^4.3.2", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^8.4.0", + "eslint-visitor-keys": "^4.2.1", + "espree": "^10.4.0", + "esquery": "^1.5.0", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^8.0.0", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "lodash.merge": "^4.6.2", + "minimatch": "^3.1.2", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "jiti": "*" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + } + } + }, + "node_modules/eslint-scope": { + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz", + "integrity": "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", + "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/espree": { + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", + "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "acorn": "^8.15.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^4.2.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.6.0.tgz", + "integrity": "sha512-ca9pw9fomFcKPvFLXhBKUK90ZvGibiGOvRJNbjljY7s7uq/5YO4BOzcYtJqExdx99rF6aAcnRxHmcUHcz6sQsg==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expect-type": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.2.2.tgz", + "integrity": "sha512-JhFGDVJ7tmDJItKhYgJCGLOWjuK9vPxiXoUFLwLDc99NlmklilbiQJwoctZtt13+xMw91MCk/REan6MWHqDjyA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true, + "license": "MIT" + }, + "node_modules/file-entry-cache": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", + "integrity": "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "flat-cache": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz", + "integrity": "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==", + "dev": true, + "license": "MIT", + "dependencies": { + "flatted": "^3.2.9", + "keyv": "^4.5.4" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/flatted": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.2.tgz", + "integrity": "sha512-AiwGJM8YcNOaobumgtng+6NHuOqC3A7MixFeDafM3X9cIUM+xUXoS5Vfgf+OihAYe20fxqNM9yPBXJzRtZ/4eA==", + "dev": true, + "license": "ISC" + }, + "node_modules/foreground-child": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.0.tgz", + "integrity": "sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg==", + "dev": true, + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.0", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/get-tsconfig": { + "version": "4.8.1", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.8.1.tgz", + "integrity": "sha512-k9PN+cFBmaLWtVz29SkUoqU5O0slLuHJXt/2P+tMVFT+phsSGXGkp9t3rQIqdz0e+06EHNGs3oM6ZX1s2zHxRg==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, + "node_modules/glob": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", + "dev": true, + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/glob/node_modules/brace-expansion": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/glob/node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/globals": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", + "integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/graphemer": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", + "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", + "dev": true, + "license": "MIT" + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", + "dev": true, + "license": "MIT" + }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/import-fresh": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/istanbul-lib-coverage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", + "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-report": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", + "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "istanbul-lib-coverage": "^3.0.0", + "make-dir": "^4.0.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-source-maps": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-5.0.6.tgz", + "integrity": "sha512-yg2d+Em4KizZC5niWhQaIomgf5WlL4vOOjZ5xGCmF8SnPE/mDWWXgvRExdcpCgh9lLRRa1/fSYp2ymmbJ1pI+A==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.23", + "debug": "^4.1.1", + "istanbul-lib-coverage": "^3.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-reports": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.1.7.tgz", + "integrity": "sha512-BewmUXImeuRk2YY0PVbxgKAysvhRPUQE0h5QRM++nVWyubKGV0l8qQ5op8+B2DOmwSe63Jivj0BjkPQVf8fP5g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "html-escaper": "^2.0.0", + "istanbul-lib-report": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, + "node_modules/js-tokens": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-9.0.1.tgz", + "integrity": "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/js-yaml": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "dev": true, + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "json-buffer": "3.0.1" + } + }, + "node_modules/kleur": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", + "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/locate-character": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-character/-/locate-character-3.0.0.tgz", + "integrity": "sha512-SW13ws7BjaeJ6p7Q6CO2nchbYEc3X3J6WrmTTDto7yMPqVSZTUyY5Tjbid+Ab8gLnATtygYtiDIJGQRRn2ZOiA==", + "dev": true, + "license": "MIT" + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/loupe": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.2.1.tgz", + "integrity": "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/magic-string": { + "version": "0.30.17", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.17.tgz", + "integrity": "sha512-sNPKHvyjVf7gyjwS4xGTaW/mCnF8wnjtifKBEhxfZ7E/S8tQ0rssrwGNn6q8JH/ohItJfSQp9mBtQYuTlH5QnA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0" + } + }, + "node_modules/magicast": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/magicast/-/magicast-0.3.5.tgz", + "integrity": "sha512-L0WhttDl+2BOsybvEOLK7fW3UA0OQ0IQ2d6Zl2x/a6vVRs3bAY0ECOSHHeL5jD+SbOpOCUEi0y1DgHEn9Qn1AQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.25.4", + "@babel/types": "^7.25.4", + "source-map-js": "^1.2.0" + } + }, + "node_modules/make-dir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", + "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/mri": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz", + "integrity": "sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT" + }, + "node_modules/optionator": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", + "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "dev": true, + "license": "BlueOak-1.0.0" + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "license": "MIT", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "dev": true, + "license": "MIT" + }, + "node_modules/pathval": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pathval/-/pathval-2.0.1.tgz", + "integrity": "sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14.16" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/postcss": { + "version": "8.5.6", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.6.tgz", + "integrity": "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.11", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/resolve-pkg-maps": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", + "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" + } + }, + "node_modules/rollup": { + "version": "4.49.0", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.49.0.tgz", + "integrity": "sha512-3IVq0cGJ6H7fKXXEdVt+RcYvRCt8beYY9K1760wGQwSAHZcS9eot1zDG5axUbcp/kWRi5zKIIDX8MoKv/TzvZA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.8" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.49.0", + "@rollup/rollup-android-arm64": "4.49.0", + "@rollup/rollup-darwin-arm64": "4.49.0", + "@rollup/rollup-darwin-x64": "4.49.0", + "@rollup/rollup-freebsd-arm64": "4.49.0", + "@rollup/rollup-freebsd-x64": "4.49.0", + "@rollup/rollup-linux-arm-gnueabihf": "4.49.0", + "@rollup/rollup-linux-arm-musleabihf": "4.49.0", + "@rollup/rollup-linux-arm64-gnu": "4.49.0", + "@rollup/rollup-linux-arm64-musl": "4.49.0", + "@rollup/rollup-linux-loongarch64-gnu": "4.49.0", + "@rollup/rollup-linux-ppc64-gnu": "4.49.0", + "@rollup/rollup-linux-riscv64-gnu": "4.49.0", + "@rollup/rollup-linux-riscv64-musl": "4.49.0", + "@rollup/rollup-linux-s390x-gnu": "4.49.0", + "@rollup/rollup-linux-x64-gnu": "4.49.0", + "@rollup/rollup-linux-x64-musl": "4.49.0", + "@rollup/rollup-win32-arm64-msvc": "4.49.0", + "@rollup/rollup-win32-ia32-msvc": "4.49.0", + "@rollup/rollup-win32-x64-msvc": "4.49.0", + "fsevents": "~2.3.2" + } + }, + "node_modules/sade": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/sade/-/sade-1.8.1.tgz", + "integrity": "sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A==", + "dev": true, + "license": "MIT", + "dependencies": { + "mri": "^1.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/semver": { + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/source-map": { + "version": "0.7.6", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.6.tgz", + "integrity": "sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">= 12" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/std-env": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.9.0.tgz", + "integrity": "sha512-UGvjygr6F6tpH7o2qyqR6QYpwraIjKSdtzyBdyytFOHmPZY917kwdwLG0RbOjWOnKmnm3PeHjaoLLMie7kPLQw==", + "dev": true, + "license": "MIT" + }, + "node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/string-width-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/strip-literal": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-literal/-/strip-literal-3.0.0.tgz", + "integrity": "sha512-TcccoMhJOM3OebGhSBEmp3UZ2SfDMZUEBdRA/9ynfLi8yYajyWX3JiXArcJt4Umh4vISpspkQIY8ZZoCqjbviA==", + "dev": true, + "license": "MIT", + "dependencies": { + "js-tokens": "^9.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/test-exclude": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-7.0.1.tgz", + "integrity": "sha512-pFYqmTw68LXVjeWJMST4+borgQP2AyMNbg1BpZh9LbyhUeNkeaPF9gzfPGUAnSMV3qPYdWUwDIjjCLiSDOl7vg==", + "dev": true, + "license": "ISC", + "dependencies": { + "@istanbuljs/schema": "^0.1.2", + "glob": "^10.4.1", + "minimatch": "^9.0.4" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/test-exclude/node_modules/brace-expansion": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/test-exclude/node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.2.tgz", + "integrity": "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyglobby": { + "version": "0.2.15", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", + "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.3" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinyglobby/node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/tinyglobby/node_modules/picomatch": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", + "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/tinypool": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-1.1.1.tgz", + "integrity": "sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.0.0 || >=20.0.0" + } + }, + "node_modules/tinyrainbow": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-2.0.0.tgz", + "integrity": "sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tinyspy": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-4.0.3.tgz", + "integrity": "sha512-t2T/WLB2WRgZ9EpE4jgPJ9w+i66UZfDc8wHh0xrwiRNN+UwH98GIJkTeZqX9rg0i0ptwzqW+uYeIF0T4F8LR7A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/ts-api-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.1.0.tgz", + "integrity": "sha512-CUgTZL1irw8u29bzrOD/nH85jqyc74D6SshFgujOIA7osm2Rz7dYH77agkx7H4FBNxDq7Cjf+IjaX/8zwFW+ZQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.12" + }, + "peerDependencies": { + "typescript": ">=4.8.4" + } + }, + "node_modules/tsx": { + "version": "4.21.0", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz", + "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "esbuild": "~0.27.0", + "get-tsconfig": "^4.7.5" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/typescript": { + "version": "5.8.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz", + "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/typescript-eslint": { + "version": "8.48.1", + "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.48.1.tgz", + "integrity": "sha512-FbOKN1fqNoXp1hIl5KYpObVrp0mCn+CLgn479nmu2IsRMrx2vyv74MmsBLVlhg8qVwNFGbXSp8fh1zp8pEoC2A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/eslint-plugin": "8.48.1", + "@typescript-eslint/parser": "8.48.1", + "@typescript-eslint/typescript-estree": "8.48.1", + "@typescript-eslint/utils": "8.48.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0", + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/undici-types": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", + "dev": true, + "license": "MIT" + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/vite": { + "version": "7.2.6", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.2.6.tgz", + "integrity": "sha512-tI2l/nFHC5rLh7+5+o7QjKjSR04ivXDF4jcgV0f/bTQ+OJiITy5S6gaynVsEM+7RqzufMnVbIon6Sr5x1SDYaQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "esbuild": "^0.25.0", + "fdir": "^6.5.0", + "picomatch": "^4.0.3", + "postcss": "^8.5.6", + "rollup": "^4.43.0", + "tinyglobby": "^0.2.15" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "lightningcss": "^1.21.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vite-node": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-3.2.4.tgz", + "integrity": "sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cac": "^6.7.14", + "debug": "^4.4.1", + "es-module-lexer": "^1.7.0", + "pathe": "^2.0.3", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" + }, + "bin": { + "vite-node": "vite-node.mjs" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/vite/node_modules/@esbuild/aix-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", + "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/android-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", + "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/android-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", + "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/android-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", + "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/darwin-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", + "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/darwin-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", + "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", + "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/freebsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", + "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", + "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", + "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", + "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-loong64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", + "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-mips64el": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", + "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", + "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-riscv64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", + "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-s390x": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", + "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", + "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", + "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/netbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", + "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", + "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/openbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", + "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/openharmony-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", + "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/sunos-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", + "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/win32-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", + "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/win32-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", + "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/win32-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", + "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/esbuild": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", + "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.12", + "@esbuild/android-arm": "0.25.12", + "@esbuild/android-arm64": "0.25.12", + "@esbuild/android-x64": "0.25.12", + "@esbuild/darwin-arm64": "0.25.12", + "@esbuild/darwin-x64": "0.25.12", + "@esbuild/freebsd-arm64": "0.25.12", + "@esbuild/freebsd-x64": "0.25.12", + "@esbuild/linux-arm": "0.25.12", + "@esbuild/linux-arm64": "0.25.12", + "@esbuild/linux-ia32": "0.25.12", + "@esbuild/linux-loong64": "0.25.12", + "@esbuild/linux-mips64el": "0.25.12", + "@esbuild/linux-ppc64": "0.25.12", + "@esbuild/linux-riscv64": "0.25.12", + "@esbuild/linux-s390x": "0.25.12", + "@esbuild/linux-x64": "0.25.12", + "@esbuild/netbsd-arm64": "0.25.12", + "@esbuild/netbsd-x64": "0.25.12", + "@esbuild/openbsd-arm64": "0.25.12", + "@esbuild/openbsd-x64": "0.25.12", + "@esbuild/openharmony-arm64": "0.25.12", + "@esbuild/sunos-x64": "0.25.12", + "@esbuild/win32-arm64": "0.25.12", + "@esbuild/win32-ia32": "0.25.12", + "@esbuild/win32-x64": "0.25.12" + } + }, + "node_modules/vite/node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/vite/node_modules/picomatch": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", + "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/vitest": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-3.2.4.tgz", + "integrity": "sha512-LUCP5ev3GURDysTWiP47wRRUpLKMOfPh+yKTx3kVIEiu5KOMeqzpnYNsKyOoVrULivR8tLcks4+lga33Whn90A==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/expect": "3.2.4", + "@vitest/mocker": "3.2.4", + "@vitest/pretty-format": "^3.2.4", + "@vitest/runner": "3.2.4", + "@vitest/snapshot": "3.2.4", + "@vitest/spy": "3.2.4", + "@vitest/utils": "3.2.4", + "chai": "^5.2.0", + "debug": "^4.4.1", + "expect-type": "^1.2.1", + "magic-string": "^0.30.17", + "pathe": "^2.0.3", + "picomatch": "^4.0.2", + "std-env": "^3.9.0", + "tinybench": "^2.9.0", + "tinyexec": "^0.3.2", + "tinyglobby": "^0.2.14", + "tinypool": "^1.1.1", + "tinyrainbow": "^2.0.0", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0", + "vite-node": "3.2.4", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@types/debug": "^4.1.12", + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "@vitest/browser": "3.2.4", + "@vitest/ui": "3.2.4", + "happy-dom": "*", + "jsdom": "*" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@types/debug": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + } + } + }, + "node_modules/vitest/node_modules/picomatch": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", + "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/word-wrap": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/wrap-ansi-cjs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/lib/binding_web/package.json b/lib/binding_web/package.json index c275c70d..1bc53aad 100644 --- a/lib/binding_web/package.json +++ b/lib/binding_web/package.json @@ -1,36 +1,100 @@ { "name": "web-tree-sitter", - "version": "0.25.0", + "version": "0.27.0", "description": "Tree-sitter bindings for the web", - "main": "tree-sitter.js", - "types": "tree-sitter-web.d.ts", - "directories": { - "test": "test" - }, - "scripts": { - "test": "mocha", - "prepack": "cp ../../LICENSE .", - "prepublishOnly": "node check-artifacts-fresh.js" - }, "repository": { "type": "git", - "url": "git+https://github.com/tree-sitter/tree-sitter.git" + "url": "git+https://github.com/tree-sitter/tree-sitter.git", + "directory": "lib/binding_web" }, + "license": "MIT", + "author": { + "name": "Max Brunsfeld", + "email": "maxbrunsfeld@gmail.com" + }, + "maintainers": [ + { + "name": "Amaan Qureshi", + "email": "amaanq12@gmail.com" + } + ], + "type": "module", + "exports": { + ".": { + "import": { + "types": "./web-tree-sitter.d.ts", + "default": "./web-tree-sitter.js" + }, + "require": { + "types": "./web-tree-sitter.d.cts", + "default": "./web-tree-sitter.cjs" + } + }, + "./web-tree-sitter.wasm": "./web-tree-sitter.wasm", + "./debug": { + "import": { + "types": "./web-tree-sitter.d.ts", + "default": "./debug/web-tree-sitter.js" + }, + "require": { + "types": "./web-tree-sitter.d.cts", + "default": "./debug/web-tree-sitter.cjs" + } + }, + "./debug/web-tree-sitter.wasm": "./debug/web-tree-sitter.wasm" + }, + "types": "web-tree-sitter.d.ts", "keywords": [ "incremental", - "parsing" + "parsing", + "tree-sitter", + "wasm" + ], + "files": [ + "web-tree-sitter.cjs", + "web-tree-sitter.cjs.map", + "web-tree-sitter.js", + "web-tree-sitter.js.map", + "web-tree-sitter.wasm", + "web-tree-sitter.wasm.map", + "debug/web-tree-sitter.cjs", + "debug/web-tree-sitter.cjs.map", + "debug/web-tree-sitter.js", + "debug/web-tree-sitter.js.map", + "debug/web-tree-sitter.wasm", + "debug/web-tree-sitter.wasm.map", + "web-tree-sitter.d.ts", + "web-tree-sitter.d.ts.map", + "web-tree-sitter.d.cts", + "web-tree-sitter.d.cts.map" ], - "author": "Max Brunsfeld", - "license": "MIT", - "bugs": { - "url": "https://github.com/tree-sitter/tree-sitter/issues" - }, - "homepage": "https://github.com/tree-sitter/tree-sitter/tree/master/lib/binding_web", "devDependencies": { - "@types/emscripten": "^1.39.10", - "chai": "^4.3.7", - "eslint": ">=8.56.0", - "eslint-config-google": "^0.14.0", - "mocha": "^10.2.0" + "@eslint/js": "^9.39.1", + "@types/emscripten": "^1.41.5", + "@types/node": "^24.10.1", + "@vitest/coverage-v8": "^3.0.5", + "dts-buddy": "^0.6.2", + "esbuild": "^0.27.1", + "eslint": "^9.39.1", + "source-map": "^0.7.4", + "tsx": "^4.21.0", + "typescript": "^5.7.3", + "typescript-eslint": "^8.48.1", + "vitest": "^3.0.5" + }, + "scripts": { + "build:ts": "tsc -b . && node script/build.js", + "build:wasm": "cd ../../ && cargo xtask build-wasm", + "build:wasm:debug": "cd ../../ && cargo xtask build-wasm --debug", + "build": "npm run build:wasm && npm run build:ts", + "build:debug": "npm run build:wasm:debug && npm run build:ts -- --debug", + "build:dts": "node script/generate-dts.js", + "lint": "eslint src/*.ts script/*.ts test/*.ts", + "lint:fix": "eslint src/*.ts script/*.ts test/*.ts --fix", + "test": "vitest run", + "test:watch": "vitest", + "prepack": "cp ../../LICENSE .", + "postpack": "rm LICENSE", + "prepublishOnly": "tsx script/check-artifacts-fresh.ts" } } diff --git a/lib/binding_web/package.nix b/lib/binding_web/package.nix new file mode 100644 index 00000000..83f286b5 --- /dev/null +++ b/lib/binding_web/package.nix @@ -0,0 +1,73 @@ +{ + wasm-test-grammars, + lib, + buildNpmPackage, + rustPlatform, + cargo, + pkg-config, + emscripten, + src, + version, +}: +buildNpmPackage { + inherit src version; + + pname = "web-tree-sitter"; + + npmDepsHash = "sha256-y0GobcskcZTmju90TM64GjeWiBmPFCrTOg0yfccdB+Q="; + + nativeBuildInputs = [ + rustPlatform.cargoSetupHook + cargo + pkg-config + emscripten + ]; + + cargoDeps = rustPlatform.importCargoLock { + lockFile = ../../Cargo.lock; + }; + + doCheck = true; + + postPatch = '' + cp lib/binding_web/package{,-lock}.json . + ''; + + buildPhase = '' + pushd lib/binding_web + + CJS=true npm run build + CJS=true npm run build:debug + npm run build:debug + npm run build + + popd + + mkdir -p target/release + + for grammar in ${wasm-test-grammars}/*.wasm; do + if [ -f "$grammar" ]; then + cp "$grammar" target/release/ + fi + done + ''; + + checkPhase = '' + cd lib/binding_web && npm test + ''; + + meta = { + description = "web-tree-sitter - WebAssembly bindings to the Tree-sitter parsing library."; + longDescription = '' + web-tree-sitter provides WebAssembly bindings to the Tree-sitter parsing library. + It can build a concrete syntax tree for a source file and efficiently update + the syntax tree as the source file is edited. This package provides the WebAssembly bindings + and a JavaScript API for using them in web browsers + ''; + homepage = "https://tree-sitter.github.io/tree-sitter"; + changelog = "https://github.com/tree-sitter/tree-sitter/releases/tag/v${version}"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ amaanq ]; + platforms = lib.platforms.all; + }; +} diff --git a/lib/binding_web/prefix.js b/lib/binding_web/prefix.js deleted file mode 100644 index c6bc7c22..00000000 --- a/lib/binding_web/prefix.js +++ /dev/null @@ -1,19 +0,0 @@ -var TreeSitter = function() { - var initPromise; - var document = typeof window == 'object' - ? {currentScript: window.document.currentScript} - : null; - - class Parser { - constructor() { - this.initialize(); - } - - initialize() { - throw new Error("cannot construct a Parser before calling `init()`"); - } - - static init(moduleOptions) { - if (initPromise) return initPromise; - Module = Object.assign({}, Module, moduleOptions); - return initPromise = new Promise((resolveInitPromise) => { diff --git a/lib/binding_web/script/build.js b/lib/binding_web/script/build.js new file mode 100644 index 00000000..737267a1 --- /dev/null +++ b/lib/binding_web/script/build.js @@ -0,0 +1,56 @@ +import esbuild from 'esbuild'; +import fs from 'fs/promises'; + +const format = process.env.CJS ? 'cjs' : 'esm'; +const debug = process.argv.includes('--debug'); +const outfile = `${debug ? 'debug/' : ''}web-tree-sitter.${format === 'esm' ? 'js' : 'cjs'}`; + +async function processWasmSourceMap(inputPath, outputPath) { + const mapContent = await fs.readFile(inputPath, 'utf8'); + const sourceMap = JSON.parse(mapContent); + + const isTreeSitterSource = (source) => + source.includes('../../src/') || source === 'tree-sitter.c'; + + const normalizePath = (source) => { + if (source.includes('../../src/')) { + return source.replace('../../src/', debug ? '../lib/' : 'lib/'); + } else if (source === 'tree-sitter.c') { + return debug ? '../lib/tree-sitter.c' : 'lib/tree-sitter.c'; + } + return source; + }; + + const filtered = sourceMap.sources + .map((source, index) => ({ source, content: sourceMap.sourcesContent?.[index] })) + .filter(item => isTreeSitterSource(item.source)) + .map(item => ({ source: normalizePath(item.source), content: item.content })); + + sourceMap.sources = filtered.map(item => item.source); + sourceMap.sourcesContent = filtered.map(item => item.content); + + await fs.writeFile(outputPath, JSON.stringify(sourceMap, null, 2)); +} + +async function build() { + await esbuild.build({ + entryPoints: ['src/index.ts'], + bundle: true, + platform: 'node', + format, + outfile, + sourcemap: true, + sourcesContent: true, + keepNames: true, + external: ['fs/*', 'fs/promises'], + resolveExtensions: ['.ts', '.js', format === 'esm' ? '.mjs' : '.cjs'], + }); + + // Copy the Wasm files to the appropriate spot, as esbuild doesn't "bundle" Wasm files + const outputWasmName = `${debug ? 'debug/' : ''}web-tree-sitter.wasm`; + await fs.copyFile('lib/web-tree-sitter.wasm', outputWasmName); + + await processWasmSourceMap('lib/web-tree-sitter.wasm.map', `${outputWasmName}.map`); +} + +build().catch(console.error); diff --git a/lib/binding_web/script/check-artifacts-fresh.ts b/lib/binding_web/script/check-artifacts-fresh.ts new file mode 100755 index 00000000..fc7d191e --- /dev/null +++ b/lib/binding_web/script/check-artifacts-fresh.ts @@ -0,0 +1,45 @@ +import fs from 'fs'; +import path from 'path'; +import { fileURLToPath } from 'node:url'; + +const scriptDir = path.dirname(fileURLToPath(import.meta.url)); + +const inputFiles = [ + '../lib/tree-sitter.c', + '../src/constants.ts', + '../src/index.ts', + '../src/language.ts', + '../src/lookahead_iterator.ts', + '../src/marshal.ts', + '../src/node.ts', + '../src/parser.ts', + '../src/query.ts', + '../src/tree.ts', + '../src/tree_cursor.ts', + '../lib/exports.txt', + '../lib/imports.js', + '../lib/prefix.js', + ...listFiles('../../include/tree_sitter'), + ...listFiles('../../src'), +]; + +const outputFiles = ['../web-tree-sitter.js', '../web-tree-sitter.wasm']; +const outputMtime = Math.min(...outputFiles.map(getMtime)); + +for (const inputFile of inputFiles) { + if (getMtime(inputFile) > outputMtime) { + console.log(`File '${inputFile}' has changed. Re-run 'npm run build:wasm'.`); + process.exit(1); + } +} + +function listFiles(dir: string): string[] { + return fs + .readdirSync(path.resolve(scriptDir, dir)) + .filter(p => !p.startsWith('.')) + .map(p => path.join(dir, p)); +} + +function getMtime(p: string): number { + return fs.statSync(path.resolve(scriptDir, p)).mtime.getTime(); +} diff --git a/lib/binding_web/script/generate-dts.js b/lib/binding_web/script/generate-dts.js new file mode 100644 index 00000000..0be5999d --- /dev/null +++ b/lib/binding_web/script/generate-dts.js @@ -0,0 +1,14 @@ +import { createBundle } from 'dts-buddy'; + +for (let ext of ['ts', 'cts']) { + await createBundle({ + project: 'tsconfig.json', + output: `web-tree-sitter.d.${ext}`, + modules: { + 'web-tree-sitter': 'src/index.ts' + }, + compilerOptions: { + stripInternal: true, + }, + }); +} diff --git a/lib/binding_web/src/bindings.ts b/lib/binding_web/src/bindings.ts new file mode 100644 index 00000000..c7b10c93 --- /dev/null +++ b/lib/binding_web/src/bindings.ts @@ -0,0 +1,23 @@ +import createModule, { type MainModule } from '../lib/web-tree-sitter'; +// eslint-disable-next-line @typescript-eslint/no-unused-vars +import { type Parser } from './parser'; + +export let Module: MainModule | null = null; + +/** + * @internal + * + * Initialize the Tree-sitter Wasm module. This should only be called by the {@link Parser} class via {@link Parser.init}. + */ +export async function initializeBinding(moduleOptions?: Partial): Promise { + return Module ??= await createModule(moduleOptions); +} + +/** + * @internal + * + * Checks if the Tree-sitter Wasm module has been initialized. + */ +export function checkModule(): boolean { + return !!Module; +} diff --git a/lib/binding_web/src/constants.ts b/lib/binding_web/src/constants.ts new file mode 100644 index 00000000..026e9f16 --- /dev/null +++ b/lib/binding_web/src/constants.ts @@ -0,0 +1,110 @@ +import { type MainModule } from '../lib/web-tree-sitter'; +// eslint-disable-next-line @typescript-eslint/no-unused-vars +import { ParseState, type Parser } from './parser'; + +/** + * A position in a multi-line text document, in terms of rows and columns. + * + * Rows and columns are zero-based. + */ +export interface Point { + /** The zero-based row number. */ + row: number; + + /** The zero-based column number. */ + column: number; +} + +/** + * A range of positions in a multi-line text document, both in terms of bytes + * and of rows and columns. + */ +export interface Range { + /** The start position of the range. */ + startPosition: Point; + + /** The end position of the range. */ + endPosition: Point; + + /** The start index of the range. */ + startIndex: number; + + /** The end index of the range. */ + endIndex: number; +} + +/** @internal */ +export const SIZE_OF_SHORT = 2; + +/** @internal */ +export const SIZE_OF_INT = 4; + +/** @internal */ +export const SIZE_OF_CURSOR = 4 * SIZE_OF_INT; + +/** @internal */ +export const SIZE_OF_NODE = 5 * SIZE_OF_INT; + +/** @internal */ +export const SIZE_OF_POINT = 2 * SIZE_OF_INT; + +/** @internal */ +export const SIZE_OF_RANGE = 2 * SIZE_OF_INT + 2 * SIZE_OF_POINT; + +/** @internal */ +export const ZERO_POINT: Point = { row: 0, column: 0 }; + +/** + * A callback for parsing that takes an index and point, and should return a string. + */ +export type ParseCallback = (index: number, position: Point) => string | undefined; + +/** + * A callback that receives the parse state during parsing. + */ +export type ProgressCallback = (progress: ParseState) => boolean; + +/** + * A callback for logging messages. + * + * If `isLex` is `true`, the message is from the lexer, otherwise it's from the parser. + */ +export type LogCallback = (message: string, isLex: boolean) => void; + +// Helper type for internal use +/** @internal */ +export const INTERNAL = Symbol('INTERNAL'); +/** @internal */ +export type Internal = typeof INTERNAL; + +// Helper functions for type checking +/** @internal */ +export function assertInternal(x: unknown): asserts x is Internal { + if (x !== INTERNAL) throw new Error('Illegal constructor'); +} + +/** @internal */ +export function isPoint(point?: Point): point is Point { + return ( + !!point && + typeof (point).row === 'number' && + typeof (point).column === 'number' + ); +} + +/** + * @internal + * + * Sets the Tree-sitter Wasm module. This should only be called by the {@link Parser} class via {@link Parser.init}. + */ +export function setModule(module: MainModule) { + C = module; +} + +/** + * @internal + * + * `C` is a convenient shorthand for the Tree-sitter Wasm module, + * which allows us to call all of the exported functions. + */ +export let C: MainModule; diff --git a/lib/binding_web/src/edit.ts b/lib/binding_web/src/edit.ts new file mode 100644 index 00000000..ee3d2d46 --- /dev/null +++ b/lib/binding_web/src/edit.ts @@ -0,0 +1,125 @@ +import { Point, Range } from "./constants"; + +export class Edit { + /** The start position of the change. */ + startPosition: Point; + + /** The end position of the change before the edit. */ + oldEndPosition: Point; + + /** The end position of the change after the edit. */ + newEndPosition: Point; + + /** The start index of the change. */ + startIndex: number; + + /** The end index of the change before the edit. */ + oldEndIndex: number; + + /** The end index of the change after the edit. */ + newEndIndex: number; + + constructor({ + startIndex, + oldEndIndex, + newEndIndex, + startPosition, + oldEndPosition, + newEndPosition, + }: { + startIndex: number; + oldEndIndex: number; + newEndIndex: number; + startPosition: Point; + oldEndPosition: Point; + newEndPosition: Point; + }) { + this.startIndex = startIndex >>> 0; + this.oldEndIndex = oldEndIndex >>> 0; + this.newEndIndex = newEndIndex >>> 0; + this.startPosition = startPosition; + this.oldEndPosition = oldEndPosition; + this.newEndPosition = newEndPosition; + } + + /** + * Edit a point and index to keep it in-sync with source code that has been edited. + * + * This function updates a single point's byte offset and row/column position + * based on an edit operation. This is useful for editing points without + * requiring a tree or node instance. + */ + editPoint(point: Point, index: number): { point: Point; index: number } { + let newIndex = index; + const newPoint = { ...point }; + + if (index >= this.oldEndIndex) { + newIndex = this.newEndIndex + (index - this.oldEndIndex); + const originalRow = point.row; + newPoint.row = this.newEndPosition.row + (point.row - this.oldEndPosition.row); + newPoint.column = originalRow === this.oldEndPosition.row + ? this.newEndPosition.column + (point.column - this.oldEndPosition.column) + : point.column; + } else if (index > this.startIndex) { + newIndex = this.newEndIndex; + newPoint.row = this.newEndPosition.row; + newPoint.column = this.newEndPosition.column; + } + + return { point: newPoint, index: newIndex }; + } + + /** + * Edit a range to keep it in-sync with source code that has been edited. + * + * This function updates a range's start and end positions based on an edit + * operation. This is useful for editing ranges without requiring a tree + * or node instance. + */ + editRange(range: Range): Range { + const newRange: Range = { + startIndex: range.startIndex, + startPosition: { ...range.startPosition }, + endIndex: range.endIndex, + endPosition: { ...range.endPosition } + }; + + if (range.endIndex >= this.oldEndIndex) { + if (range.endIndex !== Number.MAX_SAFE_INTEGER) { + newRange.endIndex = this.newEndIndex + (range.endIndex - this.oldEndIndex); + newRange.endPosition = { + row: this.newEndPosition.row + (range.endPosition.row - this.oldEndPosition.row), + column: range.endPosition.row === this.oldEndPosition.row + ? this.newEndPosition.column + (range.endPosition.column - this.oldEndPosition.column) + : range.endPosition.column, + }; + if (newRange.endIndex < this.newEndIndex) { + newRange.endIndex = Number.MAX_SAFE_INTEGER; + newRange.endPosition = { row: Number.MAX_SAFE_INTEGER, column: Number.MAX_SAFE_INTEGER }; + } + } + } else if (range.endIndex > this.startIndex) { + newRange.endIndex = this.startIndex; + newRange.endPosition = { ...this.startPosition }; + } + + if (range.startIndex >= this.oldEndIndex) { + newRange.startIndex = this.newEndIndex + (range.startIndex - this.oldEndIndex); + newRange.startPosition = { + row: this.newEndPosition.row + (range.startPosition.row - this.oldEndPosition.row), + column: range.startPosition.row === this.oldEndPosition.row + ? this.newEndPosition.column + (range.startPosition.column - this.oldEndPosition.column) + : range.startPosition.column, + }; + if (newRange.startIndex < this.newEndIndex) { + newRange.startIndex = Number.MAX_SAFE_INTEGER; + newRange.startPosition = { row: Number.MAX_SAFE_INTEGER, column: Number.MAX_SAFE_INTEGER }; + } + } else if (range.startIndex > this.startIndex) { + newRange.startIndex = this.startIndex; + newRange.startPosition = { ...this.startPosition }; + } + + return newRange; + } +} diff --git a/lib/binding_web/src/finalization_registry.ts b/lib/binding_web/src/finalization_registry.ts new file mode 100644 index 00000000..5f9c45cc --- /dev/null +++ b/lib/binding_web/src/finalization_registry.ts @@ -0,0 +1,8 @@ +export function newFinalizer(handler: (value: T) => void): FinalizationRegistry | undefined { + try { + return new FinalizationRegistry(handler); + } catch(e) { + console.error('Unsupported FinalizationRegistry:', e); + return; + } +} diff --git a/lib/binding_web/src/index.ts b/lib/binding_web/src/index.ts new file mode 100644 index 00000000..5876db92 --- /dev/null +++ b/lib/binding_web/src/index.ts @@ -0,0 +1,31 @@ +export type { + Point, + Range, + ParseCallback, + ProgressCallback, + LogCallback, +} from './constants'; +export { Edit } from './edit'; +export { + type ParseOptions, + type ParseState, + LANGUAGE_VERSION, + MIN_COMPATIBLE_VERSION, + Parser, +} from './parser'; +export { Language } from './language'; +export { Tree } from './tree'; +export { Node } from './node'; +export { TreeCursor } from './tree_cursor'; +export { + type QueryOptions, + type QueryState, + type QueryProperties, + type QueryPredicate, + type QueryCapture, + type QueryMatch, + CaptureQuantifier, + type PredicateStep, + Query, +} from './query'; +export { LookaheadIterator } from './lookahead_iterator'; diff --git a/lib/binding_web/src/language.ts b/lib/binding_web/src/language.ts new file mode 100644 index 00000000..664e355b --- /dev/null +++ b/lib/binding_web/src/language.ts @@ -0,0 +1,268 @@ +import { C, INTERNAL, Internal, assertInternal, SIZE_OF_INT, SIZE_OF_SHORT } from './constants'; +import { LookaheadIterator } from './lookahead_iterator'; +import { unmarshalLanguageMetadata } from './marshal'; +import { TRANSFER_BUFFER } from './parser'; + +const LANGUAGE_FUNCTION_REGEX = /^tree_sitter_\w+$/; + +export interface LanguageMetadata { + readonly major_version: number; + readonly minor_version: number; + readonly patch_version: number; +} + +/** + * An opaque object that defines how to parse a particular language. + * The code for each `Language` is generated by the Tree-sitter CLI. + */ +export class Language { + /** @internal */ + private [0] = 0; // Internal handle for Wasm + + /** + * A list of all node types in the language. The index of each type in this + * array is its node type id. + */ + types: string[]; + + /** + * A list of all field names in the language. The index of each field name in + * this array is its field id. + */ + fields: (string | null)[]; + + /** @internal */ + constructor(internal: Internal, address: number) { + assertInternal(internal); + this[0] = address; + this.types = new Array(C._ts_language_symbol_count(this[0])); + for (let i = 0, n = this.types.length; i < n; i++) { + if (C._ts_language_symbol_type(this[0], i) < 2) { + this.types[i] = C.UTF8ToString(C._ts_language_symbol_name(this[0], i)); + } + } + this.fields = new Array(C._ts_language_field_count(this[0]) + 1); + for (let i = 0, n = this.fields.length; i < n; i++) { + const fieldName = C._ts_language_field_name_for_id(this[0], i); + if (fieldName !== 0) { + this.fields[i] = C.UTF8ToString(fieldName); + } else { + this.fields[i] = null; + } + } + } + + + /** + * Gets the name of the language. + */ + get name(): string | null { + const ptr = C._ts_language_name(this[0]); + if (ptr === 0) return null; + return C.UTF8ToString(ptr); + } + + /** + * Gets the ABI version of the language. + */ + get abiVersion(): number { + return C._ts_language_abi_version(this[0]); + } + + /** + * Get the metadata for this language. This information is generated by the + * CLI, and relies on the language author providing the correct metadata in + * the language's `tree-sitter.json` file. + */ + get metadata(): LanguageMetadata | null { + C._ts_language_metadata_wasm(this[0]); + const length = C.getValue(TRANSFER_BUFFER, 'i32'); + if (length === 0) return null; + return unmarshalLanguageMetadata(TRANSFER_BUFFER + SIZE_OF_INT); + } + + /** + * Gets the number of fields in the language. + */ + get fieldCount(): number { + return this.fields.length - 1; + } + + /** + * Gets the number of states in the language. + */ + get stateCount(): number { + return C._ts_language_state_count(this[0]); + } + + /** + * Get the field id for a field name. + */ + fieldIdForName(fieldName: string): number | null { + const result = this.fields.indexOf(fieldName); + return result !== -1 ? result : null; + } + + /** + * Get the field name for a field id. + */ + fieldNameForId(fieldId: number): string | null { + return this.fields[fieldId] ?? null; + } + + /** + * Get the node type id for a node type name. + */ + idForNodeType(type: string, named: boolean): number | null { + const typeLength = C.lengthBytesUTF8(type); + const typeAddress = C._malloc(typeLength + 1); + C.stringToUTF8(type, typeAddress, typeLength + 1); + const result = C._ts_language_symbol_for_name(this[0], typeAddress, typeLength, named ? 1 : 0); + C._free(typeAddress); + return result || null; + } + + /** + * Gets the number of node types in the language. + */ + get nodeTypeCount(): number { + return C._ts_language_symbol_count(this[0]); + } + + /** + * Get the node type name for a node type id. + */ + nodeTypeForId(typeId: number): string | null { + const name = C._ts_language_symbol_name(this[0], typeId); + return name ? C.UTF8ToString(name) : null; + } + + /** + * Check if a node type is named. + * + * @see {@link https://tree-sitter.github.io/tree-sitter/using-parsers/2-basic-parsing.html#named-vs-anonymous-nodes} + */ + nodeTypeIsNamed(typeId: number): boolean { + return C._ts_language_type_is_named_wasm(this[0], typeId) ? true : false; + } + + /** + * Check if a node type is visible. + */ + nodeTypeIsVisible(typeId: number): boolean { + return C._ts_language_type_is_visible_wasm(this[0], typeId) ? true : false; + } + + /** + * Get the supertypes ids of this language. + * + * @see {@link https://tree-sitter.github.io/tree-sitter/using-parsers/6-static-node-types.html?highlight=supertype#supertype-nodes} + */ + get supertypes(): number[] { + C._ts_language_supertypes_wasm(this[0]); + const count = C.getValue(TRANSFER_BUFFER, 'i32'); + const buffer = C.getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); + const result = new Array(count); + + if (count > 0) { + let address = buffer; + for (let i = 0; i < count; i++) { + result[i] = C.getValue(address, 'i16'); + address += SIZE_OF_SHORT; + } + } + + return result; + } + + /** + * Get the subtype ids for a given supertype node id. + */ + subtypes(supertype: number): number[] { + C._ts_language_subtypes_wasm(this[0], supertype); + const count = C.getValue(TRANSFER_BUFFER, 'i32'); + const buffer = C.getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); + const result = new Array(count); + + if (count > 0) { + let address = buffer; + for (let i = 0; i < count; i++) { + result[i] = C.getValue(address, 'i16'); + address += SIZE_OF_SHORT; + } + } + + return result; + } + + /** + * Get the next state id for a given state id and node type id. + */ + nextState(stateId: number, typeId: number): number { + return C._ts_language_next_state(this[0], stateId, typeId); + } + + /** + * Create a new lookahead iterator for this language and parse state. + * + * This returns `null` if state is invalid for this language. + * + * Iterating {@link LookaheadIterator} will yield valid symbols in the given + * parse state. Newly created lookahead iterators will return the `ERROR` + * symbol from {@link LookaheadIterator#currentType}. + * + * Lookahead iterators can be useful for generating suggestions and improving + * syntax error diagnostics. To get symbols valid in an `ERROR` node, use the + * lookahead iterator on its first leaf node state. For `MISSING` nodes, a + * lookahead iterator created on the previous non-extra leaf node may be + * appropriate. + */ + lookaheadIterator(stateId: number): LookaheadIterator | null { + const address = C._ts_lookahead_iterator_new(this[0], stateId); + if (address) return new LookaheadIterator(INTERNAL, address, this); + return null; + } + + /** + * Load a language from a WebAssembly module. + * The module can be provided as a path to a file or as a buffer. + */ + static async load(input: string | Uint8Array): Promise { + let binary: Uint8Array | WebAssembly.Module; + if (input instanceof Uint8Array) { + binary = input; + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + } else if (globalThis.process?.versions.node) { + const fs: typeof import('fs/promises') = await import('fs/promises'); + binary = await fs.readFile(input); + } else { + const response = await fetch(input); + + if (!response.ok){ + const body = await response.text(); + throw new Error(`Language.load failed with status ${response.status}.\n\n${body}`); + } + + const retryResp = response.clone(); + try { + binary = await WebAssembly.compileStreaming(response); + } catch (reason) { + console.error('wasm streaming compile failed:', reason); + console.error('falling back to ArrayBuffer instantiation'); + // fallback, probably because of bad MIME type + binary = new Uint8Array(await retryResp.arrayBuffer()) + } + } + + const mod = await C.loadWebAssemblyModule(binary, { loadAsync: true }); + const symbolNames = Object.keys(mod); + const functionName = symbolNames.find((key) => LANGUAGE_FUNCTION_REGEX.test(key) && + !key.includes('external_scanner_')); + if (!functionName) { + console.log(`Couldn't find language function in Wasm file. Symbols:\n${JSON.stringify(symbolNames, null, 2)}`); + throw new Error('Language.load failed: no language function found in Wasm file'); + } + const languageAddress = mod[functionName](); + return new Language(INTERNAL, languageAddress); + } +} diff --git a/lib/binding_web/src/lookahead_iterator.ts b/lib/binding_web/src/lookahead_iterator.ts new file mode 100644 index 00000000..4dd6b296 --- /dev/null +++ b/lib/binding_web/src/lookahead_iterator.ts @@ -0,0 +1,82 @@ +import { C, Internal, assertInternal } from './constants'; +import { Language } from './language'; +import { newFinalizer } from './finalization_registry'; + +const finalizer = newFinalizer((address: number) => { + C._ts_lookahead_iterator_delete(address); +}); + +export class LookaheadIterator implements Iterable { + /** @internal */ + private [0] = 0; // Internal handle for Wasm + + /** @internal */ + private language: Language; + + /** @internal */ + constructor(internal: Internal, address: number, language: Language) { + assertInternal(internal); + this[0] = address; + this.language = language; + finalizer?.register(this, address, this); + } + + /** Get the current symbol of the lookahead iterator. */ + get currentTypeId(): number { + return C._ts_lookahead_iterator_current_symbol(this[0]); + } + + /** Get the current symbol name of the lookahead iterator. */ + get currentType(): string { + return this.language.types[this.currentTypeId] || 'ERROR'; + } + + /** Delete the lookahead iterator, freeing its resources. */ + delete(): void { + finalizer?.unregister(this); + C._ts_lookahead_iterator_delete(this[0]); + this[0] = 0; + } + + + /** + * Reset the lookahead iterator. + * + * This returns `true` if the language was set successfully and `false` + * otherwise. + */ + reset(language: Language, stateId: number): boolean { + if (C._ts_lookahead_iterator_reset(this[0], language[0], stateId)) { + this.language = language; + return true; + } + return false; + } + + /** + * Reset the lookahead iterator to another state. + * + * This returns `true` if the iterator was reset to the given state and + * `false` otherwise. + */ + resetState(stateId: number): boolean { + return Boolean(C._ts_lookahead_iterator_reset_state(this[0], stateId)); + } + + /** + * Returns an iterator that iterates over the symbols of the lookahead iterator. + * + * The iterator will yield the current symbol name as a string for each step + * until there are no more symbols to iterate over. + */ + [Symbol.iterator](): Iterator { + return { + next: (): IteratorResult => { + if (C._ts_lookahead_iterator_next(this[0])) { + return { done: false, value: this.currentType }; + } + return { done: true, value: '' }; + } + }; + } +} diff --git a/lib/binding_web/src/marshal.ts b/lib/binding_web/src/marshal.ts new file mode 100644 index 00000000..f09f8e23 --- /dev/null +++ b/lib/binding_web/src/marshal.ts @@ -0,0 +1,177 @@ +import { INTERNAL, Point, Range, SIZE_OF_INT, SIZE_OF_NODE, SIZE_OF_POINT, C } from "./constants"; +import { Node } from "./node"; +import { Tree } from "./tree"; +// eslint-disable-next-line @typescript-eslint/no-unused-vars +import { Query, QueryCapture, type QueryMatch } from "./query"; +import { TreeCursor } from "./tree_cursor"; +import { TRANSFER_BUFFER } from "./parser"; +import { LanguageMetadata } from "./language"; +import { Edit } from "./edit"; + +/** + * @internal + * + * Unmarshals a {@link QueryMatch} to the transfer buffer. + */ +export function unmarshalCaptures( + query: Query, + tree: Tree, + address: number, + patternIndex: number, + result: QueryCapture[] +) { + for (let i = 0, n = result.length; i < n; i++) { + const captureIndex = C.getValue(address, 'i32'); + address += SIZE_OF_INT; + const node = unmarshalNode(tree, address)!; + address += SIZE_OF_NODE; + result[i] = {patternIndex, name: query.captureNames[captureIndex], node}; + } + return address; +} + +/** + * @internal + * + * Marshals a {@link Node} to the transfer buffer. + */ +export function marshalNode(node: Node, index = 0) { + let address = TRANSFER_BUFFER + index * SIZE_OF_NODE; + C.setValue(address, node.id, 'i32'); + address += SIZE_OF_INT; + C.setValue(address, node.startIndex, 'i32'); + address += SIZE_OF_INT; + C.setValue(address, node.startPosition.row, 'i32'); + address += SIZE_OF_INT; + C.setValue(address, node.startPosition.column, 'i32'); + address += SIZE_OF_INT; + C.setValue(address, node[0], 'i32'); +} + +/** + * @internal + * + * Unmarshals a {@link Node} from the transfer buffer. + */ +export function unmarshalNode(tree: Tree, address = TRANSFER_BUFFER): Node | null { + const id = C.getValue(address, 'i32'); + address += SIZE_OF_INT; + if (id === 0) return null; + + const index = C.getValue(address, 'i32'); + address += SIZE_OF_INT; + const row = C.getValue(address, 'i32'); + address += SIZE_OF_INT; + const column = C.getValue(address, 'i32'); + address += SIZE_OF_INT; + const other = C.getValue(address, 'i32'); + + const result = new Node(INTERNAL, { + id, + tree, + startIndex: index, + startPosition: {row, column}, + other, + }); + + return result; +} + +/** + * @internal + * + * Marshals a {@link TreeCursor} to the transfer buffer. + */ +export function marshalTreeCursor(cursor: TreeCursor, address = TRANSFER_BUFFER) { + C.setValue(address + 0 * SIZE_OF_INT, cursor[0], 'i32'); + C.setValue(address + 1 * SIZE_OF_INT, cursor[1], 'i32'); + C.setValue(address + 2 * SIZE_OF_INT, cursor[2], 'i32'); + C.setValue(address + 3 * SIZE_OF_INT, cursor[3], 'i32'); +} + +/** + * @internal + * + * Unmarshals a {@link TreeCursor} from the transfer buffer. + */ +export function unmarshalTreeCursor(cursor: TreeCursor) { + cursor[0] = C.getValue(TRANSFER_BUFFER + 0 * SIZE_OF_INT, 'i32'); + cursor[1] = C.getValue(TRANSFER_BUFFER + 1 * SIZE_OF_INT, 'i32'); + cursor[2] = C.getValue(TRANSFER_BUFFER + 2 * SIZE_OF_INT, 'i32'); + cursor[3] = C.getValue(TRANSFER_BUFFER + 3 * SIZE_OF_INT, 'i32'); +} + +/** + * @internal + * + * Marshals a {@link Point} to the transfer buffer. + */ +export function marshalPoint(address: number, point: Point): void { + C.setValue(address, point.row, 'i32'); + C.setValue(address + SIZE_OF_INT, point.column, 'i32'); +} + +/** + * @internal + * + * Unmarshals a {@link Point} from the transfer buffer. + */ +export function unmarshalPoint(address: number): Point { + const result = { + row: C.getValue(address, 'i32') >>> 0, + column: C.getValue(address + SIZE_OF_INT, 'i32') >>> 0, + }; + return result; +} + +/** + * @internal + * + * Marshals a {@link Range} to the transfer buffer. + */ +export function marshalRange(address: number, range: Range): void { + marshalPoint(address, range.startPosition); address += SIZE_OF_POINT; + marshalPoint(address, range.endPosition); address += SIZE_OF_POINT; + C.setValue(address, range.startIndex, 'i32'); address += SIZE_OF_INT; + C.setValue(address, range.endIndex, 'i32'); address += SIZE_OF_INT; +} + +/** + * @internal + * + * Unmarshals a {@link Range} from the transfer buffer. + */ +export function unmarshalRange(address: number): Range { + const result = {} as Range; + result.startPosition = unmarshalPoint(address); address += SIZE_OF_POINT; + result.endPosition = unmarshalPoint(address); address += SIZE_OF_POINT; + result.startIndex = C.getValue(address, 'i32') >>> 0; address += SIZE_OF_INT; + result.endIndex = C.getValue(address, 'i32') >>> 0; + return result; +} + +/** + * @internal + * + * Marshals an {@link Edit} to the transfer buffer. + */ +export function marshalEdit(edit: Edit, address = TRANSFER_BUFFER) { + marshalPoint(address, edit.startPosition); address += SIZE_OF_POINT; + marshalPoint(address, edit.oldEndPosition); address += SIZE_OF_POINT; + marshalPoint(address, edit.newEndPosition); address += SIZE_OF_POINT; + C.setValue(address, edit.startIndex, 'i32'); address += SIZE_OF_INT; + C.setValue(address, edit.oldEndIndex, 'i32'); address += SIZE_OF_INT; + C.setValue(address, edit.newEndIndex, 'i32'); address += SIZE_OF_INT; +} + +/** + * @internal + * + * Unmarshals a {@link LanguageMetadata} from the transfer buffer. + */ +export function unmarshalLanguageMetadata(address: number): LanguageMetadata { + const major_version = C.getValue(address, 'i32'); + const minor_version = C.getValue(address += SIZE_OF_INT, 'i32'); + const patch_version = C.getValue(address += SIZE_OF_INT, 'i32'); + return { major_version, minor_version, patch_version }; +} diff --git a/lib/binding_web/src/node.ts b/lib/binding_web/src/node.ts new file mode 100644 index 00000000..1026d680 --- /dev/null +++ b/lib/binding_web/src/node.ts @@ -0,0 +1,647 @@ +import { INTERNAL, Internal, assertInternal, SIZE_OF_INT, SIZE_OF_NODE, SIZE_OF_POINT, ZERO_POINT, isPoint, C, Point } from './constants'; +import { getText, Tree } from './tree'; +import { TreeCursor } from './tree_cursor'; +// eslint-disable-next-line @typescript-eslint/no-unused-vars +import { Language } from './language'; +import { marshalNode, marshalPoint, unmarshalNode, unmarshalPoint } from './marshal'; +import { TRANSFER_BUFFER } from './parser'; +import { Edit } from './edit'; + +/** A single node within a syntax {@link Tree}. */ +export class Node { + /** @internal */ + // @ts-expect-error: never read + private [0] = 0; // Internal handle for Wasm + + /** @internal */ + private _children?: Node[]; + + /** @internal */ + private _namedChildren?: Node[]; + + /** @internal */ + constructor( + internal: Internal, + { + id, + tree, + startIndex, + startPosition, + other, + }: { + id: number; + tree: Tree; + startIndex: number; + startPosition: Point; + other: number; + } + ) { + assertInternal(internal); + this[0] = other; + this.id = id; + this.tree = tree; + this.startIndex = startIndex; + this.startPosition = startPosition; + } + + /** + * The numeric id for this node that is unique. + * + * Within a given syntax tree, no two nodes have the same id. However: + * + * * If a new tree is created based on an older tree, and a node from the old tree is reused in + * the process, then that node will have the same id in both trees. + * + * * A node not marked as having changes does not guarantee it was reused. + * + * * If a node is marked as having changed in the old tree, it will not be reused. + */ + id: number; + + /** The byte index where this node starts. */ + startIndex: number; + + /** The position where this node starts. */ + startPosition: Point; + + /** The tree that this node belongs to. */ + tree: Tree; + + /** Get this node's type as a numerical id. */ + get typeId(): number { + marshalNode(this); + return C._ts_node_symbol_wasm(this.tree[0]); + } + + /** + * Get the node's type as a numerical id as it appears in the grammar, + * ignoring aliases. + */ + get grammarId(): number { + marshalNode(this); + return C._ts_node_grammar_symbol_wasm(this.tree[0]); + } + + /** Get this node's type as a string. */ + get type(): string { + return this.tree.language.types[this.typeId] || 'ERROR'; + } + + /** + * Get this node's symbol name as it appears in the grammar, ignoring + * aliases as a string. + */ + get grammarType(): string { + return this.tree.language.types[this.grammarId] || 'ERROR'; + } + + /** + * Check if this node is *named*. + * + * Named nodes correspond to named rules in the grammar, whereas + * *anonymous* nodes correspond to string literals in the grammar. + */ + get isNamed(): boolean { + marshalNode(this); + return C._ts_node_is_named_wasm(this.tree[0]) === 1; + } + + /** + * Check if this node is *extra*. + * + * Extra nodes represent things like comments, which are not required + * by the grammar, but can appear anywhere. + */ + get isExtra(): boolean { + marshalNode(this); + return C._ts_node_is_extra_wasm(this.tree[0]) === 1; + } + + /** + * Check if this node represents a syntax error. + * + * Syntax errors represent parts of the code that could not be incorporated + * into a valid syntax tree. + */ + get isError(): boolean { + marshalNode(this); + return C._ts_node_is_error_wasm(this.tree[0]) === 1; + } + + /** + * Check if this node is *missing*. + * + * Missing nodes are inserted by the parser in order to recover from + * certain kinds of syntax errors. + */ + get isMissing(): boolean { + marshalNode(this); + return C._ts_node_is_missing_wasm(this.tree[0]) === 1; + } + + /** Check if this node has been edited. */ + get hasChanges(): boolean { + marshalNode(this); + return C._ts_node_has_changes_wasm(this.tree[0]) === 1; + } + + /** + * Check if this node represents a syntax error or contains any syntax + * errors anywhere within it. + */ + get hasError(): boolean { + marshalNode(this); + return C._ts_node_has_error_wasm(this.tree[0]) === 1; + } + + /** Get the byte index where this node ends. */ + get endIndex(): number { + marshalNode(this); + return C._ts_node_end_index_wasm(this.tree[0]); + } + + /** Get the position where this node ends. */ + get endPosition(): Point { + marshalNode(this); + C._ts_node_end_point_wasm(this.tree[0]); + return unmarshalPoint(TRANSFER_BUFFER); + } + + /** Get the string content of this node. */ + get text(): string { + return getText(this.tree, this.startIndex, this.endIndex, this.startPosition); + } + + /** Get this node's parse state. */ + get parseState(): number { + marshalNode(this); + return C._ts_node_parse_state_wasm(this.tree[0]); + } + + /** Get the parse state after this node. */ + get nextParseState(): number { + marshalNode(this); + return C._ts_node_next_parse_state_wasm(this.tree[0]); + } + + /** Check if this node is equal to another node. */ + equals(other: Node): boolean { + return this.tree === other.tree && this.id === other.id; + } + + /** + * Get the node's child at the given index, where zero represents the first child. + * + * This method is fairly fast, but its cost is technically log(n), so if + * you might be iterating over a long list of children, you should use + * {@link Node#children} instead. + */ + child(index: number): Node | null { + marshalNode(this); + C._ts_node_child_wasm(this.tree[0], index); + return unmarshalNode(this.tree); + } + + /** + * Get this node's *named* child at the given index. + * + * See also {@link Node#isNamed}. + * This method is fairly fast, but its cost is technically log(n), so if + * you might be iterating over a long list of children, you should use + * {@link Node#namedChildren} instead. + */ + namedChild(index: number): Node | null { + marshalNode(this); + C._ts_node_named_child_wasm(this.tree[0], index); + return unmarshalNode(this.tree); + } + + /** + * Get this node's child with the given numerical field id. + * + * See also {@link Node#childForFieldName}. You can + * convert a field name to an id using {@link Language#fieldIdForName}. + */ + childForFieldId(fieldId: number): Node | null { + marshalNode(this); + C._ts_node_child_by_field_id_wasm(this.tree[0], fieldId); + return unmarshalNode(this.tree); + } + + /** + * Get the first child with the given field name. + * + * If multiple children may have the same field name, access them using + * {@link Node#childrenForFieldName}. + */ + childForFieldName(fieldName: string): Node | null { + const fieldId = this.tree.language.fields.indexOf(fieldName); + if (fieldId !== -1) return this.childForFieldId(fieldId); + return null; + } + + /** Get the field name of this node's child at the given index. */ + fieldNameForChild(index: number): string | null { + marshalNode(this); + const address = C._ts_node_field_name_for_child_wasm(this.tree[0], index); + if (!address) return null; + return C.AsciiToString(address); + } + + /** Get the field name of this node's named child at the given index. */ + fieldNameForNamedChild(index: number): string | null { + marshalNode(this); + const address = C._ts_node_field_name_for_named_child_wasm(this.tree[0], index); + if (!address) return null; + return C.AsciiToString(address); + } + /** + * Get an array of this node's children with a given field name. + * + * See also {@link Node#children}. + */ + childrenForFieldName(fieldName: string): Node[] { + const fieldId = this.tree.language.fields.indexOf(fieldName); + if (fieldId !== -1 && fieldId !== 0) return this.childrenForFieldId(fieldId); + return []; + } + + /** + * Get an array of this node's children with a given field id. + * + * See also {@link Node#childrenForFieldName}. + */ + childrenForFieldId(fieldId: number): Node[] { + marshalNode(this); + C._ts_node_children_by_field_id_wasm(this.tree[0], fieldId); + const count = C.getValue(TRANSFER_BUFFER, 'i32'); + const buffer = C.getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); + const result = new Array(count); + + if (count > 0) { + let address = buffer; + for (let i = 0; i < count; i++) { + result[i] = unmarshalNode(this.tree, address)!; + address += SIZE_OF_NODE; + } + C._free(buffer); + } + return result; + } + + /** Get the node's first child that contains or starts after the given byte offset. */ + firstChildForIndex(index: number): Node | null { + marshalNode(this); + const address = TRANSFER_BUFFER + SIZE_OF_NODE; + C.setValue(address, index, 'i32'); + C._ts_node_first_child_for_byte_wasm(this.tree[0]); + return unmarshalNode(this.tree); + } + + /** Get the node's first named child that contains or starts after the given byte offset. */ + firstNamedChildForIndex(index: number): Node | null { + marshalNode(this); + const address = TRANSFER_BUFFER + SIZE_OF_NODE; + C.setValue(address, index, 'i32'); + C._ts_node_first_named_child_for_byte_wasm(this.tree[0]); + return unmarshalNode(this.tree); + } + + /** Get this node's number of children. */ + get childCount(): number { + marshalNode(this); + return C._ts_node_child_count_wasm(this.tree[0]); + } + + + /** + * Get this node's number of *named* children. + * + * See also {@link Node#isNamed}. + */ + get namedChildCount(): number { + marshalNode(this); + return C._ts_node_named_child_count_wasm(this.tree[0]); + } + + /** Get this node's first child. */ + get firstChild(): Node | null { + return this.child(0); + } + + /** + * Get this node's first named child. + * + * See also {@link Node#isNamed}. + */ + get firstNamedChild(): Node | null { + return this.namedChild(0); + } + + /** Get this node's last child. */ + get lastChild(): Node | null { + return this.child(this.childCount - 1); + } + + /** + * Get this node's last named child. + * + * See also {@link Node#isNamed}. + */ + get lastNamedChild(): Node | null { + return this.namedChild(this.namedChildCount - 1); + } + + /** + * Iterate over this node's children. + * + * If you're walking the tree recursively, you may want to use the + * {@link TreeCursor} APIs directly instead. + */ + get children(): Node[] { + if (!this._children) { + marshalNode(this); + C._ts_node_children_wasm(this.tree[0]); + const count = C.getValue(TRANSFER_BUFFER, 'i32'); + const buffer = C.getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); + this._children = new Array(count); + if (count > 0) { + let address = buffer; + for (let i = 0; i < count; i++) { + this._children[i] = unmarshalNode(this.tree, address)!; + address += SIZE_OF_NODE; + } + C._free(buffer); + } + } + return this._children; + } + + /** + * Iterate over this node's named children. + * + * See also {@link Node#children}. + */ + get namedChildren(): Node[] { + if (!this._namedChildren) { + marshalNode(this); + C._ts_node_named_children_wasm(this.tree[0]); + const count = C.getValue(TRANSFER_BUFFER, 'i32'); + const buffer = C.getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); + this._namedChildren = new Array(count); + if (count > 0) { + let address = buffer; + for (let i = 0; i < count; i++) { + this._namedChildren[i] = unmarshalNode(this.tree, address)!; + address += SIZE_OF_NODE; + } + C._free(buffer); + } + } + return this._namedChildren; + } + + /** + * Get the descendants of this node that are the given type, or in the given types array. + * + * The types array should contain node type strings, which can be retrieved from {@link Language#types}. + * + * Additionally, a `startPosition` and `endPosition` can be passed in to restrict the search to a byte range. + */ + descendantsOfType( + types: string | string[], + startPosition: Point = ZERO_POINT, + endPosition: Point = ZERO_POINT + ): Node[] { + if (!Array.isArray(types)) types = [types]; + + // Convert the type strings to numeric type symbols + const symbols: number[] = []; + const typesBySymbol = this.tree.language.types; + for (const node_type of types) { + if (node_type == "ERROR") { + symbols.push(65535); // Internally, ts_builtin_sym_error is -1, which is UINT_16MAX + } + } + for (let i = 0, n = typesBySymbol.length; i < n; i++) { + if (types.includes(typesBySymbol[i])) { + symbols.push(i); + } + } + + // Copy the array of symbols to the Wasm heap + const symbolsAddress = C._malloc(SIZE_OF_INT * symbols.length); + for (let i = 0, n = symbols.length; i < n; i++) { + C.setValue(symbolsAddress + i * SIZE_OF_INT, symbols[i], 'i32'); + } + + // Call the C API to compute the descendants + marshalNode(this); + C._ts_node_descendants_of_type_wasm( + this.tree[0], + symbolsAddress, + symbols.length, + startPosition.row, + startPosition.column, + endPosition.row, + endPosition.column + ); + + // Instantiate the nodes based on the data returned + const descendantCount = C.getValue(TRANSFER_BUFFER, 'i32'); + const descendantAddress = C.getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); + const result = new Array(descendantCount); + if (descendantCount > 0) { + let address = descendantAddress; + for (let i = 0; i < descendantCount; i++) { + result[i] = unmarshalNode(this.tree, address)!; + address += SIZE_OF_NODE; + } + } + + // Free the intermediate buffers + C._free(descendantAddress); + C._free(symbolsAddress); + return result; + } + + /** Get this node's next sibling. */ + get nextSibling(): Node | null { + marshalNode(this); + C._ts_node_next_sibling_wasm(this.tree[0]); + return unmarshalNode(this.tree); + } + + /** Get this node's previous sibling. */ + get previousSibling(): Node | null { + marshalNode(this); + C._ts_node_prev_sibling_wasm(this.tree[0]); + return unmarshalNode(this.tree); + } + + /** + * Get this node's next *named* sibling. + * + * See also {@link Node#isNamed}. + */ + get nextNamedSibling(): Node | null { + marshalNode(this); + C._ts_node_next_named_sibling_wasm(this.tree[0]); + return unmarshalNode(this.tree); + } + + /** + * Get this node's previous *named* sibling. + * + * See also {@link Node#isNamed}. + */ + get previousNamedSibling(): Node | null { + marshalNode(this); + C._ts_node_prev_named_sibling_wasm(this.tree[0]); + return unmarshalNode(this.tree); + } + + /** Get the node's number of descendants, including one for the node itself. */ + get descendantCount(): number { + marshalNode(this); + return C._ts_node_descendant_count_wasm(this.tree[0]); + } + + /** + * Get this node's immediate parent. + * Prefer {@link Node#childWithDescendant} for iterating over this node's ancestors. + */ + get parent(): Node | null { + marshalNode(this); + C._ts_node_parent_wasm(this.tree[0]); + return unmarshalNode(this.tree); + } + + /** + * Get the node that contains `descendant`. + * + * Note that this can return `descendant` itself. + */ + childWithDescendant(descendant: Node): Node | null { + marshalNode(this); + marshalNode(descendant, 1); + C._ts_node_child_with_descendant_wasm(this.tree[0]); + return unmarshalNode(this.tree); + } + + /** Get the smallest node within this node that spans the given byte range. */ + descendantForIndex(start: number, end: number = start): Node | null { + if (typeof start !== 'number' || typeof end !== 'number') { + throw new Error('Arguments must be numbers'); + } + + marshalNode(this); + const address = TRANSFER_BUFFER + SIZE_OF_NODE; + C.setValue(address, start, 'i32'); + C.setValue(address + SIZE_OF_INT, end, 'i32'); + C._ts_node_descendant_for_index_wasm(this.tree[0]); + return unmarshalNode(this.tree); + } + + /** Get the smallest named node within this node that spans the given byte range. */ + namedDescendantForIndex(start: number, end: number = start): Node | null { + if (typeof start !== 'number' || typeof end !== 'number') { + throw new Error('Arguments must be numbers'); + } + + marshalNode(this); + const address = TRANSFER_BUFFER + SIZE_OF_NODE; + C.setValue(address, start, 'i32'); + C.setValue(address + SIZE_OF_INT, end, 'i32'); + C._ts_node_named_descendant_for_index_wasm(this.tree[0]); + return unmarshalNode(this.tree); + } + + /** Get the smallest node within this node that spans the given point range. */ + descendantForPosition(start: Point, end: Point = start) { + if (!isPoint(start) || !isPoint(end)) { + throw new Error('Arguments must be {row, column} objects'); + } + + marshalNode(this); + const address = TRANSFER_BUFFER + SIZE_OF_NODE; + marshalPoint(address, start); + marshalPoint(address + SIZE_OF_POINT, end); + C._ts_node_descendant_for_position_wasm(this.tree[0]); + return unmarshalNode(this.tree); + } + + /** Get the smallest named node within this node that spans the given point range. */ + namedDescendantForPosition(start: Point, end: Point = start) { + if (!isPoint(start) || !isPoint(end)) { + throw new Error('Arguments must be {row, column} objects'); + } + + marshalNode(this); + const address = TRANSFER_BUFFER + SIZE_OF_NODE; + marshalPoint(address, start); + marshalPoint(address + SIZE_OF_POINT, end); + C._ts_node_named_descendant_for_position_wasm(this.tree[0]); + return unmarshalNode(this.tree); + } + + /** + * Create a new {@link TreeCursor} starting from this node. + * + * Note that the given node is considered the root of the cursor, + * and the cursor cannot walk outside this node. + */ + walk(): TreeCursor { + marshalNode(this); + C._ts_tree_cursor_new_wasm(this.tree[0]); + return new TreeCursor(INTERNAL, this.tree); + } + + /** + * Edit this node to keep it in-sync with source code that has been edited. + * + * This function is only rarely needed. When you edit a syntax tree with + * the {@link Tree#edit} method, all of the nodes that you retrieve from + * the tree afterward will already reflect the edit. You only need to + * use {@link Node#edit} when you have a specific {@link Node} instance that + * you want to keep and continue to use after an edit. + */ + edit(edit: Edit) { + if (this.startIndex >= edit.oldEndIndex) { + this.startIndex = edit.newEndIndex + (this.startIndex - edit.oldEndIndex); + let subbedPointRow; + let subbedPointColumn; + if (this.startPosition.row > edit.oldEndPosition.row) { + subbedPointRow = this.startPosition.row - edit.oldEndPosition.row; + subbedPointColumn = this.startPosition.column; + } else { + subbedPointRow = 0; + subbedPointColumn = this.startPosition.column; + if (this.startPosition.column >= edit.oldEndPosition.column) { + subbedPointColumn = + this.startPosition.column - edit.oldEndPosition.column; + } + } + + if (subbedPointRow > 0) { + this.startPosition.row += subbedPointRow; + this.startPosition.column = subbedPointColumn; + } else { + this.startPosition.column += subbedPointColumn; + } + } else if (this.startIndex > edit.startIndex) { + this.startIndex = edit.newEndIndex; + this.startPosition.row = edit.newEndPosition.row; + this.startPosition.column = edit.newEndPosition.column; + } + } + + /** Get the S-expression representation of this node. */ + toString(): string { + marshalNode(this); + const address = C._ts_node_to_string_wasm(this.tree[0]); + const result = C.AsciiToString(address); + C._free(address); + return result; + } +} diff --git a/lib/binding_web/src/parser.ts b/lib/binding_web/src/parser.ts new file mode 100644 index 00000000..7e3c3b4a --- /dev/null +++ b/lib/binding_web/src/parser.ts @@ -0,0 +1,309 @@ +import { C, INTERNAL, LogCallback, ParseCallback, Range, SIZE_OF_INT, SIZE_OF_RANGE, setModule } from './constants'; +import { Language } from './language'; +import { marshalRange, unmarshalRange } from './marshal'; +import { checkModule, initializeBinding } from './bindings'; +import { Tree } from './tree'; +import { newFinalizer } from './finalization_registry'; + +/** + * Options for parsing + * + * The `includedRanges` property is an array of {@link Range} objects that + * represent the ranges of text that the parser should include when parsing. + * + * The `progressCallback` property is a function that is called periodically + * during parsing to check whether parsing should be cancelled. + * + * See {@link Parser#parse} for more information. + */ +export interface ParseOptions { + /** + * An array of {@link Range} objects that + * represent the ranges of text that the parser should include when parsing. + * + * This sets the ranges of text that the parser should include when parsing. + * By default, the parser will always include entire documents. This + * function allows you to parse only a *portion* of a document but + * still return a syntax tree whose ranges match up with the document + * as a whole. You can also pass multiple disjoint ranges. + * If `ranges` is empty, then the entire document will be parsed. + * Otherwise, the given ranges must be ordered from earliest to latest + * in the document, and they must not overlap. That is, the following + * must hold for all `i` < `length - 1`: + * ```text + * ranges[i].end_byte <= ranges[i + 1].start_byte + * ``` + */ + includedRanges?: Range[]; + + /** + * A function that is called periodically during parsing to check + * whether parsing should be cancelled. If the progress callback returns + * `true`, then parsing will be cancelled. You can also use this to instrument + * parsing and check where the parser is at in the document. The progress callback + * takes a single argument, which is a {@link ParseState} representing the current + * state of the parser. + */ + progressCallback?: (state: ParseState) => void; +} + +/** + * A stateful object that is passed into the progress callback {@link ParseOptions#progressCallback} + * to provide the current state of the parser. + */ +export interface ParseState { + /** The byte offset in the document that the parser is at. */ + currentOffset: number; + + /** Indicates whether the parser has encountered an error during parsing. */ + hasError: boolean; +} + +/** + * @internal + * + * Global variable for transferring data across the FFI boundary + */ +export let TRANSFER_BUFFER: number; + +/** + * The latest ABI version that is supported by the current version of the + * library. + * + * When Languages are generated by the Tree-sitter CLI, they are + * assigned an ABI version number that corresponds to the current CLI version. + * The Tree-sitter library is generally backwards-compatible with languages + * generated using older CLI versions, but is not forwards-compatible. + */ +export let LANGUAGE_VERSION: number; + +/** + * The earliest ABI version that is supported by the current version of the + * library. + */ +export let MIN_COMPATIBLE_VERSION: number; + +const finalizer = newFinalizer((addresses: number[]) => { + C._ts_parser_delete(addresses[0]); + C._free(addresses[1]); +}); + +/** + * A stateful object that is used to produce a {@link Tree} based on some + * source code. + */ +export class Parser { + /** @internal */ + private [0] = 0; // Internal handle for Wasm + + /** @internal */ + private [1] = 0; // Internal handle for Wasm + + /** @internal */ + private logCallback: LogCallback | null = null; + + /** The parser's current language. */ + language: Language | null = null; + + /** + * This must always be called before creating a Parser. + * + * You can optionally pass in options to configure the Wasm module, the most common + * one being `locateFile` to help the module find the `.wasm` file. + */ + static async init(moduleOptions?: Partial) { + setModule(await initializeBinding(moduleOptions)); + TRANSFER_BUFFER = C._ts_init(); + LANGUAGE_VERSION = C.getValue(TRANSFER_BUFFER, 'i32'); + MIN_COMPATIBLE_VERSION = C.getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); + } + + /** + * Create a new parser. + */ + constructor() { + this.initialize(); + finalizer?.register(this, [this[0], this[1]], this); + } + + /** @internal */ + initialize() { + if (!checkModule()) { + throw new Error("cannot construct a Parser before calling `init()`"); + } + C._ts_parser_new_wasm(); + this[0] = C.getValue(TRANSFER_BUFFER, 'i32'); + this[1] = C.getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); + } + + /** Delete the parser, freeing its resources. */ + delete() { + finalizer?.unregister(this); + C._ts_parser_delete(this[0]); + C._free(this[1]); + this[0] = 0; + this[1] = 0; + } + + /** + * Set the language that the parser should use for parsing. + * + * If the language was not successfully assigned, an error will be thrown. + * This happens if the language was generated with an incompatible + * version of the Tree-sitter CLI. Check the language's version using + * {@link Language#version} and compare it to this library's + * {@link LANGUAGE_VERSION} and {@link MIN_COMPATIBLE_VERSION} constants. + */ + setLanguage(language: Language | null): this { + let address: number; + if (!language) { + address = 0; + this.language = null; + } else if (language.constructor === Language) { + address = language[0]; + const version = C._ts_language_abi_version(address); + if (version < MIN_COMPATIBLE_VERSION || LANGUAGE_VERSION < version) { + throw new Error( + `Incompatible language version ${version}. ` + + `Compatibility range ${MIN_COMPATIBLE_VERSION} through ${LANGUAGE_VERSION}.` + ); + } + this.language = language; + } else { + throw new Error('Argument must be a Language'); + } + + C._ts_parser_set_language(this[0], address); + return this; + } + + /** + * Parse a slice of UTF8 text. + * + * @param {string | ParseCallback} callback - The UTF8-encoded text to parse or a callback function. + * + * @param {Tree | null} [oldTree] - A previous syntax tree parsed from the same document. If the text of the + * document has changed since `oldTree` was created, then you must edit `oldTree` to match + * the new text using {@link Tree#edit}. + * + * @param {ParseOptions} [options] - Options for parsing the text. + * This can be used to set the included ranges, or a progress callback. + * + * @returns {Tree | null} A {@link Tree} if parsing succeeded, or `null` if: + * - The parser has not yet had a language assigned with {@link Parser#setLanguage}. + * - The progress callback returned true. + */ + parse( + callback: string | ParseCallback, + oldTree?: Tree | null, + options?: ParseOptions, + ): Tree | null { + if (typeof callback === 'string') { + C.currentParseCallback = (index: number) => callback.slice(index); + } else if (typeof callback === 'function') { + C.currentParseCallback = callback; + } else { + throw new Error('Argument must be a string or a function'); + } + + if (options?.progressCallback) { + C.currentProgressCallback = options.progressCallback; + } else { + C.currentProgressCallback = null; + } + + if (this.logCallback) { + C.currentLogCallback = this.logCallback; + C._ts_parser_enable_logger_wasm(this[0], 1); + } else { + C.currentLogCallback = null; + C._ts_parser_enable_logger_wasm(this[0], 0); + } + + let rangeCount = 0; + let rangeAddress = 0; + if (options?.includedRanges) { + rangeCount = options.includedRanges.length; + rangeAddress = C._calloc(rangeCount, SIZE_OF_RANGE); + let address = rangeAddress; + for (let i = 0; i < rangeCount; i++) { + marshalRange(address, options.includedRanges[i]); + address += SIZE_OF_RANGE; + } + } + + const treeAddress = C._ts_parser_parse_wasm( + this[0], + this[1], + oldTree ? oldTree[0] : 0, + rangeAddress, + rangeCount + ); + + if (!treeAddress) { + C.currentParseCallback = null; + C.currentLogCallback = null; + C.currentProgressCallback = null; + return null; + } + + if (!this.language) { + throw new Error('Parser must have a language to parse'); + } + + const result = new Tree(INTERNAL, treeAddress, this.language, C.currentParseCallback); + C.currentParseCallback = null; + C.currentLogCallback = null; + C.currentProgressCallback = null; + return result; + } + + /** + * Instruct the parser to start the next parse from the beginning. + * + * If the parser previously failed because of a callback, + * then by default, it will resume where it left off on the + * next call to {@link Parser#parse} or other parsing functions. + * If you don't want to resume, and instead intend to use this parser to + * parse some other document, you must call `reset` first. + */ + reset(): void { + C._ts_parser_reset(this[0]); + } + + /** Get the ranges of text that the parser will include when parsing. */ + getIncludedRanges(): Range[] { + C._ts_parser_included_ranges_wasm(this[0]); + const count = C.getValue(TRANSFER_BUFFER, 'i32'); + const buffer = C.getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); + const result = new Array(count); + + if (count > 0) { + let address = buffer; + for (let i = 0; i < count; i++) { + result[i] = unmarshalRange(address); + address += SIZE_OF_RANGE; + } + C._free(buffer); + } + + return result; + } + + /** Set the logging callback that a parser should use during parsing. */ + setLogger(callback: LogCallback | boolean | null): this { + if (!callback) { + this.logCallback = null; + } else if (typeof callback !== 'function') { + throw new Error('Logger callback must be a function'); + } else { + this.logCallback = callback; + } + return this; + } + + /** Get the parser's current logger. */ + getLogger(): LogCallback | null { + return this.logCallback; + } +} diff --git a/lib/binding_web/src/query.ts b/lib/binding_web/src/query.ts new file mode 100644 index 00000000..e2994b14 --- /dev/null +++ b/lib/binding_web/src/query.ts @@ -0,0 +1,1029 @@ +import { Point, ZERO_POINT, SIZE_OF_INT, C } from './constants'; +import { Node } from './node'; +import { marshalNode, unmarshalCaptures } from './marshal'; +import { TRANSFER_BUFFER } from './parser'; +import { Language } from './language'; +import { newFinalizer } from './finalization_registry'; + +const PREDICATE_STEP_TYPE_CAPTURE = 1; + +const PREDICATE_STEP_TYPE_STRING = 2; + +const QUERY_WORD_REGEX = /[\w-]+/g; + +/** + * Options for query execution + */ +export interface QueryOptions { + /** The start position of the range to query */ + startPosition?: Point; + + /** The end position of the range to query */ + endPosition?: Point; + + /** The start position of the range to query Only the matches that are fully + * contained within provided range will be returned. + **/ + startContainingPosition?: Point; + + /** The end position of the range to query Only the matches that are fully + * contained within provided range will be returned. + **/ + endContainingPosition?: Point; + + /** The start index of the range to query */ + startIndex?: number; + + /** The end index of the range to query */ + endIndex?: number; + + /** The start index of the range to query Only the matches that are fully + * contained within provided range will be returned. + **/ + startContainingIndex?: number; + + /** The end index of the range to query Only the matches that are fully + * contained within provided range will be returned. + **/ + endContainingIndex?: number; + + /** + * The maximum number of in-progress matches for this query. + * The limit must be > 0 and <= 65536. + */ + matchLimit?: number; + + /** + * The maximum start depth for a query cursor. + * + * This prevents cursors from exploring children nodes at a certain depth. + * Note if a pattern includes many children, then they will still be + * checked. + * + * The zero max start depth value can be used as a special behavior and + * it helps to destructure a subtree by staying on a node and using + * captures for interested parts. Note that the zero max start depth + * only limit a search depth for a pattern's root node but other nodes + * that are parts of the pattern may be searched at any depth what + * defined by the pattern structure. + * + * Set to `null` to remove the maximum start depth. + */ + maxStartDepth?: number; + + /** + * A function that will be called periodically during the execution of the query to check + * if query execution should be cancelled. You can also use this to instrument query execution + * and check where the query is at in the document. The progress callback takes a single argument, + * which is a {@link QueryState} representing the current state of the query. + */ + progressCallback?: (state: QueryState) => void; +} + +/** + * A stateful object that is passed into the progress callback {@link QueryOptions#progressCallback} + * to provide the current state of the query. + */ +export interface QueryState { + /** The byte offset in the document that the query is at. */ + currentOffset: number; +} + +/** A record of key-value pairs associated with a particular pattern in a {@link Query}. */ +export type QueryProperties = Record; + +/** + * A predicate that contains an operator and list of operands. + */ +export interface QueryPredicate { + /** The operator of the predicate, like `match?`, `eq?`, `set!`, etc. */ + operator: string; + + /** The operands of the predicate, which are either captures or strings. */ + operands: PredicateStep[]; +} + +/** + * A particular {@link Node} that has been captured with a particular name within a + * {@link Query}. + */ +export interface QueryCapture { + /** The index of the pattern that matched. */ + patternIndex: number; + + /** The name of the capture */ + name: string; + + /** The captured node */ + node: Node; + + /** The properties for predicates declared with the operator `set!`. */ + setProperties?: QueryProperties; + + /** The properties for predicates declared with the operator `is?`. */ + assertedProperties?: QueryProperties; + + /** The properties for predicates declared with the operator `is-not?`. */ + refutedProperties?: QueryProperties; +} + +/** A match of a {@link Query} to a particular set of {@link Node}s. */ +export interface QueryMatch { + /** The index of the pattern that matched. */ + patternIndex: number; + + /** The captures associated with the match. */ + captures: QueryCapture[]; + + /** The properties for predicates declared with the operator `set!`. */ + setProperties?: QueryProperties; + + /** The properties for predicates declared with the operator `is?`. */ + assertedProperties?: QueryProperties; + + /** The properties for predicates declared with the operator `is-not?`. */ + refutedProperties?: QueryProperties; +} + +/** A quantifier for captures */ +export const CaptureQuantifier = { + Zero: 0, + ZeroOrOne: 1, + ZeroOrMore: 2, + One: 3, + OneOrMore: 4 +} as const; + +/** A quantifier for captures */ +export type CaptureQuantifier = typeof CaptureQuantifier[keyof typeof CaptureQuantifier]; + +/** + * Predicates are represented as a single array of steps. There are two + * types of steps, which correspond to the two legal values for + * the `type` field: + * + * - `CapturePredicateStep` - Steps with this type represent names + * of captures. + * + * - `StringPredicateStep` - Steps with this type represent literal + * strings. + */ +export type PredicateStep = CapturePredicateStep | StringPredicateStep; + +/** + * A step in a predicate that refers to a capture. + * + * The `name` field is the name of the capture. + */ +export interface CapturePredicateStep { type: 'capture', name: string } + +/** + * A step in a predicate that refers to a string. + * + * The `value` field is the string value. + */ +export interface StringPredicateStep { type: 'string', value: string } + +const isCaptureStep = (step: PredicateStep): step is Extract => + step.type === 'capture'; + +const isStringStep = (step: PredicateStep): step is Extract => + step.type === 'string'; + +/** + * @internal + * + * A function that checks if a given set of captures matches a particular + * condition. This is used in the built-in `eq?`, `match?`, and `any-of?` + * predicates. + */ +export type TextPredicate = (captures: QueryCapture[]) => boolean; + +/** Error codes returned from tree-sitter query parsing */ +export const QueryErrorKind = { + Syntax: 1, + NodeName: 2, + FieldName: 3, + CaptureName: 4, + PatternStructure: 5, +} as const; + +/** An error that occurred while parsing a query string. */ +export type QueryErrorKind = typeof QueryErrorKind[keyof typeof QueryErrorKind]; + +/** Information about a {@link QueryError}. */ +export interface QueryErrorInfo { + [QueryErrorKind.NodeName]: { word: string }; + [QueryErrorKind.FieldName]: { word: string }; + [QueryErrorKind.CaptureName]: { word: string }; + [QueryErrorKind.PatternStructure]: { suffix: string }; + [QueryErrorKind.Syntax]: { suffix: string }; +} + +/** Error thrown when parsing a tree-sitter query fails */ +export class QueryError extends Error { + constructor( + public kind: QueryErrorKind, + public info: QueryErrorInfo[typeof kind], + public index: number, + public length: number + ) { + super(QueryError.formatMessage(kind, info)); + this.name = 'QueryError'; + } + + /** Formats an error message based on the error kind and info */ + private static formatMessage(kind: QueryErrorKind, info: QueryErrorInfo[QueryErrorKind]): string { + switch (kind) { + case QueryErrorKind.NodeName: + return `Bad node name '${(info as QueryErrorInfo[2]).word}'`; + case QueryErrorKind.FieldName: + return `Bad field name '${(info as QueryErrorInfo[3]).word}'`; + case QueryErrorKind.CaptureName: + return `Bad capture name @${(info as QueryErrorInfo[4]).word}`; + case QueryErrorKind.PatternStructure: + return `Bad pattern structure at offset ${(info as QueryErrorInfo[5]).suffix}`; + case QueryErrorKind.Syntax: + return `Bad syntax at offset ${(info as QueryErrorInfo[1]).suffix}`; + } + } +} + +/** + * Parses the `eq?` and `not-eq?` predicates in a query, and updates the text predicates. + */ +function parseAnyPredicate( + steps: PredicateStep[], + index: number, + operator: string, + textPredicates: TextPredicate[][], +) { + if (steps.length !== 3) { + throw new Error( + `Wrong number of arguments to \`#${operator}\` predicate. Expected 2, got ${steps.length - 1}` + ); + } + + if (!isCaptureStep(steps[1])) { + throw new Error( + `First argument of \`#${operator}\` predicate must be a capture. Got "${steps[1].value}"` + ); + } + + const isPositive = operator === 'eq?' || operator === 'any-eq?'; + const matchAll = !operator.startsWith('any-'); + + if (isCaptureStep(steps[2])) { + const captureName1 = steps[1].name; + const captureName2 = steps[2].name; + textPredicates[index].push((captures) => { + const nodes1: Node[] = []; + const nodes2: Node[] = []; + for (const c of captures) { + if (c.name === captureName1) nodes1.push(c.node); + if (c.name === captureName2) nodes2.push(c.node); + } + const compare = (n1: { text: string }, n2: { text: string }, positive: boolean) => { + return positive ? n1.text === n2.text : n1.text !== n2.text; + }; + return matchAll + ? nodes1.every((n1) => nodes2.some((n2) => compare(n1, n2, isPositive))) + : nodes1.some((n1) => nodes2.some((n2) => compare(n1, n2, isPositive))); + }); + } else { + const captureName = steps[1].name; + const stringValue = steps[2].value; + const matches = (n: Node) => n.text === stringValue; + const doesNotMatch = (n: Node) => n.text !== stringValue; + textPredicates[index].push((captures) => { + const nodes = []; + for (const c of captures) { + if (c.name === captureName) nodes.push(c.node); + } + const test = isPositive ? matches : doesNotMatch; + return matchAll ? nodes.every(test) : nodes.some(test); + }); + } +} + +/** + * Parses the `match?` and `not-match?` predicates in a query, and updates the text predicates. + */ +function parseMatchPredicate( + steps: PredicateStep[], + index: number, + operator: string, + textPredicates: TextPredicate[][], +) { + if (steps.length !== 3) { + throw new Error( + `Wrong number of arguments to \`#${operator}\` predicate. Expected 2, got ${steps.length - 1}.`, + ); + } + + if (steps[1].type !== 'capture') { + throw new Error( + `First argument of \`#${operator}\` predicate must be a capture. Got "${steps[1].value}".`, + ); + } + + if (steps[2].type !== 'string') { + throw new Error( + `Second argument of \`#${operator}\` predicate must be a string. Got @${steps[2].name}.`, + ); + } + + const isPositive = operator === 'match?' || operator === 'any-match?'; + const matchAll = !operator.startsWith('any-'); + const captureName = steps[1].name; + const regex = new RegExp(steps[2].value); + textPredicates[index].push((captures) => { + const nodes = []; + for (const c of captures) { + if (c.name === captureName) nodes.push(c.node.text); + } + const test = (text: string, positive: boolean) => { + return positive ? + regex.test(text) : + !regex.test(text); + }; + if (nodes.length === 0) return !isPositive; + return matchAll ? + nodes.every((text) => test(text, isPositive)) : + nodes.some((text) => test(text, isPositive)); + }); +} + +/** + * Parses the `any-of?` and `not-any-of?` predicates in a query, and updates the text predicates. + */ +function parseAnyOfPredicate( + steps: PredicateStep[], + index: number, + operator: string, + textPredicates: TextPredicate[][], +) { + if (steps.length < 2) { + throw new Error( + `Wrong number of arguments to \`#${operator}\` predicate. Expected at least 1. Got ${steps.length - 1}.`, + ); + } + + if (steps[1].type !== 'capture') { + throw new Error( + `First argument of \`#${operator}\` predicate must be a capture. Got "${steps[1].value}".`, + ); + } + + const isPositive = operator === 'any-of?'; + const captureName = steps[1].name; + + const stringSteps = steps.slice(2); + if (!stringSteps.every(isStringStep)) { + throw new Error( + `Arguments to \`#${operator}\` predicate must be strings.".`, + ); + } + const values = stringSteps.map((s) => s.value); + + textPredicates[index].push((captures) => { + const nodes = []; + for (const c of captures) { + if (c.name === captureName) nodes.push(c.node.text); + } + if (nodes.length === 0) return !isPositive; + return nodes.every((text) => values.includes(text)) === isPositive; + }); +} + +/** + * Parses the `is?` and `is-not?` predicates in a query, and updates the asserted or refuted properties, + * depending on if the operator is positive or negative. + */ +function parseIsPredicate( + steps: PredicateStep[], + index: number, + operator: string, + assertedProperties: QueryProperties[], + refutedProperties: QueryProperties[], +) { + if (steps.length < 2 || steps.length > 3) { + throw new Error( + `Wrong number of arguments to \`#${operator}\` predicate. Expected 1 or 2. Got ${steps.length - 1}.`, + ); + } + + if (!steps.every(isStringStep)) { + throw new Error( + `Arguments to \`#${operator}\` predicate must be strings.".`, + ); + } + + const properties = operator === 'is?' ? assertedProperties : refutedProperties; + if (!properties[index]) properties[index] = {}; + properties[index][steps[1].value] = steps[2]?.value ?? null; +} + +/** + * Parses the `set!` directive in a query, and updates the set properties. + */ +function parseSetDirective( + steps: PredicateStep[], + index: number, + setProperties: QueryProperties[], +) { + if (steps.length < 2 || steps.length > 3) { + throw new Error(`Wrong number of arguments to \`#set!\` predicate. Expected 1 or 2. Got ${steps.length - 1}.`); + } + if (!steps.every(isStringStep)) { + throw new Error(`Arguments to \`#set!\` predicate must be strings.".`); + } + if (!setProperties[index]) setProperties[index] = {}; + setProperties[index][steps[1].value] = steps[2]?.value ?? null; +} + +/** + * Parses the predicate at a given step in a pattern, and updates the appropriate + * predicates or properties. + */ +function parsePattern( + index: number, + stepType: number, + stepValueId: number, + captureNames: string[], + stringValues: string[], + steps: PredicateStep[], + textPredicates: TextPredicate[][], + predicates: QueryPredicate[][], + setProperties: QueryProperties[], + assertedProperties: QueryProperties[], + refutedProperties: QueryProperties[], +) { + if (stepType === PREDICATE_STEP_TYPE_CAPTURE) { + const name = captureNames[stepValueId]; + steps.push({ type: 'capture', name }); + } else if (stepType === PREDICATE_STEP_TYPE_STRING) { + steps.push({ type: 'string', value: stringValues[stepValueId] }); + } else if (steps.length > 0) { + if (steps[0].type !== 'string') { + throw new Error('Predicates must begin with a literal value'); + } + + const operator = steps[0].value; + switch (operator) { + case 'any-not-eq?': + case 'not-eq?': + case 'any-eq?': + case 'eq?': + parseAnyPredicate(steps, index, operator, textPredicates); + break; + + case 'any-not-match?': + case 'not-match?': + case 'any-match?': + case 'match?': + parseMatchPredicate(steps, index, operator, textPredicates); + break; + + case 'not-any-of?': + case 'any-of?': + parseAnyOfPredicate(steps, index, operator, textPredicates); + break; + + case 'is?': + case 'is-not?': + parseIsPredicate(steps, index, operator, assertedProperties, refutedProperties); + break; + + case 'set!': + parseSetDirective(steps, index, setProperties); + break; + + default: + predicates[index].push({ operator, operands: steps.slice(1) }); + } + + steps.length = 0; + } +} + +const finalizer = newFinalizer((address: number) => { + C._ts_query_delete(address); +}); + +export class Query { + /** @internal */ + private [0] = 0; // Internal handle for Wasm + + /** @internal */ + private exceededMatchLimit: boolean; + + /** @internal */ + private textPredicates: TextPredicate[][]; + + /** The names of the captures used in the query. */ + readonly captureNames: string[]; + + /** The quantifiers of the captures used in the query. */ + readonly captureQuantifiers: CaptureQuantifier[][]; + + /** + * The other user-defined predicates associated with the given index. + * + * This includes predicates with operators other than: + * - `match?` + * - `eq?` and `not-eq?` + * - `any-of?` and `not-any-of?` + * - `is?` and `is-not?` + * - `set!` + */ + readonly predicates: QueryPredicate[][]; + + /** The properties for predicates with the operator `set!`. */ + readonly setProperties: QueryProperties[]; + + /** The properties for predicates with the operator `is?`. */ + readonly assertedProperties: QueryProperties[]; + + /** The properties for predicates with the operator `is-not?`. */ + readonly refutedProperties: QueryProperties[]; + + /** The maximum number of in-progress matches for this cursor. */ + matchLimit?: number; + + /** + * Create a new query from a string containing one or more S-expression + * patterns. + * + * The query is associated with a particular language, and can only be run + * on syntax nodes parsed with that language. References to Queries can be + * shared between multiple threads. + * + * @link {@see https://tree-sitter.github.io/tree-sitter/using-parsers/queries} + */ + constructor(language: Language, source: string) { + const sourceLength = C.lengthBytesUTF8(source); + const sourceAddress = C._malloc(sourceLength + 1); + C.stringToUTF8(source, sourceAddress, sourceLength + 1); + const address = C._ts_query_new( + language[0], + sourceAddress, + sourceLength, + TRANSFER_BUFFER, + TRANSFER_BUFFER + SIZE_OF_INT + ); + + if (!address) { + const errorId = C.getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32') as QueryErrorKind; + const errorByte = C.getValue(TRANSFER_BUFFER, 'i32'); + const errorIndex = C.UTF8ToString(sourceAddress, errorByte).length; + const suffix = source.slice(errorIndex, errorIndex + 100).split('\n')[0]; + const word = suffix.match(QUERY_WORD_REGEX)?.[0] ?? ''; + C._free(sourceAddress); + + switch (errorId) { + case QueryErrorKind.Syntax: + throw new QueryError(QueryErrorKind.Syntax, { suffix: `${errorIndex}: '${suffix}'...` }, errorIndex, 0); + case QueryErrorKind.NodeName: + throw new QueryError(errorId, { word }, errorIndex, word.length); + case QueryErrorKind.FieldName: + throw new QueryError(errorId, { word }, errorIndex, word.length); + case QueryErrorKind.CaptureName: + throw new QueryError(errorId, { word }, errorIndex, word.length); + case QueryErrorKind.PatternStructure: + throw new QueryError(errorId, { suffix: `${errorIndex}: '${suffix}'...` }, errorIndex, 0); + } + } + + const stringCount = C._ts_query_string_count(address); + const captureCount = C._ts_query_capture_count(address); + const patternCount = C._ts_query_pattern_count(address); + const captureNames = new Array(captureCount); + const captureQuantifiers = new Array(patternCount); + const stringValues = new Array(stringCount); + + // Fill in the capture names + for (let i = 0; i < captureCount; i++) { + const nameAddress = C._ts_query_capture_name_for_id( + address, + i, + TRANSFER_BUFFER + ); + const nameLength = C.getValue(TRANSFER_BUFFER, 'i32'); + captureNames[i] = C.UTF8ToString(nameAddress, nameLength); + } + + // Fill in the capture quantifiers + for (let i = 0; i < patternCount; i++) { + const captureQuantifiersArray = new Array(captureCount); + for (let j = 0; j < captureCount; j++) { + const quantifier = C._ts_query_capture_quantifier_for_id(address, i, j); + captureQuantifiersArray[j] = quantifier as CaptureQuantifier; + } + captureQuantifiers[i] = captureQuantifiersArray; + } + + // Fill in the string values + for (let i = 0; i < stringCount; i++) { + const valueAddress = C._ts_query_string_value_for_id( + address, + i, + TRANSFER_BUFFER + ); + const nameLength = C.getValue(TRANSFER_BUFFER, 'i32'); + stringValues[i] = C.UTF8ToString(valueAddress, nameLength); + } + + const setProperties = new Array(patternCount); + const assertedProperties = new Array(patternCount); + const refutedProperties = new Array(patternCount); + const predicates = new Array(patternCount); + const textPredicates = new Array(patternCount); + + // Parse the predicates, and add the appropriate predicates or properties + for (let i = 0; i < patternCount; i++) { + const predicatesAddress = C._ts_query_predicates_for_pattern(address, i, TRANSFER_BUFFER); + const stepCount = C.getValue(TRANSFER_BUFFER, 'i32'); + + predicates[i] = []; + textPredicates[i] = []; + + const steps = new Array(); + + let stepAddress = predicatesAddress; + for (let j = 0; j < stepCount; j++) { + const stepType = C.getValue(stepAddress, 'i32'); + stepAddress += SIZE_OF_INT; + + const stepValueId = C.getValue(stepAddress, 'i32'); + stepAddress += SIZE_OF_INT; + + parsePattern( + i, + stepType, + stepValueId, + captureNames, + stringValues, + steps, + textPredicates, + predicates, + setProperties, + assertedProperties, + refutedProperties, + ); + } + + Object.freeze(textPredicates[i]); + Object.freeze(predicates[i]); + Object.freeze(setProperties[i]); + Object.freeze(assertedProperties[i]); + Object.freeze(refutedProperties[i]); + } + + C._free(sourceAddress); + + + this[0] = address; + this.captureNames = captureNames; + this.captureQuantifiers = captureQuantifiers; + this.textPredicates = textPredicates; + this.predicates = predicates; + this.setProperties = setProperties; + this.assertedProperties = assertedProperties; + this.refutedProperties = refutedProperties; + this.exceededMatchLimit = false; + finalizer?.register(this, address, this); + } + + /** Delete the query, freeing its resources. */ + delete(): void { + finalizer?.unregister(this); + C._ts_query_delete(this[0]); + this[0] = 0; + } + + /** + * Iterate over all of the matches in the order that they were found. + * + * Each match contains the index of the pattern that matched, and a list of + * captures. Because multiple patterns can match the same set of nodes, + * one match may contain captures that appear *before* some of the + * captures from a previous match. + * + * @param {Node} node - The node to execute the query on. + * + * @param {QueryOptions} options - Options for query execution. + */ + matches( + node: Node, + options: QueryOptions = {} + ): QueryMatch[] { + const startPosition = options.startPosition ?? ZERO_POINT; + const endPosition = options.endPosition ?? ZERO_POINT; + const startIndex = options.startIndex ?? 0; + const endIndex = options.endIndex ?? 0; + const startContainingPosition = options.startContainingPosition ?? ZERO_POINT; + const endContainingPosition = options.endContainingPosition ?? ZERO_POINT; + const startContainingIndex = options.startContainingIndex ?? 0; + const endContainingIndex = options.endContainingIndex ?? 0; + const matchLimit = options.matchLimit ?? 0xFFFFFFFF; + const maxStartDepth = options.maxStartDepth ?? 0xFFFFFFFF; + const progressCallback = options.progressCallback; + + if (typeof matchLimit !== 'number') { + throw new Error('Arguments must be numbers'); + } + this.matchLimit = matchLimit; + + if (endIndex !== 0 && startIndex > endIndex) { + throw new Error('`startIndex` cannot be greater than `endIndex`'); + } + + if (endPosition !== ZERO_POINT && ( + startPosition.row > endPosition.row || + (startPosition.row === endPosition.row && startPosition.column > endPosition.column) + )) { + throw new Error('`startPosition` cannot be greater than `endPosition`'); + } + + if (endContainingIndex !== 0 && startContainingIndex > endContainingIndex) { + throw new Error('`startContainingIndex` cannot be greater than `endContainingIndex`'); + } + + if (endContainingPosition !== ZERO_POINT && ( + startContainingPosition.row > endContainingPosition.row || + (startContainingPosition.row === endContainingPosition.row && + startContainingPosition.column > endContainingPosition.column) + )) { + throw new Error('`startContainingPosition` cannot be greater than `endContainingPosition`'); + } + + if (progressCallback) { + C.currentQueryProgressCallback = progressCallback; + } + + marshalNode(node); + + C._ts_query_matches_wasm( + this[0], + node.tree[0], + startPosition.row, + startPosition.column, + endPosition.row, + endPosition.column, + startIndex, + endIndex, + startContainingPosition.row, + startContainingPosition.column, + endContainingPosition.row, + endContainingPosition.column, + startContainingIndex, + endContainingIndex, + matchLimit, + maxStartDepth, + ); + + const rawCount = C.getValue(TRANSFER_BUFFER, 'i32'); + const startAddress = C.getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); + const didExceedMatchLimit = C.getValue(TRANSFER_BUFFER + 2 * SIZE_OF_INT, 'i32'); + const result = new Array(rawCount); + this.exceededMatchLimit = Boolean(didExceedMatchLimit); + + let filteredCount = 0; + let address = startAddress; + for (let i = 0; i < rawCount; i++) { + const patternIndex = C.getValue(address, 'i32'); + address += SIZE_OF_INT; + const captureCount = C.getValue(address, 'i32'); + address += SIZE_OF_INT; + + const captures = new Array(captureCount); + address = unmarshalCaptures(this, node.tree, address, patternIndex, captures); + + if (this.textPredicates[patternIndex].every((p) => p(captures))) { + result[filteredCount] = { patternIndex, captures }; + const setProperties = this.setProperties[patternIndex]; + result[filteredCount].setProperties = setProperties; + const assertedProperties = this.assertedProperties[patternIndex]; + result[filteredCount].assertedProperties = assertedProperties; + const refutedProperties = this.refutedProperties[patternIndex]; + result[filteredCount].refutedProperties = refutedProperties; + filteredCount++; + } + } + result.length = filteredCount; + + C._free(startAddress); + C.currentQueryProgressCallback = null; + return result; + } + + /** + * Iterate over all of the individual captures in the order that they + * appear. + * + * This is useful if you don't care about which pattern matched, and just + * want a single, ordered sequence of captures. + * + * @param {Node} node - The node to execute the query on. + * + * @param {QueryOptions} options - Options for query execution. + */ + captures( + node: Node, + options: QueryOptions = {} + ): QueryCapture[] { + const startPosition = options.startPosition ?? ZERO_POINT; + const endPosition = options.endPosition ?? ZERO_POINT; + const startIndex = options.startIndex ?? 0; + const endIndex = options.endIndex ?? 0; + const startContainingPosition = options.startContainingPosition ?? ZERO_POINT; + const endContainingPosition = options.endContainingPosition ?? ZERO_POINT; + const startContainingIndex = options.startContainingIndex ?? 0; + const endContainingIndex = options.endContainingIndex ?? 0; + const matchLimit = options.matchLimit ?? 0xFFFFFFFF; + const maxStartDepth = options.maxStartDepth ?? 0xFFFFFFFF; + const progressCallback = options.progressCallback; + + if (typeof matchLimit !== 'number') { + throw new Error('Arguments must be numbers'); + } + this.matchLimit = matchLimit; + + if (endIndex !== 0 && startIndex > endIndex) { + throw new Error('`startIndex` cannot be greater than `endIndex`'); + } + + if (endPosition !== ZERO_POINT && ( + startPosition.row > endPosition.row || + (startPosition.row === endPosition.row && startPosition.column > endPosition.column) + )) { + throw new Error('`startPosition` cannot be greater than `endPosition`'); + } + + if (endContainingIndex !== 0 && startContainingIndex > endContainingIndex) { + throw new Error('`startContainingIndex` cannot be greater than `endContainingIndex`'); + } + + if (endContainingPosition !== ZERO_POINT && ( + startContainingPosition.row > endContainingPosition.row || + (startContainingPosition.row === endContainingPosition.row && + startContainingPosition.column > endContainingPosition.column) + )) { + throw new Error('`startContainingPosition` cannot be greater than `endContainingPosition`'); + } + + if (progressCallback) { + C.currentQueryProgressCallback = progressCallback; + } + + marshalNode(node); + + C._ts_query_captures_wasm( + this[0], + node.tree[0], + startPosition.row, + startPosition.column, + endPosition.row, + endPosition.column, + startIndex, + endIndex, + startContainingPosition.row, + startContainingPosition.column, + endContainingPosition.row, + endContainingPosition.column, + startContainingIndex, + endContainingIndex, + matchLimit, + maxStartDepth, + ); + + const count = C.getValue(TRANSFER_BUFFER, 'i32'); + const startAddress = C.getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); + const didExceedMatchLimit = C.getValue(TRANSFER_BUFFER + 2 * SIZE_OF_INT, 'i32'); + const result = new Array(); + this.exceededMatchLimit = Boolean(didExceedMatchLimit); + + const captures = new Array(); + let address = startAddress; + for (let i = 0; i < count; i++) { + const patternIndex = C.getValue(address, 'i32'); + address += SIZE_OF_INT; + const captureCount = C.getValue(address, 'i32'); + address += SIZE_OF_INT; + const captureIndex = C.getValue(address, 'i32'); + address += SIZE_OF_INT; + + captures.length = captureCount; + address = unmarshalCaptures(this, node.tree, address, patternIndex, captures); + + if (this.textPredicates[patternIndex].every(p => p(captures))) { + const capture = captures[captureIndex]; + const setProperties = this.setProperties[patternIndex]; + capture.setProperties = setProperties; + const assertedProperties = this.assertedProperties[patternIndex]; + capture.assertedProperties = assertedProperties; + const refutedProperties = this.refutedProperties[patternIndex]; + capture.refutedProperties = refutedProperties; + result.push(capture); + } + } + + C._free(startAddress); + C.currentQueryProgressCallback = null; + return result; + } + + /** Get the predicates for a given pattern. */ + predicatesForPattern(patternIndex: number): QueryPredicate[] { + return this.predicates[patternIndex]; + } + + /** + * Disable a certain capture within a query. + * + * This prevents the capture from being returned in matches, and also + * avoids any resource usage associated with recording the capture. + */ + disableCapture(captureName: string): void { + const captureNameLength = C.lengthBytesUTF8(captureName); + const captureNameAddress = C._malloc(captureNameLength + 1); + C.stringToUTF8(captureName, captureNameAddress, captureNameLength + 1); + C._ts_query_disable_capture(this[0], captureNameAddress, captureNameLength); + C._free(captureNameAddress); + } + + /** + * Disable a certain pattern within a query. + * + * This prevents the pattern from matching, and also avoids any resource + * usage associated with the pattern. This throws an error if the pattern + * index is out of bounds. + */ + disablePattern(patternIndex: number): void { + if (patternIndex >= this.predicates.length) { + throw new Error( + `Pattern index is ${patternIndex} but the pattern count is ${this.predicates.length}` + ); + } + C._ts_query_disable_pattern(this[0], patternIndex); + } + + /** + * Check if, on its last execution, this cursor exceeded its maximum number + * of in-progress matches. + */ + didExceedMatchLimit(): boolean { + return this.exceededMatchLimit; + } + + /** Get the byte offset where the given pattern starts in the query's source. */ + startIndexForPattern(patternIndex: number): number { + if (patternIndex >= this.predicates.length) { + throw new Error( + `Pattern index is ${patternIndex} but the pattern count is ${this.predicates.length}` + ); + } + return C._ts_query_start_byte_for_pattern(this[0], patternIndex); + } + + /** Get the byte offset where the given pattern ends in the query's source. */ + endIndexForPattern(patternIndex: number): number { + if (patternIndex >= this.predicates.length) { + throw new Error( + `Pattern index is ${patternIndex} but the pattern count is ${this.predicates.length}` + ); + } + return C._ts_query_end_byte_for_pattern(this[0], patternIndex); + } + + /** Get the number of patterns in the query. */ + patternCount(): number { + return C._ts_query_pattern_count(this[0]); + } + + /** Get the index for a given capture name. */ + captureIndexForName(captureName: string): number { + return this.captureNames.indexOf(captureName); + } + + /** Check if a given pattern within a query has a single root node. */ + isPatternRooted(patternIndex: number): boolean { + return C._ts_query_is_pattern_rooted(this[0], patternIndex) === 1; + } + + /** Check if a given pattern within a query has a single root node. */ + isPatternNonLocal(patternIndex: number): boolean { + return C._ts_query_is_pattern_non_local(this[0], patternIndex) === 1; + } + + /** + * Check if a given step in a query is 'definite'. + * + * A query step is 'definite' if its parent pattern will be guaranteed to + * match successfully once it reaches the step. + */ + isPatternGuaranteedAtStep(byteIndex: number): boolean { + return C._ts_query_is_pattern_guaranteed_at_step(this[0], byteIndex) === 1; + } +} diff --git a/lib/binding_web/src/tree.ts b/lib/binding_web/src/tree.ts new file mode 100644 index 00000000..f6a7aaf3 --- /dev/null +++ b/lib/binding_web/src/tree.ts @@ -0,0 +1,153 @@ +import { INTERNAL, Internal, assertInternal, ParseCallback, Point, Range, SIZE_OF_NODE, SIZE_OF_INT, SIZE_OF_RANGE, C } from './constants'; +import { Language } from './language'; +import { Node } from './node'; +import { TreeCursor } from './tree_cursor'; +import { marshalEdit, marshalPoint, unmarshalNode, unmarshalRange } from './marshal'; +import { TRANSFER_BUFFER } from './parser'; +import { Edit } from './edit'; +import { newFinalizer } from './finalization_registry'; + +/** @internal */ +export function getText(tree: Tree, startIndex: number, endIndex: number, startPosition: Point): string { + const length = endIndex - startIndex; + let result = tree.textCallback(startIndex, startPosition); + if (result) { + startIndex += result.length; + while (startIndex < endIndex) { + const string = tree.textCallback(startIndex, startPosition); + if (string && string.length > 0) { + startIndex += string.length; + result += string; + } else { + break; + } + } + if (startIndex > endIndex) { + result = result.slice(0, length); + } + } + return result ?? ''; +} + +const finalizer = newFinalizer((address: number) => { + C._ts_tree_delete(address); +}); + +/** A tree that represents the syntactic structure of a source code file. */ +export class Tree { + /** @internal */ + private [0] = 0; // Internal handle for Wasm + + /** @internal */ + textCallback: ParseCallback; + + /** The language that was used to parse the syntax tree. */ + language: Language; + + /** @internal */ + constructor(internal: Internal, address: number, language: Language, textCallback: ParseCallback) { + assertInternal(internal); + this[0] = address; + this.language = language; + this.textCallback = textCallback; + finalizer?.register(this, address, this); + } + + /** Create a shallow copy of the syntax tree. This is very fast. */ + copy(): Tree { + const address = C._ts_tree_copy(this[0]); + return new Tree(INTERNAL, address, this.language, this.textCallback); + } + + /** Delete the syntax tree, freeing its resources. */ + delete(): void { + finalizer?.unregister(this); + C._ts_tree_delete(this[0]); + this[0] = 0; + } + + /** Get the root node of the syntax tree. */ + get rootNode(): Node { + C._ts_tree_root_node_wasm(this[0]); + return unmarshalNode(this)!; + } + + /** + * Get the root node of the syntax tree, but with its position shifted + * forward by the given offset. + */ + rootNodeWithOffset(offsetBytes: number, offsetExtent: Point): Node { + const address = TRANSFER_BUFFER + SIZE_OF_NODE; + C.setValue(address, offsetBytes, 'i32'); + marshalPoint(address + SIZE_OF_INT, offsetExtent); + C._ts_tree_root_node_with_offset_wasm(this[0]); + return unmarshalNode(this)!; + } + + /** + * Edit the syntax tree to keep it in sync with source code that has been + * edited. + * + * You must describe the edit both in terms of byte offsets and in terms of + * row/column coordinates. + */ + edit(edit: Edit): void { + marshalEdit(edit); + C._ts_tree_edit_wasm(this[0]); + } + + /** Create a new {@link TreeCursor} starting from the root of the tree. */ + walk(): TreeCursor { + return this.rootNode.walk(); + } + + /** + * Compare this old edited syntax tree to a new syntax tree representing + * the same document, returning a sequence of ranges whose syntactic + * structure has changed. + * + * For this to work correctly, this syntax tree must have been edited such + * that its ranges match up to the new tree. Generally, you'll want to + * call this method right after calling one of the [`Parser::parse`] + * functions. Call it on the old tree that was passed to parse, and + * pass the new tree that was returned from `parse`. + */ + getChangedRanges(other: Tree): Range[] { + if (!(other instanceof Tree)) { + throw new TypeError('Argument must be a Tree'); + } + + C._ts_tree_get_changed_ranges_wasm(this[0], other[0]); + const count = C.getValue(TRANSFER_BUFFER, 'i32'); + const buffer = C.getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); + const result = new Array(count); + + if (count > 0) { + let address = buffer; + for (let i = 0; i < count; i++) { + result[i] = unmarshalRange(address); + address += SIZE_OF_RANGE; + } + C._free(buffer); + } + return result; + } + + /** Get the included ranges that were used to parse the syntax tree. */ + getIncludedRanges(): Range[] { + C._ts_tree_included_ranges_wasm(this[0]); + const count = C.getValue(TRANSFER_BUFFER, 'i32'); + const buffer = C.getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); + const result = new Array(count); + + if (count > 0) { + let address = buffer; + for (let i = 0; i < count; i++) { + result[i] = unmarshalRange(address); + address += SIZE_OF_RANGE; + } + C._free(buffer); + } + return result; + } +} diff --git a/lib/binding_web/src/tree_cursor.ts b/lib/binding_web/src/tree_cursor.ts new file mode 100644 index 00000000..978a86dc --- /dev/null +++ b/lib/binding_web/src/tree_cursor.ts @@ -0,0 +1,325 @@ +import { INTERNAL, Internal, assertInternal, Point, SIZE_OF_NODE, SIZE_OF_CURSOR, C } from './constants'; +import { marshalNode, marshalPoint, marshalTreeCursor, unmarshalNode, unmarshalPoint, unmarshalTreeCursor } from './marshal'; +import { Node } from './node'; +import { TRANSFER_BUFFER } from './parser'; +import { getText, Tree } from './tree'; +import { newFinalizer } from './finalization_registry'; + +const finalizer = newFinalizer((address: number) => { + C._ts_tree_cursor_delete_wasm(address); +}); + +/** A stateful object for walking a syntax {@link Tree} efficiently. */ +export class TreeCursor { + /** @internal */ + // @ts-expect-error: never read + private [0] = 0; // Internal handle for Wasm + + /** @internal */ + // @ts-expect-error: never read + private [1] = 0; // Internal handle for Wasm + + /** @internal */ + // @ts-expect-error: never read + private [2] = 0; // Internal handle for Wasm + + /** @internal */ + // @ts-expect-error: never read + private [3] = 0; // Internal handle for Wasm + + /** @internal */ + private tree: Tree; + + /** @internal */ + constructor(internal: Internal, tree: Tree) { + assertInternal(internal); + this.tree = tree; + unmarshalTreeCursor(this); + finalizer?.register(this, this.tree[0], this); + } + + /** Creates a deep copy of the tree cursor. This allocates new memory. */ + copy(): TreeCursor { + const copy = new TreeCursor(INTERNAL, this.tree); + C._ts_tree_cursor_copy_wasm(this.tree[0]); + unmarshalTreeCursor(copy); + return copy; + } + + /** Delete the tree cursor, freeing its resources. */ + delete(): void { + finalizer?.unregister(this); + marshalTreeCursor(this); + C._ts_tree_cursor_delete_wasm(this.tree[0]); + this[0] = this[1] = this[2] = 0; + } + + /** Get the tree cursor's current {@link Node}. */ + get currentNode(): Node { + marshalTreeCursor(this); + C._ts_tree_cursor_current_node_wasm(this.tree[0]); + return unmarshalNode(this.tree)!; + } + + /** + * Get the numerical field id of this tree cursor's current node. + * + * See also {@link TreeCursor#currentFieldName}. + */ + get currentFieldId(): number { + marshalTreeCursor(this); + return C._ts_tree_cursor_current_field_id_wasm(this.tree[0]); + } + + /** Get the field name of this tree cursor's current node. */ + get currentFieldName(): string | null { + return this.tree.language.fields[this.currentFieldId]; + } + + /** + * Get the depth of the cursor's current node relative to the original + * node that the cursor was constructed with. + */ + get currentDepth(): number { + marshalTreeCursor(this); + return C._ts_tree_cursor_current_depth_wasm(this.tree[0]); + } + + /** + * Get the index of the cursor's current node out of all of the + * descendants of the original node that the cursor was constructed with. + */ + get currentDescendantIndex(): number { + marshalTreeCursor(this); + return C._ts_tree_cursor_current_descendant_index_wasm(this.tree[0]); + } + + /** Get the type of the cursor's current node. */ + get nodeType(): string { + return this.tree.language.types[this.nodeTypeId] || 'ERROR'; + } + + /** Get the type id of the cursor's current node. */ + get nodeTypeId(): number { + marshalTreeCursor(this); + return C._ts_tree_cursor_current_node_type_id_wasm(this.tree[0]); + } + + /** Get the state id of the cursor's current node. */ + get nodeStateId(): number { + marshalTreeCursor(this); + return C._ts_tree_cursor_current_node_state_id_wasm(this.tree[0]); + } + + /** Get the id of the cursor's current node. */ + get nodeId(): number { + marshalTreeCursor(this); + return C._ts_tree_cursor_current_node_id_wasm(this.tree[0]); + } + + /** + * Check if the cursor's current node is *named*. + * + * Named nodes correspond to named rules in the grammar, whereas + * *anonymous* nodes correspond to string literals in the grammar. + */ + get nodeIsNamed(): boolean { + marshalTreeCursor(this); + return C._ts_tree_cursor_current_node_is_named_wasm(this.tree[0]) === 1; + } + + /** + * Check if the cursor's current node is *missing*. + * + * Missing nodes are inserted by the parser in order to recover from + * certain kinds of syntax errors. + */ + get nodeIsMissing(): boolean { + marshalTreeCursor(this); + return C._ts_tree_cursor_current_node_is_missing_wasm(this.tree[0]) === 1; + } + + /** Get the string content of the cursor's current node. */ + get nodeText(): string { + marshalTreeCursor(this); + const startIndex = C._ts_tree_cursor_start_index_wasm(this.tree[0]); + const endIndex = C._ts_tree_cursor_end_index_wasm(this.tree[0]); + C._ts_tree_cursor_start_position_wasm(this.tree[0]); + const startPosition = unmarshalPoint(TRANSFER_BUFFER); + return getText(this.tree, startIndex, endIndex, startPosition); + } + + /** Get the start position of the cursor's current node. */ + get startPosition(): Point { + marshalTreeCursor(this); + C._ts_tree_cursor_start_position_wasm(this.tree[0]); + return unmarshalPoint(TRANSFER_BUFFER); + } + + /** Get the end position of the cursor's current node. */ + get endPosition(): Point { + marshalTreeCursor(this); + C._ts_tree_cursor_end_position_wasm(this.tree[0]); + return unmarshalPoint(TRANSFER_BUFFER); + } + + /** Get the start index of the cursor's current node. */ + get startIndex(): number { + marshalTreeCursor(this); + return C._ts_tree_cursor_start_index_wasm(this.tree[0]); + } + + /** Get the end index of the cursor's current node. */ + get endIndex(): number { + marshalTreeCursor(this); + return C._ts_tree_cursor_end_index_wasm(this.tree[0]); + } + + /** + * Move this cursor to the first child of its current node. + * + * This returns `true` if the cursor successfully moved, and returns + * `false` if there were no children. + */ + gotoFirstChild(): boolean { + marshalTreeCursor(this); + const result = C._ts_tree_cursor_goto_first_child_wasm(this.tree[0]); + unmarshalTreeCursor(this); + return result === 1; + } + + /** + * Move this cursor to the last child of its current node. + * + * This returns `true` if the cursor successfully moved, and returns + * `false` if there were no children. + * + * Note that this function may be slower than + * {@link TreeCursor#gotoFirstChild} because it needs to + * iterate through all the children to compute the child's position. + */ + gotoLastChild(): boolean { + marshalTreeCursor(this); + const result = C._ts_tree_cursor_goto_last_child_wasm(this.tree[0]); + unmarshalTreeCursor(this); + return result === 1; + } + + /** + * Move this cursor to the parent of its current node. + * + * This returns `true` if the cursor successfully moved, and returns + * `false` if there was no parent node (the cursor was already on the + * root node). + * + * Note that the node the cursor was constructed with is considered the root + * of the cursor, and the cursor cannot walk outside this node. + */ + gotoParent(): boolean { + marshalTreeCursor(this); + const result = C._ts_tree_cursor_goto_parent_wasm(this.tree[0]); + unmarshalTreeCursor(this); + return result === 1; + } + + /** + * Move this cursor to the next sibling of its current node. + * + * This returns `true` if the cursor successfully moved, and returns + * `false` if there was no next sibling node. + * + * Note that the node the cursor was constructed with is considered the root + * of the cursor, and the cursor cannot walk outside this node. + */ + gotoNextSibling(): boolean { + marshalTreeCursor(this); + const result = C._ts_tree_cursor_goto_next_sibling_wasm(this.tree[0]); + unmarshalTreeCursor(this); + return result === 1; + } + + /** + * Move this cursor to the previous sibling of its current node. + * + * This returns `true` if the cursor successfully moved, and returns + * `false` if there was no previous sibling node. + * + * Note that this function may be slower than + * {@link TreeCursor#gotoNextSibling} due to how node + * positions are stored. In the worst case, this will need to iterate + * through all the children up to the previous sibling node to recalculate + * its position. Also note that the node the cursor was constructed with is + * considered the root of the cursor, and the cursor cannot walk outside this node. + */ + gotoPreviousSibling(): boolean { + marshalTreeCursor(this); + const result = C._ts_tree_cursor_goto_previous_sibling_wasm(this.tree[0]); + unmarshalTreeCursor(this); + return result === 1; + } + + /** + * Move the cursor to the node that is the nth descendant of + * the original node that the cursor was constructed with, where + * zero represents the original node itself. + */ + gotoDescendant(goalDescendantIndex: number): void { + marshalTreeCursor(this); + C._ts_tree_cursor_goto_descendant_wasm(this.tree[0], goalDescendantIndex); + unmarshalTreeCursor(this); + } + + /** + * Move this cursor to the first child of its current node that contains or + * starts after the given byte offset. + * + * This returns `true` if the cursor successfully moved to a child node, and returns + * `false` if no such child was found. + */ + gotoFirstChildForIndex(goalIndex: number): boolean { + marshalTreeCursor(this); + C.setValue(TRANSFER_BUFFER + SIZE_OF_CURSOR, goalIndex, 'i32'); + const result = C._ts_tree_cursor_goto_first_child_for_index_wasm(this.tree[0]); + unmarshalTreeCursor(this); + return result === 1; + } + + /** + * Move this cursor to the first child of its current node that contains or + * starts after the given byte offset. + * + * This returns the index of the child node if one was found, and returns + * `null` if no such child was found. + */ + gotoFirstChildForPosition(goalPosition: Point): boolean { + marshalTreeCursor(this); + marshalPoint(TRANSFER_BUFFER + SIZE_OF_CURSOR, goalPosition); + const result = C._ts_tree_cursor_goto_first_child_for_position_wasm(this.tree[0]); + unmarshalTreeCursor(this); + return result === 1; + } + + /** + * Re-initialize this tree cursor to start at the original node that the + * cursor was constructed with. + */ + reset(node: Node): void { + marshalNode(node); + marshalTreeCursor(this, TRANSFER_BUFFER + SIZE_OF_NODE); + C._ts_tree_cursor_reset_wasm(this.tree[0]); + unmarshalTreeCursor(this); + } + + /** + * Re-initialize a tree cursor to the same position as another cursor. + * + * Unlike {@link TreeCursor#reset}, this will not lose parent + * information and allows reusing already created cursors. + */ + resetTo(cursor: TreeCursor): void { + marshalTreeCursor(this, TRANSFER_BUFFER); + marshalTreeCursor(cursor, TRANSFER_BUFFER + SIZE_OF_CURSOR); + C._ts_tree_cursor_reset_to_wasm(this.tree[0], cursor.tree[0]); + unmarshalTreeCursor(this); + } +} diff --git a/lib/binding_web/suffix.js b/lib/binding_web/suffix.js deleted file mode 100644 index 8e096f61..00000000 --- a/lib/binding_web/suffix.js +++ /dev/null @@ -1,23 +0,0 @@ - for (const name of Object.getOwnPropertyNames(ParserImpl.prototype)) { - Object.defineProperty(Parser.prototype, name, { - value: ParserImpl.prototype[name], - enumerable: false, - writable: false, - }) - } - - Parser.Language = Language; - Module.onRuntimeInitialized = () => { - ParserImpl.init(); - resolveInitPromise(); - }; - }); - } - } - - return Parser; -}(); - -if (typeof exports === 'object') { - module.exports = TreeSitter; -} diff --git a/lib/binding_web/test/edit.test.ts b/lib/binding_web/test/edit.test.ts new file mode 100644 index 00000000..bd4cb194 --- /dev/null +++ b/lib/binding_web/test/edit.test.ts @@ -0,0 +1,124 @@ +import { describe, it, expect } from 'vitest'; +import { Edit } from '../src'; + +describe('Edit', () => { + it('edits a point after the edit', () => { + const edit = new Edit({ + startIndex: 5, + oldEndIndex: 5, + newEndIndex: 10, + startPosition: { row: 0, column: 5 }, + oldEndPosition: { row: 0, column: 5 }, + newEndPosition: { row: 0, column: 10 }, + }); + + const point = { row: 0, column: 8 }; + const index = 8; + const result = edit.editPoint(point, index); + expect(result.point).toEqual({ row: 0, column: 13 }); + expect(result.index).toBe(13); + }); + + it('edits a point before the edit', () => { + const edit = new Edit({ + startIndex: 5, + oldEndIndex: 5, + newEndIndex: 10, + startPosition: { row: 0, column: 5 }, + oldEndPosition: { row: 0, column: 5 }, + newEndPosition: { row: 0, column: 10 }, + }); + + const point = { row: 0, column: 2 }; + const index = 2; + const result = edit.editPoint(point, index); + expect(result.point).toEqual({ row: 0, column: 2 }); + expect(result.index).toBe(2); + }); + + it('edits a point at the start of the edit', () => { + const edit = new Edit({ + startIndex: 5, + oldEndIndex: 5, + newEndIndex: 10, + startPosition: { row: 0, column: 5 }, + oldEndPosition: { row: 0, column: 5 }, + newEndPosition: { row: 0, column: 10 }, + }); + + const point = { row: 0, column: 5 }; + const index = 5; + const result = edit.editPoint(point, index); + expect(result.point).toEqual({ row: 0, column: 10 }); + expect(result.index).toBe(10); + }); + + it('edits a range after the edit', () => { + const edit = new Edit({ + startIndex: 10, + oldEndIndex: 15, + newEndIndex: 20, + startPosition: { row: 1, column: 0 }, + oldEndPosition: { row: 1, column: 5 }, + newEndPosition: { row: 2, column: 0 }, + }); + + const range = { + startPosition: { row: 2, column: 0 }, + endPosition: { row: 2, column: 5 }, + startIndex: 20, + endIndex: 25, + }; + const result = edit.editRange(range); + expect(result.startIndex).toBe(25); + expect(result.endIndex).toBe(30); + expect(result.startPosition).toEqual({ row: 3, column: 0 }); + expect(result.endPosition).toEqual({ row: 3, column: 5 }); + }); + + it('edits a range before the edit', () => { + const edit = new Edit({ + startIndex: 10, + oldEndIndex: 15, + newEndIndex: 20, + startPosition: { row: 1, column: 0 }, + oldEndPosition: { row: 1, column: 5 }, + newEndPosition: { row: 2, column: 0 }, + }); + + const range = { + startPosition: { row: 0, column: 5 }, + endPosition: { row: 0, column: 8 }, + startIndex: 5, + endIndex: 8, + }; + const result = edit.editRange(range); + expect(result.startIndex).toBe(5); + expect(result.endIndex).toBe(8); + expect(result.startPosition).toEqual({ row: 0, column: 5 }); + expect(result.endPosition).toEqual({ row: 0, column: 8 }); + }); + + it('edits a range overlapping the edit', () => { + const edit = new Edit({ + startIndex: 10, + oldEndIndex: 15, + newEndIndex: 20, + startPosition: { row: 1, column: 0 }, + oldEndPosition: { row: 1, column: 5 }, + newEndPosition: { row: 2, column: 0 } + }); + + const range = { + startPosition: { row: 0, column: 8 }, + endPosition: { row: 1, column: 2 }, + startIndex: 8, + endIndex: 12, + }; + const result = edit.editRange(range); + expect(result.startIndex).toBe(8); + expect(result.endIndex).toBe(10); + expect(result.startPosition).toEqual({ row: 0, column: 8 }); + expect(result.endPosition).toEqual({ row: 1, column: 0 }); + }); +}); diff --git a/lib/binding_web/test/helper.js b/lib/binding_web/test/helper.js deleted file mode 100644 index 9e0869d4..00000000 --- a/lib/binding_web/test/helper.js +++ /dev/null @@ -1,15 +0,0 @@ -const Parser = require('..'); - -function languageURL(name) { - return require.resolve(`../../../target/release/tree-sitter-${name}.wasm`); -} - -module.exports = Parser.init().then(async () => ({ - Parser, - languageURL, - EmbeddedTemplate: await Parser.Language.load(languageURL('embedded-template')), - HTML: await Parser.Language.load(languageURL('html')), - JavaScript: await Parser.Language.load(languageURL('javascript')), - JSON: await Parser.Language.load(languageURL('json')), - Python: await Parser.Language.load(languageURL('python')), -})); diff --git a/lib/binding_web/test/helper.ts b/lib/binding_web/test/helper.ts new file mode 100644 index 00000000..15a4d02c --- /dev/null +++ b/lib/binding_web/test/helper.ts @@ -0,0 +1,21 @@ +import { Parser, Language } from '../src'; +import path from 'path'; + +// https://github.com/tree-sitter/tree-sitter/blob/master/xtask/src/fetch.rs#L15 +export type LanguageName = 'bash' | 'c' | 'cpp' | 'embedded-template' | 'go' | 'html' | 'java' | 'javascript' | 'jsdoc' | 'json' | 'php' | 'python' | 'ruby' | 'rust' | 'typescript' | 'tsx'; + +function languageURL(name: LanguageName): string { + const basePath = process.cwd(); + return path.join(basePath, `../../target/release/tree-sitter-${name}.wasm`); +} + +export default Parser.init().then(async () => ({ + languageURL, + C: await Language.load(languageURL('c')), + EmbeddedTemplate: await Language.load(languageURL('embedded-template')), + HTML: await Language.load(languageURL('html')), + JavaScript: await Language.load(languageURL('javascript')), + JSON: await Language.load(languageURL('json')), + Python: await Language.load(languageURL('python')), + Rust: await Language.load(languageURL('rust')), +})); diff --git a/lib/binding_web/test/language-test.js b/lib/binding_web/test/language-test.js deleted file mode 100644 index d3591fca..00000000 --- a/lib/binding_web/test/language-test.js +++ /dev/null @@ -1,87 +0,0 @@ -const {assert} = require('chai'); -let JavaScript; - -describe('Language', () => { - before(async () => ({JavaScript} = await require('./helper'))); - - describe('.fieldIdForName, .fieldNameForId', () => { - it('converts between the string and integer representations of fields', () => { - const nameId = JavaScript.fieldIdForName('name'); - const bodyId = JavaScript.fieldIdForName('body'); - - assert.isBelow(nameId, JavaScript.fieldCount); - assert.isBelow(bodyId, JavaScript.fieldCount); - assert.equal('name', JavaScript.fieldNameForId(nameId)); - assert.equal('body', JavaScript.fieldNameForId(bodyId)); - }); - - it('handles invalid inputs', () => { - assert.equal(null, JavaScript.fieldIdForName('namezzz')); - assert.equal(null, JavaScript.fieldNameForId(-1)); - assert.equal(null, JavaScript.fieldNameForId(10000)); - }); - }); - - describe('.idForNodeType, .nodeTypeForId, .nodeTypeIsNamed', () => { - it('converts between the string and integer representations of a node type', () => { - const exportStatementId = JavaScript.idForNodeType('export_statement', true); - const starId = JavaScript.idForNodeType('*', false); - - assert.isBelow(exportStatementId, JavaScript.nodeTypeCount); - assert.isBelow(starId, JavaScript.nodeTypeCount); - assert.equal(true, JavaScript.nodeTypeIsNamed(exportStatementId)); - assert.equal('export_statement', JavaScript.nodeTypeForId(exportStatementId)); - assert.equal(false, JavaScript.nodeTypeIsNamed(starId)); - assert.equal('*', JavaScript.nodeTypeForId(starId)); - }); - - it('handles invalid inputs', () => { - assert.equal(null, JavaScript.nodeTypeForId(-1)); - assert.equal(null, JavaScript.nodeTypeForId(10000)); - assert.equal(null, JavaScript.idForNodeType('export_statement', false)); - }); - }); -}); - -describe('Lookahead iterator', () => { - let lookahead; - let state; - before(async () => { - let Parser; - ({JavaScript, Parser} = await require('./helper')); - const parser = new Parser().setLanguage(JavaScript); - const tree = parser.parse('function fn() {}'); - parser.delete(); - const cursor = tree.walk(); - assert(cursor.gotoFirstChild()); - assert(cursor.gotoFirstChild()); - state = cursor.currentNode.nextParseState; - lookahead = JavaScript.lookaheadIterator(state); - assert.exists(lookahead); - }); - - after(() => { - lookahead.delete(); - }); - - const expected = ['(', 'identifier', '*', 'formal_parameters', 'html_comment', 'comment']; - it('should iterate over valid symbols in the state', () => { - const symbols = Array.from(lookahead); - assert.includeMembers(symbols, expected); - assert.lengthOf(symbols, expected.length); - }); - - it('should reset to the initial state', () => { - assert(lookahead.resetState(state)); - const symbols = Array.from(lookahead); - assert.includeMembers(symbols, expected); - assert.lengthOf(symbols, expected.length); - }); - - it('should reset', () => { - assert(lookahead.reset(JavaScript, state)); - const symbols = Array.from(lookahead); - assert.includeMembers(symbols, expected); - assert.lengthOf(symbols, expected.length); - }); -}); diff --git a/lib/binding_web/test/language.test.ts b/lib/binding_web/test/language.test.ts new file mode 100644 index 00000000..bda09a29 --- /dev/null +++ b/lib/binding_web/test/language.test.ts @@ -0,0 +1,176 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import helper from './helper'; +import type { LookaheadIterator, Language } from '../src'; +import { Parser } from '../src'; + +let JavaScript: Language; +let Rust: Language; + +describe('Language', () => { + beforeAll(async () => ({ JavaScript, Rust } = await helper)); + + describe('.name, .version', () => { + it('returns the name and version of the language', () => { + expect(JavaScript.name).toBe('javascript'); + expect(JavaScript.abiVersion).toBe(15); + }); + }); + + describe('.fieldIdForName, .fieldNameForId', () => { + it('converts between the string and integer representations of fields', () => { + const nameId = JavaScript.fieldIdForName('name'); + const bodyId = JavaScript.fieldIdForName('body'); + + expect(nameId).toBeLessThan(JavaScript.fieldCount); + expect(bodyId).toBeLessThan(JavaScript.fieldCount); + expect(JavaScript.fieldNameForId(nameId!)).toBe('name'); + expect(JavaScript.fieldNameForId(bodyId!)).toBe('body'); + }); + + it('handles invalid inputs', () => { + expect(JavaScript.fieldIdForName('namezzz')).toBeNull(); + expect(JavaScript.fieldNameForId(-3)).toBeNull(); + expect(JavaScript.fieldNameForId(10000)).toBeNull(); + }); + }); + + describe('.idForNodeType, .nodeTypeForId, .nodeTypeIsNamed', () => { + it('converts between the string and integer representations of a node type', () => { + const exportStatementId = JavaScript.idForNodeType('export_statement', true)!; + const starId = JavaScript.idForNodeType('*', false)!; + + expect(exportStatementId).toBeLessThan(JavaScript.nodeTypeCount); + expect(starId).toBeLessThan(JavaScript.nodeTypeCount); + expect(JavaScript.nodeTypeIsNamed(exportStatementId)).toBe(true); + expect(JavaScript.nodeTypeForId(exportStatementId)).toBe('export_statement'); + expect(JavaScript.nodeTypeIsNamed(starId)).toBe(false); + expect(JavaScript.nodeTypeForId(starId)).toBe('*'); + }); + + it('handles invalid inputs', () => { + expect(JavaScript.nodeTypeForId(-3)).toBeNull(); + expect(JavaScript.nodeTypeForId(10000)).toBeNull(); + expect(JavaScript.idForNodeType('export_statement', false)).toBeNull(); + }); + }); + + describe('Supertypes', () => { + it('gets the supertypes and subtypes of a parser', () => { + const supertypes = Rust.supertypes; + const names = supertypes.map((id) => Rust.nodeTypeForId(id)); + expect(names).toEqual([ + '_expression', + '_literal', + '_literal_pattern', + '_pattern', + '_type' + ]); + + for (const id of supertypes) { + const name = Rust.nodeTypeForId(id); + const subtypes = Rust.subtypes(id); + let subtypeNames = subtypes.map((id) => Rust.nodeTypeForId(id)); + subtypeNames = [...new Set(subtypeNames)].sort(); // Remove duplicates & sort + + switch (name) { + case '_literal': + expect(subtypeNames).toEqual([ + 'boolean_literal', + 'char_literal', + 'float_literal', + 'integer_literal', + 'raw_string_literal', + 'string_literal', + ]); + break; + case '_pattern': + expect(subtypeNames).toEqual([ + '_', + '_literal_pattern', + 'captured_pattern', + 'const_block', + 'generic_pattern', + 'identifier', + 'macro_invocation', + 'mut_pattern', + 'or_pattern', + 'range_pattern', + 'ref_pattern', + 'reference_pattern', + 'remaining_field_pattern', + 'scoped_identifier', + 'slice_pattern', + 'struct_pattern', + 'tuple_pattern', + 'tuple_struct_pattern', + ]); + break; + case '_type': + expect(subtypeNames).toEqual([ + 'abstract_type', + 'array_type', + 'bounded_type', + 'dynamic_type', + 'function_type', + 'generic_type', + 'macro_invocation', + 'metavariable', + 'never_type', + 'pointer_type', + 'primitive_type', + 'reference_type', + 'removed_trait_bound', + 'scoped_type_identifier', + 'tuple_type', + 'type_identifier', + 'unit_type', + ]); + break; + } + } + }); + }); +}); + +describe('Lookahead iterator', () => { + let lookahead: LookaheadIterator; + let state: number; + + beforeAll(async () => { + ({ JavaScript } = await helper); + const parser = new Parser(); + parser.setLanguage(JavaScript); + const tree = parser.parse('function fn() {}')!; + parser.delete(); + const cursor = tree.walk(); + expect(cursor.gotoFirstChild()).toBe(true); + expect(cursor.gotoFirstChild()).toBe(true); + state = cursor.currentNode.nextParseState; + lookahead = JavaScript.lookaheadIterator(state)!; + expect(lookahead).toBeDefined(); + }); + + afterAll(() => { lookahead.delete() }); + + const expected = ['(', 'identifier', '*', 'formal_parameters', 'html_comment', 'comment']; + + it('should iterate over valid symbols in the state', () => { + const symbols = Array.from(lookahead); + expect(symbols).toEqual(expect.arrayContaining(expected)); + expect(symbols).toHaveLength(expected.length); + }); + + it('should reset to the initial state', () => { + expect(lookahead.resetState(state)).toBe(true); + const symbols = Array.from(lookahead); + expect(symbols).toEqual(expect.arrayContaining(expected)); + expect(symbols).toHaveLength(expected.length); + }); + + it('should reset', () => { + expect(lookahead.reset(JavaScript, state)).toBe(true); + const symbols = Array.from(lookahead); + expect(symbols).toEqual(expect.arrayContaining(expected)); + expect(symbols).toHaveLength(expected.length); + }); +}); diff --git a/lib/binding_web/test/memory.test.ts b/lib/binding_web/test/memory.test.ts new file mode 100644 index 00000000..46238934 --- /dev/null +++ b/lib/binding_web/test/memory.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from 'vitest'; +import { gc, event, Finalizer } from './memory'; + +// hijack finalization registry before import web-tree-sitter +globalThis.FinalizationRegistry = Finalizer; + +describe('Memory Management', () => { + describe('call .delete()', () => { + it('test free memory manually', async () => { + const timer = setInterval(() => { + gc(); + }, 100); + let done = 0; + event.on('gc', () => { + done++; + }); + await (async () => { + const { JavaScript } = await (await import('./helper')).default; + const { Parser, Query } = await import('../src'); + const parser = new Parser(); + parser.setLanguage(JavaScript); + const tree = parser.parse('1+1')!; + const copyTree = tree.copy(); + const cursor = tree.walk(); + const copyCursor = cursor.copy(); + const lookaheadIterator = JavaScript.lookaheadIterator(cursor.currentNode.nextParseState)!; + const query = new Query(JavaScript, '(identifier) @element'); + parser.delete(); + tree.delete(); + copyTree.delete(); + cursor.delete(); + copyCursor.delete(); + lookaheadIterator.delete(); + query.delete(); + })(); + // wait for gc + await new Promise((resolve) => setTimeout(resolve, 1000)); + clearInterval(timer); + // expect no gc event fired + expect(done).toBe(0); + }); + }); + + describe('do not call .delete()', () => { + it('test free memory automatically', async () => { + const timer = setInterval(() => { + gc(); + }, 100); + let done = 0; + const promise = new Promise((resolve) => { + event.on('gc', () => { + if (++done === 7) { + resolve(undefined); + clearInterval(timer); + } + console.log('free memory times: ', done); + }); + }); + await (async () => { + const { JavaScript } = await (await import('./helper')).default; + const { Parser, Query } = await import('../src'); + const parser = new Parser(); // 1 + parser.setLanguage(JavaScript); + const tree = parser.parse('1+1')!; // 2 + tree.copy(); // 3 + const cursor = tree.walk(); // 4 + cursor.copy(); // 5 + JavaScript.lookaheadIterator(cursor.currentNode.nextParseState)!; // 6 + new Query(JavaScript, '(identifier) @element'); // 7 + })(); + await promise; + }); + }); +}); diff --git a/lib/binding_web/test/memory.ts b/lib/binding_web/test/memory.ts new file mode 100644 index 00000000..62cb8b7d --- /dev/null +++ b/lib/binding_web/test/memory.ts @@ -0,0 +1,20 @@ +import { EventEmitter } from 'events'; +import { Session } from 'inspector'; + +const session = new Session(); +session.connect(); + +export function gc() { + session.post('HeapProfiler.collectGarbage'); +} + +export const event = new EventEmitter(); + +export class Finalizer extends FinalizationRegistry { + constructor(handler: (value: T) => void) { + super((value) => { + handler(value); + event.emit('gc'); + }); + } +} diff --git a/lib/binding_web/test/memory_unsupported.test.ts b/lib/binding_web/test/memory_unsupported.test.ts new file mode 100644 index 00000000..cc1f69bf --- /dev/null +++ b/lib/binding_web/test/memory_unsupported.test.ts @@ -0,0 +1,25 @@ +import { describe, it } from 'vitest'; + +describe('FinalizationRegistry is unsupported', () => { + it('test FinalizationRegistry is unsupported', async () => { + // @ts-expect-error: test FinalizationRegistry is not supported + globalThis.FinalizationRegistry = undefined; + const { JavaScript } = await (await import('./helper')).default; + const { Parser, Query } = await import('../src'); + const parser = new Parser(); + parser.setLanguage(JavaScript); + const tree = parser.parse('1+1')!; + const copyTree = tree.copy(); + const cursor = tree.walk(); + const copyCursor = cursor.copy(); + const lookaheadIterator = JavaScript.lookaheadIterator(cursor.currentNode.nextParseState)!; + const query = new Query(JavaScript, '(identifier) @element'); + parser.delete(); + tree.delete(); + copyTree.delete(); + cursor.delete(); + copyCursor.delete(); + lookaheadIterator.delete(); + query.delete(); + }); +}); diff --git a/lib/binding_web/test/node-test.js b/lib/binding_web/test/node-test.js deleted file mode 100644 index 7c3e8f7b..00000000 --- a/lib/binding_web/test/node-test.js +++ /dev/null @@ -1,636 +0,0 @@ -const {assert} = require('chai'); -let Parser; let JavaScript; let JSON; let EmbeddedTemplate; let Python; - -const JSON_EXAMPLE = ` - -[ - 123, - false, - { - "x": null - } -] -`; - -function getAllNodes(tree) { - const result = []; - let visitedChildren = false; - const cursor = tree.walk(); - while (true) { - if (!visitedChildren) { - result.push(cursor.currentNode); - if (!cursor.gotoFirstChild()) { - visitedChildren = true; - } - } else if (cursor.gotoNextSibling()) { - visitedChildren = false; - } else if (!cursor.gotoParent()) { - break; - } - } - return result; -} - -describe('Node', () => { - let parser; let tree; - - before(async () => - ({Parser, EmbeddedTemplate, JavaScript, JSON, Python} = await require('./helper')), - ); - - beforeEach(() => { - tree = null; - parser = new Parser().setLanguage(JavaScript); - }); - - afterEach(() => { - parser.delete(); - tree.delete(); - }); - - describe('.children', () => { - it('returns an array of child nodes', () => { - tree = parser.parse('x10 + 1000'); - assert.equal(1, tree.rootNode.children.length); - const sumNode = tree.rootNode.firstChild.firstChild; - assert.deepEqual( - sumNode.children.map((child) => child.type), - ['identifier', '+', 'number'], - ); - }); - }); - - describe('.namedChildren', () => { - it('returns an array of named child nodes', () => { - tree = parser.parse('x10 + 1000'); - const sumNode = tree.rootNode.firstChild.firstChild; - assert.equal(1, tree.rootNode.namedChildren.length); - assert.deepEqual( - ['identifier', 'number'], - sumNode.namedChildren.map((child) => child.type), - ); - }); - }); - - describe('.childrenForFieldName', () => { - it('returns an array of child nodes for the given field name', () => { - parser.setLanguage(Python); - const source = ` - if one: - a() - elif two: - b() - elif three: - c() - elif four: - d()`; - - tree = parser.parse(source); - const node = tree.rootNode.firstChild; - assert.equal(node.type, 'if_statement'); - const alternatives = node.childrenForFieldName('alternative'); - const alternativeTexts = alternatives.map((n) => { - const condition = n.childForFieldName('condition'); - return source.slice(condition.startIndex, condition.endIndex); - }); - assert.deepEqual(alternativeTexts, ['two', 'three', 'four']); - }); - }); - - describe('.startIndex and .endIndex', () => { - it('returns the character index where the node starts/ends in the text', () => { - tree = parser.parse('a👍👎1 / b👎c👎'); - const quotientNode = tree.rootNode.firstChild.firstChild; - - assert.equal(0, quotientNode.startIndex); - assert.equal(15, quotientNode.endIndex); - assert.deepEqual( - [0, 7, 9], - quotientNode.children.map((child) => child.startIndex), - ); - assert.deepEqual( - [6, 8, 15], - quotientNode.children.map((child) => child.endIndex), - ); - }); - }); - - describe('.startPosition and .endPosition', () => { - it('returns the row and column where the node starts/ends in the text', () => { - tree = parser.parse('x10 + 1000'); - const sumNode = tree.rootNode.firstChild.firstChild; - assert.equal('binary_expression', sumNode.type); - - assert.deepEqual({row: 0, column: 0}, sumNode.startPosition); - assert.deepEqual({row: 0, column: 10}, sumNode.endPosition); - assert.deepEqual( - [{row: 0, column: 0}, {row: 0, column: 4}, {row: 0, column: 6}], - sumNode.children.map((child) => child.startPosition), - ); - assert.deepEqual( - [{row: 0, column: 3}, {row: 0, column: 5}, {row: 0, column: 10}], - sumNode.children.map((child) => child.endPosition), - ); - }); - - it('handles characters that occupy two UTF16 code units', () => { - tree = parser.parse('a👍👎1 /\n b👎c👎'); - const sumNode = tree.rootNode.firstChild.firstChild; - assert.deepEqual( - [ - [{row: 0, column: 0}, {row: 0, column: 6}], - [{row: 0, column: 7}, {row: 0, column: 8}], - [{row: 1, column: 1}, {row: 1, column: 7}], - ], - sumNode.children.map((child) => [child.startPosition, child.endPosition]), - ); - }); - }); - - describe('.parent', () => { - it('returns the node\'s parent', () => { - tree = parser.parse('x10 + 1000'); - const sumNode = tree.rootNode.firstChild; - const variableNode = sumNode.firstChild; - assert.notEqual(sumNode.id, variableNode.id); - assert.equal(sumNode.id, variableNode.parent.id); - assert.equal(tree.rootNode.id, sumNode.parent.id); - }); - }); - - describe('.child(), .firstChild, .lastChild', () => { - it('returns null when the node has no children', () => { - tree = parser.parse('x10 + 1000'); - const sumNode = tree.rootNode.firstChild.firstChild; - const variableNode = sumNode.firstChild; - assert.equal(variableNode.firstChild, null); - assert.equal(variableNode.lastChild, null); - assert.equal(variableNode.firstNamedChild, null); - assert.equal(variableNode.lastNamedChild, null); - assert.equal(variableNode.child(1), null); - }); - }); - - describe('.childForFieldName()', () => { - it('returns null when the node has no children', () => { - tree = parser.parse('class A { b() {} }'); - - const classNode = tree.rootNode.firstChild; - assert.equal(classNode.type, 'class_declaration'); - - const classNameNode = classNode.childForFieldName('name'); - assert.equal(classNameNode.type, 'identifier'); - assert.equal(classNameNode.text, 'A'); - - const bodyNode = classNode.childForFieldName('body'); - assert.equal(bodyNode.type, 'class_body'); - assert.equal(bodyNode.text, '{ b() {} }'); - - const methodNode = bodyNode.firstNamedChild; - assert.equal(methodNode.type, 'method_definition'); - assert.equal(methodNode.text, 'b() {}'); - - const methodNameNode = methodNode.childForFieldName('name'); - assert.equal(methodNameNode.type, 'property_identifier'); - assert.equal(methodNameNode.text, 'b'); - - const paramsNode = methodNode.childForFieldName('parameters'); - assert.equal(paramsNode.type, 'formal_parameters'); - assert.equal(paramsNode.text, '()'); - }); - }); - - describe('.nextSibling and .previousSibling', () => { - it('returns the node\'s next and previous sibling', () => { - tree = parser.parse('x10 + 1000'); - const sumNode = tree.rootNode.firstChild.firstChild; - assert.equal(sumNode.children[1].id, sumNode.children[0].nextSibling.id); - assert.equal(sumNode.children[2].id, sumNode.children[1].nextSibling.id); - assert.equal( - sumNode.children[0].id, - sumNode.children[1].previousSibling.id, - ); - assert.equal( - sumNode.children[1].id, - sumNode.children[2].previousSibling.id, - ); - }); - }); - - describe('.nextNamedSibling and .previousNamedSibling', () => { - it('returns the node\'s next and previous named sibling', () => { - tree = parser.parse('x10 + 1000'); - const sumNode = tree.rootNode.firstChild.firstChild; - assert.equal( - sumNode.namedChildren[1].id, - sumNode.namedChildren[0].nextNamedSibling.id, - ); - assert.equal( - sumNode.namedChildren[0].id, - sumNode.namedChildren[1].previousNamedSibling.id, - ); - }); - }); - - describe('.descendantForIndex(min, max)', () => { - it('returns the smallest node that spans the given range', () => { - tree = parser.parse('x10 + 1000'); - const sumNode = tree.rootNode.firstChild.firstChild; - assert.equal('identifier', sumNode.descendantForIndex(1, 2).type); - assert.equal('+', sumNode.descendantForIndex(4, 4).type); - - assert.throws(() => { - sumNode.descendantForIndex(1, {}); - }, 'Arguments must be numbers'); - - assert.throws(() => { - sumNode.descendantForIndex(); - }, 'Arguments must be numbers'); - }); - }); - - describe('.namedDescendantForIndex', () => { - it('returns the smallest node that spans the given range', () => { - tree = parser.parse('x10 + 1000'); - const sumNode = tree.rootNode.firstChild; - assert.equal('identifier', sumNode.descendantForIndex(1, 2).type); - assert.equal('+', sumNode.descendantForIndex(4, 4).type); - }); - }); - - describe('.descendantForPosition(min, max)', () => { - it('returns the smallest node that spans the given range', () => { - tree = parser.parse('x10 + 1000'); - const sumNode = tree.rootNode.firstChild; - - assert.equal( - 'identifier', - sumNode.descendantForPosition( - {row: 0, column: 1}, - {row: 0, column: 2}, - ).type, - ); - - assert.equal( - '+', - sumNode.descendantForPosition({row: 0, column: 4}).type, - ); - - assert.throws(() => { - sumNode.descendantForPosition(1, {}); - }, 'Arguments must be {row, column} objects'); - - assert.throws(() => { - sumNode.descendantForPosition(); - }, 'Arguments must be {row, column} objects'); - }); - }); - - describe('.namedDescendantForPosition(min, max)', () => { - it('returns the smallest named node that spans the given range', () => { - tree = parser.parse('x10 + 1000'); - const sumNode = tree.rootNode.firstChild; - - assert.equal( - sumNode.namedDescendantForPosition( - {row: 0, column: 1}, - {row: 0, column: 2}, - ).type, - 'identifier', - ); - - assert.equal( - sumNode.namedDescendantForPosition({row: 0, column: 4}).type, - 'binary_expression', - ); - }); - }); - - describe('.hasError', () => { - it('returns true if the node contains an error', () => { - tree = parser.parse('1 + 2 * * 3'); - const node = tree.rootNode; - assert.equal( - node.toString(), - '(program (expression_statement (binary_expression left: (number) right: (binary_expression left: (number) (ERROR) right: (number)))))', - ); - - const sum = node.firstChild.firstChild; - assert(sum.hasError); - assert(!sum.children[0].hasError); - assert(!sum.children[1].hasError); - assert(sum.children[2].hasError); - }); - }); - - describe('.isError', () => { - it('returns true if the node is an error', () => { - tree = parser.parse('2 * * 3'); - const node = tree.rootNode; - assert.equal( - node.toString(), - '(program (expression_statement (binary_expression left: (number) (ERROR) right: (number))))', - ); - - const multi = node.firstChild.firstChild; - assert(multi.hasError); - assert(!multi.children[0].isError); - assert(!multi.children[1].isError); - assert(multi.children[2].isError); - assert(!multi.children[3].isError); - }); - }); - - describe('.isMissing', () => { - it('returns true if the node is missing from the source and was inserted via error recovery', () => { - tree = parser.parse('(2 ||)'); - const node = tree.rootNode; - assert.equal( - node.toString(), - '(program (expression_statement (parenthesized_expression (binary_expression left: (number) right: (MISSING identifier)))))', - ); - - const sum = node.firstChild.firstChild.firstNamedChild; - assert.equal(sum.type, 'binary_expression'); - assert(sum.hasError); - assert(!sum.children[0].isMissing); - assert(!sum.children[1].isMissing); - assert(sum.children[2].isMissing); - }); - }); - - describe('.isExtra', () => { - it('returns true if the node is an extra node like comments', () => { - tree = parser.parse('foo(/* hi */);'); - const node = tree.rootNode; - const commentNode = node.descendantForIndex(7, 7); - - assert.equal(node.type, 'program'); - assert.equal(commentNode.type, 'comment'); - assert(!node.isExtra); - assert(commentNode.isExtra); - }); - }); - - describe('.text', () => { - const text = 'α0 / b👎c👎'; - - Object.entries({ - '.parse(String)': text, - '.parse(Function)': (offset) => text.slice(offset, 4), - }).forEach(([method, _parse]) => - it(`returns the text of a node generated by ${method}`, async () => { - const [numeratorSrc, denominatorSrc] = text.split(/\s*\/\s+/); - tree = await parser.parse(text); - const quotientNode = tree.rootNode.firstChild.firstChild; - const [numerator, slash, denominator] = quotientNode.children; - - assert.equal(text, tree.rootNode.text, 'root node text'); - assert.equal(denominatorSrc, denominator.text, 'denominator text'); - assert.equal(text, quotientNode.text, 'quotient text'); - assert.equal(numeratorSrc, numerator.text, 'numerator text'); - assert.equal('/', slash.text, '"/" text'); - }), - ); - }); - - describe('.descendantCount', () => { - it('returns the number of descendants', () => { - parser.setLanguage(JSON); - tree = parser.parse(JSON_EXAMPLE); - const valueNode = tree.rootNode; - const allNodes = getAllNodes(tree); - - assert.equal(valueNode.descendantCount, allNodes.length); - - const cursor = tree.walk(); - for (let i = 0; i < allNodes.length; i++) { - const node = allNodes[i]; - cursor.gotoDescendant(i); - assert.equal(cursor.currentNode.id, node.id, `index ${i}`); - } - - for (let i = allNodes.length - 1; i >= 0; i--) { - const node = allNodes[i]; - cursor.gotoDescendant(i); - assert.equal(cursor.currentNode.id, node.id, `rev index ${i}`); - } - }); - - it('tests a single node tree', () => { - parser.setLanguage(EmbeddedTemplate); - tree = parser.parse('hello'); - - const nodes = getAllNodes(tree); - assert.equal(nodes.length, 2); - assert.equal(tree.rootNode.descendantCount, 2); - - const cursor = tree.walk(); - - cursor.gotoDescendant(0); - assert.equal(cursor.currentDepth, 0); - assert.equal(cursor.currentNode.id, nodes[0].id); - - cursor.gotoDescendant(1); - assert.equal(cursor.currentDepth, 1); - assert.equal(cursor.currentNode.id, nodes[1].id); - }); - }); - - describe('.rootNodeWithOffset', () => { - it('returns the root node of the tree, offset by the given byte offset', () => { - tree = parser.parse(' if (a) b'); - const node = tree.rootNodeWithOffset(6, {row: 2, column: 2}); - assert.equal(node.startIndex, 8); - assert.equal(node.endIndex, 16); - assert.deepEqual(node.startPosition, {row: 2, column: 4}); - assert.deepEqual(node.endPosition, {row: 2, column: 12}); - - let child = node.firstChild.child(2); - assert.equal(child.type, 'expression_statement'); - assert.equal(child.startIndex, 15); - assert.equal(child.endIndex, 16); - assert.deepEqual(child.startPosition, {row: 2, column: 11}); - assert.deepEqual(child.endPosition, {row: 2, column: 12}); - - const cursor = node.walk(); - cursor.gotoFirstChild(); - cursor.gotoFirstChild(); - cursor.gotoNextSibling(); - child = cursor.currentNode; - assert.equal(child.type, 'parenthesized_expression'); - assert.equal(child.startIndex, 11); - assert.equal(child.endIndex, 14); - assert.deepEqual(child.startPosition, {row: 2, column: 7}); - assert.deepEqual(child.endPosition, {row: 2, column: 10}); - }); - }); - - describe('.parseState, .nextParseState', () => { - const text = '10 / 5'; - - it('returns node parse state ids', async () => { - tree = await parser.parse(text); - const quotientNode = tree.rootNode.firstChild.firstChild; - const [numerator, slash, denominator] = quotientNode.children; - - assert.equal(tree.rootNode.parseState, 0); - // parse states will change on any change to the grammar so test that it - // returns something instead - assert.isAbove(numerator.parseState, 0); - assert.isAbove(slash.parseState, 0); - assert.isAbove(denominator.parseState, 0); - }); - - it('returns next parse state equal to the language', async () => { - tree = await parser.parse(text); - const quotientNode = tree.rootNode.firstChild.firstChild; - quotientNode.children.forEach((node) => { - assert.equal( - node.nextParseState, - JavaScript.nextState(node.parseState, node.grammarId), - ); - }); - }); - }); - - describe('.descendantsOfType(type, min, max)', () => { - it('finds all of the descendants of the given type in the given range', () => { - tree = parser.parse('a + 1 * b * 2 + c + 3'); - const outerSum = tree.rootNode.firstChild.firstChild; - let descendants = outerSum.descendantsOfType('number', {row: 0, column: 2}, {row: 0, column: 15}); - assert.deepEqual( - descendants.map((node) => node.startIndex), - [4, 12], - ); - assert.deepEqual( - descendants.map((node) => node.endPosition), - [{row: 0, column: 5}, {row: 0, column: 13}], - ); - - descendants = outerSum.descendantsOfType('identifier', {row: 0, column: 2}, {row: 0, column: 15}); - assert.deepEqual( - descendants.map((node) => node.startIndex), - [8], - ); - - descendants = outerSum.descendantsOfType('identifier', {row: 0, column: 0}, {row: 0, column: 30}); - assert.deepEqual( - descendants.map((node) => node.startIndex), - [0, 8, 16], - ); - - descendants = outerSum.descendantsOfType('number', {row: 0, column: 0}, {row: 0, column: 30}); - assert.deepEqual( - descendants.map((node) => node.startIndex), - [4, 12, 20], - ); - - descendants = outerSum.descendantsOfType( - ['identifier', 'number'], - {row: 0, column: 0}, - {row: 0, column: 30}, - ); - assert.deepEqual( - descendants.map((node) => node.startIndex), - [0, 4, 8, 12, 16, 20], - ); - - descendants = outerSum.descendantsOfType('number'); - assert.deepEqual( - descendants.map((node) => node.startIndex), - [4, 12, 20], - ); - - descendants = outerSum.firstChild.descendantsOfType('number', {row: 0, column: 0}, {row: 0, column: 30}); - assert.deepEqual( - descendants.map((node) => node.startIndex), - [4, 12], - ); - }); - }); - - describe.skip('.closest(type)', () => { - it('returns the closest ancestor of the given type', () => { - tree = parser.parse('a(b + -d.e)'); - const property = tree.rootNode.descendantForIndex('a(b + -d.'.length); - assert.equal(property.type, 'property_identifier'); - - const unary = property.closest('unary_expression'); - assert.equal(unary.type, 'unary_expression'); - assert.equal(unary.startIndex, 'a(b + '.length); - assert.equal(unary.endIndex, 'a(b + -d.e'.length); - - const sum = property.closest(['binary_expression', 'call_expression']); - assert.equal(sum.type, 'binary_expression'); - assert.equal(sum.startIndex, 2); - assert.equal(sum.endIndex, 'a(b + -d.e'.length); - }); - - it('throws an exception when an invalid argument is given', () => { - tree = parser.parse('a + 1 * b * 2 + c + 3'); - const number = tree.rootNode.descendantForIndex(4); - - assert.throws(() => number.closest({a: 1}), /Argument must be a string or array of strings/); - }); - }); - - describe('.firstChildForIndex(index)', () => { - it('returns the first child that extends beyond the given index', () => { - tree = parser.parse('x10 + 1000'); - const sumNode = tree.rootNode.firstChild.firstChild; - - assert.equal('identifier', sumNode.firstChildForIndex(0).type); - assert.equal('identifier', sumNode.firstChildForIndex(1).type); - assert.equal('+', sumNode.firstChildForIndex(3).type); - assert.equal('number', sumNode.firstChildForIndex(5).type); - }); - }); - - describe('.firstNamedChildForIndex(index)', () => { - it('returns the first child that extends beyond the given index', () => { - tree = parser.parse('x10 + 1000'); - const sumNode = tree.rootNode.firstChild.firstChild; - - assert.equal('identifier', sumNode.firstNamedChildForIndex(0).type); - assert.equal('identifier', sumNode.firstNamedChildForIndex(1).type); - assert.equal('number', sumNode.firstNamedChildForIndex(3).type); - }); - }); - - describe('.equals(other)', () => { - it('returns true if the nodes are the same', () => { - tree = parser.parse('1 + 2'); - - const sumNode = tree.rootNode.firstChild.firstChild; - const node1 = sumNode.firstChild; - const node2 = sumNode.firstChild; - assert(node1.equals(node2)); - }); - - it('returns false if the nodes are not the same', () => { - tree = parser.parse('1 + 2'); - - const sumNode = tree.rootNode.firstChild.firstChild; - const node1 = sumNode.firstChild; - const node2 = node1.nextSibling; - assert(!node1.equals(node2)); - }); - }); - - describe('.fieldNameForChild(index)', () => { - it('returns the field of a child or null', () => { - tree = parser.parse('let a = 5'); - - const noField = tree.rootNode.fieldNameForChild(0); - const name = tree.rootNode.firstChild.children[1].fieldNameForChild(0); - const value = tree.rootNode.firstChild.children[1].fieldNameForChild(2); - const overflow = tree.rootNode.firstChild.children[1].fieldNameForChild(3); - - assert.equal(noField, null); - assert.equal(name, 'name'); - assert.equal(value, 'value'); - assert.equal(overflow, null); - }); - }); -}); diff --git a/lib/binding_web/test/node.test.ts b/lib/binding_web/test/node.test.ts new file mode 100644 index 00000000..d89dc7e4 --- /dev/null +++ b/lib/binding_web/test/node.test.ts @@ -0,0 +1,596 @@ +import { describe, it, expect, beforeAll, beforeEach, afterEach } from 'vitest'; +import type { Language, Tree, Node } from '../src'; +import { Parser } from '../src'; +import helper from './helper'; + +let C: Language; +let JavaScript: Language; +let JSON: Language; +let EmbeddedTemplate: Language; +let Python: Language; + +const JSON_EXAMPLE = ` +[ + 123, + false, + { + "x": null + } +] +`; + +function getAllNodes(tree: Tree): Node[] { + const result: Node[] = []; + let visitedChildren = false; + const cursor = tree.walk(); + + while (true) { + if (!visitedChildren) { + result.push(cursor.currentNode); + if (!cursor.gotoFirstChild()) { + visitedChildren = true; + } + } else if (cursor.gotoNextSibling()) { + visitedChildren = false; + } else if (!cursor.gotoParent()) { + break; + } + } + return result; +} + +describe('Node', () => { + let parser: Parser; + let tree: Tree | null; + + beforeAll(async () => { + ({ C, EmbeddedTemplate, JavaScript, JSON, Python } = await helper); + }); + + beforeEach(() => { + tree = null; + parser = new Parser(); + parser.setLanguage(JavaScript); + }); + + afterEach(() => { + parser.delete(); + tree!.delete(); + }); + + describe('.children', () => { + it('returns an array of child nodes', () => { + tree = parser.parse('x10 + 1000')!; + expect(tree.rootNode.children).toHaveLength(1); + const sumNode = tree.rootNode.firstChild!.firstChild!; + expect(sumNode.children.map(child => child.type)).toEqual(['identifier', '+', 'number']); + }); + }); + + describe('.namedChildren', () => { + it('returns an array of named child nodes', () => { + tree = parser.parse('x10 + 1000')!; + const sumNode = tree.rootNode.firstChild!.firstChild!; + expect(tree.rootNode.namedChildren).toHaveLength(1); + expect(sumNode.namedChildren.map(child => child.type)).toEqual(['identifier', 'number']); + }); + }); + + describe('.childrenForFieldName', () => { + it('returns an array of child nodes for the given field name', () => { + parser.setLanguage(Python); + const source = ` + if one: + a() + elif two: + b() + elif three: + c() + elif four: + d()`; + + tree = parser.parse(source)!; + const node = tree.rootNode.firstChild!; + expect(node.type).toBe('if_statement'); + const alternatives = node.childrenForFieldName('alternative'); + const alternativeTexts = alternatives.map(n => { + const condition = n.childForFieldName('condition')!; + return source.slice(condition.startIndex, condition.endIndex); + }); + expect(alternativeTexts).toEqual(['two', 'three', 'four']); + }); + }); + + describe('.startIndex and .endIndex', () => { + it('returns the character index where the node starts/ends in the text', () => { + tree = parser.parse('a👍👎1 / b👎c👎')!; + const quotientNode = tree.rootNode.firstChild!.firstChild!; + + expect(quotientNode.startIndex).toBe(0); + expect(quotientNode.endIndex).toBe(15); + expect(quotientNode.children.map(child => child.startIndex)).toEqual([0, 7, 9]); + expect(quotientNode.children.map(child => child.endIndex)).toEqual([6, 8, 15]); + }); + }); + + describe('.startPosition and .endPosition', () => { + it('returns the row and column where the node starts/ends in the text', () => { + tree = parser.parse('x10 + 1000')!; + const sumNode = tree.rootNode.firstChild!.firstChild!; + expect(sumNode.type).toBe('binary_expression'); + + expect(sumNode.startPosition).toEqual({ row: 0, column: 0 }); + expect(sumNode.endPosition).toEqual({ row: 0, column: 10 }); + expect(sumNode.children.map((child) => child.startPosition)).toEqual([ + { row: 0, column: 0 }, + { row: 0, column: 4 }, + { row: 0, column: 6 }, + ]); + expect(sumNode.children.map((child) => child.endPosition)).toEqual([ + { row: 0, column: 3 }, + { row: 0, column: 5 }, + { row: 0, column: 10 }, + ]); + }); + + it('handles characters that occupy two UTF16 code units', () => { + tree = parser.parse('a👍👎1 /\n b👎c👎')!; + const sumNode = tree.rootNode.firstChild!.firstChild!; + expect(sumNode.children.map(child => [child.startPosition, child.endPosition])).toEqual([ + [{ row: 0, column: 0 }, { row: 0, column: 6 }], + [{ row: 0, column: 7 }, { row: 0, column: 8 }], + [{ row: 1, column: 1 }, { row: 1, column: 7 }] + ]); + }); + }); + + describe('.parent', () => { + it('returns the node\'s parent', () => { + tree = parser.parse('x10 + 1000')!; + const sumNode = tree.rootNode.firstChild!; + const variableNode = sumNode.firstChild!; + expect(sumNode.id).not.toBe(variableNode.id); + expect(sumNode.id).toBe(variableNode.parent!.id); + expect(tree.rootNode.id).toBe(sumNode.parent!.id); + }); + }); + + describe('.child(), .firstChild, .lastChild', () => { + it('returns null when the node has no children', () => { + tree = parser.parse('x10 + 1000')!; + const sumNode = tree.rootNode.firstChild!.firstChild!; + const variableNode = sumNode.firstChild!; + expect(variableNode.firstChild).toBeNull(); + expect(variableNode.lastChild).toBeNull(); + expect(variableNode.firstNamedChild).toBeNull(); + expect(variableNode.lastNamedChild).toBeNull(); + expect(variableNode.child(1)).toBeNull(); + }); + }); + + describe('.childForFieldName()', () => { + it('returns node for the given field name', () => { + tree = parser.parse('class A { b() {} }')!; + + const classNode = tree.rootNode.firstChild!; + expect(classNode.type).toBe('class_declaration'); + + const classNameNode = classNode.childForFieldName('name')!; + expect(classNameNode.type).toBe('identifier'); + expect(classNameNode.text).toBe('A'); + + const bodyNode = classNode.childForFieldName('body')!; + expect(bodyNode.type).toBe('class_body'); + expect(bodyNode.text).toBe('{ b() {} }'); + + const methodNode = bodyNode.firstNamedChild!; + expect(methodNode.type).toBe('method_definition'); + expect(methodNode.text).toBe('b() {}'); + }); + }); + + describe('.childWithDescendant()', () => { + it('correctly retrieves immediate children', () => { + const sourceCode = 'let x = 1; console.log(x);'; + tree = parser.parse(sourceCode)!; + const root = tree.rootNode + const child = root.children[0].children[0] + const a = root.childWithDescendant(child) + expect(a!.startIndex).toBe(0) + const b = a!.childWithDescendant(child) + expect(b).toEqual(child) + const c = b!.childWithDescendant(child) + expect(c).toBeNull() + }); + }); + + describe('.nextSibling and .previousSibling', () => { + it('returns the node\'s next and previous sibling', () => { + tree = parser.parse('x10 + 1000')!; + const sumNode = tree.rootNode.firstChild!.firstChild!; + expect(sumNode.children[1].id).toBe(sumNode.children[0].nextSibling!.id); + expect(sumNode.children[2].id).toBe(sumNode.children[1].nextSibling!.id); + expect(sumNode.children[0].id).toBe(sumNode.children[1].previousSibling!.id); + expect(sumNode.children[1].id).toBe(sumNode.children[2].previousSibling!.id); + }); + }); + + describe('.nextNamedSibling and .previousNamedSibling', () => { + it('returns the node\'s next and previous named sibling', () => { + tree = parser.parse('x10 + 1000')!; + const sumNode = tree.rootNode.firstChild!.firstChild!; + expect(sumNode.namedChildren[1].id).toBe(sumNode.namedChildren[0].nextNamedSibling!.id); + expect(sumNode.namedChildren[0].id).toBe(sumNode.namedChildren[1].previousNamedSibling!.id); + }); + }); + + describe('.descendantForIndex(min, max)', () => { + it('returns the smallest node that spans the given range', () => { + tree = parser.parse('x10 + 1000')!; + const sumNode = tree.rootNode.firstChild!.firstChild!; + expect(sumNode.descendantForIndex(1, 2)!.type).toBe('identifier'); + expect(sumNode.descendantForIndex(4, 4)!.type).toBe('+'); + + expect(() => { + // @ts-expect-error Testing invalid arguments + sumNode.descendantForIndex(1, {}); + }).toThrow('Arguments must be numbers'); + + expect(() => { + // @ts-expect-error Testing invalid arguments + sumNode.descendantForIndex(undefined); + }).toThrow('Arguments must be numbers'); + }); + }); + + describe('.namedDescendantForIndex', () => { + it('returns the smallest named node that spans the given range', () => { + tree = parser.parse('x10 + 1000')!; + const sumNode = tree.rootNode.firstChild!; + expect(sumNode.descendantForIndex(1, 2)!.type).toBe('identifier'); + expect(sumNode.descendantForIndex(4, 4)!.type).toBe('+'); + }); + }); + + describe('.descendantForPosition', () => { + it('returns the smallest node that spans the given range', () => { + tree = parser.parse('x10 + 1000')!; + const sumNode = tree.rootNode.firstChild!; + + expect(sumNode.descendantForPosition({ row: 0, column: 1 }, { row: 0, column: 2 })!.type).toBe('identifier'); + expect(sumNode.descendantForPosition({ row: 0, column: 4 })!.type).toBe('+'); + + expect(() => { + // @ts-expect-error Testing invalid arguments + sumNode.descendantForPosition(1, {}); + }).toThrow('Arguments must be {row, column} objects'); + + expect(() => { + // @ts-expect-error Testing invalid arguments + sumNode.descendantForPosition(undefined); + }).toThrow('Arguments must be {row, column} objects'); + }); + }); + + describe('.namedDescendantForPosition(min, max)', () => { + it('returns the smallest named node that spans the given range', () => { + tree = parser.parse('x10 + 1000')!; + const sumNode = tree.rootNode.firstChild!; + + expect(sumNode.namedDescendantForPosition({ row: 0, column: 1 }, { row: 0, column: 2 })!.type).toBe('identifier') + expect(sumNode.namedDescendantForPosition({ row: 0, column: 4 })!.type).toBe('binary_expression'); + }); + }); + + describe('.hasError', () => { + it('returns true if the node contains an error', () => { + tree = parser.parse('1 + 2 * * 3')!; + const node = tree.rootNode; + expect(node.toString()).toBe( + '(program (expression_statement (binary_expression left: (number) right: (binary_expression left: (number) (ERROR) right: (number)))))' + ); + + const sum = node.firstChild!.firstChild!; + expect(sum.hasError).toBe(true); + expect(sum.children[0].hasError).toBe(false); + expect(sum.children[1].hasError).toBe(false); + expect(sum.children[2].hasError).toBe(true); + }); + }); + + describe('.isError', () => { + it('returns true if the node is an error', () => { + tree = parser.parse('2 * * 3')!; + const node = tree.rootNode; + expect(node.toString()).toBe( + '(program (expression_statement (binary_expression left: (number) (ERROR) right: (number))))' + ); + + const multi = node.firstChild!.firstChild!; + expect(multi.hasError).toBe(true); + expect(multi.children[0].isError).toBe(false); + expect(multi.children[1].isError).toBe(false); + expect(multi.children[2].isError).toBe(true); + expect(multi.children[3].isError).toBe(false); + }); + }); + + describe('.isMissing', () => { + it('returns true if the node was inserted via error recovery', () => { + tree = parser.parse('(2 ||)')!; + const node = tree.rootNode; + expect(node.toString()).toBe( + '(program (expression_statement (parenthesized_expression (binary_expression left: (number) right: (MISSING identifier)))))' + ); + + const sum = node.firstChild!.firstChild!.firstNamedChild!; + expect(sum.type).toBe('binary_expression'); + expect(sum.hasError).toBe(true); + expect(sum.children[0].isMissing).toBe(false); + expect(sum.children[1].isMissing).toBe(false); + expect(sum.children[2].isMissing).toBe(true); + }); + }); + + describe('.isExtra', () => { + it('returns true if the node is an extra node like comments', () => { + tree = parser.parse('foo(/* hi */);')!; + const node = tree.rootNode; + const commentNode = node.descendantForIndex(7, 7)!; + + expect(node.type).toBe('program'); + expect(commentNode.type).toBe('comment'); + expect(node.isExtra).toBe(false); + expect(commentNode.isExtra).toBe(true); + }); + }); + + describe('.text', () => { + const text = 'α0 / b👎c👎'; + + Object.entries({ + '.parse(String)': text, + '.parse(Function)': (offset: number) => text.slice(offset, offset + 4), + }).forEach(([method, _parse]) => { + it(`returns the text of a node generated by ${method}`, () => { + const [numeratorSrc, denominatorSrc] = text.split(/\s*\/\s+/); + tree = parser.parse(_parse)!; + const quotientNode = tree.rootNode.firstChild!.firstChild!; + const [numerator, slash, denominator] = quotientNode.children; + + expect(tree.rootNode.text).toBe(text); + expect(denominator.text).toBe(denominatorSrc); + expect(quotientNode.text).toBe(text); + expect(numerator.text).toBe(numeratorSrc); + expect(slash.text).toBe('/'); + }); + }); + }); + + describe('.descendantCount', () => { + it('returns the number of descendants', () => { + parser.setLanguage(JSON); + tree = parser.parse(JSON_EXAMPLE)!; + const valueNode = tree.rootNode; + const allNodes = getAllNodes(tree); + + expect(valueNode.descendantCount).toBe(allNodes.length); + + const cursor = tree.walk(); + for (let i = 0; i < allNodes.length; i++) { + const node = allNodes[i]; + cursor.gotoDescendant(i); + expect(cursor.currentNode.id).toBe(node.id); + } + + for (let i = allNodes.length - 1; i >= 0; i--) { + const node = allNodes[i]; + cursor.gotoDescendant(i); + expect(cursor.currentNode.id).toBe(node.id); + } + }); + + it('tests a single node tree', () => { + parser.setLanguage(EmbeddedTemplate); + tree = parser.parse('hello')!; + + const nodes = getAllNodes(tree); + expect(nodes).toHaveLength(2); + expect(tree.rootNode.descendantCount).toBe(2); + + const cursor = tree.walk(); + + cursor.gotoDescendant(0); + expect(cursor.currentDepth).toBe(0); + expect(cursor.currentNode.id).toBe(nodes[0].id); + + cursor.gotoDescendant(1); + expect(cursor.currentDepth).toBe(1); + expect(cursor.currentNode.id).toBe(nodes[1].id); + }); + }); + + describe('.rootNodeWithOffset', () => { + it('returns the root node of the tree, offset by the given byte offset', () => { + tree = parser.parse(' if (a) b')!; + const node = tree.rootNodeWithOffset(6, { row: 2, column: 2 }); + expect(node.startIndex).toBe(8); + expect(node.endIndex).toBe(16); + expect(node.startPosition).toEqual({ row: 2, column: 4 }); + expect(node.endPosition).toEqual({ row: 2, column: 12 }); + + let child = node.firstChild!.child(2)!; + expect(child.type).toBe('expression_statement'); + expect(child.startIndex).toBe(15); + expect(child.endIndex).toBe(16); + expect(child.startPosition).toEqual({ row: 2, column: 11 }); + expect(child.endPosition).toEqual({ row: 2, column: 12 }); + + const cursor = node.walk(); + cursor.gotoFirstChild(); + cursor.gotoFirstChild(); + cursor.gotoNextSibling(); + child = cursor.currentNode; + expect(child.type).toBe('parenthesized_expression'); + expect(child.startIndex).toBe(11); + expect(child.endIndex).toBe(14); + expect(child.startPosition).toEqual({ row: 2, column: 7 }); + expect(child.endPosition).toEqual({ row: 2, column: 10 }); + }); + }); + + describe('.parseState, .nextParseState', () => { + const text = '10 / 5'; + + it('returns node parse state ids', () => { + tree = parser.parse(text)!; + const quotientNode = tree.rootNode.firstChild!.firstChild!; + const [numerator, slash, denominator] = quotientNode.children; + + expect(tree.rootNode.parseState).toBe(0); + // parse states will change on any change to the grammar so test that it + // returns something instead + expect(numerator.parseState).toBeGreaterThan(0); + expect(slash.parseState).toBeGreaterThan(0); + expect(denominator.parseState).toBeGreaterThan(0); + }); + + it('returns next parse state equal to the language', () => { + tree = parser.parse(text)!; + const quotientNode = tree.rootNode.firstChild!.firstChild!; + quotientNode.children.forEach((node) => { + expect(node.nextParseState).toBe(JavaScript.nextState(node.parseState, node.grammarId)); + }); + }); + }); + + describe('.descendantsOfType("ERROR")', () => { + it('finds all of the descendants of an ERROR node', () => { + tree = parser.parse( + `if ({a: 'b'} {c: 'd'}) { + // ^ ERROR + x = function(a) { b; } function(c) { d; } + }` + )!; + const errorNode = tree.rootNode; + const descendants = errorNode.descendantsOfType('ERROR'); + expect( + descendants.map((node) => node.startIndex) + ).toEqual( + [4] + ); + }); + }); + + describe('.descendantsOfType', () => { + it('finds all descendants of a given type in the given range', () => { + tree = parser.parse('a + 1 * b * 2 + c + 3')!; + const outerSum = tree.rootNode.firstChild!.firstChild!; + + const descendants = outerSum.descendantsOfType('number', { row: 0, column: 2 }, { row: 0, column: 15 }); + expect(descendants.map(node => node.startIndex)).toEqual([4, 12]); + expect(descendants.map(node => node.endPosition)).toEqual([{ row: 0, column: 5 }, { row: 0, column: 13 }]); + }); + }); + + + + describe('.firstChildForIndex(index)', () => { + it('returns the first child that contains or starts after the given index', () => { + tree = parser.parse('x10 + 1000')!; + const sumNode = tree.rootNode.firstChild!.firstChild!; + + expect(sumNode.firstChildForIndex(0)!.type).toBe('identifier'); + expect(sumNode.firstChildForIndex(1)!.type).toBe('identifier'); + expect(sumNode.firstChildForIndex(3)!.type).toBe('+'); + expect(sumNode.firstChildForIndex(5)!.type).toBe('number'); + }); + }); + + describe('.firstNamedChildForIndex(index)', () => { + it('returns the first child that contains or starts after the given index', () => { + tree = parser.parse('x10 + 1000')!; + const sumNode = tree.rootNode.firstChild!.firstChild!; + + expect(sumNode.firstNamedChildForIndex(0)!.type).toBe('identifier'); + expect(sumNode.firstNamedChildForIndex(1)!.type).toBe('identifier'); + expect(sumNode.firstNamedChildForIndex(3)!.type).toBe('number'); + }); + }); + + describe('.equals(other)', () => { + it('returns true if the nodes are the same', () => { + tree = parser.parse('1 + 2')!; + + const sumNode = tree.rootNode.firstChild!.firstChild!; + const node1 = sumNode.firstChild!; + const node2 = sumNode.firstChild!; + expect(node1.equals(node2)).toBe(true); + }); + + it('returns false if the nodes are not the same', () => { + tree = parser.parse('1 + 2')!; + + const sumNode = tree.rootNode.firstChild!.firstChild!; + const node1 = sumNode.firstChild!; + const node2 = node1.nextSibling!; + expect(node1.equals(node2)).toBe(false); + }); + }); + + describe('.fieldNameForChild(index)', () => { + it('returns the field of a child or null', () => { + parser.setLanguage(C); + tree = parser.parse('int w = x + /* y is special! */ y;')!; + + const translationUnitNode = tree.rootNode; + const declarationNode = translationUnitNode.firstChild; + const binaryExpressionNode = declarationNode! + .childForFieldName('declarator')! + .childForFieldName('value')!; + + // ------------------- + // left: (identifier) 0 + // operator: "+" 1 <--- (not a named child) + // (comment) 2 <--- (is an extra) + // right: (identifier) 3 + // ------------------- + + expect(binaryExpressionNode.fieldNameForChild(0)).toBe('left'); + expect(binaryExpressionNode.fieldNameForChild(1)).toBe('operator'); + // The comment should not have a field name, as it's just an extra + expect(binaryExpressionNode.fieldNameForChild(2)).toBeNull(); + expect(binaryExpressionNode.fieldNameForChild(3)).toBe('right'); + // Negative test - Not a valid child index + expect(binaryExpressionNode.fieldNameForChild(4)).toBeNull(); + }); + }); + + describe('.fieldNameForNamedChild(index)', () => { + it('returns the field of a named child or null', () => { + parser.setLanguage(C); + tree = parser.parse('int w = x + /* y is special! */ y;')!; + + const translationUnitNode = tree.rootNode; + const declarationNode = translationUnitNode.firstNamedChild; + const binaryExpressionNode = declarationNode! + .childForFieldName('declarator')! + .childForFieldName('value')!; + + // ------------------- + // left: (identifier) 0 + // operator: "+" _ <--- (not a named child) + // (comment) 1 <--- (is an extra) + // right: (identifier) 2 + // ------------------- + + expect(binaryExpressionNode.fieldNameForNamedChild(0)).toBe('left'); + // The comment should not have a field name, as it's just an extra + expect(binaryExpressionNode.fieldNameForNamedChild(1)).toBeNull(); + // The operator is not a named child, so the named child at index 2 is the right child + expect(binaryExpressionNode.fieldNameForNamedChild(2)).toBe('right'); + // Negative test - Not a valid child index + expect(binaryExpressionNode.fieldNameForNamedChild(3)).toBeNull(); + }); + }); +}); diff --git a/lib/binding_web/test/parser-test.js b/lib/binding_web/test/parser-test.js deleted file mode 100644 index 4c58d020..00000000 --- a/lib/binding_web/test/parser-test.js +++ /dev/null @@ -1,392 +0,0 @@ -const {assert} = require('chai'); -let Parser; let JavaScript; let HTML; let languageURL; - -describe('Parser', () => { - let parser; - - before(async () => - ({Parser, JavaScript, HTML, languageURL} = await require('./helper')), - ); - - beforeEach(() => { - parser = new Parser(); - }); - - afterEach(() => { - parser.delete(); - }); - - describe('.setLanguage', () => { - it('allows setting the language to null', () => { - assert.equal(parser.getLanguage(), null); - parser.setLanguage(JavaScript); - assert.equal(parser.getLanguage(), JavaScript); - parser.setLanguage(null); - assert.equal(parser.getLanguage(), null); - }); - - it('throws an exception when the given object is not a tree-sitter language', () => { - assert.throws(() => parser.setLanguage({}), /Argument must be a Language/); - assert.throws(() => parser.setLanguage(1), /Argument must be a Language/); - }); - }); - - describe('.setLogger', () => { - beforeEach(() => { - parser.setLanguage(JavaScript); - }); - - it('calls the given callback for each parse event', () => { - const debugMessages = []; - parser.setLogger((message) => debugMessages.push(message)); - parser.parse('a + b + c'); - assert.includeMembers(debugMessages, [ - 'skip character:\' \'', - 'consume character:\'b\'', - 'reduce sym:program, child_count:1', - 'accept', - ]); - }); - - it('allows the callback to be retrieved later', () => { - const callback = () => {}; - parser.setLogger(callback); - assert.equal(parser.getLogger(), callback); - parser.setLogger(false); - assert.equal(parser.getLogger(), null); - }); - - it('disables debugging when given a falsy value', () => { - const debugMessages = []; - parser.setLogger((message) => debugMessages.push(message)); - parser.setLogger(false); - parser.parse('a + b * c'); - assert.equal(debugMessages.length, 0); - }); - - it('throws an error when given a truthy value that isn\'t a function ', () => { - assert.throws( - () => parser.setLogger('5'), - 'Logger callback must be a function', - ); - }); - - it('rethrows errors thrown by the logging callback', () => { - const error = new Error('The error message'); - parser.setLogger((_msg, _params) => { - throw error; - }); - assert.throws( - () => parser.parse('ok;'), - 'The error message', - ); - }); - }); - - describe('one included range', () => { - it('parses the text within a range', () => { - parser.setLanguage(HTML); - const sourceCode = 'hi'; - const htmlTree = parser.parse(sourceCode); - const scriptContentNode = htmlTree.rootNode.child(1).child(1); - assert.equal(scriptContentNode.type, 'raw_text'); - - parser.setLanguage(JavaScript); - assert.deepEqual(parser.getIncludedRanges(), [{ - startIndex: 0, - endIndex: 2147483647, - startPosition: {row: 0, column: 0}, - endPosition: {row: 4294967295, column: 2147483647}, - }]); - const ranges = [{ - startIndex: scriptContentNode.startIndex, - endIndex: scriptContentNode.endIndex, - startPosition: scriptContentNode.startPosition, - endPosition: scriptContentNode.endPosition, - }]; - const jsTree = parser.parse( - sourceCode, - null, - {includedRanges: ranges}, - ); - assert.deepEqual(parser.getIncludedRanges(), ranges); - - assert.equal( - jsTree.rootNode.toString(), - '(program (expression_statement (call_expression ' + - 'function: (member_expression object: (identifier) property: (property_identifier)) ' + - 'arguments: (arguments (string (string_fragment))))))', - ); - assert.deepEqual(jsTree.rootNode.startPosition, {row: 0, column: sourceCode.indexOf('console')}); - }); - }); - - describe('multiple included ranges', () => { - it('parses the text within multiple ranges', () => { - parser.setLanguage(JavaScript); - const sourceCode = 'html `
Hello, ${name.toUpperCase()}, it\'s ${now()}.
`'; - const jsTree = parser.parse(sourceCode); - const templateStringNode = jsTree.rootNode.descendantForIndex(sourceCode.indexOf('`<'), sourceCode.indexOf('>`')); - assert.equal(templateStringNode.type, 'template_string'); - - const openQuoteNode = templateStringNode.child(0); - const interpolationNode1 = templateStringNode.child(2); - const interpolationNode2 = templateStringNode.child(4); - const closeQuoteNode = templateStringNode.child(6); - - parser.setLanguage(HTML); - const htmlRanges = [ - { - startIndex: openQuoteNode.endIndex, - startPosition: openQuoteNode.endPosition, - endIndex: interpolationNode1.startIndex, - endPosition: interpolationNode1.startPosition, - }, - { - startIndex: interpolationNode1.endIndex, - startPosition: interpolationNode1.endPosition, - endIndex: interpolationNode2.startIndex, - endPosition: interpolationNode2.startPosition, - }, - { - startIndex: interpolationNode2.endIndex, - startPosition: interpolationNode2.endPosition, - endIndex: closeQuoteNode.startIndex, - endPosition: closeQuoteNode.startPosition, - }, - ]; - const htmlTree = parser.parse(sourceCode, null, {includedRanges: htmlRanges}); - - assert.equal( - htmlTree.rootNode.toString(), - '(document (element' + - ' (start_tag (tag_name))' + - ' (text)' + - ' (element (start_tag (tag_name)) (end_tag (tag_name)))' + - ' (text)' + - ' (end_tag (tag_name))))', - ); - assert.deepEqual(htmlTree.getIncludedRanges(), htmlRanges); - - const divElementNode = htmlTree.rootNode.child(0); - const helloTextNode = divElementNode.child(1); - const bElementNode = divElementNode.child(2); - const bStartTagNode = bElementNode.child(0); - const bEndTagNode = bElementNode.child(1); - - assert.equal(helloTextNode.type, 'text'); - assert.equal(helloTextNode.startIndex, sourceCode.indexOf('Hello')); - assert.equal(helloTextNode.endIndex, sourceCode.indexOf(' ')); - - assert.equal(bStartTagNode.type, 'start_tag'); - assert.equal(bStartTagNode.startIndex, sourceCode.indexOf('')); - assert.equal(bStartTagNode.endIndex, sourceCode.indexOf('${now()}')); - - assert.equal(bEndTagNode.type, 'end_tag'); - assert.equal(bEndTagNode.startIndex, sourceCode.indexOf('')); - assert.equal(bEndTagNode.endIndex, sourceCode.indexOf('.')); - }); - }); - - describe('an included range containing mismatched positions', () => { - it('parses the text within the range', () => { - const sourceCode = '
test
{_ignore_this_part_}'; - - parser.setLanguage(HTML); - - const endIndex = sourceCode.indexOf('{_ignore_this_part_'); - - const rangeToParse = { - startIndex: 0, - startPosition: {row: 10, column: 12}, - endIndex, - endPosition: {row: 10, column: 12 + endIndex}, - }; - - const htmlTree = parser.parse(sourceCode, null, {includedRanges: [rangeToParse]}); - - assert.deepEqual(htmlTree.getIncludedRanges()[0], rangeToParse); - - assert.equal( - htmlTree.rootNode.toString(), - '(document (element (start_tag (tag_name)) (text) (end_tag (tag_name))))', - ); - }); - }); - - describe('.parse', () => { - let tree; - - beforeEach(() => { - tree = null; - parser.setLanguage(JavaScript); - }); - - afterEach(() => { - if (tree) tree.delete(); - }); - - it('reads from the given input', () => { - const parts = ['first', '_', 'second', '_', 'third']; - tree = parser.parse(() => parts.shift()); - assert.equal(tree.rootNode.toString(), '(program (expression_statement (identifier)))'); - }); - - it('stops reading when the input callback return something that\'s not a string', () => { - const parts = ['abc', 'def', 'ghi', {}, {}, {}, 'second-word', ' ']; - tree = parser.parse(() => parts.shift()); - assert.equal( - tree.rootNode.toString(), - '(program (expression_statement (identifier)))', - ); - assert.equal(tree.rootNode.endIndex, 9); - assert.equal(parts.length, 2); - }); - - it('throws an exception when the given input is not a function', () => { - assert.throws(() => parser.parse(null), 'Argument must be a string or a function'); - assert.throws(() => parser.parse(5), 'Argument must be a string or a function'); - assert.throws(() => parser.parse({}), 'Argument must be a string or a function'); - }); - - it('handles long input strings', () => { - const repeatCount = 10000; - const inputString = `[${Array(repeatCount).fill('0').join(',')}]`; - - tree = parser.parse(inputString); - assert.equal(tree.rootNode.type, 'program'); - assert.equal(tree.rootNode.firstChild.firstChild.namedChildCount, repeatCount); - }).timeout(5000); - - it('can use the bash parser', async () => { - parser.setLanguage(await Parser.Language.load(languageURL('bash'))); - tree = parser.parse('FOO=bar echo < err.txt > hello.txt \nhello${FOO}\nEOF'); - assert.equal( - tree.rootNode.toString(), - '(program ' + - '(redirected_statement ' + - 'body: (command ' + - '(variable_assignment name: (variable_name) value: (word)) ' + - 'name: (command_name (word))) ' + - 'redirect: (heredoc_redirect (heredoc_start) ' + - 'redirect: (file_redirect descriptor: (file_descriptor) destination: (word)) ' + - 'redirect: (file_redirect destination: (word)) ' + - '(heredoc_body ' + - '(expansion (variable_name)) (heredoc_content)) (heredoc_end))))', - ); - }).timeout(5000); - - it('can use the c++ parser', async () => { - parser.setLanguage(await Parser.Language.load(languageURL('cpp'))); - tree = parser.parse('const char *s = R"EOF(HELLO WORLD)EOF";'); - assert.equal( - tree.rootNode.toString(), - '(translation_unit (declaration ' + - '(type_qualifier) ' + - 'type: (primitive_type) ' + - 'declarator: (init_declarator ' + - 'declarator: (pointer_declarator declarator: (identifier)) ' + - 'value: (raw_string_literal delimiter: (raw_string_delimiter) (raw_string_content) (raw_string_delimiter)))))', - ); - }).timeout(5000); - - it('can use the HTML parser', async () => { - parser.setLanguage(await Parser.Language.load(languageURL('html'))); - tree = parser.parse('
'); - assert.equal( - tree.rootNode.toString(), - '(document (element (start_tag (tag_name)) (element (start_tag (tag_name)) (element (start_tag (tag_name)) (end_tag (tag_name))) (end_tag (tag_name))) (end_tag (tag_name))))', - ); - }).timeout(5000); - - it('can use the python parser', async () => { - parser.setLanguage(await Parser.Language.load(languageURL('python'))); - tree = parser.parse('class A:\n def b():\n c()'); - assert.equal( - tree.rootNode.toString(), - '(module (class_definition ' + - 'name: (identifier) ' + - 'body: (block ' + - '(function_definition ' + - 'name: (identifier) ' + - 'parameters: (parameters) ' + - 'body: (block (expression_statement (call ' + - 'function: (identifier) ' + - 'arguments: (argument_list))))))))', - ); - }).timeout(5000); - - it('can use the rust parser', async () => { - parser.setLanguage(await Parser.Language.load(languageURL('rust'))); - tree = parser.parse('const x: &\'static str = r###"hello"###;'); - assert.equal( - tree.rootNode.toString(), - '(source_file (const_item ' + - 'name: (identifier) ' + - 'type: (reference_type (lifetime (identifier)) type: (primitive_type)) ' + - 'value: (raw_string_literal (string_content))))', - ); - }).timeout(5000); - - it('can use the typescript parser', async () => { - parser.setLanguage(await Parser.Language.load(languageURL('typescript'))); - tree = parser.parse('a()\nb()\n[c]'); - assert.equal( - tree.rootNode.toString(), - '(program ' + - '(expression_statement (call_expression function: (identifier) arguments: (arguments))) ' + - '(expression_statement (subscript_expression ' + - 'object: (call_expression ' + - 'function: (identifier) ' + - 'arguments: (arguments)) ' + - 'index: (identifier))))', - ); - }).timeout(5000); - - it('can use the tsx parser', async () => { - parser.setLanguage(await Parser.Language.load(languageURL('tsx'))); - tree = parser.parse('a()\nb()\n[c]'); - assert.equal( - tree.rootNode.toString(), - '(program ' + - '(expression_statement (call_expression function: (identifier) arguments: (arguments))) ' + - '(expression_statement (subscript_expression ' + - 'object: (call_expression ' + - 'function: (identifier) ' + - 'arguments: (arguments)) ' + - 'index: (identifier))))', - ); - }).timeout(5000); - - it('parses only the text within the `includedRanges` if they are specified', () => { - const sourceCode = '<% foo() %> <% bar %>'; - - const start1 = sourceCode.indexOf('foo'); - const end1 = start1 + 5; - const start2 = sourceCode.indexOf('bar'); - const end2 = start2 + 3; - - const tree = parser.parse(sourceCode, null, { - includedRanges: [ - { - startIndex: start1, - endIndex: end1, - startPosition: {row: 0, column: start1}, - endPosition: {row: 0, column: end1}, - }, - { - startIndex: start2, - endIndex: end2, - startPosition: {row: 0, column: start2}, - endPosition: {row: 0, column: end2}, - }, - ], - }); - - assert.equal( - tree.rootNode.toString(), - '(program (expression_statement (call_expression function: (identifier) arguments: (arguments))) (expression_statement (identifier)))', - ); - }); - }); -}); diff --git a/lib/binding_web/test/parser.test.ts b/lib/binding_web/test/parser.test.ts new file mode 100644 index 00000000..7d859827 --- /dev/null +++ b/lib/binding_web/test/parser.test.ts @@ -0,0 +1,446 @@ +import { describe, it, expect, beforeAll, beforeEach, afterEach } from 'vitest'; +import helper, { type LanguageName } from './helper'; +import type { ParseState, Tree } from '../src'; +import { Parser, Language } from '../src'; + +let JavaScript: Language; +let HTML: Language; +let JSON: Language; +let languageURL: (name: LanguageName) => string; + +describe('Parser', () => { + let parser: Parser; + + beforeAll(async () => { + ({ JavaScript, HTML, JSON, languageURL } = await helper); + }); + + beforeEach(() => { + parser = new Parser(); + }); + + afterEach(() => { + parser.delete(); + }); + + describe('.setLanguage', () => { + it('allows setting the language to null', () => { + expect(parser.language).toBeNull(); + parser.setLanguage(JavaScript); + expect(parser.language).toBe(JavaScript); + parser.setLanguage(null); + expect(parser.language).toBeNull(); + }); + + it('throws an exception when the given object is not a tree-sitter language', () => { + // @ts-expect-error Testing invalid arguments + expect(() => { parser.setLanguage({}); }).toThrow(/Argument must be a Language/); + // @ts-expect-error Testing invalid arguments + expect(() => { parser.setLanguage(1); }).toThrow(/Argument must be a Language/); + }); + }); + + describe('.setLogger', () => { + beforeEach(() => { + parser.setLanguage(JavaScript); + }); + + it('calls the given callback for each parse event', () => { + const debugMessages: string[] = []; + parser.setLogger((message) => debugMessages.push(message)); + parser.parse('a + b + c')!; + expect(debugMessages).toEqual(expect.arrayContaining([ + 'skip character:\' \'', + 'consume character:\'b\'', + 'reduce sym:program, child_count:1', + 'accept' + ])); + }); + + it('allows the callback to be retrieved later', () => { + const callback = () => { return; }; + parser.setLogger(callback); + expect(parser.getLogger()).toBe(callback); + parser.setLogger(false); + expect(parser.getLogger()).toBeNull(); + }); + + it('disables debugging when given a falsy value', () => { + const debugMessages: string[] = []; + parser.setLogger((message) => debugMessages.push(message)); + parser.setLogger(false); + parser.parse('a + b * c')!; + expect(debugMessages).toHaveLength(0); + }); + + it('throws an error when given a truthy value that isn\'t a function', () => { + // @ts-expect-error Testing invalid arguments + expect(() => { parser.setLogger('5'); }).toThrow('Logger callback must be a function'); + }); + + it('rethrows errors thrown by the logging callback', () => { + const error = new Error('The error message'); + parser.setLogger(() => { + throw error; + }); + expect(() => parser.parse('ok;')).toThrow('The error message'); + }); + }); + + describe('one included range', () => { + it('parses the text within a range', () => { + parser.setLanguage(HTML); + const sourceCode = 'hi'; + const htmlTree = parser.parse(sourceCode)!; + const scriptContentNode = htmlTree.rootNode.child(1)!.child(1)!; + expect(scriptContentNode.type).toBe('raw_text'); + + parser.setLanguage(JavaScript); + expect(parser.getIncludedRanges()).toEqual([{ + startIndex: 0, + endIndex: 2147483647, + startPosition: { row: 0, column: 0 }, + endPosition: { row: 4294967295, column: 2147483647 } + }]); + + const ranges = [{ + startIndex: scriptContentNode.startIndex, + endIndex: scriptContentNode.endIndex, + startPosition: scriptContentNode.startPosition, + endPosition: scriptContentNode.endPosition, + }]; + + const jsTree = parser.parse( + sourceCode, + null, + { includedRanges: ranges } + )!; + expect(parser.getIncludedRanges()).toEqual(ranges); + + expect(jsTree.rootNode.toString()).toBe( + '(program (expression_statement (call_expression ' + + 'function: (member_expression object: (identifier) property: (property_identifier)) ' + + 'arguments: (arguments (string (string_fragment))))))' + ); + expect(jsTree.rootNode.startPosition).toEqual({ row: 0, column: sourceCode.indexOf('console') }); + }); + }); + + describe('multiple included ranges', () => { + it('parses the text within multiple ranges', () => { + parser.setLanguage(JavaScript); + const sourceCode = 'html `
Hello, ${name.toUpperCase()}, it\'s ${now()}.
`'; + const jsTree = parser.parse(sourceCode)!; + const templateStringNode = jsTree.rootNode.descendantForIndex( + sourceCode.indexOf('`<'), + sourceCode.indexOf('>`') + )!; + expect(templateStringNode.type).toBe('template_string'); + + const openQuoteNode = templateStringNode.child(0)!; + const interpolationNode1 = templateStringNode.child(2)!; + const interpolationNode2 = templateStringNode.child(4)!; + const closeQuoteNode = templateStringNode.child(6)!; + + parser.setLanguage(HTML); + const htmlRanges = [ + { + startIndex: openQuoteNode.endIndex, + startPosition: openQuoteNode.endPosition, + endIndex: interpolationNode1.startIndex, + endPosition: interpolationNode1.startPosition, + }, + { + startIndex: interpolationNode1.endIndex, + startPosition: interpolationNode1.endPosition, + endIndex: interpolationNode2.startIndex, + endPosition: interpolationNode2.startPosition, + }, + { + startIndex: interpolationNode2.endIndex, + startPosition: interpolationNode2.endPosition, + endIndex: closeQuoteNode.startIndex, + endPosition: closeQuoteNode.startPosition, + }, + ]; + + const htmlTree = parser.parse(sourceCode, null, { includedRanges: htmlRanges })!; + + expect(htmlTree.rootNode.toString()).toBe( + '(document (element' + + ' (start_tag (tag_name))' + + ' (text)' + + ' (element (start_tag (tag_name)) (end_tag (tag_name)))' + + ' (text)' + + ' (end_tag (tag_name))))' + ); + expect(htmlTree.getIncludedRanges()).toEqual(htmlRanges); + + const divElementNode = htmlTree.rootNode.child(0)!; + const helloTextNode = divElementNode.child(1)!; + const bElementNode = divElementNode.child(2)!; + const bStartTagNode = bElementNode.child(0)!; + const bEndTagNode = bElementNode.child(1)!; + + expect(helloTextNode.type).toBe('text'); + expect(helloTextNode.startIndex).toBe(sourceCode.indexOf('Hello')); + expect(helloTextNode.endIndex).toBe(sourceCode.indexOf(' ')); + + expect(bStartTagNode.type).toBe('start_tag'); + expect(bStartTagNode.startIndex).toBe(sourceCode.indexOf('')); + expect(bStartTagNode.endIndex).toBe(sourceCode.indexOf('${now()}')); + + expect(bEndTagNode.type).toBe('end_tag'); + expect(bEndTagNode.startIndex).toBe(sourceCode.indexOf('')); + expect(bEndTagNode.endIndex).toBe(sourceCode.indexOf('.')); + }); + }); + + describe('an included range containing mismatched positions', () => { + it('parses the text within the range', () => { + const sourceCode = '
test
{_ignore_this_part_}'; + + parser.setLanguage(HTML); + + const endIndex = sourceCode.indexOf('{_ignore_this_part_'); + + const rangeToParse = { + startIndex: 0, + startPosition: { row: 10, column: 12 }, + endIndex, + endPosition: { row: 10, column: 12 + endIndex }, + }; + + const htmlTree = parser.parse(sourceCode, null, { includedRanges: [rangeToParse] })!; + + expect(htmlTree.getIncludedRanges()[0]).toEqual(rangeToParse); + + expect(htmlTree.rootNode.toString()).toBe( + '(document (element (start_tag (tag_name)) (text) (end_tag (tag_name))))' + ); + }); + }); + + describe('.parse', () => { + let tree: Tree | null; + + beforeEach(() => { + tree = null; + parser.setLanguage(JavaScript); + }); + + afterEach(() => { + if (tree) tree.delete(); + }); + + it('reads from the given input', () => { + const parts = ['first', '_', 'second', '_', 'third']; + tree = parser.parse(() => parts.shift())!; + expect(tree.rootNode.toString()).toBe('(program (expression_statement (identifier)))'); + }); + + it('stops reading when the input callback returns something that\'s not a string', () => { + const parts = ['abc', 'def', 'ghi', {}, {}, {}, 'second-word', ' ']; + tree = parser.parse(() => parts.shift() as string)!; + expect(tree.rootNode.toString()).toBe('(program (expression_statement (identifier)))'); + expect(tree.rootNode.endIndex).toBe(9); + expect(parts).toHaveLength(2); + }); + + it('throws an exception when the given input is not a function', () => { + // @ts-expect-error Testing invalid arguments + expect(() => parser.parse(null)).toThrow('Argument must be a string or a function'); + // @ts-expect-error Testing invalid arguments + expect(() => parser.parse(5)).toThrow('Argument must be a string or a function'); + // @ts-expect-error Testing invalid arguments + expect(() => parser.parse({})).toThrow('Argument must be a string or a function'); + }); + + it('handles long input strings', { timeout: 10000 }, () => { + const repeatCount = 10000; + const inputString = `[${Array(repeatCount).fill('0').join(',')}]`; + + tree = parser.parse(inputString)!; + expect(tree.rootNode.type).toBe('program'); + expect(tree.rootNode.firstChild!.firstChild!.namedChildCount).toBe(repeatCount); + }); + + it('can use the bash parser', { timeout: 5000 }, async () => { + parser.setLanguage(await Language.load(languageURL('bash'))); + tree = parser.parse('FOO=bar echo < err.txt > hello.txt \nhello${FOO}\nEOF')!; + expect(tree.rootNode.toString()).toBe( + '(program ' + + '(redirected_statement ' + + 'body: (command ' + + '(variable_assignment name: (variable_name) value: (word)) ' + + 'name: (command_name (word))) ' + + 'redirect: (heredoc_redirect (heredoc_start) ' + + 'redirect: (file_redirect descriptor: (file_descriptor) destination: (word)) ' + + 'redirect: (file_redirect destination: (word)) ' + + '(heredoc_body ' + + '(expansion (variable_name)) (heredoc_content)) (heredoc_end))))' + ); + }); + + it('can use the c++ parser', { timeout: 5000 }, async () => { + parser.setLanguage(await Language.load(languageURL('cpp'))); + tree = parser.parse('const char *s = R"EOF(HELLO WORLD)EOF";')!; + expect(tree.rootNode.toString()).toBe( + '(translation_unit (declaration ' + + '(type_qualifier) ' + + 'type: (primitive_type) ' + + 'declarator: (init_declarator ' + + 'declarator: (pointer_declarator declarator: (identifier)) ' + + 'value: (raw_string_literal delimiter: (raw_string_delimiter) (raw_string_content) (raw_string_delimiter)))))' + ); + }); + + it('can use the HTML parser', { timeout: 5000 }, async () => { + parser.setLanguage(await Language.load(languageURL('html'))); + tree = parser.parse('
')!; + expect(tree.rootNode.toString()).toBe( + '(document (element (start_tag (tag_name)) (element (start_tag (tag_name)) ' + + '(element (start_tag (tag_name)) (end_tag (tag_name))) (end_tag (tag_name))) (end_tag (tag_name))))' + ); + }); + + it('can use the python parser', { timeout: 5000 }, async () => { + parser.setLanguage(await Language.load(languageURL('python'))); + tree = parser.parse('class A:\n def b():\n c()')!; + expect(tree.rootNode.toString()).toBe( + '(module (class_definition ' + + 'name: (identifier) ' + + 'body: (block ' + + '(function_definition ' + + 'name: (identifier) ' + + 'parameters: (parameters) ' + + 'body: (block (expression_statement (call ' + + 'function: (identifier) ' + + 'arguments: (argument_list))))))))' + ); + }); + + it('can use the rust parser', { timeout: 5000 }, async () => { + parser.setLanguage(await Language.load(languageURL('rust'))); + tree = parser.parse('const x: &\'static str = r###"hello"###;')!; + expect(tree.rootNode.toString()).toBe( + '(source_file (const_item ' + + 'name: (identifier) ' + + 'type: (reference_type (lifetime (identifier)) type: (primitive_type)) ' + + 'value: (raw_string_literal (string_content))))' + ); + }); + + it('can use the typescript parser', { timeout: 5000 }, async () => { + parser.setLanguage(await Language.load(languageURL('typescript'))); + tree = parser.parse('a()\nb()\n[c]')!; + expect(tree.rootNode.toString()).toBe( + '(program ' + + '(expression_statement (call_expression function: (identifier) arguments: (arguments))) ' + + '(expression_statement (subscript_expression ' + + 'object: (call_expression ' + + 'function: (identifier) ' + + 'arguments: (arguments)) ' + + 'index: (identifier))))' + ); + }); + + it('can use the tsx parser', { timeout: 5000 }, async () => { + parser.setLanguage(await Language.load(languageURL('tsx'))); + tree = parser.parse('a()\nb()\n[c]')!; + expect(tree.rootNode.toString()).toBe( + '(program ' + + '(expression_statement (call_expression function: (identifier) arguments: (arguments))) ' + + '(expression_statement (subscript_expression ' + + 'object: (call_expression ' + + 'function: (identifier) ' + + 'arguments: (arguments)) ' + + 'index: (identifier))))', + + ); + }); + + it('parses only the text within the `includedRanges` if they are specified', () => { + const sourceCode = '<% foo() %> <% bar %>'; + + const start1 = sourceCode.indexOf('foo'); + const end1 = start1 + 5; + const start2 = sourceCode.indexOf('bar'); + const end2 = start2 + 3; + + const tree = parser.parse(sourceCode, null, { + includedRanges: [ + { + startIndex: start1, + endIndex: end1, + startPosition: { row: 0, column: start1 }, + endPosition: { row: 0, column: end1 }, + }, + { + startIndex: start2, + endIndex: end2, + startPosition: { row: 0, column: start2 }, + endPosition: { row: 0, column: end2 }, + }, + ], + })!; + + expect(tree.rootNode.toString()).toBe( + '(program ' + + '(expression_statement (call_expression function: (identifier) arguments: (arguments))) ' + + '(expression_statement (identifier)))' + ); + }); + + it('parses with a timeout', { timeout: 5000 }, () => { + parser.setLanguage(JSON); + + const startTime = performance.now(); + let currentByteOffset = 0; + const progressCallback = (state: ParseState) => { + expect(state.currentOffset).toBeGreaterThanOrEqual(currentByteOffset); + currentByteOffset = state.currentOffset; + + if (performance.now() - startTime > 1) { + return true; + } + return false; + }; + + expect(parser.parse( + (offset) => offset === 0 ? '[' : ',0', + null, + { progressCallback }, + )).toBeNull(); + }); + + it('times out when an error is detected', { timeout: 5000 }, () => { + parser.setLanguage(JSON); + + let offset = 0; + const erroneousCode = '!,'; + const progressCallback = (state: ParseState) => { + offset = state.currentOffset; + return state.hasError; + }; + + const tree = parser.parse( + (offset) => { + if (offset === 0) return '['; + if (offset >= 1 && offset < 1000) return '0,'; + return erroneousCode; + }, + null, + { progressCallback }, + ); + + // The callback is called at the end of parsing, however, what we're asserting here is that + // parsing ends immediately as the error is detected. This is verified by checking the offset + // of the last byte processed is the length of the erroneous code we inserted, aka, 1002, or + // 1000 + the length of the erroneous code. Note that in this Wasm test, we multiply the offset + // by 2 because JavaScript strings are UTF-16 encoded. + expect(offset).toBe((1000 + erroneousCode.length) * 2); + expect(tree).toBeNull(); + }); + }); +}); diff --git a/lib/binding_web/test/query-test.js b/lib/binding_web/test/query-test.js deleted file mode 100644 index db4c10f8..00000000 --- a/lib/binding_web/test/query-test.js +++ /dev/null @@ -1,481 +0,0 @@ -const {assert} = require('chai'); -let Parser; let JavaScript; - -describe('Query', () => { - let parser; let tree; let query; - - before(async () => ({Parser, JavaScript} = await require('./helper'))); - - beforeEach(() => { - parser = new Parser().setLanguage(JavaScript); - }); - - afterEach(() => { - parser.delete(); - if (tree) tree.delete(); - if (query) query.delete(); - }); - - describe('construction', () => { - it('throws an error on invalid patterns', () => { - assert.throws(() => { - JavaScript.query('(function_declaration wat)'); - }, 'Bad syntax at offset 22: \'wat)\'...'); - assert.throws(() => { - JavaScript.query('(non_existent)'); - }, 'Bad node name \'non_existent\''); - assert.throws(() => { - JavaScript.query('(a)'); - }, 'Bad node name \'a\''); - assert.throws(() => { - JavaScript.query('(function_declaration non_existent:(identifier))'); - }, 'Bad field name \'non_existent\''); - assert.throws(() => { - JavaScript.query('(function_declaration name:(statement_block))'); - }, 'Bad pattern structure at offset 22: \'name:(statement_block))\''); - }); - - it('throws an error on invalid predicates', () => { - assert.throws(() => { - JavaScript.query('((identifier) @abc (#eq? @ab hi))'); - }, 'Bad capture name @ab'); - assert.throws(() => { - JavaScript.query('((identifier) @abc (#eq? @ab hi))'); - }, 'Bad capture name @ab'); - assert.throws(() => { - JavaScript.query('((identifier) @abc (#eq?))'); - }, 'Wrong number of arguments to `#eq?` predicate. Expected 2, got 0'); - assert.throws(() => { - JavaScript.query('((identifier) @a (#eq? @a @a @a))'); - }, 'Wrong number of arguments to `#eq?` predicate. Expected 2, got 3'); - }); - }); - - describe('.matches', () => { - it('returns all of the matches for the given query', () => { - tree = parser.parse('function one() { two(); function three() {} }'); - query = JavaScript.query(` - (function_declaration name: (identifier) @fn-def) - (call_expression function: (identifier) @fn-ref) - `); - const matches = query.matches(tree.rootNode); - assert.deepEqual(formatMatches(matches), [ - {pattern: 0, captures: [{name: 'fn-def', text: 'one'}]}, - {pattern: 1, captures: [{name: 'fn-ref', text: 'two'}]}, - {pattern: 0, captures: [{name: 'fn-def', text: 'three'}]}, - ]); - }); - - it('can search in a specified ranges', () => { - tree = parser.parse('[a, b,\nc, d,\ne, f,\ng, h]'); - query = JavaScript.query('(identifier) @element'); - const matches = query.matches( - tree.rootNode, - { - startPosition: {row: 1, column: 1}, - endPosition: {row: 3, column: 1}, - }, - ); - assert.deepEqual(formatMatches(matches), [ - {pattern: 0, captures: [{name: 'element', text: 'd'}]}, - {pattern: 0, captures: [{name: 'element', text: 'e'}]}, - {pattern: 0, captures: [{name: 'element', text: 'f'}]}, - {pattern: 0, captures: [{name: 'element', text: 'g'}]}, - ]); - }); - - it('handles predicates that compare the text of capture to literal strings', () => { - tree = parser.parse(` - giraffe(1, 2, []); - helment([false]); - goat(false); - gross(3, []); - hiccup([]); - gaff(5); - `); - - // Find all calls to functions beginning with 'g', where one argument - // is an array literal. - query = JavaScript.query(` - (call_expression - function: (identifier) @name - arguments: (arguments (array)) - (#match? @name "^g")) - `); - - const matches = query.matches(tree.rootNode); - assert.deepEqual(formatMatches(matches), [ - {pattern: 0, captures: [{name: 'name', text: 'giraffe'}]}, - {pattern: 0, captures: [{name: 'name', text: 'gross'}]}, - ]); - }); - - it('handles multiple matches where the first one is filtered', () => { - tree = parser.parse(` - const a = window.b; - `); - - query = JavaScript.query(` - ((identifier) @variable.builtin - (#match? @variable.builtin "^(arguments|module|console|window|document)$") - (#is-not? local)) - `); - - const matches = query.matches(tree.rootNode); - assert.deepEqual(formatMatches(matches), [ - {pattern: 0, captures: [{name: 'variable.builtin', text: 'window'}]}, - ]); - }); - }); - - describe('.captures', () => { - it('returns all of the captures for the given query, in order', () => { - tree = parser.parse(` - a({ - bc: function de() { - const fg = function hi() {} - }, - jk: function lm() { - const no = function pq() {} - }, - }); - `); - query = JavaScript.query(` - (pair - key: _ @method.def - (function_expression - name: (identifier) @method.alias)) - - (variable_declarator - name: _ @function.def - value: (function_expression - name: (identifier) @function.alias)) - - ":" @delimiter - "=" @operator - `); - - const captures = query.captures(tree.rootNode); - assert.deepEqual(formatCaptures(captures), [ - {name: 'method.def', text: 'bc'}, - {name: 'delimiter', text: ':'}, - {name: 'method.alias', text: 'de'}, - {name: 'function.def', text: 'fg'}, - {name: 'operator', text: '='}, - {name: 'function.alias', text: 'hi'}, - {name: 'method.def', text: 'jk'}, - {name: 'delimiter', text: ':'}, - {name: 'method.alias', text: 'lm'}, - {name: 'function.def', text: 'no'}, - {name: 'operator', text: '='}, - {name: 'function.alias', text: 'pq'}, - ]); - }); - - it('handles conditions that compare the text of capture to literal strings', () => { - tree = parser.parse(` - lambda - panda - load - toad - const ab = require('./ab'); - new Cd(EF); - `); - - query = JavaScript.query(` - ((identifier) @variable - (#not-match? @variable "^(lambda|load)$")) - - ((identifier) @function.builtin - (#eq? @function.builtin "require")) - - ((identifier) @constructor - (#match? @constructor "^[A-Z]")) - - ((identifier) @constant - (#match? @constant "^[A-Z]{2,}$")) - `); - - const captures = query.captures(tree.rootNode); - assert.deepEqual(formatCaptures(captures), [ - {name: 'variable', text: 'panda'}, - {name: 'variable', text: 'toad'}, - {name: 'variable', text: 'ab'}, - {name: 'variable', text: 'require'}, - {name: 'function.builtin', text: 'require'}, - {name: 'variable', text: 'Cd'}, - {name: 'constructor', text: 'Cd'}, - {name: 'variable', text: 'EF'}, - {name: 'constructor', text: 'EF'}, - {name: 'constant', text: 'EF'}, - ]); - }); - - it('handles conditions that compare the text of capture to each other', () => { - tree = parser.parse(` - ab = abc + 1; - def = de + 1; - ghi = ghi + 1; - `); - - query = JavaScript.query(` - ( - (assignment_expression - left: (identifier) @id1 - right: (binary_expression - left: (identifier) @id2)) - (#eq? @id1 @id2) - ) - `); - - const captures = query.captures(tree.rootNode); - assert.deepEqual(formatCaptures(captures), [ - {name: 'id1', text: 'ghi'}, - {name: 'id2', text: 'ghi'}, - ]); - }); - - it('handles patterns with properties', () => { - tree = parser.parse(`a(b.c);`); - query = JavaScript.query(` - ((call_expression (identifier) @func) - (#set! foo) - (#set! bar baz)) - - ((property_identifier) @prop - (#is? foo) - (#is-not? bar baz)) - `); - - const captures = query.captures(tree.rootNode); - assert.deepEqual(formatCaptures(captures), [ - {name: 'func', text: 'a', setProperties: {foo: null, bar: 'baz'}}, - { - name: 'prop', - text: 'c', - assertedProperties: {foo: null}, - refutedProperties: {bar: 'baz'}, - }, - ]); - assert.ok(!query.didExceedMatchLimit()); - }); - - it('detects queries with too many permutations to track', () => { - tree = parser.parse(` - [ - hello, hello, hello, hello, hello, hello, hello, hello, hello, hello, - hello, hello, hello, hello, hello, hello, hello, hello, hello, hello, - hello, hello, hello, hello, hello, hello, hello, hello, hello, hello, - hello, hello, hello, hello, hello, hello, hello, hello, hello, hello, - hello, hello, hello, hello, hello, hello, hello, hello, hello, hello, - ]; - `); - - query = JavaScript.query(` - (array (identifier) @pre (identifier) @post) - `); - - query.captures(tree.rootNode, {matchLimit: 32}); - assert.ok(query.didExceedMatchLimit()); - }); - - it('handles quantified captures properly', () => { - let captures; - - tree = parser.parse(` - /// foo - /// bar - /// baz - `); - - query = JavaScript.query(` - ( - (comment)+ @foo - (#any-eq? @foo "/// foo") - ) - `); - - const expectCount = (tree, queryText, expectedCount) => { - query = JavaScript.query(queryText); - captures = query.captures(tree.rootNode); - assert.equal(captures.length, expectedCount); - }; - - expectCount( - tree, - `((comment)+ @foo (#any-eq? @foo "/// foo"))`, - 3, - ); - - expectCount( - tree, - `((comment)+ @foo (#eq? @foo "/// foo"))`, - 0, - ); - - expectCount( - tree, - `((comment)+ @foo (#any-not-eq? @foo "/// foo"))`, - 3, - ); - - expectCount( - tree, - `((comment)+ @foo (#not-eq? @foo "/// foo"))`, - 0, - ); - - expectCount( - tree, - `((comment)+ @foo (#match? @foo "^/// foo"))`, - 0, - ); - - expectCount( - tree, - `((comment)+ @foo (#any-match? @foo "^/// foo"))`, - 3, - ); - - expectCount( - tree, - `((comment)+ @foo (#not-match? @foo "^/// foo"))`, - 0, - ); - - expectCount( - tree, - `((comment)+ @foo (#not-match? @foo "fsdfsdafdfs"))`, - 3, - ); - - expectCount( - tree, - `((comment)+ @foo (#any-not-match? @foo "^///"))`, - 0, - ); - - expectCount( - tree, - `((comment)+ @foo (#any-not-match? @foo "^/// foo"))`, - 3, - ); - }); - }); - - describe('.predicatesForPattern(index)', () => { - it('returns all of the predicates as objects', () => { - query = JavaScript.query(` - ( - (binary_expression - left: (identifier) @a - right: (identifier) @b) - (#something? @a @b) - (#match? @a "c") - (#something-else? @a "A" @b "B") - ) - - ((identifier) @c - (#hello! @c)) - - "if" @d - `); - - assert.deepEqual(query.predicatesForPattern(0), [ - { - operator: 'something?', - operands: [ - {type: 'capture', name: 'a'}, - {type: 'capture', name: 'b'}, - ], - }, - { - operator: 'something-else?', - operands: [ - {type: 'capture', name: 'a'}, - {type: 'string', value: 'A'}, - {type: 'capture', name: 'b'}, - {type: 'string', value: 'B'}, - ], - }, - ]); - assert.deepEqual(query.predicatesForPattern(1), [ - { - operator: 'hello!', - operands: [{type: 'capture', name: 'c'}], - }, - ]); - assert.deepEqual(query.predicatesForPattern(2), []); - }); - }); - - describe('.disableCapture', () => { - it('disables a capture', () => { - const query = JavaScript.query(` - (function_declaration - (identifier) @name1 @name2 @name3 - (statement_block) @body1 @body2) - `); - - const source = 'function foo() { return 1; }'; - const tree = parser.parse(source); - - let matches = query.matches(tree.rootNode); - assert.deepEqual(formatMatches(matches), [ - { - pattern: 0, - captures: [ - {name: 'name1', text: 'foo'}, - {name: 'name2', text: 'foo'}, - {name: 'name3', text: 'foo'}, - {name: 'body1', text: '{ return 1; }'}, - {name: 'body2', text: '{ return 1; }'}, - ], - }, - ]); - - // disabling captures still works when there are multiple captures on a - // single node. - query.disableCapture('name2'); - matches = query.matches(tree.rootNode); - assert.deepEqual(formatMatches(matches), [ - { - pattern: 0, - captures: [ - {name: 'name1', text: 'foo'}, - {name: 'name3', text: 'foo'}, - {name: 'body1', text: '{ return 1; }'}, - {name: 'body2', text: '{ return 1; }'}, - ], - }, - ]); - }); - }); - - describe('Set a timeout', () => - it('returns less than the expected matches', () => { - tree = parser.parse('function foo() while (true) { } }\n'.repeat(1000)); - query = JavaScript.query('(function_declaration name: (identifier) @function)'); - const matches = query.matches(tree.rootNode, { timeoutMicros: 1000 }); - assert.isBelow(matches.length, 1000); - const matches2 = query.matches(tree.rootNode, { timeoutMicros: 0 }); - assert.equal(matches2.length, 1000); - }) - ); -}); - -function formatMatches(matches) { - return matches.map(({pattern, captures}) => ({ - pattern, - captures: formatCaptures(captures), - })); -} - -function formatCaptures(captures) { - return captures.map((c) => { - const node = c.node; - delete c.node; - c.text = node.text; - return c; - }); -} diff --git a/lib/binding_web/test/query.test.ts b/lib/binding_web/test/query.test.ts new file mode 100644 index 00000000..f90e9464 --- /dev/null +++ b/lib/binding_web/test/query.test.ts @@ -0,0 +1,629 @@ +import { describe, it, expect, beforeAll, beforeEach, afterEach } from 'vitest'; +import type { Language, Tree, QueryMatch, QueryCapture } from '../src'; +import { Parser, Query } from '../src'; +import helper from './helper'; + +let JavaScript: Language; + +describe('Query', () => { + let parser: Parser; + let tree: Tree | null; + let query: Query | null; + + beforeAll(async () => { + ({ JavaScript } = await helper); + }); + + beforeEach(() => { + parser = new Parser(); + parser.setLanguage(JavaScript); + }); + + afterEach(() => { + parser.delete(); + if (tree) tree.delete(); + if (query) query.delete(); + }); + + describe('construction', () => { + it('throws an error on invalid patterns', () => { + expect(() => { + new Query(JavaScript, '(function_declaration wat)'); + }).toThrow('Bad syntax at offset 22: \'wat)\'...'); + + expect(() => { + new Query(JavaScript, '(non_existent)'); + }).toThrow('Bad node name \'non_existent\''); + + expect(() => { + new Query(JavaScript, '(a)'); + }).toThrow('Bad node name \'a\''); + + expect(() => { + new Query(JavaScript, '(function_declaration non_existent:(identifier))'); + }).toThrow('Bad field name \'non_existent\''); + + expect(() => { + new Query(JavaScript, '(function_declaration name:(statement_block))'); + }).toThrow('Bad pattern structure at offset 22: \'name:(statement_block))\''); + }); + + it('throws an error on invalid predicates', () => { + expect(() => { + new Query(JavaScript, '((identifier) @abc (#eq? @ab hi))'); + }).toThrow('Bad capture name @ab'); + + expect(() => { + new Query(JavaScript, '((identifier) @abc (#eq?))'); + }).toThrow('Wrong number of arguments to `#eq?` predicate. Expected 2, got 0'); + + expect(() => { + new Query(JavaScript, '((identifier) @a (#eq? @a @a @a))'); + }).toThrow('Wrong number of arguments to `#eq?` predicate. Expected 2, got 3'); + }); + }); + + describe('.matches', () => { + it('returns all of the matches for the given query', { timeout: 10000 }, () => { + tree = parser.parse('function one() { two(); function three() {} }')!; + query = new Query(JavaScript, ` + (function_declaration name: (identifier) @fn-def) + (call_expression function: (identifier) @fn-ref) + `); + const matches = query.matches(tree.rootNode); + expect(formatMatches(matches)).toEqual([ + { patternIndex: 0, captures: [{ patternIndex: 0, name: 'fn-def', text: 'one' }] }, + { patternIndex: 1, captures: [{ patternIndex: 1, name: 'fn-ref', text: 'two' }] }, + { patternIndex: 0, captures: [{ patternIndex: 0, name: 'fn-def', text: 'three' }] }, + ]); + }); + + it('can search in specified ranges', () => { + tree = parser.parse('[a, b,\nc, d,\ne, f,\ng, h]')!; + query = new Query(JavaScript, '(identifier) @element'); + const matches = query.matches( + tree.rootNode, + { + startPosition: { row: 1, column: 1 }, + endPosition: { row: 3, column: 1 }, + } + ); + expect(formatMatches(matches)).toEqual([ + { patternIndex: 0, captures: [{ patternIndex: 0, name: 'element', text: 'd' }] }, + { patternIndex: 0, captures: [{ patternIndex: 0, name: 'element', text: 'e' }] }, + { patternIndex: 0, captures: [{ patternIndex: 0, name: 'element', text: 'f' }] }, + { patternIndex: 0, captures: [{ patternIndex: 0, name: 'element', text: 'g' }] }, + ]); + }); + + it('can search in contained within point ranges', () => { + tree = parser.parse(`[ + {"key1": "value1"}, + {"key2": "value2"}, + {"key3": "value3"}, + {"key4": "value4"}, + {"key5": "value5"}, + {"key6": "value6"}, + {"key7": "value7"}, + {"key8": "value8"}, + {"key9": "value9"}, + {"key10": "value10"}, + {"key11": "value11"}, + {"key12": "value12"}, +]`)!; + query = new Query(JavaScript, '("[" @l_bracket "]" @r_bracket) ("{" @l_brace "}" @r_brace)'); + const matches = query.matches( + tree.rootNode, + { + startContainingPosition: { row: 5, column: 0 }, + endContainingPosition: { row: 7, column: 0 }, + } + ); + expect(formatMatches(matches)).toEqual([ + { patternIndex: 1, captures: [{ patternIndex: 1, name: 'l_brace', text: '{' }, { patternIndex: 1, name: 'r_brace', text: '}' },] }, + { patternIndex: 1, captures: [{ patternIndex: 1, name: 'l_brace', text: '{' }, { patternIndex: 1, name: 'r_brace', text: '}' },] }, + ]); + }); + + it('can search in contained within byte ranges', () => { + tree = parser.parse(`[ + {"key1": "value1"}, + {"key2": "value2"}, + {"key3": "value3"}, + {"key4": "value4"}, + {"key5": "value5"}, + {"key6": "value6"}, + {"key7": "value7"}, + {"key8": "value8"}, + {"key9": "value9"}, + {"key10": "value10"}, + {"key11": "value11"}, + {"key12": "value12"}, +]`)!; + query = new Query(JavaScript, '("[" @l_bracket "]" @r_bracket) ("{" @l_brace "}" @r_brace)'); + const matches = query.matches( + tree.rootNode, + { + startContainingIndex: 290, + endContainingIndex: 432, + } + ); + expect(formatMatches(matches)).toEqual([ + { patternIndex: 1, captures: [{ patternIndex: 1, name: 'l_brace', text: '{' }, { patternIndex: 1, name: 'r_brace', text: '}' },] }, + { patternIndex: 1, captures: [{ patternIndex: 1, name: 'l_brace', text: '{' }, { patternIndex: 1, name: 'r_brace', text: '}' },] }, + ]); + }); + + it('handles predicates that compare the text of capture to literal strings', () => { + tree = parser.parse(` + giraffe(1, 2, []); + helment([false]); + goat(false); + gross(3, []); + hiccup([]); + gaff(5); + `)!; + + // Find all calls to functions beginning with 'g', where one argument + // is an array literal. + query = new Query(JavaScript, ` + (call_expression + function: (identifier) @name + arguments: (arguments (array)) + (#match? @name "^g")) + `); + + const matches = query.matches(tree.rootNode); + expect(formatMatches(matches)).toEqual([ + { patternIndex: 0, captures: [{ patternIndex: 0, name: 'name', text: 'giraffe' }] }, + { patternIndex: 0, captures: [{ patternIndex: 0, name: 'name', text: 'gross' }] }, + ]); + }); + + it('handles multiple matches where the first one is filtered', () => { + tree = parser.parse(` + const a = window.b; + `)!; + + query = new Query(JavaScript, ` + ((identifier) @variable.builtin + (#match? @variable.builtin "^(arguments|module|console|window|document)$") + (#is-not? local)) + `); + + const matches = query.matches(tree.rootNode); + expect(formatMatches(matches)).toEqual([ + { patternIndex: 0, captures: [{ patternIndex: 0, name: 'variable.builtin', text: 'window' }] }, + ]); + }); + }); + + describe('.captures', () => { + it('returns all of the captures for the given query, in order', () => { + tree = parser.parse(` + a({ + bc: function de() { + const fg = function hi() {} + }, + jk: function lm() { + const no = function pq() {} + }, + }); + `)!; + query = new Query(JavaScript, ` + (pair + key: _ @method.def + (function_expression + name: (identifier) @method.alias)) + + (variable_declarator + name: _ @function.def + value: (function_expression + name: (identifier) @function.alias)) + + ":" @delimiter + "=" @operator + `); + + const captures = query.captures(tree.rootNode); + expect(formatCaptures(captures)).toEqual([ + { patternIndex: 0, name: 'method.def', text: 'bc' }, + { patternIndex: 2, name: 'delimiter', text: ':' }, + { patternIndex: 0, name: 'method.alias', text: 'de' }, + { patternIndex: 1, name: 'function.def', text: 'fg' }, + { patternIndex: 3, name: 'operator', text: '=' }, + { patternIndex: 1, name: 'function.alias', text: 'hi' }, + { patternIndex: 0, name: 'method.def', text: 'jk' }, + { patternIndex: 2, name: 'delimiter', text: ':' }, + { patternIndex: 0, name: 'method.alias', text: 'lm' }, + { patternIndex: 1, name: 'function.def', text: 'no' }, + { patternIndex: 3, name: 'operator', text: '=' }, + { patternIndex: 1, name: 'function.alias', text: 'pq' }, + ]); + }); + + it('handles conditions that compare the text of capture to literal strings', () => { + tree = parser.parse(` + lambda + panda + load + toad + const ab = require('./ab'); + new Cd(EF); + `)!; + + query = new Query(JavaScript, ` + ((identifier) @variable + (#not-match? @variable "^(lambda|load)$")) + + ((identifier) @function.builtin + (#eq? @function.builtin "require")) + + ((identifier) @constructor + (#match? @constructor "^[A-Z]")) + + ((identifier) @constant + (#match? @constant "^[A-Z]{2,}$")) + `); + + const captures = query.captures(tree.rootNode); + expect(formatCaptures(captures)).toEqual([ + { patternIndex: 0, name: 'variable', text: 'panda' }, + { patternIndex: 0, name: 'variable', text: 'toad' }, + { patternIndex: 0, name: 'variable', text: 'ab' }, + { patternIndex: 0, name: 'variable', text: 'require' }, + { patternIndex: 1, name: 'function.builtin', text: 'require' }, + { patternIndex: 0, name: 'variable', text: 'Cd' }, + { patternIndex: 2, name: 'constructor', text: 'Cd' }, + { patternIndex: 0, name: 'variable', text: 'EF' }, + { patternIndex: 2, name: 'constructor', text: 'EF' }, + { patternIndex: 3, name: 'constant', text: 'EF' }, + ]); + }); + + it('handles conditions that compare the text of captures to each other', () => { + tree = parser.parse(` + ab = abc + 1; + def = de + 1; + ghi = ghi + 1; + `)!; + + query = new Query(JavaScript, ` + ( + (assignment_expression + left: (identifier) @id1 + right: (binary_expression + left: (identifier) @id2)) + (#eq? @id1 @id2) + ) + `); + + const captures = query.captures(tree.rootNode); + expect(formatCaptures(captures)).toEqual([ + { patternIndex: 0, name: 'id1', text: 'ghi' }, + { patternIndex: 0, name: 'id2', text: 'ghi' }, + ]); + }); + + it('handles patterns with properties', () => { + tree = parser.parse(`a(b.c);`)!; + query = new Query(JavaScript, ` + ((call_expression (identifier) @func) + (#set! foo) + (#set! bar baz)) + + ((property_identifier) @prop + (#is? foo) + (#is-not? bar baz)) + `); + + const captures = query.captures(tree.rootNode); + expect(formatCaptures(captures)).toEqual([ + { + patternIndex: 0, + name: 'func', + text: 'a', + setProperties: { foo: null, bar: 'baz' } + }, + { + patternIndex: 1, + name: 'prop', + text: 'c', + assertedProperties: { foo: null }, + refutedProperties: { bar: 'baz' }, + }, + ]); + expect(query.didExceedMatchLimit()).toBe(false); + }); + + it('detects queries with too many permutations to track', () => { + tree = parser.parse(` + [ + hello, hello, hello, hello, hello, hello, hello, hello, hello, hello, + hello, hello, hello, hello, hello, hello, hello, hello, hello, hello, + hello, hello, hello, hello, hello, hello, hello, hello, hello, hello, + hello, hello, hello, hello, hello, hello, hello, hello, hello, hello, + hello, hello, hello, hello, hello, hello, hello, hello, hello, hello, + ]; + `)!; + + query = new Query(JavaScript, `(array (identifier) @pre (identifier) @post)`); + + query.captures(tree.rootNode, { matchLimit: 32 }); + expect(query.didExceedMatchLimit()).toBe(true); + }); + + it('handles quantified captures properly', () => { + tree = parser.parse(` + /// foo + /// bar + /// baz + `)!; + + const expectCount = (tree: Tree, queryText: string, expectedCount: number) => { + query = new Query(JavaScript, queryText); + const captures = query.captures(tree.rootNode); + expect(captures).toHaveLength(expectedCount); + }; + + expectCount( + tree, + `((comment)+ @foo (#any-eq? @foo "/// foo"))`, + 3 + ); + + expectCount( + tree, + `((comment)+ @foo (#eq? @foo "/// foo"))`, + 0 + ); + + expectCount( + tree, + `((comment)+ @foo (#any-not-eq? @foo "/// foo"))`, + 3 + ); + + expectCount( + tree, + `((comment)+ @foo (#not-eq? @foo "/// foo"))`, + 0 + ); + + expectCount( + tree, + `((comment)+ @foo (#match? @foo "^/// foo"))`, + 0 + ); + + expectCount( + tree, + `((comment)+ @foo (#any-match? @foo "^/// foo"))`, + 3 + ); + + expectCount( + tree, + `((comment)+ @foo (#not-match? @foo "^/// foo"))`, + 0 + ); + + expectCount( + tree, + `((comment)+ @foo (#not-match? @foo "fsdfsdafdfs"))`, + 3 + ); + + expectCount( + tree, + `((comment)+ @foo (#any-not-match? @foo "^///"))`, + 0 + ); + + expectCount( + tree, + `((comment)+ @foo (#any-not-match? @foo "^/// foo"))`, + 3 + ); + }); + }); + + describe('.predicatesForPattern(index)', () => { + it('returns all of the predicates as objects', () => { + query = new Query(JavaScript, ` + ( + (binary_expression + left: (identifier) @a + right: (identifier) @b) + (#something? @a @b) + (#match? @a "c") + (#something-else? @a "A" @b "B") + ) + + ((identifier) @c + (#hello! @c)) + + "if" @d + `); + + expect(query.predicatesForPattern(0)).toStrictEqual([ + { + operator: 'something?', + operands: [ + { type: 'capture', name: 'a' }, + { type: 'capture', name: 'b' }, + ], + }, + { + operator: 'something-else?', + operands: [ + { type: 'capture', name: 'a' }, + { type: 'string', value: 'A' }, + { type: 'capture', name: 'b' }, + { type: 'string', value: 'B' }, + ], + }, + ]); + + expect(query.predicatesForPattern(1)).toStrictEqual([ + { + operator: 'hello!', + operands: [{ type: 'capture', name: 'c' }], + }, + ]); + + expect(query.predicatesForPattern(2)).toEqual([]); + }); + }); + + describe('.disableCapture', () => { + it('disables a capture', () => { + query = new Query(JavaScript, ` + (function_declaration + (identifier) @name1 @name2 @name3 + (statement_block) @body1 @body2) + `); + + const source = 'function foo() { return 1; }'; + const tree = parser.parse(source)!; + + let matches = query.matches(tree.rootNode); + expect(formatMatches(matches)).toEqual([ + { + patternIndex: 0, + captures: [ + { patternIndex: 0, name: 'name1', text: 'foo' }, + { patternIndex: 0, name: 'name2', text: 'foo' }, + { patternIndex: 0, name: 'name3', text: 'foo' }, + { patternIndex: 0, name: 'body1', text: '{ return 1; }' }, + { patternIndex: 0, name: 'body2', text: '{ return 1; }' }, + ], + }, + ]); + + // disabling captures still works when there are multiple captures on a + // single node. + query.disableCapture('name2'); + matches = query.matches(tree.rootNode); + expect(formatMatches(matches)).toEqual([ + { + patternIndex: 0, + captures: [ + { patternIndex: 0, name: 'name1', text: 'foo' }, + { patternIndex: 0, name: 'name3', text: 'foo' }, + { patternIndex: 0, name: 'body1', text: '{ return 1; }' }, + { patternIndex: 0, name: 'body2', text: '{ return 1; }' }, + ], + }, + ]); + }); + }); + + describe('Start and end indices for patterns', () => { + it('Returns the start and end indices for a pattern', () => { + const patterns1 = ` +"+" @operator +"-" @operator +"*" @operator +"=" @operator +"=>" @operator + `.trim(); + + const patterns2 = ` +(identifier) @a +(string) @b +`.trim(); + + const patterns3 = ` +((identifier) @b (#match? @b i)) +(function_declaration name: (identifier) @c) +(method_definition name: (property_identifier) @d) +`.trim(); + + const source = patterns1 + patterns2 + patterns3; + + const query = new Query(JavaScript, source); + + expect(query.startIndexForPattern(0)).toBe(0); + expect(query.endIndexForPattern(0)).toBe('"+" @operator\n'.length); + expect(query.startIndexForPattern(5)).toBe(patterns1.length); + expect(query.endIndexForPattern(5)).toBe( + patterns1.length + '(identifier) @a\n'.length + ); + expect(query.startIndexForPattern(7)).toBe(patterns1.length + patterns2.length); + expect(query.endIndexForPattern(7)).toBe( + patterns1.length + + patterns2.length + + '((identifier) @b (#match? @b i))\n'.length + ); + }); + }); + + describe('Disable pattern', () => { + it('Disables patterns in the query', () => { + const query = new Query(JavaScript, ` + (function_declaration name: (identifier) @name) + (function_declaration body: (statement_block) @body) + (class_declaration name: (identifier) @name) + (class_declaration body: (class_body) @body) + `); + + // disable the patterns that match names + query.disablePattern(0); + query.disablePattern(2); + + const source = 'class A { constructor() {} } function b() { return 1; }'; + tree = parser.parse(source)!; + const matches = query.matches(tree.rootNode); + expect(formatMatches(matches)).toEqual([ + { + patternIndex: 3, + captures: [{ patternIndex: 3, name: 'body', text: '{ constructor() {} }' }], + }, + { patternIndex: 1, captures: [{ patternIndex: 1, name: 'body', text: '{ return 1; }' }] }, + ]); + }); + }); + + describe('Executes with a timeout', { timeout: 10000 }, () => { + it('Returns less than the expected matches', () => { + tree = parser.parse('function foo() while (true) { } }\n'.repeat(1000))!; + query = new Query(JavaScript, '(function_declaration) @function'); + + const startTime = performance.now(); + + const matches = query.matches( + tree.rootNode, + { + progressCallback: () => { + if (performance.now() - startTime > 1) { + return true; + } + return false; + }, + } + ); + expect(matches.length).toBeLessThan(1000); + + const matches2 = query.matches(tree.rootNode); + expect(matches2).toHaveLength(1000); + }); + }); +}); + +// Helper functions +function formatMatches(matches: QueryMatch[]): Omit[] { + return matches.map(({ patternIndex, captures }) => ({ + patternIndex, + captures: formatCaptures(captures), + })); +} + +function formatCaptures(captures: QueryCapture[]): (QueryCapture & { text: string })[] { + return captures.map((c) => { + const node = c.node; + // @ts-expect-error We're not interested in the node object for these tests + delete c.node; + return { ...c, text: node.text }; + }); +} diff --git a/lib/binding_web/test/tree-test.js b/lib/binding_web/test/tree-test.js deleted file mode 100644 index c9216eb1..00000000 --- a/lib/binding_web/test/tree-test.js +++ /dev/null @@ -1,424 +0,0 @@ -const {assert} = require('chai'); -let Parser; let JavaScript; - -describe('Tree', () => { - let parser; let tree; - - before(async () => - ({Parser, JavaScript} = await require('./helper')), - ); - - beforeEach(() => { - parser = new Parser().setLanguage(JavaScript); - }); - - afterEach(() => { - parser.delete(); - tree.delete(); - }); - - describe('.edit', () => { - let input; let edit; - - it('updates the positions of nodes', () => { - input = 'abc + cde'; - tree = parser.parse(input); - assert.equal( - tree.rootNode.toString(), - '(program (expression_statement (binary_expression left: (identifier) right: (identifier))))', - ); - - let sumNode = tree.rootNode.firstChild.firstChild; - let variableNode1 = sumNode.firstChild; - let variableNode2 = sumNode.lastChild; - assert.equal(variableNode1.startIndex, 0); - assert.equal(variableNode1.endIndex, 3); - assert.equal(variableNode2.startIndex, 6); - assert.equal(variableNode2.endIndex, 9); - - ([input, edit] = spliceInput(input, input.indexOf('bc'), 0, ' * ')); - assert.equal(input, 'a * bc + cde'); - tree.edit(edit); - - sumNode = tree.rootNode.firstChild.firstChild; - variableNode1 = sumNode.firstChild; - variableNode2 = sumNode.lastChild; - assert.equal(variableNode1.startIndex, 0); - assert.equal(variableNode1.endIndex, 6); - assert.equal(variableNode2.startIndex, 9); - assert.equal(variableNode2.endIndex, 12); - - tree = parser.parse(input, tree); - assert.equal( - tree.rootNode.toString(), - '(program (expression_statement (binary_expression left: (binary_expression left: (identifier) right: (identifier)) right: (identifier))))', - ); - }); - - it('handles non-ascii characters', () => { - input = 'αβδ + cde'; - - tree = parser.parse(input); - assert.equal( - tree.rootNode.toString(), - '(program (expression_statement (binary_expression left: (identifier) right: (identifier))))', - ); - - let variableNode = tree.rootNode.firstChild.firstChild.lastChild; - - ([input, edit] = spliceInput(input, input.indexOf('δ'), 0, '👍 * ')); - assert.equal(input, 'αβ👍 * δ + cde'); - tree.edit(edit); - - variableNode = tree.rootNode.firstChild.firstChild.lastChild; - assert.equal(variableNode.startIndex, input.indexOf('cde')); - - tree = parser.parse(input, tree); - assert.equal( - tree.rootNode.toString(), - '(program (expression_statement (binary_expression left: (binary_expression left: (identifier) right: (identifier)) right: (identifier))))', - ); - }); - }); - - describe('.getChangedRanges(previous)', () => { - it('reports the ranges of text whose syntactic meaning has changed', () => { - let sourceCode = 'abcdefg + hij'; - tree = parser.parse(sourceCode); - - assert.equal( - tree.rootNode.toString(), - '(program (expression_statement (binary_expression left: (identifier) right: (identifier))))', - ); - - sourceCode = 'abc + defg + hij'; - tree.edit({ - startIndex: 2, - oldEndIndex: 2, - newEndIndex: 5, - startPosition: {row: 0, column: 2}, - oldEndPosition: {row: 0, column: 2}, - newEndPosition: {row: 0, column: 5}, - }); - - const tree2 = parser.parse(sourceCode, tree); - assert.equal( - tree2.rootNode.toString(), - '(program (expression_statement (binary_expression left: (binary_expression left: (identifier) right: (identifier)) right: (identifier))))', - ); - - const ranges = tree.getChangedRanges(tree2); - assert.deepEqual(ranges, [ - { - startIndex: 0, - endIndex: 'abc + defg'.length, - startPosition: {row: 0, column: 0}, - endPosition: {row: 0, column: 'abc + defg'.length}, - }, - ]); - - tree2.delete(); - }); - - it('throws an exception if the argument is not a tree', () => { - tree = parser.parse('abcdefg + hij'); - - assert.throws(() => { - tree.getChangedRanges({}); - }, /Argument must be a Tree/); - }); - }); - - describe('.walk()', () => { - let cursor; - - afterEach(() => { - cursor.delete(); - }); - - it('returns a cursor that can be used to walk the tree', () => { - tree = parser.parse('a * b + c / d'); - cursor = tree.walk(); - - assertCursorState(cursor, { - nodeType: 'program', - nodeIsNamed: true, - startPosition: {row: 0, column: 0}, - endPosition: {row: 0, column: 13}, - startIndex: 0, - endIndex: 13, - }); - - assert(cursor.gotoFirstChild()); - assertCursorState(cursor, { - nodeType: 'expression_statement', - nodeIsNamed: true, - startPosition: {row: 0, column: 0}, - endPosition: {row: 0, column: 13}, - startIndex: 0, - endIndex: 13, - }); - - assert(cursor.gotoFirstChild()); - assertCursorState(cursor, { - nodeType: 'binary_expression', - nodeIsNamed: true, - startPosition: {row: 0, column: 0}, - endPosition: {row: 0, column: 13}, - startIndex: 0, - endIndex: 13, - }); - - assert(cursor.gotoFirstChild()); - assertCursorState(cursor, { - nodeType: 'binary_expression', - nodeIsNamed: true, - startPosition: {row: 0, column: 0}, - endPosition: {row: 0, column: 5}, - startIndex: 0, - endIndex: 5, - }); - - assert(cursor.gotoFirstChild()); - assert.equal(cursor.nodeText, 'a'); - assertCursorState(cursor, { - nodeType: 'identifier', - nodeIsNamed: true, - startPosition: {row: 0, column: 0}, - endPosition: {row: 0, column: 1}, - startIndex: 0, - endIndex: 1, - }); - - assert(!cursor.gotoFirstChild()); - assert(cursor.gotoNextSibling()); - assert.equal(cursor.nodeText, '*'); - assertCursorState(cursor, { - nodeType: '*', - nodeIsNamed: false, - startPosition: {row: 0, column: 2}, - endPosition: {row: 0, column: 3}, - startIndex: 2, - endIndex: 3, - }); - - assert(cursor.gotoNextSibling()); - assert.equal(cursor.nodeText, 'b'); - assertCursorState(cursor, { - nodeType: 'identifier', - nodeIsNamed: true, - startPosition: {row: 0, column: 4}, - endPosition: {row: 0, column: 5}, - startIndex: 4, - endIndex: 5, - }); - - assert(!cursor.gotoNextSibling()); - assert(cursor.gotoParent()); - assertCursorState(cursor, { - nodeType: 'binary_expression', - nodeIsNamed: true, - startPosition: {row: 0, column: 0}, - endPosition: {row: 0, column: 5}, - startIndex: 0, - endIndex: 5, - }); - - assert(cursor.gotoNextSibling()); - assertCursorState(cursor, { - nodeType: '+', - nodeIsNamed: false, - startPosition: {row: 0, column: 6}, - endPosition: {row: 0, column: 7}, - startIndex: 6, - endIndex: 7, - }); - - assert(cursor.gotoNextSibling()); - assertCursorState(cursor, { - nodeType: 'binary_expression', - nodeIsNamed: true, - startPosition: {row: 0, column: 8}, - endPosition: {row: 0, column: 13}, - startIndex: 8, - endIndex: 13, - }); - - const copy = tree.walk(); - copy.resetTo(cursor); - - assert(copy.gotoPreviousSibling()); - assertCursorState(copy, { - nodeType: '+', - nodeIsNamed: false, - startPosition: {row: 0, column: 6}, - endPosition: {row: 0, column: 7}, - startIndex: 6, - endIndex: 7, - }); - - assert(copy.gotoPreviousSibling()); - assertCursorState(copy, { - nodeType: 'binary_expression', - nodeIsNamed: true, - startPosition: {row: 0, column: 0}, - endPosition: {row: 0, column: 5}, - startIndex: 0, - endIndex: 5, - }); - - assert(copy.gotoLastChild()); - assertCursorState(copy, { - nodeType: 'identifier', - nodeIsNamed: true, - startPosition: {row: 0, column: 4}, - endPosition: {row: 0, column: 5}, - startIndex: 4, - endIndex: 5, - }); - - assert(copy.gotoParent()); - assert(copy.gotoParent()); - assert.equal(copy.nodeType, 'binary_expression'); - assert(copy.gotoParent()); - assert.equal(copy.nodeType, 'expression_statement'); - assert(copy.gotoParent()); - assert.equal(copy.nodeType, 'program'); - assert(!copy.gotoParent()); - - assert(cursor.gotoParent()); - assert.equal(cursor.nodeType, 'binary_expression'); - assert(cursor.gotoParent()); - assert.equal(cursor.nodeType, 'expression_statement'); - assert(cursor.gotoParent()); - assert.equal(cursor.nodeType, 'program'); - assert(!cursor.gotoParent()); - }); - - it('keeps track of the field name associated with each node', () => { - tree = parser.parse('a.b();'); - cursor = tree.walk(); - cursor.gotoFirstChild(); - cursor.gotoFirstChild(); - - assert.equal(cursor.currentNode.type, 'call_expression'); - assert.equal(cursor.currentFieldName, null); - - cursor.gotoFirstChild(); - assert.equal(cursor.currentNode.type, 'member_expression'); - assert.equal(cursor.currentFieldName, 'function'); - - cursor.gotoFirstChild(); - assert.equal(cursor.currentNode.type, 'identifier'); - assert.equal(cursor.currentFieldName, 'object'); - - cursor.gotoNextSibling(); - cursor.gotoNextSibling(); - assert.equal(cursor.currentNode.type, 'property_identifier'); - assert.equal(cursor.currentFieldName, 'property'); - - cursor.gotoParent(); - cursor.gotoNextSibling(); - assert.equal(cursor.currentNode.type, 'arguments'); - assert.equal(cursor.currentFieldName, 'arguments'); - }); - - it('returns a cursor that can be reset anywhere in the tree', () => { - tree = parser.parse('a * b + c / d'); - cursor = tree.walk(); - const root = tree.rootNode.firstChild; - - cursor.reset(root.firstChild.firstChild); - assertCursorState(cursor, { - nodeType: 'binary_expression', - nodeIsNamed: true, - startPosition: {row: 0, column: 0}, - endPosition: {row: 0, column: 5}, - startIndex: 0, - endIndex: 5, - }); - - cursor.gotoFirstChild(); - assertCursorState(cursor, { - nodeType: 'identifier', - nodeIsNamed: true, - startPosition: {row: 0, column: 0}, - endPosition: {row: 0, column: 1}, - startIndex: 0, - endIndex: 1, - }); - - assert(cursor.gotoParent()); - assert(!cursor.gotoParent()); - }); - }); - - describe('.copy', () => { - it('creates another tree that remains stable if the original tree is edited', () => { - input = 'abc + cde'; - tree = parser.parse(input); - assert.equal( - tree.rootNode.toString(), - '(program (expression_statement (binary_expression left: (identifier) right: (identifier))))', - ); - - const tree2 = tree.copy(); - ([input, edit] = spliceInput(input, 3, 0, '123')); - assert.equal(input, 'abc123 + cde'); - tree.edit(edit); - - const leftNode = tree.rootNode.firstChild.firstChild.firstChild; - const leftNode2 = tree2.rootNode.firstChild.firstChild.firstChild; - const rightNode = tree.rootNode.firstChild.firstChild.lastChild; - const rightNode2 = tree2.rootNode.firstChild.firstChild.lastChild; - assert.equal(leftNode.endIndex, 6); - assert.equal(leftNode2.endIndex, 3); - assert.equal(rightNode.startIndex, 9); - assert.equal(rightNode2.startIndex, 6); - }); - }); -}); - -function spliceInput(input, startIndex, lengthRemoved, newText) { - const oldEndIndex = startIndex + lengthRemoved; - const newEndIndex = startIndex + newText.length; - const startPosition = getExtent(input.slice(0, startIndex)); - const oldEndPosition = getExtent(input.slice(0, oldEndIndex)); - input = input.slice(0, startIndex) + newText + input.slice(oldEndIndex); - const newEndPosition = getExtent(input.slice(0, newEndIndex)); - return [ - input, - { - startIndex, startPosition, - oldEndIndex, oldEndPosition, - newEndIndex, newEndPosition, - }, - ]; -} - -function getExtent(text) { - let row = 0; - let index; - for (index = 0; index !== -1; index = text.indexOf('\n', index)) { - index++; - row++; - } - return {row, column: text.length - index}; -} - -function assertCursorState(cursor, params) { - assert.equal(cursor.nodeType, params.nodeType); - assert.equal(cursor.nodeIsNamed, params.nodeIsNamed); - assert.deepEqual(cursor.startPosition, params.startPosition); - assert.deepEqual(cursor.endPosition, params.endPosition); - assert.deepEqual(cursor.startIndex, params.startIndex); - assert.deepEqual(cursor.endIndex, params.endIndex); - - const node = cursor.currentNode; - assert.equal(node.type, params.nodeType); - assert.equal(node.isNamed, params.nodeIsNamed); - assert.deepEqual(node.startPosition, params.startPosition); - assert.deepEqual(node.endPosition, params.endPosition); - assert.deepEqual(node.startIndex, params.startIndex); - assert.deepEqual(node.endIndex, params.endIndex); -} diff --git a/lib/binding_web/test/tree.test.ts b/lib/binding_web/test/tree.test.ts new file mode 100644 index 00000000..85f895a7 --- /dev/null +++ b/lib/binding_web/test/tree.test.ts @@ -0,0 +1,443 @@ +import { describe, it, expect, beforeAll, beforeEach, afterEach } from 'vitest'; +import type { Point, Language, Tree, TreeCursor } from '../src'; +import { Parser, Edit } from '../src'; +import helper from './helper'; + +let JavaScript: Language; + +interface CursorState { + nodeType: string; + nodeIsNamed: boolean; + startPosition: Point; + endPosition: Point; + startIndex: number; + endIndex: number; +} + +describe('Tree', () => { + let parser: Parser; + let tree: Tree; + + beforeAll(async () => { + ({ JavaScript } = await helper); + }); + + beforeEach(() => { + parser = new Parser(); + parser.setLanguage(JavaScript); + }); + + afterEach(() => { + parser.delete(); + tree.delete(); + }); + + describe('.edit', () => { + let input: string; + let edit: Edit; + + it('updates the positions of nodes', () => { + input = 'abc + cde'; + tree = parser.parse(input)!; + expect(tree.rootNode.toString()).toBe( + '(program (expression_statement (binary_expression left: (identifier) right: (identifier))))' + ); + + let sumNode = tree.rootNode.firstChild!.firstChild; + let variableNode1 = sumNode!.firstChild; + let variableNode2 = sumNode!.lastChild; + expect(variableNode1!.startIndex).toBe(0); + expect(variableNode1!.endIndex).toBe(3); + expect(variableNode2!.startIndex).toBe(6); + expect(variableNode2!.endIndex).toBe(9); + + [input, edit] = spliceInput(input, input.indexOf('bc'), 0, ' * '); + expect(input).toBe('a * bc + cde'); + tree.edit(edit); + + sumNode = tree.rootNode.firstChild!.firstChild; + variableNode1 = sumNode!.firstChild; + variableNode2 = sumNode!.lastChild; + expect(variableNode1!.startIndex).toBe(0); + expect(variableNode1!.endIndex).toBe(6); + expect(variableNode2!.startIndex).toBe(9); + expect(variableNode2!.endIndex).toBe(12); + + tree = parser.parse(input, tree)!; + expect(tree.rootNode.toString()).toBe( + '(program (expression_statement (binary_expression left: (binary_expression left: (identifier) right: (identifier)) right: (identifier))))' + ); + }); + + it('handles non-ascii characters', () => { + input = 'αβδ + cde'; + + tree = parser.parse(input)!; + expect(tree.rootNode.toString()).toBe( + '(program (expression_statement (binary_expression left: (identifier) right: (identifier))))' + ); + + let variableNode = tree.rootNode.firstChild!.firstChild!.lastChild; + + [input, edit] = spliceInput(input, input.indexOf('δ'), 0, '👍 * '); + expect(input).toBe('αβ👍 * δ + cde'); + tree.edit(edit); + + variableNode = tree.rootNode.firstChild!.firstChild!.lastChild; + expect(variableNode!.startIndex).toBe(input.indexOf('cde')); + + tree = parser.parse(input, tree)!; + expect(tree.rootNode.toString()).toBe( + '(program (expression_statement (binary_expression left: (binary_expression left: (identifier) right: (identifier)) right: (identifier))))' + ); + }); + }); + + describe('.getChangedRanges(previous)', () => { + it('reports the ranges of text whose syntactic meaning has changed', () => { + let sourceCode = 'abcdefg + hij'; + tree = parser.parse(sourceCode)!; + + expect(tree.rootNode.toString()).toBe( + '(program (expression_statement (binary_expression left: (identifier) right: (identifier))))' + ); + + sourceCode = 'abc + defg + hij'; + tree.edit(new Edit({ + startIndex: 2, + oldEndIndex: 2, + newEndIndex: 5, + startPosition: { row: 0, column: 2 }, + oldEndPosition: { row: 0, column: 2 }, + newEndPosition: { row: 0, column: 5 }, + })); + + const tree2 = parser.parse(sourceCode, tree)!; + expect(tree2.rootNode.toString()).toBe( + '(program (expression_statement (binary_expression left: (binary_expression left: (identifier) right: (identifier)) right: (identifier))))' + ); + + const ranges = tree.getChangedRanges(tree2); + expect(ranges).toEqual([ + { + startIndex: 0, + endIndex: 'abc + defg'.length, + startPosition: { row: 0, column: 0 }, + endPosition: { row: 0, column: 'abc + defg'.length }, + }, + ]); + + tree2.delete(); + }); + + it('throws an exception if the argument is not a tree', () => { + tree = parser.parse('abcdefg + hij')!; + + expect(() => { + tree.getChangedRanges({} as Tree); + }).toThrow(/Argument must be a Tree/); + }); + }); + + describe('.walk()', () => { + let cursor: TreeCursor; + + afterEach(() => { + cursor.delete(); + }); + + it('returns a cursor that can be used to walk the tree', () => { + tree = parser.parse('a * b + c / d')!; + cursor = tree.walk(); + + assertCursorState(cursor, { + nodeType: 'program', + nodeIsNamed: true, + startPosition: { row: 0, column: 0 }, + endPosition: { row: 0, column: 13 }, + startIndex: 0, + endIndex: 13, + }); + + expect(cursor.gotoFirstChild()).toBe(true); + assertCursorState(cursor, { + nodeType: 'expression_statement', + nodeIsNamed: true, + startPosition: { row: 0, column: 0 }, + endPosition: { row: 0, column: 13 }, + startIndex: 0, + endIndex: 13, + }); + + expect(cursor.gotoFirstChild()).toBe(true); + assertCursorState(cursor, { + nodeType: 'binary_expression', + nodeIsNamed: true, + startPosition: { row: 0, column: 0 }, + endPosition: { row: 0, column: 13 }, + startIndex: 0, + endIndex: 13, + }); + + expect(cursor.gotoFirstChild()).toBe(true); + assertCursorState(cursor, { + nodeType: 'binary_expression', + nodeIsNamed: true, + startPosition: { row: 0, column: 0 }, + endPosition: { row: 0, column: 5 }, + startIndex: 0, + endIndex: 5, + }); + + expect(cursor.gotoFirstChild()).toBe(true); + expect(cursor.nodeText).toBe('a'); + assertCursorState(cursor, { + nodeType: 'identifier', + nodeIsNamed: true, + startPosition: { row: 0, column: 0 }, + endPosition: { row: 0, column: 1 }, + startIndex: 0, + endIndex: 1, + }); + + expect(cursor.gotoFirstChild()).toBe(false); + expect(cursor.gotoNextSibling()).toBe(true); + expect(cursor.nodeText).toBe('*'); + assertCursorState(cursor, { + nodeType: '*', + nodeIsNamed: false, + startPosition: { row: 0, column: 2 }, + endPosition: { row: 0, column: 3 }, + startIndex: 2, + endIndex: 3, + }); + + expect(cursor.gotoNextSibling()).toBe(true); + expect(cursor.nodeText).toBe('b'); + assertCursorState(cursor, { + nodeType: 'identifier', + nodeIsNamed: true, + startPosition: { row: 0, column: 4 }, + endPosition: { row: 0, column: 5 }, + startIndex: 4, + endIndex: 5, + }); + + expect(cursor.gotoNextSibling()).toBe(false); + expect(cursor.gotoParent()).toBe(true); + assertCursorState(cursor, { + nodeType: 'binary_expression', + nodeIsNamed: true, + startPosition: { row: 0, column: 0 }, + endPosition: { row: 0, column: 5 }, + startIndex: 0, + endIndex: 5, + }); + + expect(cursor.gotoNextSibling()).toBe(true); + assertCursorState(cursor, { + nodeType: '+', + nodeIsNamed: false, + startPosition: { row: 0, column: 6 }, + endPosition: { row: 0, column: 7 }, + startIndex: 6, + endIndex: 7, + }); + + expect(cursor.gotoNextSibling()).toBe(true); + assertCursorState(cursor, { + nodeType: 'binary_expression', + nodeIsNamed: true, + startPosition: { row: 0, column: 8 }, + endPosition: { row: 0, column: 13 }, + startIndex: 8, + endIndex: 13, + }); + + const copy = tree.walk(); + copy.resetTo(cursor); + + expect(copy.gotoPreviousSibling()).toBe(true); + assertCursorState(copy, { + nodeType: '+', + nodeIsNamed: false, + startPosition: { row: 0, column: 6 }, + endPosition: { row: 0, column: 7 }, + startIndex: 6, + endIndex: 7, + }); + + expect(copy.gotoPreviousSibling()).toBe(true); + assertCursorState(copy, { + nodeType: 'binary_expression', + nodeIsNamed: true, + startPosition: { row: 0, column: 0 }, + endPosition: { row: 0, column: 5 }, + startIndex: 0, + endIndex: 5, + }); + + expect(copy.gotoLastChild()).toBe(true); + assertCursorState(copy, { + nodeType: 'identifier', + nodeIsNamed: true, + startPosition: { row: 0, column: 4 }, + endPosition: { row: 0, column: 5 }, + startIndex: 4, + endIndex: 5, + }); + + expect(copy.gotoParent()).toBe(true); + expect(copy.gotoParent()).toBe(true); + expect(copy.nodeType).toBe('binary_expression'); + expect(copy.gotoParent()).toBe(true); + expect(copy.nodeType).toBe('expression_statement'); + expect(copy.gotoParent()).toBe(true); + expect(copy.nodeType).toBe('program'); + expect(copy.gotoParent()).toBe(false); + copy.delete(); + + expect(cursor.gotoParent()).toBe(true); + expect(cursor.nodeType).toBe('binary_expression'); + expect(cursor.gotoParent()).toBe(true); + expect(cursor.nodeType).toBe('expression_statement'); + expect(cursor.gotoParent()).toBe(true); + expect(cursor.nodeType).toBe('program'); + }); + + it('keeps track of the field name associated with each node', () => { + tree = parser.parse('a.b();')!; + cursor = tree.walk(); + cursor.gotoFirstChild(); + cursor.gotoFirstChild(); + + expect(cursor.currentNode.type).toBe('call_expression'); + expect(cursor.currentFieldName).toBeNull(); + + cursor.gotoFirstChild(); + expect(cursor.currentNode.type).toBe('member_expression'); + expect(cursor.currentFieldName).toBe('function'); + + cursor.gotoFirstChild(); + expect(cursor.currentNode.type).toBe('identifier'); + expect(cursor.currentFieldName).toBe('object'); + + cursor.gotoNextSibling(); + cursor.gotoNextSibling(); + expect(cursor.currentNode.type).toBe('property_identifier'); + expect(cursor.currentFieldName).toBe('property'); + + cursor.gotoParent(); + cursor.gotoNextSibling(); + expect(cursor.currentNode.type).toBe('arguments'); + expect(cursor.currentFieldName).toBe('arguments'); + }); + + it('returns a cursor that can be reset anywhere in the tree', () => { + tree = parser.parse('a * b + c / d')!; + cursor = tree.walk(); + const root = tree.rootNode.firstChild; + + cursor.reset(root!.firstChild!.firstChild!); + assertCursorState(cursor, { + nodeType: 'binary_expression', + nodeIsNamed: true, + startPosition: { row: 0, column: 0 }, + endPosition: { row: 0, column: 5 }, + startIndex: 0, + endIndex: 5, + }); + + cursor.gotoFirstChild(); + assertCursorState(cursor, { + nodeType: 'identifier', + nodeIsNamed: true, + startPosition: { row: 0, column: 0 }, + endPosition: { row: 0, column: 1 }, + startIndex: 0, + endIndex: 1, + }); + + expect(cursor.gotoParent()).toBe(true); + expect(cursor.gotoParent()).toBe(false); + }); + }); + + describe('.copy', () => { + let input: string; + let edit: Edit; + + it('creates another tree that remains stable if the original tree is edited', () => { + input = 'abc + cde'; + tree = parser.parse(input)!; + expect(tree.rootNode.toString()).toBe( + '(program (expression_statement (binary_expression left: (identifier) right: (identifier))))' + ); + + const tree2 = tree.copy(); + [input, edit] = spliceInput(input, 3, 0, '123'); + expect(input).toBe('abc123 + cde'); + tree.edit(edit); + + const leftNode = tree.rootNode.firstChild!.firstChild!.firstChild; + const leftNode2 = tree2.rootNode.firstChild!.firstChild!.firstChild; + const rightNode = tree.rootNode.firstChild!.firstChild!.lastChild; + const rightNode2 = tree2.rootNode.firstChild!.firstChild!.lastChild; + expect(leftNode!.endIndex).toBe(6); + expect(leftNode2!.endIndex).toBe(3); + expect(rightNode!.startIndex).toBe(9); + expect(rightNode2!.startIndex).toBe(6); + + tree2.delete(); + }); + }); +}); + +function spliceInput(input: string, startIndex: number, lengthRemoved: number, newText: string): [string, Edit] { + const oldEndIndex = startIndex + lengthRemoved; + const newEndIndex = startIndex + newText.length; + const startPosition = getExtent(input.slice(0, startIndex)); + const oldEndPosition = getExtent(input.slice(0, oldEndIndex)); + input = input.slice(0, startIndex) + newText + input.slice(oldEndIndex); + const newEndPosition = getExtent(input.slice(0, newEndIndex)); + return [ + input, + new Edit({ + startIndex, + startPosition, + oldEndIndex, + oldEndPosition, + newEndIndex, + newEndPosition, + }), + ]; +} + +// Gets the extent of the text in terms of zero-based row and column numbers. +function getExtent(text: string): Point { + let row = 0; + let index = -1; + let lastIndex = 0; + while ((index = text.indexOf('\n', index + 1)) !== -1) { + row++; + lastIndex = index + 1; + } + return { row, column: text.length - lastIndex }; +} + +function assertCursorState(cursor: TreeCursor, params: CursorState): void { + expect(cursor.nodeType).toBe(params.nodeType); + expect(cursor.nodeIsNamed).toBe(params.nodeIsNamed); + expect(cursor.startPosition).toEqual(params.startPosition); + expect(cursor.endPosition).toEqual(params.endPosition); + expect(cursor.startIndex).toEqual(params.startIndex); + expect(cursor.endIndex).toEqual(params.endIndex); + + const node = cursor.currentNode; + expect(node.type).toBe(params.nodeType); + expect(node.isNamed).toBe(params.nodeIsNamed); + expect(node.startPosition).toEqual(params.startPosition); + expect(node.endPosition).toEqual(params.endPosition); + expect(node.startIndex).toEqual(params.startIndex); + expect(node.endIndex).toEqual(params.endIndex); +} diff --git a/lib/binding_web/tree-sitter-web.d.ts b/lib/binding_web/tree-sitter-web.d.ts deleted file mode 100644 index cfd8a102..00000000 --- a/lib/binding_web/tree-sitter-web.d.ts +++ /dev/null @@ -1,242 +0,0 @@ -declare module 'web-tree-sitter' { - class Parser { - /** - * - * @param moduleOptions Optional emscripten module-object, see https://emscripten.org/docs/api_reference/module.html - */ - static init(moduleOptions?: object): Promise; - delete(): void; - parse(input: string | Parser.Input, oldTree?: Parser.Tree, options?: Parser.Options): Parser.Tree; - getIncludedRanges(): Parser.Range[]; - getTimeoutMicros(): number; - setTimeoutMicros(timeout: number): void; - reset(): void; - getLanguage(): Parser.Language; - setLanguage(language?: Parser.Language | null): void; - getLogger(): Parser.Logger; - setLogger(logFunc?: Parser.Logger | false | null): void; - } - - namespace Parser { - export type Options = { - includedRanges?: Range[]; - }; - - export type Point = { - row: number; - column: number; - }; - - export type Range = { - startIndex: number, - endIndex: number, - startPosition: Point, - endPosition: Point - }; - - export type Edit = { - startIndex: number; - oldEndIndex: number; - newEndIndex: number; - startPosition: Point; - oldEndPosition: Point; - newEndPosition: Point; - }; - - export type Logger = ( - message: string, - params: { [param: string]: string }, - type: "parse" | "lex" - ) => void; - - export interface Input { - (index: number, position?: Point): string | null; - } - - export interface SyntaxNode { - tree: Tree; - id: number; - typeId: number; - grammarId: number; - type: string; - grammarType: string; - isNamed: boolean; - isMissing: boolean; - isExtra: boolean; - hasChanges: boolean; - hasError: boolean; - isError: boolean; - text: string; - parseState: number; - nextParseState: number; - startPosition: Point; - endPosition: Point; - startIndex: number; - endIndex: number; - parent: SyntaxNode | null; - children: Array; - namedChildren: Array; - childCount: number; - namedChildCount: number; - firstChild: SyntaxNode | null; - firstNamedChild: SyntaxNode | null; - lastChild: SyntaxNode | null; - lastNamedChild: SyntaxNode | null; - nextSibling: SyntaxNode | null; - nextNamedSibling: SyntaxNode | null; - previousSibling: SyntaxNode | null; - previousNamedSibling: SyntaxNode | null; - descendantCount: number; - - equals(other: SyntaxNode): boolean; - toString(): string; - child(index: number): SyntaxNode | null; - namedChild(index: number): SyntaxNode | null; - childForFieldName(fieldName: string): SyntaxNode | null; - childForFieldId(fieldId: number): SyntaxNode | null; - fieldNameForChild(childIndex: number): string | null; - childrenForFieldName(fieldName: string): Array; - childrenForFieldId(fieldId: number): Array; - firstChildForIndex(index: number): SyntaxNode | null; - firstNamedChildForIndex(index: number): SyntaxNode | null; - - descendantForIndex(index: number): SyntaxNode; - descendantForIndex(startIndex: number, endIndex: number): SyntaxNode; - namedDescendantForIndex(index: number): SyntaxNode; - namedDescendantForIndex(startIndex: number, endIndex: number): SyntaxNode; - descendantForPosition(position: Point): SyntaxNode; - descendantForPosition(startPosition: Point, endPosition: Point): SyntaxNode; - namedDescendantForPosition(position: Point): SyntaxNode; - namedDescendantForPosition(startPosition: Point, endPosition: Point): SyntaxNode; - descendantsOfType(types: String | Array, startPosition?: Point, endPosition?: Point): Array; - - walk(): TreeCursor; - } - - export interface TreeCursor { - nodeType: string; - nodeTypeId: number; - nodeStateId: number; - nodeText: string; - nodeId: number; - nodeIsNamed: boolean; - nodeIsMissing: boolean; - startPosition: Point; - endPosition: Point; - startIndex: number; - endIndex: number; - readonly currentNode: SyntaxNode; - readonly currentFieldName: string; - readonly currentFieldId: number; - readonly currentDepth: number; - readonly currentDescendantIndex: number; - - reset(node: SyntaxNode): void; - resetTo(cursor: TreeCursor): void; - delete(): void; - gotoParent(): boolean; - gotoFirstChild(): boolean; - gotoLastChild(): boolean; - gotoFirstChildForIndex(goalIndex: number): boolean; - gotoFirstChildForPosition(goalPosition: Point): boolean; - gotoNextSibling(): boolean; - gotoPreviousSibling(): boolean; - gotoDescendant(goalDescendantIndex: number): void; - } - - export interface Tree { - readonly rootNode: SyntaxNode; - - rootNodeWithOffset(offsetBytes: number, offsetExtent: Point): SyntaxNode; - copy(): Tree; - delete(): void; - edit(edit: Edit): void; - walk(): TreeCursor; - getChangedRanges(other: Tree): Range[]; - getIncludedRanges(): Range[]; - getLanguage(): Language; - } - - export interface QueryCapture { - name: string; - text?: string; - node: SyntaxNode; - setProperties?: { [prop: string]: string | null }; - assertedProperties?: { [prop: string]: string | null }; - refutedProperties?: { [prop: string]: string | null }; - } - - export interface QueryMatch { - pattern: number; - captures: QueryCapture[]; - } - - export type QueryOptions = { - startPosition?: Point; - endPosition?: Point; - startIndex?: number; - endIndex?: number; - matchLimit?: number; - maxStartDepth?: number; - timeoutMicros?: number; - }; - - export interface PredicateResult { - operator: string; - operands: { name: string; type: string }[]; - } - - export class Query { - captureNames: string[]; - readonly predicates: { [name: string]: Function }[]; - readonly setProperties: any[]; - readonly assertedProperties: any[]; - readonly refutedProperties: any[]; - readonly matchLimit: number; - - delete(): void; - captures(node: SyntaxNode, options?: QueryOptions): QueryCapture[]; - matches(node: SyntaxNode, options?: QueryOptions): QueryMatch[]; - predicatesForPattern(patternIndex: number): PredicateResult[]; - disableCapture(captureName: string): void; - disablePattern(patternIndex: number): void; - isPatternGuaranteedAtStep(byteOffset: number): boolean; - isPatternRooted(patternIndex: number): boolean; - isPatternNonLocal(patternIndex: number): boolean; - startIndexForPattern(patternIndex: number): number; - didExceedMatchLimit(): boolean; - } - - class Language { - static load(input: string | Uint8Array): Promise; - - readonly version: number; - readonly fieldCount: number; - readonly stateCount: number; - readonly nodeTypeCount: number; - - fieldNameForId(fieldId: number): string | null; - fieldIdForName(fieldName: string): number | null; - idForNodeType(type: string, named: boolean): number; - nodeTypeForId(typeId: number): string | null; - nodeTypeIsNamed(typeId: number): boolean; - nodeTypeIsVisible(typeId: number): boolean; - nextState(stateId: number, typeId: number): number; - query(source: string): Query; - lookaheadIterator(stateId: number): LookaheadIterable | null; - } - - export class LookaheadIterable { - readonly language: Language; - readonly currentTypeId: number; - readonly currentType: string; - - delete(): void; - reset(language: Language, stateId: number): boolean; - resetState(stateId: number): boolean; - [Symbol.iterator](): Iterator; - } - } - - export = Parser -} diff --git a/lib/binding_web/tsconfig.json b/lib/binding_web/tsconfig.json new file mode 100644 index 00000000..b7ed356f --- /dev/null +++ b/lib/binding_web/tsconfig.json @@ -0,0 +1,41 @@ +{ + "compilerOptions": { + "target": "es2022", + "module": "es2022", + "lib": [ + "es2022", + "dom" + ], + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "rootDir": "./", + "outDir": "./dist", + "strict": true, + "noImplicitAny": true, + "strictNullChecks": true, + "strictFunctionTypes": true, + "strictPropertyInitialization": true, + "noImplicitThis": true, + "alwaysStrict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noImplicitReturns": true, + "moduleResolution": "node", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "skipLibCheck": true, + "composite": true, + "isolatedModules": true, + }, + "include": [ + "src/*.ts", + "script/*", + "test/*", + "lib/*.ts" + ], + "exclude": [ + "node_modules", + "dist", + ] +} diff --git a/lib/binding_web/vitest.config.ts b/lib/binding_web/vitest.config.ts new file mode 100644 index 00000000..64ed3bf2 --- /dev/null +++ b/lib/binding_web/vitest.config.ts @@ -0,0 +1,19 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + globals: true, + environment: 'node', + coverage: { + include: [ + 'web-tree-sitter.js', + ], + exclude: [ + 'test/**', + 'dist/**', + 'lib/**', + 'wasm/**' + ], + }, + } +}) diff --git a/lib/binding_web/wasm-test-grammars.nix b/lib/binding_web/wasm-test-grammars.nix new file mode 100644 index 00000000..536359d3 --- /dev/null +++ b/lib/binding_web/wasm-test-grammars.nix @@ -0,0 +1,67 @@ +{ + cli, + lib, + nodejs_22, + pkgsCross, + src, + stdenv, + test-grammars, + version, +}: +let + grammars = [ + "bash" + "c" + "cpp" + "embedded-template" + "html" + "javascript" + "json" + "python" + "rust" + "typescript" + ]; +in +stdenv.mkDerivation { + inherit src version; + + pname = "wasm-test-grammars"; + + nativeBuildInputs = [ + cli + pkgsCross.wasi32.stdenv.cc + nodejs_22 + ]; + + buildPhase = '' + export HOME=$TMPDIR + export TREE_SITTER_WASI_SDK_PATH=${pkgsCross.wasi32.stdenv.cc} + export NIX_LDFLAGS="" + + cp -r ${test-grammars}/fixtures . + chmod -R u+w fixtures + + for grammar in ${lib.concatStringsSep " " grammars}; do + if [ -d "fixtures/grammars/$grammar" ]; then + echo "Building WASM for $grammar" + + if [ "$grammar" = "typescript" ]; then + tree-sitter build --wasm -o "tree-sitter-typescript.wasm" "fixtures/grammars/$grammar/typescript" + tree-sitter build --wasm -o "tree-sitter-tsx.wasm" "fixtures/grammars/$grammar/tsx" + else + tree-sitter build --wasm -o "tree-sitter-$grammar.wasm" "fixtures/grammars/$grammar" + fi + fi + done + ''; + + installPhase = '' + mkdir -p $out + for wasm in *.wasm; do + if [ -f "$wasm" ]; then + echo "Installing $wasm" + cp "$wasm" $out/ + fi + done + ''; +} diff --git a/lib/include/tree_sitter/api.h b/lib/include/tree_sitter/api.h index 01ccae14..22c85d48 100644 --- a/lib/include/tree_sitter/api.h +++ b/lib/include/tree_sitter/api.h @@ -51,12 +51,15 @@ typedef struct TSLookaheadIterator TSLookaheadIterator; // This function signature reads one code point from the given string, // returning the number of bytes consumed. It should write the code point // to the `code_point` pointer, or write -1 if the input is invalid. -typedef uint32_t (*DecodeFunction)( +typedef uint32_t (*TSDecodeFunction)( const uint8_t *string, uint32_t length, int32_t *code_point ); +// Deprecated alias to be removed in ABI 16 +typedef TSDecodeFunction DecodeFunction; + typedef enum TSInputEncoding { TSInputEncodingUTF8, TSInputEncodingUTF16LE, @@ -87,12 +90,13 @@ typedef struct TSInput { void *payload; const char *(*read)(void *payload, uint32_t byte_index, TSPoint position, uint32_t *bytes_read); TSInputEncoding encoding; - DecodeFunction decode; + TSDecodeFunction decode; } TSInput; typedef struct TSParseState { void *payload; uint32_t current_byte_offset; + bool has_error; } TSParseState; typedef struct TSParseOptions { @@ -182,6 +186,20 @@ typedef struct TSQueryCursorOptions { bool (*progress_callback)(TSQueryCursorState *state); } TSQueryCursorOptions; +/** + * The metadata associated with a language. + * + * Currently, this metadata can be used to check the [Semantic Version](https://semver.org/) + * of the language. This version information should be used to signal if a given parser might + * be incompatible with existing queries when upgrading between major versions, or minor versions + * if it's in zerover. + */ +typedef struct TSLanguageMetadata { + uint8_t major_version; + uint8_t minor_version; + uint8_t patch_version; +} TSLanguageMetadata; + /********************/ /* Section - Parser */ /********************/ @@ -207,7 +225,7 @@ const TSLanguage *ts_parser_language(const TSParser *self); * Returns a boolean indicating whether or not the language was successfully * assigned. True means assignment succeeded. False means there was a version * mismatch: the language was generated with an incompatible version of the - * Tree-sitter CLI. Check the language's version using [`ts_language_version`] + * Tree-sitter CLI. Check the language's ABI version using [`ts_language_abi_version`] * and compare it to this library's [`TREE_SITTER_LANGUAGE_VERSION`] and * [`TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION`] constants. */ @@ -281,16 +299,7 @@ const TSRange *ts_parser_included_ranges( * are four possible reasons for failure: * 1. The parser does not have a language assigned. Check for this using the [`ts_parser_language`] function. - * 2. Parsing was cancelled due to a timeout that was set by an earlier call to - * the [`ts_parser_set_timeout_micros`] function. You can resume parsing from - * where the parser left out by calling [`ts_parser_parse`] again with the - * same arguments. Or you can start parsing from scratch by first calling - * [`ts_parser_reset`]. - * 3. Parsing was cancelled using a cancellation flag that was set by an - * earlier call to [`ts_parser_set_cancellation_flag`]. You can resume parsing - * from where the parser left out by calling [`ts_parser_parse`] again with - * the same arguments. - * 4. Parsing was cancelled due to the progress callback returning true. This callback + * 2. Parsing was cancelled due to the progress callback returning true. This callback * is passed in [`ts_parser_parse_with_options`] inside the [`TSParseOptions`] struct. * * [`read`]: TSInput::read @@ -348,7 +357,7 @@ TSTree *ts_parser_parse_string_encoding( /** * Instruct the parser to start the next parse from the beginning. * - * If the parser previously failed because of a timeout or a cancellation, then + * If the parser previously failed because of the progress callback, then * by default, it will resume where it left off on the next call to * [`ts_parser_parse`] or other parsing functions. If you don't want to resume, * and instead intend to use this parser to parse some other document, you must @@ -356,42 +365,6 @@ TSTree *ts_parser_parse_string_encoding( */ void ts_parser_reset(TSParser *self); -/** - * @deprecated use [`ts_parser_parse_with_options`] and pass in a callback instead, this will be removed in 0.26. - * - * Set the maximum duration in microseconds that parsing should be allowed to - * take before halting. - * - * If parsing takes longer than this, it will halt early, returning NULL. - * See [`ts_parser_parse`] for more information. - */ -void ts_parser_set_timeout_micros(TSParser *self, uint64_t timeout_micros); - -/** - * @deprecated use [`ts_parser_parse_with_options`] and pass in a callback instead, this will be removed in 0.26. - * - * Get the duration in microseconds that parsing is allowed to take. - */ -uint64_t ts_parser_timeout_micros(const TSParser *self); - -/** - * @deprecated use [`ts_parser_parse_with_options`] and pass in a callback instead, this will be removed in 0.26. - * - * Set the parser's current cancellation flag pointer. - * - * If a non-null pointer is assigned, then the parser will periodically read - * from this pointer during parsing. If it reads a non-zero value, it will - * halt early, returning NULL. See [`ts_parser_parse`] for more information. - */ -void ts_parser_set_cancellation_flag(TSParser *self, const size_t *flag); - -/** - * @deprecated use [`ts_parser_parse_with_options`] and pass in a callback instead, this will be removed in 0.26. - * - * Get the parser's current cancellation flag pointer. - */ -const size_t *ts_parser_cancellation_flag(const TSParser *self); - /** * Set the logger that a parser should use during parsing. * @@ -477,6 +450,13 @@ void ts_tree_edit(TSTree *self, const TSInputEdit *edit); * You need to pass the old tree that was passed to parse, as well as the new * tree that was returned from that function. * + * The returned ranges indicate areas where the hierarchical structure of syntax + * nodes (from root to leaf) has changed between the old and new trees. Characters + * outside these ranges have identical ancestor nodes in both trees. + * + * Note that the returned ranges may be slightly larger than the exact changed areas, + * but Tree-sitter attempts to make them as small as possible. + * * The returned array is allocated using `malloc` and the caller is responsible * for freeing it using `free`. The length of the array will be written to the * given `length` pointer. @@ -605,25 +585,15 @@ TSStateId ts_node_next_parse_state(TSNode self); /** * Get the node's immediate parent. - * Prefer [`ts_node_child_containing_descendant`] for + * Prefer [`ts_node_child_with_descendant`] for * iterating over the node's ancestors. */ TSNode ts_node_parent(TSNode self); -/** - * @deprecated use [`ts_node_contains_descendant`] instead, this will be removed in 0.25 - * - * Get the node's child containing `descendant`. This will not return - * the descendant if it is a direct child of `self`, for that use - * `ts_node_contains_descendant`. - */ -TSNode ts_node_child_containing_descendant(TSNode self, TSNode descendant); - /** * Get the node that contains `descendant`. * - * Note that this can return `descendant` itself, unlike the deprecated function - * [`ts_node_child_containing_descendant`]. + * Note that this can return `descendant` itself. */ TSNode ts_node_child_with_descendant(TSNode self, TSNode descendant); @@ -694,12 +664,12 @@ TSNode ts_node_next_named_sibling(TSNode self); TSNode ts_node_prev_named_sibling(TSNode self); /** - * Get the node's first child that extends beyond the given byte offset. + * Get the node's first child that contains or starts after the given byte offset. */ TSNode ts_node_first_child_for_byte(TSNode self, uint32_t byte); /** - * Get the node's first named child that extends beyond the given byte offset. + * Get the node's first named child that contains or starts after the given byte offset. */ TSNode ts_node_first_named_child_for_byte(TSNode self, uint32_t byte); @@ -738,6 +708,24 @@ void ts_node_edit(TSNode *self, const TSInputEdit *edit); */ bool ts_node_eq(TSNode self, TSNode other); +/** + * Edit a point to keep it in-sync with source code that has been edited. + * + * This function updates a single point's byte offset and row/column position + * based on an edit operation. This is useful for editing points without + * requiring a tree or node instance. + */ +void ts_point_edit(TSPoint *point, uint32_t *point_byte, const TSInputEdit *edit); + +/** + * Edit a range to keep it in-sync with source code that has been edited. + * + * This function updates a range's start and end positions based on an edit + * operation. This is useful for editing ranges without requiring a tree + * or node instance. + */ +void ts_range_edit(TSRange *range, const TSInputEdit *edit); + /************************/ /* Section - TreeCursor */ /************************/ @@ -748,6 +736,9 @@ bool ts_node_eq(TSNode self, TSNode other); * A tree cursor allows you to walk a syntax tree more efficiently than is * possible using the [`TSNode`] functions. It is a mutable object that is always * on a certain syntax node, and can be moved imperatively to different nodes. + * + * Note that the given node is considered the root of the cursor, + * and the cursor cannot walk outside this node. */ TSTreeCursor ts_tree_cursor_new(TSNode node); @@ -796,6 +787,9 @@ TSFieldId ts_tree_cursor_current_field_id(const TSTreeCursor *self); * * This returns `true` if the cursor successfully moved, and returns `false` * if there was no parent node (the cursor was already on the root node). + * + * Note that the node the cursor was constructed with is considered the root + * of the cursor, and the cursor cannot walk outside this node. */ bool ts_tree_cursor_goto_parent(TSTreeCursor *self); @@ -804,6 +798,9 @@ bool ts_tree_cursor_goto_parent(TSTreeCursor *self); * * This returns `true` if the cursor successfully moved, and returns `false` * if there was no next sibling node. + * + * Note that the node the cursor was constructed with is considered the root + * of the cursor, and the cursor cannot walk outside this node. */ bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self); @@ -815,8 +812,10 @@ bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self); * * Note, that this function may be slower than * [`ts_tree_cursor_goto_next_sibling`] due to how node positions are stored. In - * the worst case, this will need to iterate through all the children upto the - * previous sibling node to recalculate its position. + * the worst case, this will need to iterate through all the children up to the + * previous sibling node to recalculate its position. Also note that the node the cursor + * was constructed with is considered the root of the cursor, and the cursor cannot + * walk outside this node. */ bool ts_tree_cursor_goto_previous_sibling(TSTreeCursor *self); @@ -860,7 +859,7 @@ uint32_t ts_tree_cursor_current_descendant_index(const TSTreeCursor *self); uint32_t ts_tree_cursor_current_depth(const TSTreeCursor *self); /** - * Move the cursor to the first child of its current node that extends beyond + * Move the cursor to the first child of its current node that contains or starts after * the given byte offset or point. * * This returns the index of the child node if one was found, and returns -1 @@ -1044,7 +1043,7 @@ void ts_query_cursor_delete(TSQueryCursor *self); void ts_query_cursor_exec(TSQueryCursor *self, const TSQuery *query, TSNode node); /** - * Start running a gievn query on a given node, with some options. + * Start running a given query on a given node, with some options. */ void ts_query_cursor_exec_with_options( TSQueryCursor *self, @@ -1068,29 +1067,18 @@ bool ts_query_cursor_did_exceed_match_limit(const TSQueryCursor *self); uint32_t ts_query_cursor_match_limit(const TSQueryCursor *self); void ts_query_cursor_set_match_limit(TSQueryCursor *self, uint32_t limit); -/** - * @deprecated use [`ts_query_cursor_exec_with_options`] and pass in a callback instead, this will be removed in 0.26. - * - * Set the maximum duration in microseconds that query execution should be allowed to - * take before halting. - * - * If query execution takes longer than this, it will halt early, returning NULL. - * See [`ts_query_cursor_next_match`] or [`ts_query_cursor_next_capture`] for more information. - */ -void ts_query_cursor_set_timeout_micros(TSQueryCursor *self, uint64_t timeout_micros); - -/** - * @deprecated use [`ts_query_cursor_exec_with_options`] and pass in a callback instead, this will be removed in 0.26. - * - * Get the duration in microseconds that query execution is allowed to take. - * - * This is set via [`ts_query_cursor_set_timeout_micros`]. - */ -uint64_t ts_query_cursor_timeout_micros(const TSQueryCursor *self); - /** * Set the range of bytes in which the query will be executed. * + * The query cursor will return matches that intersect with the given point range. + * This means that a match may be returned even if some of its captures fall + * outside the specified range, as long as at least part of the match + * overlaps with the range. + * + * For example, if a query pattern matches a node that spans a larger area + * than the specified range, but part of that node intersects with the range, + * the entire match will be returned. + * * This will return `false` if the start byte is greater than the end byte, otherwise * it will return `true`. */ @@ -1099,11 +1087,42 @@ bool ts_query_cursor_set_byte_range(TSQueryCursor *self, uint32_t start_byte, ui /** * Set the range of (row, column) positions in which the query will be executed. * + * The query cursor will return matches that intersect with the given point range. + * This means that a match may be returned even if some of its captures fall + * outside the specified range, as long as at least part of the match + * overlaps with the range. + * + * For example, if a query pattern matches a node that spans a larger area + * than the specified range, but part of that node intersects with the range, + * the entire match will be returned. + * * This will return `false` if the start point is greater than the end point, otherwise * it will return `true`. */ bool ts_query_cursor_set_point_range(TSQueryCursor *self, TSPoint start_point, TSPoint end_point); +/** + * Set the byte range within which all matches must be fully contained. + * + * Set the range of bytes in which matches will be searched for. In contrast to + * `ts_query_cursor_set_byte_range`, this will restrict the query cursor to only return + * matches where _all_ nodes are _fully_ contained within the given range. Both functions + * can be used together, e.g. to search for any matches that intersect line 5000, as + * long as they are fully contained within lines 4500-5500 + */ +bool ts_query_cursor_set_containing_byte_range(TSQueryCursor *self, uint32_t start_byte, uint32_t end_byte); + +/** + * Set the point range within which all matches must be fully contained. + * + * Set the range of bytes in which matches will be searched for. In contrast to + * `ts_query_cursor_set_point_range`, this will restrict the query cursor to only return + * matches where _all_ nodes are _fully_ contained within the given range. Both functions + * can be used together, e.g. to search for any matches that intersect line 5000, as + * long as they are fully contained within lines 4500-5500 + */ +bool ts_query_cursor_set_containing_point_range(TSQueryCursor *self, TSPoint start_point, TSPoint end_point); + /** * Advance to the next match of the currently running query. * @@ -1166,11 +1185,6 @@ uint32_t ts_language_symbol_count(const TSLanguage *self); */ uint32_t ts_language_state_count(const TSLanguage *self); -/** - * Get a node type string for the given numerical id. - */ -const char *ts_language_symbol_name(const TSLanguage *self, TSSymbol symbol); - /** * Get the numerical id for the given node type string. */ @@ -1196,6 +1210,27 @@ const char *ts_language_field_name_for_id(const TSLanguage *self, TSFieldId id); */ TSFieldId ts_language_field_id_for_name(const TSLanguage *self, const char *name, uint32_t name_length); +/** + * Get a list of all supertype symbols for the language. +*/ +const TSSymbol *ts_language_supertypes(const TSLanguage *self, uint32_t *length); + +/** + * Get a list of all subtype symbol ids for a given supertype symbol. + * + * See [`ts_language_supertypes`] for fetching all supertype symbols. + */ +const TSSymbol *ts_language_subtypes( + const TSLanguage *self, + TSSymbol supertype, + uint32_t *length +); + +/** + * Get a node type string for the given numerical id. + */ +const char *ts_language_symbol_name(const TSLanguage *self, TSSymbol symbol); + /** * Check whether the given node type id belongs to named nodes, anonymous nodes, * or a hidden nodes. @@ -1211,7 +1246,16 @@ TSSymbolType ts_language_symbol_type(const TSLanguage *self, TSSymbol symbol); * * See also [`ts_parser_set_language`]. */ -uint32_t ts_language_version(const TSLanguage *self); +uint32_t ts_language_abi_version(const TSLanguage *self); + +/** + * Get the metadata for this language. This information is generated by the + * CLI, and relies on the language author providing the correct metadata in + * the language's `tree-sitter.json` file. + * + * See also [`TSMetadata`]. + */ +const TSLanguageMetadata *ts_language_metadata(const TSLanguage *self); /** * Get the next parse state. Combine this with lookahead iterators to generate @@ -1339,7 +1383,7 @@ const TSLanguage *ts_wasm_store_load_language( ); /** - * Get the number of languages instantiated in the given wasm store. + * Get the number of languages instantiated in the given Wasm store. */ size_t ts_wasm_store_language_count(const TSWasmStore *); diff --git a/lib/lldb_pretty_printers/table_entry.py b/lib/lldb_pretty_printers/table_entry.py new file mode 100644 index 00000000..f46b9d43 --- /dev/null +++ b/lib/lldb_pretty_printers/table_entry.py @@ -0,0 +1,60 @@ +from lldb import SBValue + +# typedef struct { +# const TSParseAction *actions; +# uint32_t action_count; +# bool is_reusable; +# } TableEntry; + +# TODO: Same inline issue as with `TSTreeSyntheticProvider`. + + +class TableEntrySyntheticProvider: + def __init__(self, valobj: SBValue, _dict): + self.valobj: SBValue = valobj + self.update() + + def num_children(self) -> int: + # is_reusable, action_count, actions + return 2 + max(1, self.action_count.GetValueAsUnsigned()) + + def get_child_index(self, name: str) -> int: + if name == "is_reusable": + return 0 + elif name == "action_count": + return 1 + else: + if self.action_count.GetValueAsUnsigned() == 0: + return 2 + index = name.lstrip("actions[").rstrip("]") + if index.isdigit(): + return int(index) + else: + return -1 + + def get_child_at_index(self, index: int) -> SBValue: + if index == 0: + return self.is_reusable + elif index == 1: + return self.action_count + else: + if self.action_count.GetValueAsUnsigned() == 0: + return self.actions + offset: int = index - 3 + start: int = self.actions.GetValueAsUnsigned() + address: int = start + offset * self.element_type_size + element: SBValue = self.actions.CreateValueFromAddress( + "action[%s]" % (offset), address, self.element_type + ) + return element + + def update(self): + self.is_reusable: SBValue = self.valobj.GetChildMemberWithName("is_reusable") + self.action_count: SBValue = self.valobj.GetChildMemberWithName("action_count") + self.actions: SBValue = self.valobj.GetChildMemberWithName("actions") + + self.element_type: SBType = self.actions.GetType().GetPointeeType() + self.element_type_size: int = self.element_type.GetByteSize() + + def has_children(self) -> bool: + return True diff --git a/lib/lldb_pretty_printers/tree_sitter_types.py b/lib/lldb_pretty_printers/tree_sitter_types.py new file mode 100644 index 00000000..b26c67ac --- /dev/null +++ b/lib/lldb_pretty_printers/tree_sitter_types.py @@ -0,0 +1,64 @@ +import lldb + +# Even though these are "unused", we still need them in scope in order for the classes +# to exist when we register them with the debugger +from ts_tree import TSTreeSyntheticProvider +from table_entry import TableEntrySyntheticProvider +from ts_array import ArraySyntheticProvider, anon_array_recognizer + + +class TreeSitterType(object): + TS_TREE: str = "TSTree" + SUBTREE_ARRAY: str = "SubtreeArray" + MUTABLE_SUBTREE_ARRAY: str = "MutableSubtreeArray" + STACK_SLICE_ARRAY: str = "StackSliceArray" + STACK_SUMMARY: str = "StackSummary" + STACK_ENTRY: str = "StackEntry" + REUSABLE_NODE: str = "ReusableNode" + REDUCE_ACTION_SET: str = "ReduceActionSet" + TABLE_ENTRY: str = "TableEntry" + TS_RANGE_ARRAY: str = "TSRangeArray" + CAPTURE_QUANTIFIERS: str = "CaptureQuantifiers" + CAPTURE_LIST: str = "CaptureList" + ANALYSIS_STATE_SET: str = "AnalysisStateSet" + ANALYSIS_SUBGRAPH_ARRAY: str = "AnalysisSubgraphArray" + STACK_NODE_ARRAY: str = "StackNodeArray" + STRING_DATA: str = "StringData" + + +def ts_type_to_regex(type: str) -> str: + return f"^{type}$|^struct {type}$|^typedef {type}$" + + +# Holds all tree-sitter types defined via the `Array` macro. These types will +# all share the same `ArrayTypeSyntheticProvider` synthetic provider +TS_ARRAY_TYPES = [ + TreeSitterType.REDUCE_ACTION_SET, + TreeSitterType.TS_RANGE_ARRAY, + TreeSitterType.CAPTURE_QUANTIFIERS, + TreeSitterType.ANALYSIS_STATE_SET, + TreeSitterType.CAPTURE_LIST, + TreeSitterType.ANALYSIS_SUBGRAPH_ARRAY, + TreeSitterType.STACK_SLICE_ARRAY, + TreeSitterType.STACK_SUMMARY, + TreeSitterType.SUBTREE_ARRAY, + TreeSitterType.MUTABLE_SUBTREE_ARRAY, + TreeSitterType.STRING_DATA, + TreeSitterType.STACK_NODE_ARRAY, +] + + +def __lldb_init_module(debugger: lldb.SBDebugger, _dict): + debugger.HandleCommand( + f"type synthetic add -l tree_sitter_types.TSTreeSyntheticProvider -x '{ts_type_to_regex(TreeSitterType.TS_TREE)}'" + ) + debugger.HandleCommand( + f"type synthetic add -l tree_sitter_types.TableEntrySyntheticProvider -x '{ts_type_to_regex(TreeSitterType.TABLE_ENTRY)}'" + ) + debugger.HandleCommand( + f"type synthetic add -l tree_sitter_types.ArraySyntheticProvider --recognizer-function tree_sitter_types.anon_array_recognizer" + ) + for type in TS_ARRAY_TYPES: + debugger.HandleCommand( + f"type synthetic add -l tree_sitter_types.ArraySyntheticProvider -x '{ts_type_to_regex(type)}'" + ) diff --git a/lib/lldb_pretty_printers/ts_array.py b/lib/lldb_pretty_printers/ts_array.py new file mode 100644 index 00000000..f51cd0a9 --- /dev/null +++ b/lib/lldb_pretty_printers/ts_array.py @@ -0,0 +1,78 @@ +from lldb import SBValue, SBType +import re + +# define Array(T) \ +# struct { \ +# T *contents; \ +# uint32_t size; \ +# uint32_t capacity; \ +# } + + +class ArraySyntheticProvider: + def __init__(self, valobj: SBValue, _dict): + self.valobj: SBValue = valobj + self.update() + + def num_children(self) -> int: + return 2 + self.size.GetValueAsUnsigned() # size, capacity, and elements + + def get_child_index(self, name: str) -> int: + if name == "size": + return 0 + elif name == "capacity": + return 1 + else: + if self.size.GetValueAsUnsigned() == 0: + return 2 + index = name.lstrip("[").rstrip("]") + if index.isdigit(): + return int(index) + else: + return -1 + + def get_child_at_index(self, index: int) -> SBValue: + if index == 0: + return self.size + elif index == 1: + return self.capacity + else: + if self.size.GetValueAsUnsigned() == 0: + return self.contents + offset: int = index - 2 + start: int = self.contents.GetValueAsUnsigned() + address: int = start + offset * self.element_type_size + element: SBValue = self.contents.CreateValueFromAddress( + "[%s]" % (offset), address, self.element_type + ) + return element + + def update(self): + self.contents: SBValue = self.valobj.GetChildMemberWithName("contents") + self.size: SBValue = self.valobj.GetChildMemberWithName("size") + self.capacity: SBValue = self.valobj.GetChildMemberWithName("capacity") + + self.element_type: SBType = self.contents.GetType().GetPointeeType() + self.element_type_size: int = self.element_type.GetByteSize() + + def has_children(self) -> bool: + return True + + +anon_re = re.compile( + r"struct\s*{$\s*\w+ \*contents;$\s*uint32_t size;$\s*uint32_t capacity;$\s*}", + re.MULTILINE, +) + + +# Used to recognize "anonymous" `Array(T)` types, i.e.: +# struct Foo { +# Array(Bar) bars; // Render this field usign `ArraySyntheticProvider` +# }; +def anon_array_recognizer(valobj: SBType, _dict) -> bool: + type_name = valobj.GetName() + if type_name == "(unnamed struct)": + type_str = str(valobj) + return anon_re.search(type_str) is not None + else: + return False diff --git a/lib/lldb_pretty_printers/ts_tree.py b/lib/lldb_pretty_printers/ts_tree.py new file mode 100644 index 00000000..1f690ebd --- /dev/null +++ b/lib/lldb_pretty_printers/ts_tree.py @@ -0,0 +1,101 @@ +from lldb import SBType, SBValue + +# struct TSTree { +# Subtree root; +# const TSLanguage *language; +# TSRange *included_ranges; +# unsigned included_range_count; +# }; + +# TODO: Ideally, we'd display the elements of `included_ranges` as +# children of `included_ranges` rather than separate items, i.e.: + +# (TSTree) { +# root = ... +# language = ... +# included_range_count = ... +# included_ranges = { +# [0] = { +# ... +# } +# [1] = { +# ... +# } +# ... +# } +# } +# +# instead of the current behavior: +# +# (TSTree) { +# root = ... +# language = ... +# included_range_count = ... +# included_ranges[0] = { +# ... +# } +# included_ranges[1] = { +# ... +# } +# } +# + + +class TSTreeSyntheticProvider: + def __init__(self, valobj: SBValue, _dict): + self.valobj: SBValue = valobj + self.update() + + def num_children(self) -> int: + # root, language, included_range_count, included_ranges + return 3 + self.included_range_count.GetValueAsUnsigned() + + def get_child_index(self, name: str) -> int: + if name == "root": + return 0 + elif name == "language": + return 1 + elif name == "included_range_count": + return 2 + else: + if self.included_range_count.GetValueAsUnsigned() == 0: + return 3 + index = name.lstrip("included_ranges[").rstrip("]") + if index.isdigit(): + return int(index) + else: + return -1 + + def get_child_at_index(self, index: int) -> SBValue: + if index == 0: + return self.root + elif index == 1: + return self.language + elif index == 2: + return self.included_range_count + else: + if self.included_range_count.GetValueAsUnsigned() == 0: + return self.included_ranges + offset: int = index - 3 + start: int = self.included_ranges.GetValueAsUnsigned() + address: int = start + offset * self.element_type_size + element: SBValue = self.included_ranges.CreateValueFromAddress( + "included_ranges[%s]" % (offset), address, self.element_type + ) + return element + + def update(self): + self.root: SBValue = self.valobj.GetChildMemberWithName("root") + self.language: SBValue = self.valobj.GetChildMemberWithName("language") + self.included_range_count: SBValue = self.valobj.GetChildMemberWithName( + "included_range_count" + ) + self.included_ranges: SBValue = self.valobj.GetChildMemberWithName( + "included_ranges" + ) + + self.element_type: SBType = self.included_ranges.GetType().GetPointeeType() + self.element_type_size: int = self.element_type.GetByteSize() + + def has_children(self) -> bool: + return True diff --git a/lib/package.nix b/lib/package.nix new file mode 100644 index 00000000..c94d1e8c --- /dev/null +++ b/lib/package.nix @@ -0,0 +1,49 @@ +{ + stdenv, + cmake, + pkg-config, + src, + version, + lib, +}: +stdenv.mkDerivation { + inherit src version; + pname = "tree-sitter"; + + nativeBuildInputs = [ + cmake + pkg-config + ]; + + sourceRoot = "source"; + + cmakeFlags = [ + "-DBUILD_SHARED_LIBS=ON" + "-DCMAKE_INSTALL_LIBDIR=lib" + "-DCMAKE_INSTALL_INCLUDEDIR=include" + "-DTREE_SITTER_FEATURE_WASM=OFF" + ]; + + enableParallelBuilding = true; + + postInstall = '' + mkdir -p $out/{lib/pkgconfig,share/tree-sitter} + substituteInPlace $out/lib/pkgconfig/tree-sitter.pc \ + --replace-fail "\''${prefix}" "$out" 2>/dev/null + ''; + + meta = { + description = "Tree-sitter incremental parsing library"; + longDescription = '' + Tree-sitter is a parser generator tool and an incremental parsing library. + It can build a concrete syntax tree for a source file and efficiently update + the syntax tree as the source file is edited. This package provides the core + C library that can be used to parse source code using Tree-sitter grammars. + ''; + homepage = "https://tree-sitter.github.io/tree-sitter"; + changelog = "https://github.com/tree-sitter/tree-sitter/releases/tag/v${version}"; + license = lib.licenses.mit; + maintainers = [ lib.maintainers.amaanq ]; + platforms = lib.platforms.all; + }; +} diff --git a/lib/src/clock.h b/lib/src/clock.h deleted file mode 100644 index 5d246ca7..00000000 --- a/lib/src/clock.h +++ /dev/null @@ -1,146 +0,0 @@ -#ifndef TREE_SITTER_CLOCK_H_ -#define TREE_SITTER_CLOCK_H_ - -#include -#include - -typedef uint64_t TSDuration; - -#ifdef _WIN32 - -// Windows: -// * Represent a time as a performance counter value. -// * Represent a duration as a number of performance counter ticks. - -#include -typedef uint64_t TSClock; - -static inline TSDuration duration_from_micros(uint64_t micros) { - LARGE_INTEGER frequency; - QueryPerformanceFrequency(&frequency); - return micros * (uint64_t)frequency.QuadPart / 1000000; -} - -static inline uint64_t duration_to_micros(TSDuration self) { - LARGE_INTEGER frequency; - QueryPerformanceFrequency(&frequency); - return self * 1000000 / (uint64_t)frequency.QuadPart; -} - -static inline TSClock clock_null(void) { - return 0; -} - -static inline TSClock clock_now(void) { - LARGE_INTEGER result; - QueryPerformanceCounter(&result); - return (uint64_t)result.QuadPart; -} - -static inline TSClock clock_after(TSClock base, TSDuration duration) { - return base + duration; -} - -static inline bool clock_is_null(TSClock self) { - return !self; -} - -static inline bool clock_is_gt(TSClock self, TSClock other) { - return self > other; -} - -#elif defined(CLOCK_MONOTONIC) && !defined(__APPLE__) - -// POSIX with monotonic clock support (Linux) -// * Represent a time as a monotonic (seconds, nanoseconds) pair. -// * Represent a duration as a number of microseconds. -// -// On these platforms, parse timeouts will correspond accurately to -// real time, regardless of what other processes are running. - -#include -typedef struct timespec TSClock; - -static inline TSDuration duration_from_micros(uint64_t micros) { - return micros; -} - -static inline uint64_t duration_to_micros(TSDuration self) { - return self; -} - -static inline TSClock clock_now(void) { - TSClock result; - clock_gettime(CLOCK_MONOTONIC, &result); - return result; -} - -static inline TSClock clock_null(void) { - return (TSClock) {0, 0}; -} - -static inline TSClock clock_after(TSClock base, TSDuration duration) { - TSClock result = base; - result.tv_sec += duration / 1000000; - result.tv_nsec += (duration % 1000000) * 1000; - if (result.tv_nsec >= 1000000000) { - result.tv_nsec -= 1000000000; - ++(result.tv_sec); - } - return result; -} - -static inline bool clock_is_null(TSClock self) { - return !self.tv_sec && !self.tv_nsec; -} - -static inline bool clock_is_gt(TSClock self, TSClock other) { - if (self.tv_sec > other.tv_sec) return true; - if (self.tv_sec < other.tv_sec) return false; - return self.tv_nsec > other.tv_nsec; -} - -#else - -// macOS or POSIX without monotonic clock support -// * Represent a time as a process clock value. -// * Represent a duration as a number of process clock ticks. -// -// On these platforms, parse timeouts may be affected by other processes, -// which is not ideal, but is better than using a non-monotonic time API -// like `gettimeofday`. - -#include -typedef uint64_t TSClock; - -static inline TSDuration duration_from_micros(uint64_t micros) { - return micros * (uint64_t)CLOCKS_PER_SEC / 1000000; -} - -static inline uint64_t duration_to_micros(TSDuration self) { - return self * 1000000 / (uint64_t)CLOCKS_PER_SEC; -} - -static inline TSClock clock_null(void) { - return 0; -} - -static inline TSClock clock_now(void) { - return (uint64_t)clock(); -} - -static inline TSClock clock_after(TSClock base, TSDuration duration) { - return base + duration; -} - -static inline bool clock_is_null(TSClock self) { - return !self; -} - -static inline bool clock_is_gt(TSClock self, TSClock other) { - return self > other; -} - -#endif - -#endif // TREE_SITTER_CLOCK_H_ diff --git a/lib/src/get_changed_ranges.c b/lib/src/get_changed_ranges.c index 8ca5bab3..c4c63653 100644 --- a/lib/src/get_changed_ranges.c +++ b/lib/src/get_changed_ranges.c @@ -34,7 +34,7 @@ bool ts_range_array_intersects( uint32_t end_byte ) { for (unsigned i = start_index; i < self->size; i++) { - TSRange *range = &self->contents[i]; + TSRange *range = array_get(self, i); if (range->end_byte > start_byte) { if (range->start_byte >= end_byte) break; return true; @@ -103,11 +103,46 @@ void ts_range_array_get_changed_ranges( } } +void ts_range_edit(TSRange *range, const TSInputEdit *edit) { + if (range->end_byte >= edit->old_end_byte) { + if (range->end_byte != UINT32_MAX) { + range->end_byte = edit->new_end_byte + (range->end_byte - edit->old_end_byte); + range->end_point = point_add( + edit->new_end_point, + point_sub(range->end_point, edit->old_end_point) + ); + if (range->end_byte < edit->new_end_byte) { + range->end_byte = UINT32_MAX; + range->end_point = POINT_MAX; + } + } + } else if (range->end_byte > edit->start_byte) { + range->end_byte = edit->start_byte; + range->end_point = edit->start_point; + } + + if (range->start_byte >= edit->old_end_byte) { + range->start_byte = edit->new_end_byte + (range->start_byte - edit->old_end_byte); + range->start_point = point_add( + edit->new_end_point, + point_sub(range->start_point, edit->old_end_point) + ); + if (range->start_byte < edit->new_end_byte) { + range->start_byte = UINT32_MAX; + range->start_point = POINT_MAX; + } + } else if (range->start_byte > edit->start_byte) { + range->start_byte = edit->start_byte; + range->start_point = edit->start_point; + } +} + typedef struct { TreeCursor cursor; const TSLanguage *language; unsigned visible_depth; bool in_padding; + Subtree prev_external_token; } Iterator; static Iterator iterator_new( @@ -127,6 +162,7 @@ static Iterator iterator_new( .language = language, .visible_depth = 1, .in_padding = false, + .prev_external_token = NULL_SUBTREE, }; } @@ -157,7 +193,7 @@ static bool iterator_tree_is_visible(const Iterator *self) { TreeCursorEntry entry = *array_back(&self->cursor.stack); if (ts_subtree_visible(*entry.subtree)) return true; if (self->cursor.stack.size > 1) { - Subtree parent = *self->cursor.stack.contents[self->cursor.stack.size - 2].subtree; + Subtree parent = *array_get(&self->cursor.stack, self->cursor.stack.size - 2)->subtree; return ts_language_alias_at( self->language, parent.ptr->production_id, @@ -181,10 +217,10 @@ static void iterator_get_visible_state( } for (; i + 1 > 0; i--) { - TreeCursorEntry entry = self->cursor.stack.contents[i]; + TreeCursorEntry entry = *array_get(&self->cursor.stack, i); if (i > 0) { - const Subtree *parent = self->cursor.stack.contents[i - 1].subtree; + const Subtree *parent = array_get(&self->cursor.stack, i - 1)->subtree; *alias_symbol = ts_language_alias_at( self->language, parent->ptr->production_id, @@ -244,6 +280,10 @@ static bool iterator_descend(Iterator *self, uint32_t goal_position) { position = child_right; if (!ts_subtree_extra(*child)) structural_child_index++; + Subtree last_external_token = ts_subtree_last_external_token(*child); + if (last_external_token.ptr) { + self->prev_external_token = last_external_token; + } } } while (did_descend); @@ -268,6 +308,10 @@ static void iterator_advance(Iterator *self) { const Subtree *parent = array_back(&self->cursor.stack)->subtree; uint32_t child_index = entry.child_index + 1; + Subtree last_external_token = ts_subtree_last_external_token(*entry.subtree); + if (last_external_token.ptr) { + self->prev_external_token = last_external_token; + } if (ts_subtree_child_count(*parent) > child_index) { Length position = length_add(entry.position, ts_subtree_total_size(*entry.subtree)); uint32_t structural_child_index = entry.structural_child_index; @@ -313,29 +357,41 @@ static IteratorComparison iterator_compare( TSSymbol new_alias_symbol = 0; iterator_get_visible_state(old_iter, &old_tree, &old_alias_symbol, &old_start); iterator_get_visible_state(new_iter, &new_tree, &new_alias_symbol, &new_start); + TSSymbol old_symbol = ts_subtree_symbol(old_tree); + TSSymbol new_symbol = ts_subtree_symbol(new_tree); if (!old_tree.ptr && !new_tree.ptr) return IteratorMatches; if (!old_tree.ptr || !new_tree.ptr) return IteratorDiffers; + if (old_alias_symbol != new_alias_symbol || old_symbol != new_symbol) return IteratorDiffers; + + uint32_t old_size = ts_subtree_size(old_tree).bytes; + uint32_t new_size = ts_subtree_size(new_tree).bytes; + TSStateId old_state = ts_subtree_parse_state(old_tree); + TSStateId new_state = ts_subtree_parse_state(new_tree); + bool old_has_external_tokens = ts_subtree_has_external_tokens(old_tree); + bool new_has_external_tokens = ts_subtree_has_external_tokens(new_tree); + uint32_t old_error_cost = ts_subtree_error_cost(old_tree); + uint32_t new_error_cost = ts_subtree_error_cost(new_tree); if ( - old_alias_symbol == new_alias_symbol && - ts_subtree_symbol(old_tree) == ts_subtree_symbol(new_tree) + old_start != new_start || + old_symbol == ts_builtin_sym_error || + old_size != new_size || + old_state == TS_TREE_STATE_NONE || + new_state == TS_TREE_STATE_NONE || + ((old_state == ERROR_STATE) != (new_state == ERROR_STATE)) || + old_error_cost != new_error_cost || + old_has_external_tokens != new_has_external_tokens || + ts_subtree_has_changes(old_tree) || + ( + old_has_external_tokens && + !ts_subtree_external_scanner_state_eq(old_iter->prev_external_token, new_iter->prev_external_token) + ) ) { - if (old_start == new_start && - !ts_subtree_has_changes(old_tree) && - ts_subtree_symbol(old_tree) != ts_builtin_sym_error && - ts_subtree_size(old_tree).bytes == ts_subtree_size(new_tree).bytes && - ts_subtree_parse_state(old_tree) != TS_TREE_STATE_NONE && - ts_subtree_parse_state(new_tree) != TS_TREE_STATE_NONE && - (ts_subtree_parse_state(old_tree) == ERROR_STATE) == - (ts_subtree_parse_state(new_tree) == ERROR_STATE)) { - return IteratorMatches; - } else { - return IteratorMayDiffer; - } + return IteratorMayDiffer; } - return IteratorDiffers; + return IteratorMatches; } #ifdef DEBUG_GET_CHANGED_RANGES @@ -348,8 +404,8 @@ static inline void iterator_print_state(Iterator *self) { "(%-25s %s\t depth:%u [%u, %u] - [%u, %u])", name, self->in_padding ? "(p)" : " ", self->visible_depth, - start.row + 1, start.column, - end.row + 1, end.column + start.row, start.column, + end.row, end.column ); } #endif @@ -380,7 +436,7 @@ unsigned ts_subtree_get_changed_ranges( do { #ifdef DEBUG_GET_CHANGED_RANGES - printf("At [%-2u, %-2u] Compare ", position.extent.row + 1, position.extent.column); + printf("At [%-2u, %-2u] Compare ", position.extent.row, position.extent.column); iterator_print_state(&old_iter); printf("\tvs\t"); iterator_print_state(&new_iter); @@ -475,9 +531,9 @@ unsigned ts_subtree_get_changed_ranges( // Keep track of the current position in the included range differences // array in order to avoid scanning the entire array on each iteration. while (included_range_difference_index < included_range_differences->size) { - const TSRange *range = &included_range_differences->contents[ + const TSRange *range = array_get(included_range_differences, included_range_difference_index - ]; + ); if (range->end_byte <= position.bytes) { included_range_difference_index++; } else { diff --git a/lib/src/language.c b/lib/src/language.c index c783e0db..de5e1f91 100644 --- a/lib/src/language.c +++ b/lib/src/language.c @@ -24,12 +24,41 @@ uint32_t ts_language_state_count(const TSLanguage *self) { return self->state_count; } -uint32_t ts_language_version(const TSLanguage *self) { - return self->version; +const TSSymbol *ts_language_supertypes(const TSLanguage *self, uint32_t *length) { + if (self->abi_version >= LANGUAGE_VERSION_WITH_RESERVED_WORDS) { + *length = self->supertype_count; + return self->supertype_symbols; + } else { + *length = 0; + return NULL; + } +} + +const TSSymbol *ts_language_subtypes( + const TSLanguage *self, + TSSymbol supertype, + uint32_t *length +) { + if (self->abi_version < LANGUAGE_VERSION_WITH_RESERVED_WORDS || !ts_language_symbol_metadata(self, supertype).supertype) { + *length = 0; + return NULL; + } + + TSMapSlice slice = self->supertype_map_slices[supertype]; + *length = slice.length; + return &self->supertype_map_entries[slice.index]; +} + +uint32_t ts_language_abi_version(const TSLanguage *self) { + return self->abi_version; +} + +const TSLanguageMetadata *ts_language_metadata(const TSLanguage *self) { + return self->abi_version >= LANGUAGE_VERSION_WITH_RESERVED_WORDS ? &self->metadata : NULL; } const char *ts_language_name(const TSLanguage *self) { - return self->version >= LANGUAGE_VERSION_WITH_METADATA ? self->name : NULL; + return self->abi_version >= LANGUAGE_VERSION_WITH_RESERVED_WORDS ? self->name : NULL; } uint32_t ts_language_field_count(const TSLanguage *self) { @@ -56,6 +85,39 @@ void ts_language_table_entry( } } +TSLexerMode ts_language_lex_mode_for_state( + const TSLanguage *self, + TSStateId state +) { + if (self->abi_version < 15) { + TSLexMode mode = ((const TSLexMode *)self->lex_modes)[state]; + return (TSLexerMode) { + .lex_state = mode.lex_state, + .external_lex_state = mode.external_lex_state, + .reserved_word_set_id = 0, + }; + } else { + return self->lex_modes[state]; + } +} + +bool ts_language_is_reserved_word( + const TSLanguage *self, + TSStateId state, + TSSymbol symbol +) { + TSLexerMode lex_mode = ts_language_lex_mode_for_state(self, state); + if (lex_mode.reserved_word_set_id > 0) { + unsigned start = lex_mode.reserved_word_set_id * self->max_reserved_word_set_size; + unsigned end = start + self->max_reserved_word_set_size; + for (unsigned i = start; i < end; i++) { + if (self->reserved_words[i] == symbol) return true; + if (self->reserved_words[i] == 0) break; + } + } + return false; +} + TSSymbolMetadata ts_language_symbol_metadata( const TSLanguage *self, TSSymbol symbol @@ -120,7 +182,7 @@ TSSymbol ts_language_symbol_for_name( uint32_t length, bool is_named ) { - if (!strncmp(string, "ERROR", length)) return ts_builtin_sym_error; + if (is_named && !strncmp(string, "ERROR", length)) return ts_builtin_sym_error; uint16_t count = (uint16_t)ts_language_symbol_count(self); for (TSSymbol i = 0; i < count; i++) { TSSymbolMetadata metadata = ts_language_symbol_metadata(self, i); diff --git a/lib/src/language.h b/lib/src/language.h index a19ec0f6..518c06bf 100644 --- a/lib/src/language.h +++ b/lib/src/language.h @@ -10,7 +10,7 @@ extern "C" { #define ts_builtin_sym_error_repeat (ts_builtin_sym_error - 1) -#define LANGUAGE_VERSION_WITH_METADATA 15 +#define LANGUAGE_VERSION_WITH_RESERVED_WORDS 15 #define LANGUAGE_VERSION_WITH_PRIMARY_STATES 14 typedef struct { @@ -36,15 +36,11 @@ typedef struct { } LookaheadIterator; void ts_language_table_entry(const TSLanguage *self, TSStateId state, TSSymbol symbol, TableEntry *result); - +TSLexerMode ts_language_lex_mode_for_state(const TSLanguage *self, TSStateId state); +bool ts_language_is_reserved_word(const TSLanguage *self, TSStateId state, TSSymbol symbol); TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *self, TSSymbol symbol); - TSSymbol ts_language_public_symbol(const TSLanguage *self, TSSymbol symbol); -static inline bool ts_language_is_symbol_external(const TSLanguage *self, TSSymbol symbol) { - return 0 < symbol && symbol < self->external_token_count + 1; -} - static inline const TSParseAction *ts_language_actions( const TSLanguage *self, TSStateId state, @@ -187,7 +183,7 @@ static inline bool ts_language_state_is_primary( const TSLanguage *self, TSStateId state ) { - if (self->version >= LANGUAGE_VERSION_WITH_PRIMARY_STATES) { + if (self->abi_version >= LANGUAGE_VERSION_WITH_PRIMARY_STATES) { return state == self->primary_state_ids[state]; } else { return true; @@ -236,7 +232,7 @@ static inline void ts_language_field_map( return; } - TSFieldMapSlice slice = self->field_map_slices[production_id]; + TSMapSlice slice = self->field_map_slices[production_id]; *start = &self->field_map_entries[slice.index]; *end = &self->field_map_entries[slice.index] + slice.length; } diff --git a/lib/src/length.h b/lib/src/length.h index 42d61ef3..ddf156ce 100644 --- a/lib/src/length.h +++ b/lib/src/length.h @@ -31,7 +31,7 @@ static inline Length length_add(Length len1, Length len2) { static inline Length length_sub(Length len1, Length len2) { Length result; - result.bytes = len1.bytes - len2.bytes; + result.bytes = (len1.bytes >= len2.bytes) ? len1.bytes - len2.bytes : 0; result.extent = point_sub(len1.extent, len2.extent); return result; } diff --git a/lib/src/lexer.c b/lib/src/lexer.c index f181946a..1709c418 100644 --- a/lib/src/lexer.c +++ b/lib/src/lexer.c @@ -114,7 +114,7 @@ static void ts_lexer__get_lookahead(Lexer *self) { } const uint8_t *chunk = (const uint8_t *)self->chunk + position_in_chunk; - DecodeFunction decode = + TSDecodeFunction decode = self->input.encoding == TSInputEncodingUTF8 ? ts_decode_utf8 : self->input.encoding == TSInputEncodingUTF16LE ? ts_decode_utf16_le : self->input.encoding == TSInputEncodingUTF16BE ? ts_decode_utf16_be : self->input.decode; @@ -424,10 +424,7 @@ void ts_lexer_finish(Lexer *self, uint32_t *lookahead_end_byte) { // If the token ended at an included range boundary, then its end position // will have been reset to the end of the preceding range. Reset the start // position to match. - if ( - self->token_end_position.bytes < self->token_start_position.bytes || - point_lt(self->token_end_position.extent, self->token_start_position.extent) - ) { + if (self->token_end_position.bytes < self->token_start_position.bytes) { self->token_start_position = self->token_end_position; } @@ -446,12 +443,6 @@ void ts_lexer_finish(Lexer *self, uint32_t *lookahead_end_byte) { } } -void ts_lexer_advance_to_end(Lexer *self) { - while (self->chunk) { - ts_lexer__advance(&self->data, false); - } -} - void ts_lexer_mark_end(Lexer *self) { ts_lexer__mark_end(&self->data); } diff --git a/lib/src/lexer.h b/lib/src/lexer.h index 6ad663fa..7f451e3f 100644 --- a/lib/src/lexer.h +++ b/lib/src/lexer.h @@ -43,7 +43,6 @@ void ts_lexer_set_input(Lexer *self, TSInput input); void ts_lexer_reset(Lexer *self, Length position); void ts_lexer_start(Lexer *self); void ts_lexer_finish(Lexer *self, uint32_t *lookahead_end_byte); -void ts_lexer_advance_to_end(Lexer *self); void ts_lexer_mark_end(Lexer *self); bool ts_lexer_set_included_ranges(Lexer *self, const TSRange *ranges, uint32_t count); TSRange *ts_lexer_included_ranges(const Lexer *self, uint32_t *count); diff --git a/lib/src/lib.c b/lib/src/lib.c index 9bfb69f0..f56be97a 100644 --- a/lib/src/lib.c +++ b/lib/src/lib.c @@ -4,6 +4,7 @@ #include "./lexer.c" #include "./node.c" #include "./parser.c" +#include "./point.c" #include "./query.c" #include "./stack.c" #include "./subtree.c" diff --git a/lib/src/node.c b/lib/src/node.c index c5500ba4..84ffed28 100644 --- a/lib/src/node.c +++ b/lib/src/node.c @@ -264,7 +264,15 @@ static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous) NodeChildIterator iterator = ts_node_iterate_children(&node); while (ts_node_child_iterator_next(&iterator, &child)) { if (iterator.position.bytes <= target_end_byte) continue; - if (ts_node_start_byte(child) < ts_node_start_byte(self)) { + uint32_t start_byte = ts_node_start_byte(self); + uint32_t child_start_byte = ts_node_start_byte(child); + + bool is_empty = start_byte == target_end_byte; + bool contains_target = is_empty ? + child_start_byte < start_byte : + child_start_byte <= start_byte; + + if (contains_target) { if (ts_node__subtree(child).ptr != ts_node__subtree(self).ptr) { child_containing_target = child; } @@ -549,37 +557,6 @@ TSNode ts_node_parent(TSNode self) { return node; } -TSNode ts_node_child_containing_descendant(TSNode self, TSNode descendant) { - uint32_t start_byte = ts_node_start_byte(descendant); - uint32_t end_byte = ts_node_end_byte(descendant); - bool is_empty = start_byte == end_byte; - - do { - NodeChildIterator iter = ts_node_iterate_children(&self); - do { - if ( - !ts_node_child_iterator_next(&iter, &self) - || ts_node_start_byte(self) > start_byte - || self.id == descendant.id - ) { - return ts_node__null(); - } - - // If the descendant is empty, and the end byte is within `self`, - // we check whether `self` contains it or not. - if (is_empty && iter.position.bytes >= end_byte && ts_node_child_count(self) > 0) { - TSNode child = ts_node_child_with_descendant(self, descendant); - // If the child is not null, return self if it's relevant, else return the child - if (!ts_node_is_null(child)) { - return ts_node__is_relevant(self, true) ? self : child; - } - } - } while ((is_empty ? iter.position.bytes <= end_byte : iter.position.bytes < end_byte) || ts_node_child_count(self) == 0); - } while (!ts_node__is_relevant(self, true)); - - return self; -} - TSNode ts_node_child_with_descendant(TSNode self, TSNode descendant) { uint32_t start_byte = ts_node_start_byte(descendant); uint32_t end_byte = ts_node_end_byte(descendant); @@ -884,13 +861,7 @@ void ts_node_edit(TSNode *self, const TSInputEdit *edit) { uint32_t start_byte = ts_node_start_byte(*self); TSPoint start_point = ts_node_start_point(*self); - if (start_byte >= edit->old_end_byte) { - start_byte = edit->new_end_byte + (start_byte - edit->old_end_byte); - start_point = point_add(edit->new_end_point, point_sub(start_point, edit->old_end_point)); - } else if (start_byte > edit->start_byte) { - start_byte = edit->new_end_byte; - start_point = edit->new_end_point; - } + ts_point_edit(&start_point, &start_byte, edit); self->context[0] = start_byte; self->context[1] = start_point.row; diff --git a/lib/src/parser.c b/lib/src/parser.c index 4c2976cd..bb247911 100644 --- a/lib/src/parser.c +++ b/lib/src/parser.c @@ -1,4 +1,3 @@ -#include #include #include #include @@ -6,8 +5,6 @@ #include "tree_sitter/api.h" #include "./alloc.h" #include "./array.h" -#include "./atomic.h" -#include "./clock.h" #include "./error_costs.h" #include "./get_changed_ranges.h" #include "./language.h" @@ -80,8 +77,8 @@ static const unsigned MAX_VERSION_COUNT = 6; static const unsigned MAX_VERSION_COUNT_OVERFLOW = 4; static const unsigned MAX_SUMMARY_DEPTH = 16; -static const unsigned MAX_COST_DIFFERENCE = 16 * ERROR_COST_PER_SKIPPED_TREE; -static const unsigned OP_COUNT_PER_PARSER_TIMEOUT_CHECK = 100; +static const unsigned MAX_COST_DIFFERENCE = 18 * ERROR_COST_PER_SKIPPED_TREE; +static const unsigned OP_COUNT_PER_PARSER_CALLBACK_CHECK = 100; typedef struct { Subtree token; @@ -104,17 +101,16 @@ struct TSParser { ReusableNode reusable_node; void *external_scanner_payload; FILE *dot_graph_file; - TSClock end_clock; - TSDuration timeout_duration; unsigned accept_count; unsigned operation_count; - const volatile size_t *cancellation_flag; Subtree old_tree; TSRangeArray included_range_differences; TSParseOptions parse_options; TSParseState parse_state; unsigned included_range_difference_index; bool has_scanner_error; + bool canceled_balancing; + bool has_error; }; typedef struct { @@ -191,7 +187,7 @@ static bool ts_parser__breakdown_top_of_stack( did_break_down = true; pending = false; for (uint32_t i = 0; i < pop.size; i++) { - StackSlice slice = pop.contents[i]; + StackSlice slice = *array_get(&pop, i); TSStateId state = ts_stack_state(self->stack, slice.version); Subtree parent = *array_front(&slice.subtrees); @@ -210,7 +206,7 @@ static bool ts_parser__breakdown_top_of_stack( } for (uint32_t j = 1; j < slice.subtrees.size; j++) { - Subtree tree = slice.subtrees.contents[j]; + Subtree tree = *array_get(&slice.subtrees, j); ts_stack_push(self->stack, slice.version, tree, false, state); } @@ -342,7 +338,7 @@ static bool ts_parser__better_version_exists( return false; } -static bool ts_parser__call_main_lex_fn(TSParser *self, TSLexMode lex_mode) { +static bool ts_parser__call_main_lex_fn(TSParser *self, TSLexerMode lex_mode) { if (ts_language_is_wasm(self->language)) { return ts_wasm_store_call_lex_main(self->wasm_store, lex_mode.lex_state); } else { @@ -394,20 +390,24 @@ static void ts_parser__external_scanner_destroy( static unsigned ts_parser__external_scanner_serialize( TSParser *self ) { + uint32_t length; if (ts_language_is_wasm(self->language)) { - return ts_wasm_store_call_scanner_serialize( + length = ts_wasm_store_call_scanner_serialize( self->wasm_store, (uintptr_t)self->external_scanner_payload, self->lexer.debug_buffer ); + if (ts_wasm_store_has_error(self->wasm_store)) { + self->has_scanner_error = true; + } } else { - uint32_t length = self->language->external_scanner.serialize( + length = self->language->external_scanner.serialize( self->external_scanner_payload, self->lexer.debug_buffer ); - ts_assert(length <= TREE_SITTER_SERIALIZATION_BUFFER_SIZE); - return length; } + ts_assert(length <= TREE_SITTER_SERIALIZATION_BUFFER_SIZE); + return length; } static void ts_parser__external_scanner_deserialize( @@ -473,10 +473,10 @@ static bool ts_parser__can_reuse_first_leaf( Subtree tree, TableEntry *table_entry ) { - TSLexMode current_lex_mode = self->language->lex_modes[state]; TSSymbol leaf_symbol = ts_subtree_leaf_symbol(tree); TSStateId leaf_state = ts_subtree_leaf_parse_state(tree); - TSLexMode leaf_lex_mode = self->language->lex_modes[leaf_state]; + TSLexerMode current_lex_mode = ts_language_lex_mode_for_state(self->language, state); + TSLexerMode leaf_lex_mode = ts_language_lex_mode_for_state(self->language, leaf_state); // At the end of a non-terminal extra node, the lexer normally returns // NULL, which indicates that the parser should look for a reduce action @@ -487,7 +487,7 @@ static bool ts_parser__can_reuse_first_leaf( // If the token was created in a state with the same set of lookaheads, it is reusable. if ( table_entry->action_count > 0 && - memcmp(&leaf_lex_mode, ¤t_lex_mode, sizeof(TSLexMode)) == 0 && + memcmp(&leaf_lex_mode, ¤t_lex_mode, sizeof(TSLexerMode)) == 0 && ( leaf_symbol != self->language->keyword_capture_token || (!ts_subtree_is_keyword(tree) && ts_subtree_parse_state(tree) == state) @@ -507,7 +507,7 @@ static Subtree ts_parser__lex( StackVersion version, TSStateId parse_state ) { - TSLexMode lex_mode = self->language->lex_modes[parse_state]; + TSLexerMode lex_mode = ts_language_lex_mode_for_state(self->language, parse_state); if (lex_mode.lex_state == (uint16_t)-1) { LOG("no_lookahead_after_non_terminal_extra"); return NULL_SUBTREE; @@ -554,27 +554,29 @@ static Subtree ts_parser__lex( external_scanner_state_len ); - // When recovering from an error, ignore any zero-length external tokens - // unless they have changed the external scanner's state. This helps to - // avoid infinite loops which could otherwise occur, because the lexer is - // looking for any possible token, instead of looking for the specific set of - // tokens that are valid in some parse state. + // Avoid infinite loops caused by the external scanner returning empty tokens. + // Empty tokens are needed in some circumstances, e.g. indent/dedent tokens + // in Python. Ignore the following classes of empty tokens: // - // Note that it's possible that the token end position may be *before* the - // original position of the lexer because of the way that tokens are positioned - // at included range boundaries: when a token is terminated at the start of - // an included range, it is marked as ending at the *end* of the preceding - // included range. + // * Tokens produced during error recovery. When recovering from an error, + // all tokens are allowed, so it's easy to accidentally return unwanted + // empty tokens. + // * Tokens that are marked as 'extra' in the grammar. These don't change + // the parse state, so they would definitely cause an infinite loop. if ( self->lexer.token_end_position.bytes <= current_position.bytes && - (error_mode || !ts_stack_has_advanced_since_error(self->stack, version)) && !external_scanner_state_changed ) { - LOG( - "ignore_empty_external_token symbol:%s", - SYM_NAME(self->language->external_scanner.symbol_map[self->lexer.data.result_symbol]) - ) - found_token = false; + TSSymbol symbol = self->language->external_scanner.symbol_map[self->lexer.data.result_symbol]; + TSStateId next_parse_state = ts_language_next_state(self->language, parse_state, symbol); + bool token_is_extra = (next_parse_state == parse_state); + if (error_mode || !ts_stack_has_advanced_since_error(self->stack, version) || token_is_extra) { + LOG( + "ignore_empty_external_token symbol:%s", + SYM_NAME(self->language->external_scanner.symbol_map[self->lexer.data.result_symbol]) + ); + found_token = false; + } } } @@ -601,7 +603,7 @@ static Subtree ts_parser__lex( if (!error_mode) { error_mode = true; - lex_mode = self->language->lex_modes[ERROR_STATE]; + lex_mode = ts_language_lex_mode_for_state(self->language, ERROR_STATE); ts_lexer_reset(&self->lexer, start_position); continue; } @@ -658,7 +660,10 @@ static Subtree ts_parser__lex( if ( is_keyword && self->lexer.token_end_position.bytes == end_byte && - ts_language_has_actions(self->language, parse_state, self->lexer.data.result_symbol) + ( + ts_language_has_actions(self->language, parse_state, self->lexer.data.result_symbol) || + ts_language_is_reserved_word(self->language, parse_state, self->lexer.data.result_symbol) + ) ) { symbol = self->lexer.data.result_symbol; } @@ -942,20 +947,22 @@ static StackVersion ts_parser__reduce( // children. StackSliceArray pop = ts_stack_pop_count(self->stack, version, count); uint32_t removed_version_count = 0; + uint32_t halted_version_count = ts_stack_halted_version_count(self->stack); for (uint32_t i = 0; i < pop.size; i++) { - StackSlice slice = pop.contents[i]; + StackSlice slice = *array_get(&pop, i); StackVersion slice_version = slice.version - removed_version_count; // This is where new versions are added to the parse stack. The versions // will all be sorted and truncated at the end of the outer parsing loop. // Allow the maximum version count to be temporarily exceeded, but only // by a limited threshold. - if (slice_version > MAX_VERSION_COUNT + MAX_VERSION_COUNT_OVERFLOW) { + if (slice_version > MAX_VERSION_COUNT + MAX_VERSION_COUNT_OVERFLOW + halted_version_count) { ts_stack_remove_version(self->stack, slice_version); ts_subtree_array_delete(&self->tree_pool, &slice.subtrees); removed_version_count++; while (i + 1 < pop.size) { - StackSlice next_slice = pop.contents[i + 1]; + LOG("aborting reduce with too many versions") + StackSlice next_slice = *array_get(&pop, i + 1); if (next_slice.version != slice.version) break; ts_subtree_array_delete(&self->tree_pool, &next_slice.subtrees); i++; @@ -978,7 +985,7 @@ static StackVersion ts_parser__reduce( // choose one of the arrays of trees to be the parent node's children, and // delete the rest of the tree arrays. while (i + 1 < pop.size) { - StackSlice next_slice = pop.contents[i + 1]; + StackSlice next_slice = *array_get(&pop, i + 1); if (next_slice.version != slice.version) break; i++; @@ -1020,7 +1027,7 @@ static StackVersion ts_parser__reduce( // were previously on top of the stack. ts_stack_push(self->stack, slice_version, ts_subtree_from_mut(parent), false, next_state); for (uint32_t j = 0; j < self->trailing_extras.size; j++) { - ts_stack_push(self->stack, slice_version, self->trailing_extras.contents[j], false, next_state); + ts_stack_push(self->stack, slice_version, *array_get(&self->trailing_extras, j), false, next_state); } for (StackVersion j = 0; j < slice_version; j++) { @@ -1048,11 +1055,11 @@ static void ts_parser__accept( StackSliceArray pop = ts_stack_pop_all(self->stack, version); for (uint32_t i = 0; i < pop.size; i++) { - SubtreeArray trees = pop.contents[i].subtrees; + SubtreeArray trees = array_get(&pop, i)->subtrees; Subtree root = NULL_SUBTREE; for (uint32_t j = trees.size - 1; j + 1 > 0; j--) { - Subtree tree = trees.contents[j]; + Subtree tree = *array_get(&trees, j); if (!ts_subtree_extra(tree)) { ts_assert(!tree.data.is_inline); uint32_t child_count = ts_subtree_child_count(tree); @@ -1087,7 +1094,7 @@ static void ts_parser__accept( } } - ts_stack_remove_version(self->stack, pop.contents[0].version); + ts_stack_remove_version(self->stack, array_get(&pop, 0)->version); ts_stack_halt(self->stack, version); } @@ -1153,7 +1160,7 @@ static bool ts_parser__do_all_potential_reductions( StackVersion reduction_version = STACK_VERSION_NONE; for (uint32_t j = 0; j < self->reduce_actions.size; j++) { - ReduceAction action = self->reduce_actions.contents[j]; + ReduceAction action = *array_get(&self->reduce_actions, j); reduction_version = ts_parser__reduce( self, version, action.symbol, action.count, @@ -1191,7 +1198,7 @@ static bool ts_parser__recover_to_state( StackVersion previous_version = STACK_VERSION_NONE; for (unsigned i = 0; i < pop.size; i++) { - StackSlice slice = pop.contents[i]; + StackSlice slice = *array_get(&pop, i); if (slice.version == previous_version) { ts_subtree_array_delete(&self->tree_pool, &slice.subtrees); @@ -1209,12 +1216,12 @@ static bool ts_parser__recover_to_state( SubtreeArray error_trees = ts_stack_pop_error(self->stack, slice.version); if (error_trees.size > 0) { ts_assert(error_trees.size == 1); - Subtree error_tree = error_trees.contents[0]; + Subtree error_tree = *array_get(&error_trees, 0); uint32_t error_child_count = ts_subtree_child_count(error_tree); if (error_child_count > 0) { array_splice(&slice.subtrees, 0, 0, error_child_count, ts_subtree_children(error_tree)); for (unsigned j = 0; j < error_child_count; j++) { - ts_subtree_retain(slice.subtrees.contents[j]); + ts_subtree_retain(*array_get(&slice.subtrees, j)); } } ts_subtree_array_delete(&self->tree_pool, &error_trees); @@ -1230,7 +1237,7 @@ static bool ts_parser__recover_to_state( } for (unsigned j = 0; j < self->trailing_extras.size; j++) { - Subtree tree = self->trailing_extras.contents[j]; + Subtree tree = *array_get(&self->trailing_extras, j); ts_stack_push(self->stack, slice.version, tree, false, goal_state); } @@ -1266,7 +1273,7 @@ static void ts_parser__recover( // if the current lookahead token would be valid in that state. if (summary && !ts_subtree_is_error(lookahead)) { for (unsigned i = 0; i < summary->size; i++) { - StackSummaryEntry entry = summary->contents[i]; + StackSummaryEntry entry = *array_get(summary, i); if (entry.state == ERROR_STATE) continue; if (entry.position.bytes == position.bytes) continue; @@ -1311,10 +1318,23 @@ static void ts_parser__recover( // and subsequently halted. Remove those versions. for (unsigned i = previous_version_count; i < ts_stack_version_count(self->stack); i++) { if (!ts_stack_is_active(self->stack, i)) { + LOG("removed paused version:%u", i); ts_stack_remove_version(self->stack, i--); + LOG_STACK(); } } + // If the parser is still in the error state at the end of the file, just wrap everything + // in an ERROR node and terminate. + if (ts_subtree_is_eof(lookahead)) { + LOG("recover_eof"); + SubtreeArray children = array_new(); + Subtree parent = ts_subtree_new_error_node(&children, false, self->language); + ts_stack_push(self->stack, version, parent, false, 1); + ts_parser__accept(self, version, lookahead); + return; + } + // If strategy 1 succeeded, a new stack version will have been created which is able to handle // the current lookahead token. Now, in addition, try strategy 2 described above: skip the // current lookahead token by wrapping it in an ERROR node. @@ -1335,17 +1355,6 @@ static void ts_parser__recover( return; } - // If the parser is still in the error state at the end of the file, just wrap everything - // in an ERROR node and terminate. - if (ts_subtree_is_eof(lookahead)) { - LOG("recover_eof"); - SubtreeArray children = array_new(); - Subtree parent = ts_subtree_new_error_node(&children, false, self->language); - ts_stack_push(self->stack, version, parent, false, 1); - ts_parser__accept(self, version, lookahead); - return; - } - // Do not recover if the result would clearly be worse than some existing stack version. unsigned new_cost = current_error_cost + ERROR_COST_PER_SKIPPED_TREE + @@ -1391,18 +1400,18 @@ static void ts_parser__recover( // arbitrarily and discard the rest. if (pop.size > 1) { for (unsigned i = 1; i < pop.size; i++) { - ts_subtree_array_delete(&self->tree_pool, &pop.contents[i].subtrees); + ts_subtree_array_delete(&self->tree_pool, &array_get(&pop, i)->subtrees); } - while (ts_stack_version_count(self->stack) > pop.contents[0].version + 1) { - ts_stack_remove_version(self->stack, pop.contents[0].version + 1); + while (ts_stack_version_count(self->stack) > array_get(&pop, 0)->version + 1) { + ts_stack_remove_version(self->stack, array_get(&pop, 0)->version + 1); } } - ts_stack_renumber_version(self->stack, pop.contents[0].version, version); - array_push(&pop.contents[0].subtrees, ts_subtree_from_mut(error_repeat)); + ts_stack_renumber_version(self->stack, array_get(&pop, 0)->version, version); + array_push(&array_get(&pop, 0)->subtrees, ts_subtree_from_mut(error_repeat)); error_repeat = ts_subtree_new_node( ts_builtin_sym_error_repeat, - &pop.contents[0].subtrees, + &array_get(&pop, 0)->subtrees, 0, self->language ); @@ -1415,6 +1424,16 @@ static void ts_parser__recover( self->stack, version, ts_subtree_last_external_token(lookahead) ); } + + bool has_error = true; + for (unsigned i = 0; i < ts_stack_version_count(self->stack); i++) { + ErrorStatus status = ts_parser__version_status(self, i); + if (!status.is_in_error) { + has_error = false; + break; + } + } + self->has_error = has_error; } static void ts_parser__handle_error( @@ -1514,6 +1533,27 @@ static void ts_parser__handle_error( LOG_STACK(); } +static bool ts_parser__check_progress(TSParser *self, Subtree *lookahead, const uint32_t *position, unsigned operations) { + self->operation_count += operations; + if (self->operation_count >= OP_COUNT_PER_PARSER_CALLBACK_CHECK) { + self->operation_count = 0; + } + if (position != NULL) { + self->parse_state.current_byte_offset = *position; + self->parse_state.has_error = self->has_error; + } + if ( + self->operation_count == 0 && + (self->parse_options.progress_callback && self->parse_options.progress_callback(&self->parse_state)) + ) { + if (lookahead && lookahead->ptr) { + ts_subtree_release(&self->tree_pool, *lookahead); + } + return false; + } + return true; +} + static bool ts_parser__advance( TSParser *self, StackVersion version, @@ -1564,26 +1604,10 @@ static bool ts_parser__advance( } } - // If a cancellation flag, timeout, or progress callback was provided, then check every + // If a progress callback was provided, then check every // time a fixed number of parse actions has been processed. - if (++self->operation_count == OP_COUNT_PER_PARSER_TIMEOUT_CHECK) { - self->operation_count = 0; - } - if (self->parse_options.progress_callback) { - self->parse_state.current_byte_offset = position; - } - if ( - self->operation_count == 0 && - ( - (self->cancellation_flag && atomic_load(self->cancellation_flag)) || - (!clock_is_null(self->end_clock) && clock_is_gt(clock_now(), self->end_clock)) || - (self->parse_options.progress_callback && self->parse_options.progress_callback(&self->parse_state)) - ) - ) { - if (lookahead.ptr) { - ts_subtree_release(&self->tree_pool, lookahead); - } - return false; + if (!ts_parser__check_progress(self, &lookahead, &position, 1)) { + return false; } // Process each parse action for the current lookahead token in @@ -1591,6 +1615,7 @@ static bool ts_parser__advance( // an ambiguous state. REDUCE actions always create a new stack // version, whereas SHIFT actions update the existing stack version // and terminate this loop. + bool did_reduce = false; StackVersion last_reduction_version = STACK_VERSION_NONE; for (uint32_t i = 0; i < table_entry.action_count; i++) { TSParseAction action = table_entry.actions[i]; @@ -1626,6 +1651,7 @@ static bool ts_parser__advance( action.reduce.dynamic_precedence, action.reduce.production_id, is_fragile, end_of_non_terminal_extra ); + did_reduce = true; if (reduction_version != STACK_VERSION_NONE) { last_reduction_version = reduction_version; } @@ -1677,22 +1703,30 @@ static bool ts_parser__advance( continue; } - // A non-terminal extra rule was reduced and merged into an existing - // stack version. This version can be discarded. - if (!lookahead.ptr) { + // A reduction was performed, but was merged into an existing stack version. + // This version can be discarded. + if (did_reduce) { + if (lookahead.ptr) { + ts_subtree_release(&self->tree_pool, lookahead); + } ts_stack_halt(self->stack, version); return true; } - // If there were no parse actions for the current lookahead token, then - // it is not valid in this state. If the current lookahead token is a - // keyword, then switch to treating it as the normal word token if that - // token is valid in this state. + // If the current lookahead token is a keyword that is not valid, but the + // default word token *is* valid, then treat the lookahead token as the word + // token instead. if ( ts_subtree_is_keyword(lookahead) && - ts_subtree_symbol(lookahead) != self->language->keyword_capture_token + ts_subtree_symbol(lookahead) != self->language->keyword_capture_token && + !ts_language_is_reserved_word(self->language, state, ts_subtree_symbol(lookahead)) ) { - ts_language_table_entry(self->language, state, self->language->keyword_capture_token, &table_entry); + ts_language_table_entry( + self->language, + state, + self->language->keyword_capture_token, + &table_entry + ); if (table_entry.action_count > 0) { LOG( "switch from_keyword:%s, to_word_token:%s", @@ -1707,19 +1741,10 @@ static bool ts_parser__advance( } } - // If the current lookahead token is not valid and the parser is - // already in the error state, restart the error recovery process. - // TODO - can this be unified with the other `RECOVER` case above? - if (state == ERROR_STATE) { - ts_parser__recover(self, version, lookahead); - return true; - } - - // If the current lookahead token is not valid and the previous - // subtree on the stack was reused from an old tree, it isn't actually - // valid to reuse it. Remove it from the stack, and in its place, - // push each of its children. Then try again to process the current - // lookahead. + // If the current lookahead token is not valid and the previous subtree on + // the stack was reused from an old tree, then it wasn't actually valid to + // reuse that previous subtree. Remove it from the stack, and in its place, + // push each of its children. Then try again to process the current lookahead. if (ts_parser__breakdown_top_of_stack(self, version)) { state = ts_stack_state(self->stack, version); ts_subtree_release(&self->tree_pool, lookahead); @@ -1727,12 +1752,12 @@ static bool ts_parser__advance( continue; } - // At this point, the current lookahead token is definitely not valid - // for this parse stack version. Mark this version as paused and continue - // processing any other stack versions that might exist. If some other - // version advances successfully, then this version can simply be removed. - // But if all versions end up paused, then error recovery is needed. - LOG("detect_error"); + // Otherwise, there is definitely an error in this version of the parse stack. + // Mark this version as paused and continue processing any other stack + // versions that exist. If some other version advances successfully, then + // this version can simply be removed. But if all versions end up paused, + // then error recovery is needed. + LOG("detect_error lookahead:%s", TREE_NAME(lookahead)); ts_stack_pause(self->stack, version, lookahead); return true; } @@ -1821,6 +1846,7 @@ static unsigned ts_parser__condense_stack(TSParser *self) { has_unpaused_version = true; } else { ts_stack_remove_version(self->stack, i); + made_changes = true; i--; n--; } @@ -1838,8 +1864,66 @@ static unsigned ts_parser__condense_stack(TSParser *self) { return min_error_cost; } +static bool ts_parser__balance_subtree(TSParser *self) { + Subtree finished_tree = self->finished_tree; + + // If we haven't canceled balancing in progress before, then we want to clear the tree stack and + // push the initial finished tree onto it. Otherwise, if we're resuming balancing after a + // cancellation, we don't want to clear the tree stack. + if (!self->canceled_balancing) { + array_clear(&self->tree_pool.tree_stack); + if (ts_subtree_child_count(finished_tree) > 0 && finished_tree.ptr->ref_count == 1) { + array_push(&self->tree_pool.tree_stack, ts_subtree_to_mut_unsafe(finished_tree)); + } + } + + while (self->tree_pool.tree_stack.size > 0) { + if (!ts_parser__check_progress(self, NULL, NULL, 1)) { + return false; + } + + MutableSubtree tree = *array_get(&self->tree_pool.tree_stack, + self->tree_pool.tree_stack.size - 1 + ); + + if (tree.ptr->repeat_depth > 0) { + Subtree child1 = ts_subtree_children(tree)[0]; + Subtree child2 = ts_subtree_children(tree)[tree.ptr->child_count - 1]; + long repeat_delta = (long)ts_subtree_repeat_depth(child1) - (long)ts_subtree_repeat_depth(child2); + if (repeat_delta > 0) { + unsigned n = (unsigned)repeat_delta; + + for (unsigned i = n / 2; i > 0; i /= 2) { + ts_subtree_compress(tree, i, self->language, &self->tree_pool.tree_stack); + n -= i; + + // We scale the operation count increment in `ts_parser__check_progress` proportionately to the compression + // size since larger values of i take longer to process. Shifting by 4 empirically provides good check + // intervals (e.g. 193 operations when i=3100) to prevent blocking during large compressions. + uint8_t operations = i >> 4 > 0 ? i >> 4 : 1; + if (!ts_parser__check_progress(self, NULL, NULL, operations)) { + return false; + } + } + } + } + + (void)array_pop(&self->tree_pool.tree_stack); + + for (uint32_t i = 0; i < tree.ptr->child_count; i++) { + Subtree child = ts_subtree_children(tree)[i]; + if (ts_subtree_child_count(child) > 0 && child.ptr->ref_count == 1) { + array_push(&self->tree_pool.tree_stack, ts_subtree_to_mut_unsafe(child)); + } + } + } + + return true; +} + static bool ts_parser_has_outstanding_parse(TSParser *self) { return ( + self->canceled_balancing || self->external_scanner_payload || ts_stack_state(self->stack, 0) != 1 || ts_stack_node_count_since_error(self->stack, 0) != 0 @@ -1858,12 +1942,11 @@ TSParser *ts_parser_new(void) { self->finished_tree = NULL_SUBTREE; self->reusable_node = reusable_node_new(); self->dot_graph_file = NULL; - self->cancellation_flag = NULL; - self->timeout_duration = 0; self->language = NULL; self->has_scanner_error = false; + self->has_error = false; + self->canceled_balancing = false; self->external_scanner_payload = NULL; - self->end_clock = clock_null(); self->operation_count = 0; self->old_tree = NULL_SUBTREE; self->included_range_differences = (TSRangeArray) array_new(); @@ -1909,8 +1992,8 @@ bool ts_parser_set_language(TSParser *self, const TSLanguage *language) { if (language) { if ( - language->version > TREE_SITTER_LANGUAGE_VERSION || - language->version < TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION + language->abi_version > TREE_SITTER_LANGUAGE_VERSION || + language->abi_version < TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION ) return false; if (ts_language_is_wasm(language)) { @@ -1949,22 +2032,6 @@ void ts_parser_print_dot_graphs(TSParser *self, int fd) { } } -const size_t *ts_parser_cancellation_flag(const TSParser *self) { - return (const size_t *)self->cancellation_flag; -} - -void ts_parser_set_cancellation_flag(TSParser *self, const size_t *flag) { - self->cancellation_flag = (const volatile size_t *)flag; -} - -uint64_t ts_parser_timeout_micros(const TSParser *self) { - return duration_to_micros(self->timeout_duration); -} - -void ts_parser_set_timeout_micros(TSParser *self, uint64_t timeout_micros) { - self->timeout_duration = duration_from_micros(timeout_micros); -} - bool ts_parser_set_included_ranges( TSParser *self, const TSRange *ranges, @@ -1998,6 +2065,10 @@ void ts_parser_reset(TSParser *self) { } self->accept_count = 0; self->has_scanner_error = false; + self->has_error = false; + self->canceled_balancing = false; + self->parse_options = (TSParseOptions) {0}; + self->parse_state = (TSParseState) {0}; } TSTree *ts_parser_parse( @@ -2017,8 +2088,11 @@ TSTree *ts_parser_parse( array_clear(&self->included_range_differences); self->included_range_difference_index = 0; + self->operation_count = 0; + if (ts_parser_has_outstanding_parse(self)) { LOG("resume_parsing"); + if (self->canceled_balancing) goto balance; } else { ts_parser__external_scanner_create(self); if (self->has_scanner_error) goto exit; @@ -2035,7 +2109,7 @@ TSTree *ts_parser_parse( LOG("parse_after_edit"); LOG_TREE(self->old_tree); for (unsigned i = 0; i < self->included_range_differences.size; i++) { - TSRange *range = &self->included_range_differences.contents[i]; + TSRange *range = array_get(&self->included_range_differences, i); LOG("different_included_range %u - %u", range->start_byte, range->end_byte); } } else { @@ -2044,13 +2118,6 @@ TSTree *ts_parser_parse( } } - self->operation_count = 0; - if (self->timeout_duration) { - self->end_clock = clock_after(clock_now(), self->timeout_duration); - } else { - self->end_clock = clock_null(); - } - uint32_t position = 0, last_position = 0, version_count = 0; do { for ( @@ -2099,7 +2166,7 @@ TSTree *ts_parser_parse( } while (self->included_range_difference_index < self->included_range_differences.size) { - TSRange *range = &self->included_range_differences.contents[self->included_range_difference_index]; + TSRange *range = array_get(&self->included_range_differences, self->included_range_difference_index); if (range->end_byte <= position) { self->included_range_difference_index++; } else { @@ -2108,8 +2175,13 @@ TSTree *ts_parser_parse( } } while (version_count != 0); +balance: ts_assert(self->finished_tree.ptr); - ts_subtree_balance(self->finished_tree, &self->tree_pool, self->language); + if (!ts_parser__balance_subtree(self)) { + self->canceled_balancing = true; + return false; + } + self->canceled_balancing = false; LOG("done"); LOG_TREE(self->finished_tree); @@ -2133,12 +2205,10 @@ TSTree *ts_parser_parse_with_options( TSParseOptions parse_options ) { self->parse_options = parse_options; - self->parse_state = (TSParseState) { - .payload = parse_options.payload, - }; + self->parse_state.payload = parse_options.payload; TSTree *result = ts_parser_parse(self, old_tree, input); + // Reset parser options before further parse calls. self->parse_options = (TSParseOptions) {0}; - self->parse_state = (TSParseState) {0}; return result; } diff --git a/lib/src/parser.h b/lib/src/parser.h index 2338b4a2..858107de 100644 --- a/lib/src/parser.h +++ b/lib/src/parser.h @@ -18,6 +18,11 @@ typedef uint16_t TSStateId; typedef uint16_t TSSymbol; typedef uint16_t TSFieldId; typedef struct TSLanguage TSLanguage; +typedef struct TSLanguageMetadata { + uint8_t major_version; + uint8_t minor_version; + uint8_t patch_version; +} TSLanguageMetadata; #endif typedef struct { @@ -26,10 +31,11 @@ typedef struct { bool inherited; } TSFieldMapEntry; +// Used to index the field and supertype maps. typedef struct { uint16_t index; uint16_t length; -} TSFieldMapSlice; +} TSMapSlice; typedef struct { bool visible; @@ -79,6 +85,12 @@ typedef struct { uint16_t external_lex_state; } TSLexMode; +typedef struct { + uint16_t lex_state; + uint16_t external_lex_state; + uint16_t reserved_word_set_id; +} TSLexerMode; + typedef union { TSParseAction action; struct { @@ -93,7 +105,7 @@ typedef struct { } TSCharacterRange; struct TSLanguage { - uint32_t version; + uint32_t abi_version; uint32_t symbol_count; uint32_t alias_count; uint32_t token_count; @@ -109,13 +121,13 @@ struct TSLanguage { const TSParseActionEntry *parse_actions; const char * const *symbol_names; const char * const *field_names; - const TSFieldMapSlice *field_map_slices; + const TSMapSlice *field_map_slices; const TSFieldMapEntry *field_map_entries; const TSSymbolMetadata *symbol_metadata; const TSSymbol *public_symbol_map; const uint16_t *alias_map; const TSSymbol *alias_sequences; - const TSLexMode *lex_modes; + const TSLexerMode *lex_modes; bool (*lex_fn)(TSLexer *, TSStateId); bool (*keyword_lex_fn)(TSLexer *, TSStateId); TSSymbol keyword_capture_token; @@ -130,15 +142,22 @@ struct TSLanguage { } external_scanner; const TSStateId *primary_state_ids; const char *name; + const TSSymbol *reserved_words; + uint16_t max_reserved_word_set_size; + uint32_t supertype_count; + const TSSymbol *supertype_symbols; + const TSMapSlice *supertype_map_slices; + const TSSymbol *supertype_map_entries; + TSLanguageMetadata metadata; }; -static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t lookahead) { +static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, int32_t lookahead) { uint32_t index = 0; uint32_t size = len - index; while (size > 1) { uint32_t half_size = size / 2; uint32_t mid_index = index + half_size; - TSCharacterRange *range = &ranges[mid_index]; + const TSCharacterRange *range = &ranges[mid_index]; if (lookahead >= range->start && lookahead <= range->end) { return true; } else if (lookahead > range->end) { @@ -146,7 +165,7 @@ static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t } size -= half_size; } - TSCharacterRange *range = &ranges[index]; + const TSCharacterRange *range = &ranges[index]; return (lookahead >= range->start && lookahead <= range->end); } diff --git a/lib/src/point.c b/lib/src/point.c new file mode 100644 index 00000000..da46f61b --- /dev/null +++ b/lib/src/point.c @@ -0,0 +1,17 @@ +#include "point.h" + +void ts_point_edit(TSPoint *point, uint32_t *byte, const TSInputEdit *edit) { + uint32_t start_byte = *byte; + TSPoint start_point = *point; + + if (start_byte >= edit->old_end_byte) { + start_byte = edit->new_end_byte + (start_byte - edit->old_end_byte); + start_point = point_add(edit->new_end_point, point_sub(start_point, edit->old_end_point)); + } else if (start_byte > edit->start_byte) { + start_byte = edit->new_end_byte; + start_point = edit->new_end_point; + } + + *point = start_point; + *byte = start_byte; +} diff --git a/lib/src/point.h b/lib/src/point.h index 37346c8d..39581988 100644 --- a/lib/src/point.h +++ b/lib/src/point.h @@ -22,7 +22,7 @@ static inline TSPoint point_sub(TSPoint a, TSPoint b) { if (a.row > b.row) return point__new(a.row - b.row, a.column); else - return point__new(0, a.column - b.column); + return point__new(0, (a.column >= b.column) ? a.column - b.column : 0); } static inline bool point_lte(TSPoint a, TSPoint b) { @@ -45,18 +45,4 @@ static inline bool point_eq(TSPoint a, TSPoint b) { return a.row == b.row && a.column == b.column; } -static inline TSPoint point_min(TSPoint a, TSPoint b) { - if (a.row < b.row || (a.row == b.row && a.column < b.column)) - return a; - else - return b; -} - -static inline TSPoint point_max(TSPoint a, TSPoint b) { - if (a.row > b.row || (a.row == b.row && a.column > b.column)) - return a; - else - return b; -} - #endif diff --git a/lib/src/portable/endian.h b/lib/src/portable/endian.h index ead3e340..a6560826 100644 --- a/lib/src/portable/endian.h +++ b/lib/src/portable/endian.h @@ -4,8 +4,10 @@ // be "dual licensed" under the BSD, MIT and Apache licenses, if you want to. This code is trivial anyway. Consider it // an example on how to get the endian conversion functions on different platforms. -#ifndef PORTABLE_ENDIAN_H__ -#define PORTABLE_ENDIAN_H__ +// updates from https://github.com/mikepb/endian.h/issues/4 + +#ifndef ENDIAN_H +#define ENDIAN_H #if (defined(_WIN16) || defined(_WIN32) || defined(_WIN64)) && !defined(__WINDOWS__) @@ -13,59 +15,92 @@ #endif -#if defined(__linux__) || defined(__CYGWIN__) || defined(__GNU__) || defined(__EMSCRIPTEN__) +#if defined(HAVE_ENDIAN_H) || \ + defined(__linux__) || \ + defined(__GNU__) || \ + defined(__HAIKU__) || \ + defined(__illumos__) || \ + defined(__NetBSD__) || \ + defined(__OpenBSD__) || \ + defined(__CYGWIN__) || \ + defined(__MSYS__) || \ + defined(__EMSCRIPTEN__) || \ + defined(__wasi__) || \ + defined(__wasm__) -# include +#if defined(__NetBSD__) +#define _NETBSD_SOURCE 1 +#endif + +# include + +#elif defined(HAVE_SYS_ENDIAN_H) || \ + defined(__FreeBSD__) || \ + defined(__DragonFly__) + +# include #elif defined(__APPLE__) - -# include - -# define htobe16(x) OSSwapHostToBigInt16(x) -# define htole16(x) OSSwapHostToLittleInt16(x) -# define be16toh(x) OSSwapBigToHostInt16(x) -# define le16toh(x) OSSwapLittleToHostInt16(x) - -# define htobe32(x) OSSwapHostToBigInt32(x) -# define htole32(x) OSSwapHostToLittleInt32(x) -# define be32toh(x) OSSwapBigToHostInt32(x) -# define le32toh(x) OSSwapLittleToHostInt32(x) - -# define htobe64(x) OSSwapHostToBigInt64(x) -# define htole64(x) OSSwapHostToLittleInt64(x) -# define be64toh(x) OSSwapBigToHostInt64(x) -# define le64toh(x) OSSwapLittleToHostInt64(x) - # define __BYTE_ORDER BYTE_ORDER # define __BIG_ENDIAN BIG_ENDIAN # define __LITTLE_ENDIAN LITTLE_ENDIAN # define __PDP_ENDIAN PDP_ENDIAN -#elif defined(__OpenBSD__) +# if !defined(_POSIX_C_SOURCE) +# include -# include +# define htobe16(x) OSSwapHostToBigInt16(x) +# define htole16(x) OSSwapHostToLittleInt16(x) +# define be16toh(x) OSSwapBigToHostInt16(x) +# define le16toh(x) OSSwapLittleToHostInt16(x) -# define __BYTE_ORDER BYTE_ORDER -# define __BIG_ENDIAN BIG_ENDIAN -# define __LITTLE_ENDIAN LITTLE_ENDIAN -# define __PDP_ENDIAN PDP_ENDIAN +# define htobe32(x) OSSwapHostToBigInt32(x) +# define htole32(x) OSSwapHostToLittleInt32(x) +# define be32toh(x) OSSwapBigToHostInt32(x) +# define le32toh(x) OSSwapLittleToHostInt32(x) -#elif defined(__NetBSD__) || defined(__FreeBSD__) || defined(__DragonFly__) +# define htobe64(x) OSSwapHostToBigInt64(x) +# define htole64(x) OSSwapHostToLittleInt64(x) +# define be64toh(x) OSSwapBigToHostInt64(x) +# define le64toh(x) OSSwapLittleToHostInt64(x) +# else +# if BYTE_ORDER == LITTLE_ENDIAN +# define htobe16(x) __builtin_bswap16(x) +# define htole16(x) (x) +# define be16toh(x) __builtin_bswap16(x) +# define le16toh(x) (x) -# include +# define htobe32(x) __builtin_bswap32(x) +# define htole32(x) (x) +# define be32toh(x) __builtin_bswap32(x) +# define le32toh(x) (x) -# define be16toh(x) betoh16(x) -# define le16toh(x) letoh16(x) +# define htobe64(x) __builtin_bswap64(x) +# define htole64(x) (x) +# define be64toh(x) __builtin_bswap64(x) +# define le64toh(x) (x) +# elif BYTE_ORDER == BIG_ENDIAN +# define htobe16(x) (x) +# define htole16(x) __builtin_bswap16(x) +# define be16toh(x) (x) +# define le16toh(x) __builtin_bswap16(x) -# define be32toh(x) betoh32(x) -# define le32toh(x) letoh32(x) +# define htobe32(x) (x) +# define htole32(x) __builtin_bswap32(x) +# define be32toh(x) (x) +# define le32toh(x) __builtin_bswap32(x) -# define be64toh(x) betoh64(x) -# define le64toh(x) letoh64(x) +# define htobe64(x) (x) +# define htole64(x) __builtin_bswap64(x) +# define be64toh(x) (x) +# define le64toh(x) __builtin_bswap64(x) +# else +# error byte order not supported +# endif +# endif #elif defined(__WINDOWS__) - # if defined(_MSC_VER) && !defined(__clang__) # include # define B_SWAP_16(x) _byteswap_ushort(x) diff --git a/lib/src/query.c b/lib/src/query.c index 9a83254e..e378910a 100644 --- a/lib/src/query.c +++ b/lib/src/query.c @@ -1,7 +1,15 @@ +/* + * On NetBSD, defining standard requirements like this removes symbols + * from the namespace; however, we need non-standard symbols for + * endian.h. + */ +#if defined(__NetBSD__) && defined(_POSIX_C_SOURCE) +#undef _POSIX_C_SOURCE +#endif + #include "tree_sitter/api.h" #include "./alloc.h" #include "./array.h" -#include "./clock.h" #include "./language.h" #include "./point.h" #include "./tree_cursor.h" @@ -81,7 +89,7 @@ typedef struct { * for the entire top-level pattern. When iterating through a query's * captures using `ts_query_cursor_next_capture`, this field is used to * detect that a capture can safely be returned from a match that has not - * even completed yet. + * even completed yet. */ typedef struct { TSSymbol symbol; @@ -100,6 +108,7 @@ typedef struct { bool contains_captures: 1; bool root_pattern_guaranteed: 1; bool parent_pattern_guaranteed: 1; + bool is_missing: 1; } QueryStep; /* @@ -122,7 +131,7 @@ typedef struct { } SymbolTable; /** - * CaptureQuantififers - a data structure holding the quantifiers of pattern captures. + * CaptureQuantifiers - a data structure holding the quantifiers of pattern captures. */ typedef Array(uint8_t) CaptureQuantifiers; @@ -173,7 +182,8 @@ typedef struct { * list of captures from the `CaptureListPool`. * - `seeking_immediate_match` - A flag that indicates that the state's next * step must be matched by the very next sibling. This is used when - * processing repetitions. + * processing repetitions, or when processing a wildcard node followed by + * an anchor. * - `has_in_progress_alternatives` - A flag that indicates that there is are * other states that have the same captures as this state, but are at * different steps in their pattern. This means that in order to obey the @@ -308,13 +318,9 @@ struct TSQueryCursor { CaptureListPool capture_list_pool; uint32_t depth; uint32_t max_start_depth; - uint32_t start_byte; - uint32_t end_byte; - TSPoint start_point; - TSPoint end_point; + TSRange included_range; + TSRange containing_range; uint32_t next_state_id; - TSClock end_clock; - TSDuration timeout_duration; const TSQueryCursorOptions *query_options; TSQueryCursorState query_state; unsigned operation_count; @@ -328,7 +334,7 @@ static const TSQueryError PARENT_DONE = -1; static const uint16_t PATTERN_DONE_MARKER = UINT16_MAX; static const uint16_t NONE = UINT16_MAX; static const TSSymbol WILDCARD_SYMBOL = 0; -static const unsigned OP_COUNT_PER_QUERY_TIMEOUT_CHECK = 100; +static const unsigned OP_COUNT_PER_QUERY_CALLBACK_CHECK = 100; /********** * Stream @@ -400,9 +406,7 @@ static void stream_scan_identifier(Stream *stream) { iswalnum(stream->next) || stream->next == '_' || stream->next == '-' || - stream->next == '.' || - stream->next == '?' || - stream->next == '!' + stream->next == '.' ); } @@ -426,26 +430,26 @@ static CaptureListPool capture_list_pool_new(void) { static void capture_list_pool_reset(CaptureListPool *self) { for (uint16_t i = 0; i < (uint16_t)self->list.size; i++) { // This invalid size means that the list is not in use. - self->list.contents[i].size = UINT32_MAX; + array_get(&self->list, i)->size = UINT32_MAX; } self->free_capture_list_count = self->list.size; } static void capture_list_pool_delete(CaptureListPool *self) { for (uint16_t i = 0; i < (uint16_t)self->list.size; i++) { - array_delete(&self->list.contents[i]); + array_delete(array_get(&self->list, i)); } array_delete(&self->list); } static const CaptureList *capture_list_pool_get(const CaptureListPool *self, uint16_t id) { if (id >= self->list.size) return &self->empty_list; - return &self->list.contents[id]; + return array_get(&self->list, id); } static CaptureList *capture_list_pool_get_mut(CaptureListPool *self, uint16_t id) { ts_assert(id < self->list.size); - return &self->list.contents[id]; + return array_get(&self->list, id); } static bool capture_list_pool_is_empty(const CaptureListPool *self) { @@ -458,8 +462,8 @@ static uint16_t capture_list_pool_acquire(CaptureListPool *self) { // First see if any already allocated capture list is currently unused. if (self->free_capture_list_count > 0) { for (uint16_t i = 0; i < (uint16_t)self->list.size; i++) { - if (self->list.contents[i].size == UINT32_MAX) { - array_clear(&self->list.contents[i]); + if (array_get(&self->list, i)->size == UINT32_MAX) { + array_clear(array_get(&self->list, i)); self->free_capture_list_count--; return i; } @@ -480,7 +484,7 @@ static uint16_t capture_list_pool_acquire(CaptureListPool *self) { static void capture_list_pool_release(CaptureListPool *self, uint16_t id) { if (id >= self->list.size) return; - self->list.contents[id].size = UINT32_MAX; + array_get(&self->list, id)->size = UINT32_MAX; self->free_capture_list_count++; } @@ -763,10 +767,10 @@ static int symbol_table_id_for_name( uint32_t length ) { for (unsigned i = 0; i < self->slices.size; i++) { - Slice slice = self->slices.contents[i]; + Slice slice = *array_get(&self->slices, i); if ( slice.length == length && - !strncmp(&self->characters.contents[slice.offset], name, length) + !strncmp(array_get(&self->characters, slice.offset), name, length) ) return i; } return -1; @@ -777,9 +781,9 @@ static const char *symbol_table_name_for_id( uint16_t id, uint32_t *length ) { - Slice slice = self->slices.contents[id]; + Slice slice = *(array_get(&self->slices,id)); *length = slice.length; - return &self->characters.contents[slice.offset]; + return array_get(&self->characters, slice.offset); } static uint16_t symbol_table_insert_name( @@ -794,8 +798,8 @@ static uint16_t symbol_table_insert_name( .length = length, }; array_grow_by(&self->characters, length + 1); - memcpy(&self->characters.contents[slice.offset], name, length); - self->characters.contents[self->characters.size - 1] = 0; + memcpy(array_get(&self->characters, slice.offset), name, length); + *array_get(&self->characters, self->characters.size - 1) = 0; array_push(&self->slices, slice); return self->slices.size - 1; } @@ -917,35 +921,26 @@ static unsigned analysis_state__recursion_depth(const AnalysisState *self) { return result; } -static inline int analysis_state__compare_position( - AnalysisState *const *self, - AnalysisState *const *other -) { - for (unsigned i = 0; i < (*self)->depth; i++) { - if (i >= (*other)->depth) return -1; - if ((*self)->stack[i].child_index < (*other)->stack[i].child_index) return -1; - if ((*self)->stack[i].child_index > (*other)->stack[i].child_index) return 1; - } - if ((*self)->depth < (*other)->depth) return 1; - if ((*self)->step_index < (*other)->step_index) return -1; - if ((*self)->step_index > (*other)->step_index) return 1; - return 0; -} - static inline int analysis_state__compare( AnalysisState *const *self, AnalysisState *const *other ) { - int result = analysis_state__compare_position(self, other); - if (result != 0) return result; + if ((*self)->depth < (*other)->depth) return 1; for (unsigned i = 0; i < (*self)->depth; i++) { - if ((*self)->stack[i].parent_symbol < (*other)->stack[i].parent_symbol) return -1; - if ((*self)->stack[i].parent_symbol > (*other)->stack[i].parent_symbol) return 1; - if ((*self)->stack[i].parse_state < (*other)->stack[i].parse_state) return -1; - if ((*self)->stack[i].parse_state > (*other)->stack[i].parse_state) return 1; - if ((*self)->stack[i].field_id < (*other)->stack[i].field_id) return -1; - if ((*self)->stack[i].field_id > (*other)->stack[i].field_id) return 1; + if (i >= (*other)->depth) return -1; + AnalysisStateEntry s1 = (*self)->stack[i]; + AnalysisStateEntry s2 = (*other)->stack[i]; + if (s1.child_index < s2.child_index) return -1; + if (s1.child_index > s2.child_index) return 1; + if (s1.parent_symbol < s2.parent_symbol) return -1; + if (s1.parent_symbol > s2.parent_symbol) return 1; + if (s1.parse_state < s2.parse_state) return -1; + if (s1.parse_state > s2.parse_state) return 1; + if (s1.field_id < s2.field_id) return -1; + if (s1.field_id > s2.field_id) return 1; } + if ((*self)->step_index < (*other)->step_index) return -1; + if ((*self)->step_index > (*other)->step_index) return 1; return 0; } @@ -1107,23 +1102,23 @@ static inline bool ts_query__pattern_map_search( while (size > 1) { uint32_t half_size = size / 2; uint32_t mid_index = base_index + half_size; - TSSymbol mid_symbol = self->steps.contents[ - self->pattern_map.contents[mid_index].step_index - ].symbol; + TSSymbol mid_symbol = array_get(&self->steps, + array_get(&self->pattern_map, mid_index)->step_index + )->symbol; if (needle > mid_symbol) base_index = mid_index; size -= half_size; } - TSSymbol symbol = self->steps.contents[ - self->pattern_map.contents[base_index].step_index - ].symbol; + TSSymbol symbol = array_get(&self->steps, + array_get(&self->pattern_map, base_index)->step_index + )->symbol; if (needle > symbol) { base_index++; if (base_index < self->pattern_map.size) { - symbol = self->steps.contents[ - self->pattern_map.contents[base_index].step_index - ].symbol; + symbol = array_get(&self->steps, + array_get(&self->pattern_map, base_index)->step_index + )->symbol; } } @@ -1146,9 +1141,9 @@ static inline void ts_query__pattern_map_insert( // initiated first, which allows the ordering of the states array // to be maintained more efficiently. while (index < self->pattern_map.size) { - PatternEntry *entry = &self->pattern_map.contents[index]; + PatternEntry *entry = array_get(&self->pattern_map, index); if ( - self->steps.contents[entry->step_index].symbol == symbol && + array_get(&self->steps, entry->step_index)->symbol == symbol && entry->pattern_index < new_entry.pattern_index ) { index++; @@ -1181,11 +1176,11 @@ static void ts_query__perform_analysis( #ifdef DEBUG_ANALYZE_QUERY printf("Iteration: %u. Final step indices:", iteration); for (unsigned j = 0; j < analysis->final_step_indices.size; j++) { - printf(" %4u", analysis->final_step_indices.contents[j]); + printf(" %4u", *array_get(&analysis->final_step_indices, j)); } printf("\n"); for (unsigned j = 0; j < analysis->states.size; j++) { - AnalysisState *state = analysis->states.contents[j]; + AnalysisState *state = *array_get(&analysis->states, j); printf(" %3u: step: %u, stack: [", j, state->step_index); for (unsigned k = 0; k < state->depth; k++) { printf( @@ -1228,7 +1223,7 @@ static void ts_query__perform_analysis( analysis_state_set__clear(&analysis->next_states, &analysis->state_pool); for (unsigned j = 0; j < analysis->states.size; j++) { - AnalysisState * const state = analysis->states.contents[j]; + AnalysisState * const state = *array_get(&analysis->states, j); // For efficiency, it's important to avoid processing the same analysis state more // than once. To achieve this, keep the states in order of ascending position within @@ -1236,7 +1231,7 @@ static void ts_query__perform_analysis( // the states that have made the least progress. Avoid advancing states that have already // made more progress. if (analysis->next_states.size > 0) { - int comparison = analysis_state__compare_position( + int comparison = analysis_state__compare( &state, array_back(&analysis->next_states) ); @@ -1251,7 +1246,7 @@ static void ts_query__perform_analysis( analysis_state_set__push( &analysis->next_states, &analysis->state_pool, - analysis->states.contents[j] + *array_get(&analysis->states, j) ); j++; } @@ -1263,12 +1258,12 @@ static void ts_query__perform_analysis( const TSSymbol parent_symbol = analysis_state__top(state)->parent_symbol; const TSFieldId parent_field_id = analysis_state__top(state)->field_id; const unsigned child_index = analysis_state__top(state)->child_index; - const QueryStep * const step = &self->steps.contents[state->step_index]; + const QueryStep * const step = array_get(&self->steps, state->step_index); unsigned subgraph_index, exists; array_search_sorted_by(subgraphs, .symbol, parent_symbol, &subgraph_index, &exists); if (!exists) continue; - const AnalysisSubgraph *subgraph = &subgraphs->contents[subgraph_index]; + const AnalysisSubgraph *subgraph = array_get(subgraphs, subgraph_index); // Follow every possible path in the parse table, but only visit states that // are part of the subgraph for the current symbol. @@ -1304,7 +1299,8 @@ static void ts_query__perform_analysis( &node_index, &exists ); while (node_index < subgraph->nodes.size) { - AnalysisSubgraphNode *node = &subgraph->nodes.contents[node_index++]; + AnalysisSubgraphNode *node = array_get(&subgraph->nodes, node_index); + node_index++; if (node->state != successor.state || node->child_index != successor.child_index) break; // Use the subgraph to determine what alias and field will eventually be applied @@ -1337,7 +1333,12 @@ static void ts_query__perform_analysis( // Determine if this hypothetical child node would match the current step // of the query pattern. bool does_match = false; - if (visible_symbol) { + + // ERROR nodes can appear anywhere, so if the step is + // looking for an ERROR node, consider it potentially matchable. + if (step->symbol == ts_builtin_sym_error) { + does_match = true; + } else if (visible_symbol) { does_match = true; if (step->symbol == WILDCARD_SYMBOL) { if ( @@ -1405,7 +1406,7 @@ static void ts_query__perform_analysis( if (does_match) { for (;;) { next_state.step_index++; - next_step = &self->steps.contents[next_state.step_index]; + next_step = array_get(&self->steps, next_state.step_index); if ( next_step->depth == PATTERN_DONE_MARKER || next_step->depth <= step->depth @@ -1429,7 +1430,7 @@ static void ts_query__perform_analysis( // record that matching can terminate at this step of the pattern. Otherwise, // add this state to the list of states to process on the next iteration. if (!next_step->is_dead_end) { - bool did_finish_pattern = self->steps.contents[next_state.step_index].depth != step->depth; + bool did_finish_pattern = array_get(&self->steps, next_state.step_index)->depth != step->depth; if (did_finish_pattern) { array_insert_sorted_by(&analysis->finished_parent_symbols, , state->root_symbol); } else if (next_state.depth == 0) { @@ -1449,7 +1450,7 @@ static void ts_query__perform_analysis( next_step->alternative_index > next_state.step_index ) { next_state.step_index = next_step->alternative_index; - next_step = &self->steps.contents[next_state.step_index]; + next_step = array_get(&self->steps, next_state.step_index); } else { break; } @@ -1467,9 +1468,9 @@ static void ts_query__perform_analysis( static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { Array(uint16_t) non_rooted_pattern_start_steps = array_new(); for (unsigned i = 0; i < self->pattern_map.size; i++) { - PatternEntry *pattern = &self->pattern_map.contents[i]; + PatternEntry *pattern = array_get(&self->pattern_map, i); if (!pattern->is_rooted) { - QueryStep *step = &self->steps.contents[pattern->step_index]; + QueryStep *step = array_get(&self->steps, pattern->step_index); if (step->symbol != WILDCARD_SYMBOL) { array_push(&non_rooted_pattern_start_steps, i); } @@ -1480,8 +1481,9 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { // basic information about each step. Mark all of the steps that contain // captures, and record the indices of all of the steps that have child steps. Array(uint32_t) parent_step_indices = array_new(); + bool all_patterns_are_valid = true; for (unsigned i = 0; i < self->steps.size; i++) { - QueryStep *step = &self->steps.contents[i]; + QueryStep *step = array_get(&self->steps, i); if (step->depth == PATTERN_DONE_MARKER) { step->parent_pattern_guaranteed = true; step->root_pattern_guaranteed = true; @@ -1492,7 +1494,7 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { bool is_wildcard = step->symbol == WILDCARD_SYMBOL; step->contains_captures = step->capture_ids[0] != NONE; for (unsigned j = i + 1; j < self->steps.size; j++) { - QueryStep *next_step = &self->steps.contents[j]; + QueryStep *next_step = array_get(&self->steps, j); if ( next_step->depth == PATTERN_DONE_MARKER || next_step->depth <= step->depth @@ -1507,8 +1509,45 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { has_children = true; } - if (has_children && !is_wildcard) { - array_push(&parent_step_indices, i); + if (has_children) { + if (!is_wildcard) { + array_push(&parent_step_indices, i); + } else if (step->supertype_symbol && self->language->abi_version >= LANGUAGE_VERSION_WITH_RESERVED_WORDS) { + // Look at the child steps to see if any aren't valid subtypes for this supertype. + uint32_t subtype_length; + const TSSymbol *subtypes = ts_language_subtypes( + self->language, + step->supertype_symbol, + &subtype_length + ); + + for (unsigned j = i + 1; j < self->steps.size; j++) { + QueryStep *child_step = array_get(&self->steps, j); + if (child_step->depth == PATTERN_DONE_MARKER || child_step->depth <= step->depth) { + break; + } + if (child_step->depth == step->depth + 1 && child_step->symbol != WILDCARD_SYMBOL) { + bool is_valid_subtype = false; + for (uint32_t k = 0; k < subtype_length; k++) { + if (child_step->symbol == subtypes[k]) { + is_valid_subtype = true; + break; + } + } + + if (!is_valid_subtype) { + for (unsigned offset_idx = 0; offset_idx < self->step_offsets.size; offset_idx++) { + StepOffset *step_offset = array_get(&self->step_offsets, offset_idx); + if (step_offset->step_index >= j) { + *error_offset = step_offset->byte_offset; + all_patterns_are_valid = false; + goto supertype_cleanup; + } + } + } + } + } + } } } @@ -1522,8 +1561,8 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { // parent. AnalysisSubgraphArray subgraphs = array_new(); for (unsigned i = 0; i < parent_step_indices.size; i++) { - uint32_t parent_step_index = parent_step_indices.contents[i]; - TSSymbol parent_symbol = self->steps.contents[parent_step_index].symbol; + uint32_t parent_step_index = *array_get(&parent_step_indices, i); + TSSymbol parent_symbol = array_get(&self->steps, parent_step_index)->symbol; AnalysisSubgraph subgraph = { .symbol = parent_symbol }; array_insert_sorted_by(&subgraphs, .symbol, subgraph); } @@ -1565,7 +1604,7 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { &exists ); if (exists) { - AnalysisSubgraph *subgraph = &subgraphs.contents[subgraph_index]; + AnalysisSubgraph *subgraph = array_get(&subgraphs, subgraph_index); if (subgraph->nodes.size == 0 || array_back(&subgraph->nodes)->state != state) { array_push(&subgraph->nodes, ((AnalysisSubgraphNode) { .state = state, @@ -1602,7 +1641,7 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { &exists ); if (exists) { - AnalysisSubgraph *subgraph = &subgraphs.contents[subgraph_index]; + AnalysisSubgraph *subgraph = array_get(&subgraphs, subgraph_index); if ( subgraph->start_states.size == 0 || *array_back(&subgraph->start_states) != state @@ -1619,7 +1658,7 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { // from the end states using the predecessor map. Array(AnalysisSubgraphNode) next_nodes = array_new(); for (unsigned i = 0; i < subgraphs.size; i++) { - AnalysisSubgraph *subgraph = &subgraphs.contents[i]; + AnalysisSubgraph *subgraph = array_get(&subgraphs, i); if (subgraph->nodes.size == 0) { array_delete(&subgraph->start_states); array_erase(&subgraphs, i); @@ -1660,16 +1699,16 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { #ifdef DEBUG_ANALYZE_QUERY printf("\nSubgraphs:\n"); for (unsigned i = 0; i < subgraphs.size; i++) { - AnalysisSubgraph *subgraph = &subgraphs.contents[i]; + AnalysisSubgraph *subgraph = array_get(&subgraphs, i); printf(" %u, %s:\n", subgraph->symbol, ts_language_symbol_name(self->language, subgraph->symbol)); for (unsigned j = 0; j < subgraph->start_states.size; j++) { printf( " {state: %u}\n", - subgraph->start_states.contents[j] + *array_get(&subgraph->start_states, j) ); } for (unsigned j = 0; j < subgraph->nodes.size; j++) { - AnalysisSubgraphNode *node = &subgraph->nodes.contents[j]; + AnalysisSubgraphNode *node = array_get(&subgraph->nodes, j); printf( " {state: %u, child_index: %u, production_id: %u, done: %d}\n", node->state, node->child_index, node->production_id, node->done @@ -1681,12 +1720,11 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { // For each non-terminal pattern, determine if the pattern can successfully match, // and identify all of the possible children within the pattern where matching could fail. - bool all_patterns_are_valid = true; QueryAnalysis analysis = query_analysis__new(); for (unsigned i = 0; i < parent_step_indices.size; i++) { - uint16_t parent_step_index = parent_step_indices.contents[i]; - uint16_t parent_depth = self->steps.contents[parent_step_index].depth; - TSSymbol parent_symbol = self->steps.contents[parent_step_index].symbol; + uint16_t parent_step_index = *array_get(&parent_step_indices, i); + uint16_t parent_depth = array_get(&self->steps, parent_step_index)->depth; + TSSymbol parent_symbol = array_get(&self->steps, parent_step_index)->symbol; if (parent_symbol == ts_builtin_sym_error) continue; // Find the subgraph that corresponds to this pattern's root symbol. If the pattern's @@ -1698,18 +1736,18 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { uint32_t j, child_exists; array_search_sorted_by(&self->step_offsets, .step_index, first_child_step_index, &j, &child_exists); ts_assert(child_exists); - *error_offset = self->step_offsets.contents[j].byte_offset; + *error_offset = array_get(&self->step_offsets, j)->byte_offset; all_patterns_are_valid = false; break; } // Initialize an analysis state at every parse state in the table where // this parent symbol can occur. - AnalysisSubgraph *subgraph = &subgraphs.contents[subgraph_index]; + AnalysisSubgraph *subgraph = array_get(&subgraphs, subgraph_index); analysis_state_set__clear(&analysis.states, &analysis.state_pool); analysis_state_set__clear(&analysis.deeper_states, &analysis.state_pool); for (unsigned j = 0; j < subgraph->start_states.size; j++) { - TSStateId parse_state = subgraph->start_states.contents[j]; + TSStateId parse_state = *array_get(&subgraph->start_states, j); analysis_state_set__push(&analysis.states, &analysis.state_pool, &((AnalysisState) { .step_index = parent_step_index + 1, .stack = { @@ -1729,7 +1767,7 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { #ifdef DEBUG_ANALYZE_QUERY printf( "\nWalk states for %s:\n", - ts_language_symbol_name(self->language, analysis.states.contents[0]->stack[0].parent_symbol) + ts_language_symbol_name(self->language, (*array_get(&analysis.states, 0))->stack[0].parent_symbol) ); #endif @@ -1740,7 +1778,7 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { // be considered fallible. if (analysis.did_abort) { for (unsigned j = parent_step_index + 1; j < self->steps.size; j++) { - QueryStep *step = &self->steps.contents[j]; + QueryStep *step = array_get(&self->steps, j); if ( step->depth <= parent_depth || step->depth == PATTERN_DONE_MARKER @@ -1756,12 +1794,17 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { // If this pattern cannot match, store the pattern index so that it can be // returned to the caller. if (analysis.finished_parent_symbols.size == 0) { - ts_assert(analysis.final_step_indices.size > 0); - uint16_t impossible_step_index = *array_back(&analysis.final_step_indices); + uint16_t impossible_step_index; + if (analysis.final_step_indices.size > 0) { + impossible_step_index = *array_back(&analysis.final_step_indices); + } else { + // If there isn't a final step, then that means the parent step itself is unreachable. + impossible_step_index = parent_step_index; + } uint32_t j, impossible_exists; array_search_sorted_by(&self->step_offsets, .step_index, impossible_step_index, &j, &impossible_exists); if (j >= self->step_offsets.size) j = self->step_offsets.size - 1; - *error_offset = self->step_offsets.contents[j].byte_offset; + *error_offset = array_get(&self->step_offsets, j)->byte_offset; all_patterns_are_valid = false; break; } @@ -1769,8 +1812,8 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { // Mark as fallible any step where a match terminated. // Later, this property will be propagated to all of the step's predecessors. for (unsigned j = 0; j < analysis.final_step_indices.size; j++) { - uint32_t final_step_index = analysis.final_step_indices.contents[j]; - QueryStep *step = &self->steps.contents[final_step_index]; + uint32_t final_step_index = *array_get(&analysis.final_step_indices, j); + QueryStep *step = array_get(&self->steps, final_step_index); if ( step->depth != PATTERN_DONE_MARKER && step->depth > parent_depth && @@ -1785,7 +1828,7 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { // Mark as indefinite any step with captures that are used in predicates. Array(uint16_t) predicate_capture_ids = array_new(); for (unsigned i = 0; i < self->patterns.size; i++) { - QueryPattern *pattern = &self->patterns.contents[i]; + QueryPattern *pattern = array_get(&self->patterns, i); // Gather all of the captures that are used in predicates for this pattern. array_clear(&predicate_capture_ids); @@ -1794,7 +1837,7 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { end = start + pattern->predicate_steps.length, j = start; j < end; j++ ) { - TSQueryPredicateStep *step = &self->predicate_steps.contents[j]; + TSQueryPredicateStep *step = array_get(&self->predicate_steps, j); if (step->type == TSQueryPredicateStepTypeCapture) { uint16_t value_id = step->value_id; array_insert_sorted_by(&predicate_capture_ids, , value_id); @@ -1807,7 +1850,7 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { end = start + pattern->steps.length, j = start; j < end; j++ ) { - QueryStep *step = &self->steps.contents[j]; + QueryStep *step = array_get(&self->steps, j); for (unsigned k = 0; k < MAX_STEP_CAPTURE_COUNT; k++) { uint16_t capture_id = step->capture_ids[k]; if (capture_id == NONE) break; @@ -1827,7 +1870,7 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { while (!done) { done = true; for (unsigned i = self->steps.size - 1; i > 0; i--) { - QueryStep *step = &self->steps.contents[i]; + QueryStep *step = array_get(&self->steps, i); if (step->depth == PATTERN_DONE_MARKER) continue; // Determine if this step is definite or has definite alternatives. @@ -1840,12 +1883,12 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { if (step->alternative_index == NONE || step->alternative_index < i) { break; } - step = &self->steps.contents[step->alternative_index]; + step = array_get(&self->steps, step->alternative_index); } // If not, mark its predecessor as indefinite. if (!parent_pattern_guaranteed) { - QueryStep *prev_step = &self->steps.contents[i - 1]; + QueryStep *prev_step = array_get(&self->steps, i - 1); if ( !prev_step->is_dead_end && prev_step->depth != PATTERN_DONE_MARKER && @@ -1861,7 +1904,7 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { #ifdef DEBUG_ANALYZE_QUERY printf("Steps:\n"); for (unsigned i = 0; i < self->steps.size; i++) { - QueryStep *step = &self->steps.contents[i]; + QueryStep *step = array_get(&self->steps, i); if (step->depth == PATTERN_DONE_MARKER) { printf(" %u: DONE\n", i); } else { @@ -1885,18 +1928,18 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { // prevent certain optimizations with range restrictions. analysis.did_abort = false; for (uint32_t i = 0; i < non_rooted_pattern_start_steps.size; i++) { - uint16_t pattern_entry_index = non_rooted_pattern_start_steps.contents[i]; - PatternEntry *pattern_entry = &self->pattern_map.contents[pattern_entry_index]; + uint16_t pattern_entry_index = *array_get(&non_rooted_pattern_start_steps, i); + PatternEntry *pattern_entry = array_get(&self->pattern_map, pattern_entry_index); analysis_state_set__clear(&analysis.states, &analysis.state_pool); analysis_state_set__clear(&analysis.deeper_states, &analysis.state_pool); for (unsigned j = 0; j < subgraphs.size; j++) { - AnalysisSubgraph *subgraph = &subgraphs.contents[j]; + AnalysisSubgraph *subgraph = array_get(&subgraphs, j); TSSymbolMetadata metadata = ts_language_symbol_metadata(self->language, subgraph->symbol); if (metadata.visible || metadata.named) continue; for (uint32_t k = 0; k < subgraph->start_states.size; k++) { - TSStateId parse_state = subgraph->start_states.contents[k]; + TSStateId parse_state = *array_get(&subgraph->start_states, k); analysis_state_set__push(&analysis.states, &analysis.state_pool, &((AnalysisState) { .step_index = pattern_entry->step_index, .stack = { @@ -1925,11 +1968,11 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { ); if (analysis.finished_parent_symbols.size > 0) { - self->patterns.contents[pattern_entry->pattern_index].is_non_local = true; + array_get(&self->patterns, pattern_entry->pattern_index)->is_non_local = true; } for (unsigned k = 0; k < analysis.finished_parent_symbols.size; k++) { - TSSymbol symbol = analysis.finished_parent_symbols.contents[k]; + TSSymbol symbol = *array_get(&analysis.finished_parent_symbols, k); array_insert_sorted_by(&self->repeat_symbols_with_rootless_patterns, , symbol); } } @@ -1939,7 +1982,7 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { printf("\nRepetition symbols with rootless patterns:\n"); printf("aborted analysis: %d\n", analysis.did_abort); for (unsigned i = 0; i < self->repeat_symbols_with_rootless_patterns.size; i++) { - TSSymbol symbol = self->repeat_symbols_with_rootless_patterns.contents[i]; + TSSymbol symbol = *array_get(&self->repeat_symbols_with_rootless_patterns, i); printf(" %u, %s\n", symbol, ts_language_symbol_name(self->language, symbol)); } printf("\n"); @@ -1948,17 +1991,19 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { // Cleanup for (unsigned i = 0; i < subgraphs.size; i++) { - array_delete(&subgraphs.contents[i].start_states); - array_delete(&subgraphs.contents[i].nodes); + array_delete(&array_get(&subgraphs, i)->start_states); + array_delete(&array_get(&subgraphs, i)->nodes); } array_delete(&subgraphs); query_analysis__delete(&analysis); array_delete(&next_nodes); - array_delete(&non_rooted_pattern_start_steps); - array_delete(&parent_step_indices); array_delete(&predicate_capture_ids); state_predecessor_map_delete(&predecessor_map); +supertype_cleanup: + array_delete(&non_rooted_pattern_start_steps); + array_delete(&parent_step_indices); + return all_patterns_are_valid; } @@ -1968,7 +2013,7 @@ static void ts_query__add_negated_fields( TSFieldId *field_ids, uint16_t field_count ) { - QueryStep *step = &self->steps.contents[step_index]; + QueryStep *step = array_get(&self->steps, step_index); // The negated field array stores a list of field lists, separated by zeros. // Try to find the start index of an existing list that matches this new list. @@ -1976,7 +2021,7 @@ static void ts_query__add_negated_fields( unsigned match_count = 0; unsigned start_i = 0; for (unsigned i = 0; i < self->negated_fields.size; i++) { - TSFieldId existing_field_id = self->negated_fields.contents[i]; + TSFieldId existing_field_id = *array_get(&self->negated_fields, i); // At each zero value, terminate the match attempt. If we've exactly // matched the new field list, then reuse this index. Otherwise, @@ -2080,6 +2125,10 @@ static TSQueryError ts_query__parse_predicate( if (!stream_is_ident_start(stream)) return TSQueryErrorSyntax; const char *predicate_name = stream->input; stream_scan_identifier(stream); + if (stream->next != '?' && stream->next != '!') { + return TSQueryErrorSyntax; + } + stream_advance(stream); uint32_t length = (uint32_t)(stream->input - predicate_name); uint16_t id = symbol_table_insert_name( &self->predicate_values, @@ -2246,10 +2295,10 @@ static TSQueryError ts_query__parse_pattern( // For all of the branches except for the last one, add the subsequent branch as an // alternative, and link the end of the branch to the current end of the steps. for (unsigned i = 0; i < branch_step_indices.size - 1; i++) { - uint32_t step_index = branch_step_indices.contents[i]; - uint32_t next_step_index = branch_step_indices.contents[i + 1]; - QueryStep *start_step = &self->steps.contents[step_index]; - QueryStep *end_step = &self->steps.contents[next_step_index - 1]; + uint32_t step_index = *array_get(&branch_step_indices, i); + uint32_t next_step_index = *array_get(&branch_step_indices, i + 1); + QueryStep *start_step = array_get(&self->steps, step_index); + QueryStep *end_step = array_get(&self->steps, next_step_index - 1); start_step->alternative_index = next_step_index; end_step->alternative_index = self->steps.size; end_step->is_dead_end = true; @@ -2313,16 +2362,62 @@ static TSQueryError ts_query__parse_pattern( // Otherwise, this parenthesis is the start of a named node. else { TSSymbol symbol; + bool is_missing = false; + const char *node_name = stream->input; // Parse a normal node name if (stream_is_ident_start(stream)) { - const char *node_name = stream->input; stream_scan_identifier(stream); uint32_t length = (uint32_t)(stream->input - node_name); // Parse the wildcard symbol if (length == 1 && node_name[0] == '_') { symbol = WILDCARD_SYMBOL; + } else if (!strncmp(node_name, "MISSING", length)) { + is_missing = true; + stream_skip_whitespace(stream); + + if (stream_is_ident_start(stream)) { + const char *missing_node_name = stream->input; + stream_scan_identifier(stream); + uint32_t missing_node_length = (uint32_t)(stream->input - missing_node_name); + symbol = ts_language_symbol_for_name( + self->language, + missing_node_name, + missing_node_length, + true + ); + if (!symbol) { + stream_reset(stream, missing_node_name); + return TSQueryErrorNodeType; + } + } + + else if (stream->next == '"') { + const char *string_start = stream->input; + TSQueryError e = ts_query__parse_string_literal(self, stream); + if (e) return e; + + symbol = ts_language_symbol_for_name( + self->language, + self->string_buffer.contents, + self->string_buffer.size, + false + ); + if (!symbol) { + stream_reset(stream, string_start + 1); + return TSQueryErrorNodeType; + } + } + + else if (stream->next == ')') { + symbol = WILDCARD_SYMBOL; + } + + else { + stream_reset(stream, stream->input); + return TSQueryErrorSyntax; + } } else { @@ -2348,36 +2443,79 @@ static TSQueryError ts_query__parse_pattern( step->supertype_symbol = step->symbol; step->symbol = WILDCARD_SYMBOL; } + if (is_missing) { + step->is_missing = true; + } if (symbol == WILDCARD_SYMBOL) { step->is_named = true; } - stream_skip_whitespace(stream); - + // Parse a supertype symbol if (stream->next == '/') { + if (!step->supertype_symbol) { + stream_reset(stream, node_name - 1); // reset to the start of the node + return TSQueryErrorStructure; + } + stream_advance(stream); - if (!stream_is_ident_start(stream)) { + + const char *subtype_node_name = stream->input; + + if (stream_is_ident_start(stream)) { // Named node + stream_scan_identifier(stream); + uint32_t length = (uint32_t)(stream->input - subtype_node_name); + step->symbol = ts_language_symbol_for_name( + self->language, + subtype_node_name, + length, + true + ); + } else if (stream->next == '"') { // Anonymous leaf node + TSQueryError e = ts_query__parse_string_literal(self, stream); + if (e) return e; + step->symbol = ts_language_symbol_for_name( + self->language, + self->string_buffer.contents, + self->string_buffer.size, + false + ); + } else { return TSQueryErrorSyntax; } - const char *node_name = stream->input; - stream_scan_identifier(stream); - uint32_t length = (uint32_t)(stream->input - node_name); - - step->symbol = ts_language_symbol_for_name( - self->language, - node_name, - length, - true - ); if (!step->symbol) { - stream_reset(stream, node_name); + stream_reset(stream, subtype_node_name); return TSQueryErrorNodeType; } - stream_skip_whitespace(stream); + // Get all the possible subtypes for the given supertype, + // and check if the given subtype is valid. + if (self->language->abi_version >= LANGUAGE_VERSION_WITH_RESERVED_WORDS) { + uint32_t subtype_length; + const TSSymbol *subtypes = ts_language_subtypes( + self->language, + step->supertype_symbol, + &subtype_length + ); + + bool subtype_is_valid = false; + for (uint32_t i = 0; i < subtype_length; i++) { + if (subtypes[i] == step->symbol) { + subtype_is_valid = true; + break; + } + } + + // This subtype is not valid for the given supertype. + if (!subtype_is_valid) { + stream_reset(stream, node_name - 1); // reset to the start of the node + return TSQueryErrorStructure; + } + } } + stream_skip_whitespace(stream); + // Parse the child patterns bool child_is_immediate = false; uint16_t last_child_step_index = 0; @@ -2433,6 +2571,9 @@ static TSQueryError ts_query__parse_pattern( child_is_immediate, &child_capture_quantifiers ); + // In the event we only parsed a predicate, meaning no new steps were added, + // then subtract one so we're not indexing past the end of the array + if (step_index == self->steps.size) step_index--; if (e == PARENT_DONE) { if (stream->next == ')') { if (child_is_immediate) { @@ -2440,7 +2581,23 @@ static TSQueryError ts_query__parse_pattern( capture_quantifiers_delete(&child_capture_quantifiers); return TSQueryErrorSyntax; } - self->steps.contents[last_child_step_index].is_last_child = true; + // Mark this step *and* its alternatives as the last child of the parent. + QueryStep *last_child_step = array_get(&self->steps, last_child_step_index); + last_child_step->is_last_child = true; + if ( + last_child_step->alternative_index != NONE && + last_child_step->alternative_index < self->steps.size + ) { + QueryStep *alternative_step = array_get(&self->steps, last_child_step->alternative_index); + alternative_step->is_last_child = true; + while ( + alternative_step->alternative_index != NONE && + alternative_step->alternative_index < self->steps.size + ) { + alternative_step = array_get(&self->steps, alternative_step->alternative_index); + alternative_step->is_last_child = true; + } + } } if (negated_field_count) { @@ -2543,7 +2700,7 @@ static TSQueryError ts_query__parse_pattern( } uint32_t step_index = starting_step_index; - QueryStep *step = &self->steps.contents[step_index]; + QueryStep *step = array_get(&self->steps, step_index); for (;;) { step->field = field_id; if ( @@ -2552,7 +2709,7 @@ static TSQueryError ts_query__parse_pattern( step->alternative_index < self->steps.size ) { step_index = step->alternative_index; - step = &self->steps.contents[step_index]; + step = array_get(&self->steps, step_index); } else { break; } @@ -2577,12 +2734,6 @@ static TSQueryError ts_query__parse_pattern( stream_advance(stream); stream_skip_whitespace(stream); - - QueryStep repeat_step = query_step__new(WILDCARD_SYMBOL, depth, false); - repeat_step.alternative_index = starting_step_index; - repeat_step.is_pass_through = true; - repeat_step.alternative_is_immediate = true; - array_push(&self->steps, repeat_step); } // Parse the zero-or-more repetition operator. @@ -2591,21 +2742,6 @@ static TSQueryError ts_query__parse_pattern( stream_advance(stream); stream_skip_whitespace(stream); - - QueryStep repeat_step = query_step__new(WILDCARD_SYMBOL, depth, false); - repeat_step.alternative_index = starting_step_index; - repeat_step.is_pass_through = true; - repeat_step.alternative_is_immediate = true; - array_push(&self->steps, repeat_step); - - // Stop when `step->alternative_index` is `NONE` or it points to - // `repeat_step` or beyond. Note that having just been pushed, - // `repeat_step` occupies slot `self->steps.size - 1`. - QueryStep *step = &self->steps.contents[starting_step_index]; - while (step->alternative_index != NONE && step->alternative_index < self->steps.size - 1) { - step = &self->steps.contents[step->alternative_index]; - } - step->alternative_index = self->steps.size; } // Parse the optional operator. @@ -2614,12 +2750,6 @@ static TSQueryError ts_query__parse_pattern( stream_advance(stream); stream_skip_whitespace(stream); - - QueryStep *step = &self->steps.contents[starting_step_index]; - while (step->alternative_index != NONE && step->alternative_index < self->steps.size) { - step = &self->steps.contents[step->alternative_index]; - } - step->alternative_index = self->steps.size; } // Parse an '@'-prefixed capture pattern @@ -2643,7 +2773,7 @@ static TSQueryError ts_query__parse_pattern( uint32_t step_index = starting_step_index; for (;;) { - QueryStep *step = &self->steps.contents[step_index]; + QueryStep *step = array_get(&self->steps, step_index); query_step__add_capture(step, capture_id); if ( step->alternative_index != NONE && @@ -2663,6 +2793,43 @@ static TSQueryError ts_query__parse_pattern( } } + QueryStep repeat_step; + QueryStep *step; + switch (quantifier) { + case TSQuantifierOneOrMore: + repeat_step = query_step__new(WILDCARD_SYMBOL, depth, false); + repeat_step.alternative_index = starting_step_index; + repeat_step.is_pass_through = true; + repeat_step.alternative_is_immediate = true; + array_push(&self->steps, repeat_step); + break; + case TSQuantifierZeroOrMore: + repeat_step = query_step__new(WILDCARD_SYMBOL, depth, false); + repeat_step.alternative_index = starting_step_index; + repeat_step.is_pass_through = true; + repeat_step.alternative_is_immediate = true; + array_push(&self->steps, repeat_step); + + // Stop when `step->alternative_index` is `NONE` or it points to + // `repeat_step` or beyond. Note that having just been pushed, + // `repeat_step` occupies slot `self->steps.size - 1`. + step = array_get(&self->steps, starting_step_index); + while (step->alternative_index != NONE && step->alternative_index < self->steps.size - 1) { + step = array_get(&self->steps, step->alternative_index); + } + step->alternative_index = self->steps.size; + break; + case TSQuantifierZeroOrOne: + step = array_get(&self->steps, starting_step_index); + while (step->alternative_index != NONE && step->alternative_index < self->steps.size) { + step = array_get(&self->steps, step->alternative_index); + } + step->alternative_index = self->steps.size; + break; + default: + break; + } + capture_quantifiers_mul(capture_quantifiers, quantifier); return 0; @@ -2677,8 +2844,8 @@ TSQuery *ts_query_new( ) { if ( !language || - language->version > TREE_SITTER_LANGUAGE_VERSION || - language->version < TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION + language->abi_version > TREE_SITTER_LANGUAGE_VERSION || + language->abi_version < TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION ) { *error_type = TSQueryErrorLanguage; return NULL; @@ -2741,14 +2908,14 @@ TSQuery *ts_query_new( // Maintain a map that can look up patterns for a given root symbol. uint16_t wildcard_root_alternative_index = NONE; for (;;) { - QueryStep *step = &self->steps.contents[start_step_index]; + QueryStep *step = array_get(&self->steps, start_step_index); // If a pattern has a wildcard at its root, but it has a non-wildcard child, // then optimize the matching process by skipping matching the wildcard. // Later, during the matching process, the query cursor will check that // there is a parent node, and capture it if necessary. if (step->symbol == WILDCARD_SYMBOL && step->depth == 0 && !step->field) { - QueryStep *second_step = &self->steps.contents[start_step_index + 1]; + QueryStep *second_step = array_get(&self->steps, start_step_index + 1); if (second_step->symbol != WILDCARD_SYMBOL && second_step->depth == 1 && !second_step->is_immediate) { wildcard_root_alternative_index = step->alternative_index; start_step_index += 1; @@ -2763,7 +2930,7 @@ TSQuery *ts_query_new( uint32_t start_depth = step->depth; bool is_rooted = start_depth == 0; for (uint32_t step_index = start_step_index + 1; step_index < self->steps.size; step_index++) { - QueryStep *child_step = &self->steps.contents[step_index]; + QueryStep *child_step = array_get(&self->steps, step_index); if (child_step->is_dead_end) break; if (child_step->depth == start_depth) { is_rooted = false; @@ -2867,26 +3034,24 @@ const TSQueryPredicateStep *ts_query_predicates_for_pattern( uint32_t pattern_index, uint32_t *step_count ) { - Slice slice = self->patterns.contents[pattern_index].predicate_steps; + Slice slice = array_get(&self->patterns, pattern_index)->predicate_steps; *step_count = slice.length; - if (self->predicate_steps.contents == NULL) { - return NULL; - } - return &self->predicate_steps.contents[slice.offset]; + if (slice.length == 0) return NULL; + return array_get(&self->predicate_steps, slice.offset); } uint32_t ts_query_start_byte_for_pattern( const TSQuery *self, uint32_t pattern_index ) { - return self->patterns.contents[pattern_index].start_byte; + return array_get(&self->patterns, pattern_index)->start_byte; } uint32_t ts_query_end_byte_for_pattern( const TSQuery *self, uint32_t pattern_index ) { - return self->patterns.contents[pattern_index].end_byte; + return array_get(&self->patterns, pattern_index)->end_byte; } bool ts_query_is_pattern_rooted( @@ -2894,7 +3059,7 @@ bool ts_query_is_pattern_rooted( uint32_t pattern_index ) { for (unsigned i = 0; i < self->pattern_map.size; i++) { - PatternEntry *entry = &self->pattern_map.contents[i]; + PatternEntry *entry = array_get(&self->pattern_map, i); if (entry->pattern_index == pattern_index) { if (!entry->is_rooted) return false; } @@ -2907,7 +3072,7 @@ bool ts_query_is_pattern_non_local( uint32_t pattern_index ) { if (pattern_index < self->patterns.size) { - return self->patterns.contents[pattern_index].is_non_local; + return array_get(&self->patterns, pattern_index)->is_non_local; } else { return false; } @@ -2919,12 +3084,12 @@ bool ts_query_is_pattern_guaranteed_at_step( ) { uint32_t step_index = UINT32_MAX; for (unsigned i = 0; i < self->step_offsets.size; i++) { - StepOffset *step_offset = &self->step_offsets.contents[i]; + StepOffset *step_offset = array_get(&self->step_offsets, i); if (step_offset->byte_offset > byte_offset) break; step_index = step_offset->step_index; } if (step_index < self->steps.size) { - return self->steps.contents[step_index].root_pattern_guaranteed; + return array_get(&self->steps, step_index)->root_pattern_guaranteed; } else { return false; } @@ -2935,12 +3100,12 @@ bool ts_query__step_is_fallible( uint16_t step_index ) { ts_assert((uint32_t)step_index + 1 < self->steps.size); - QueryStep *step = &self->steps.contents[step_index]; - QueryStep *next_step = &self->steps.contents[step_index + 1]; + QueryStep *step = array_get(&self->steps, step_index); + QueryStep *next_step = array_get(&self->steps, step_index + 1); return ( next_step->depth != PATTERN_DONE_MARKER && next_step->depth > step->depth && - !next_step->parent_pattern_guaranteed + (!next_step->parent_pattern_guaranteed || step->symbol == WILDCARD_SYMBOL) ); } @@ -2954,7 +3119,7 @@ void ts_query_disable_capture( int id = symbol_table_id_for_name(&self->captures, name, length); if (id != -1) { for (unsigned i = 0; i < self->steps.size; i++) { - QueryStep *step = &self->steps.contents[i]; + QueryStep *step = array_get(&self->steps, i); query_step__remove_capture(step, id); } } @@ -2967,7 +3132,7 @@ void ts_query_disable_pattern( // Remove the given pattern from the pattern map. Its steps will still // be in the `steps` array, but they will never be read. for (unsigned i = 0; i < self->pattern_map.size; i++) { - PatternEntry *pattern = &self->pattern_map.contents[i]; + PatternEntry *pattern = array_get(&self->pattern_map, i); if (pattern->pattern_index == pattern_index) { array_erase(&self->pattern_map, i); i--; @@ -2988,13 +3153,19 @@ TSQueryCursor *ts_query_cursor_new(void) { .states = array_new(), .finished_states = array_new(), .capture_list_pool = capture_list_pool_new(), - .start_byte = 0, - .end_byte = UINT32_MAX, - .start_point = {0, 0}, - .end_point = POINT_MAX, + .included_range = { + .start_point = {0, 0}, + .end_point = POINT_MAX, + .start_byte = 0, + .end_byte = UINT32_MAX, + }, + .containing_range = { + .start_point = {0, 0}, + .end_point = POINT_MAX, + .start_byte = 0, + .end_byte = UINT32_MAX, + }, .max_start_depth = UINT32_MAX, - .timeout_duration = 0, - .end_clock = clock_null(), .operation_count = 0, }; array_reserve(&self->states, 8); @@ -3022,14 +3193,6 @@ void ts_query_cursor_set_match_limit(TSQueryCursor *self, uint32_t limit) { self->capture_list_pool.max_capture_list_count = limit; } -uint64_t ts_query_cursor_timeout_micros(const TSQueryCursor *self) { - return duration_to_micros(self->timeout_duration); -} - -void ts_query_cursor_set_timeout_micros(TSQueryCursor *self, uint64_t timeout_micros) { - self->timeout_duration = duration_from_micros(timeout_micros); -} - #ifdef DEBUG_EXECUTE_QUERY #define LOG(...) fprintf(stderr, __VA_ARGS__) #else @@ -3044,7 +3207,7 @@ void ts_query_cursor_exec( if (query) { LOG("query steps:\n"); for (unsigned i = 0; i < query->steps.size; i++) { - QueryStep *step = &query->steps.contents[i]; + QueryStep *step = array_get(&query->steps, i); LOG(" %u: {", i); if (step->depth == PATTERN_DONE_MARKER) { LOG("DONE"); @@ -3079,11 +3242,6 @@ void ts_query_cursor_exec( self->query = query; self->did_exceed_match_limit = false; self->operation_count = 0; - if (self->timeout_duration) { - self->end_clock = clock_after(clock_now(), self->timeout_duration); - } else { - self->end_clock = clock_null(); - } self->query_options = NULL; self->query_state = (TSQueryCursorState) {0}; } @@ -3114,8 +3272,8 @@ bool ts_query_cursor_set_byte_range( if (start_byte > end_byte) { return false; } - self->start_byte = start_byte; - self->end_byte = end_byte; + self->included_range.start_byte = start_byte; + self->included_range.end_byte = end_byte; return true; } @@ -3130,8 +3288,40 @@ bool ts_query_cursor_set_point_range( if (point_gt(start_point, end_point)) { return false; } - self->start_point = start_point; - self->end_point = end_point; + self->included_range.start_point = start_point; + self->included_range.end_point = end_point; + return true; +} + +bool ts_query_cursor_set_containing_byte_range( + TSQueryCursor *self, + uint32_t start_byte, + uint32_t end_byte +) { + if (end_byte == 0) { + end_byte = UINT32_MAX; + } + if (start_byte > end_byte) { + return false; + } + self->containing_range.start_byte = start_byte; + self->containing_range.end_byte = end_byte; + return true; +} + +bool ts_query_cursor_set_containing_point_range( + TSQueryCursor *self, + TSPoint start_point, + TSPoint end_point +) { + if (end_point.row == 0 && end_point.column == 0) { + end_point = POINT_MAX; + } + if (point_gt(start_point, end_point)) { + return false; + } + self->containing_range.start_point = start_point; + self->containing_range.end_point = end_point; return true; } @@ -3142,14 +3332,14 @@ static bool ts_query_cursor__first_in_progress_capture( uint32_t *state_index, uint32_t *byte_offset, uint32_t *pattern_index, - bool *root_pattern_guaranteed + bool *is_definite ) { bool result = false; *state_index = UINT32_MAX; *byte_offset = UINT32_MAX; *pattern_index = UINT32_MAX; for (unsigned i = 0; i < self->states.size; i++) { - QueryState *state = &self->states.contents[i]; + QueryState *state = array_get(&self->states, i); if (state->dead) continue; const CaptureList *captures = capture_list_pool_get( @@ -3160,10 +3350,10 @@ static bool ts_query_cursor__first_in_progress_capture( continue; } - TSNode node = captures->contents[state->consumed_capture_count].node; + TSNode node = array_get(captures, state->consumed_capture_count)->node; if ( - ts_node_end_byte(node) <= self->start_byte || - point_lte(ts_node_end_point(node), self->start_point) + ts_node_end_byte(node) <= self->included_range.start_byte || + point_lte(ts_node_end_point(node), self->included_range.start_point) ) { state->consumed_capture_count++; i--; @@ -3176,9 +3366,12 @@ static bool ts_query_cursor__first_in_progress_capture( node_start_byte < *byte_offset || (node_start_byte == *byte_offset && state->pattern_index < *pattern_index) ) { - QueryStep *step = &self->query->steps.contents[state->step_index]; - if (root_pattern_guaranteed) { - *root_pattern_guaranteed = step->root_pattern_guaranteed; + QueryStep *step = array_get(&self->query->steps, state->step_index); + if (is_definite) { + // We're being a bit conservative here by asserting that the following step + // is not immediate, because this capture might end up being discarded if the + // following symbol in the tree isn't the required symbol for this step. + *is_definite = step->root_pattern_guaranteed && !step->is_immediate; } else if (step->root_pattern_guaranteed) { continue; } @@ -3229,8 +3422,8 @@ void ts_query_cursor__compare_captures( for (;;) { if (i < left_captures->size) { if (j < right_captures->size) { - TSQueryCapture *left = &left_captures->contents[i]; - TSQueryCapture *right = &right_captures->contents[j]; + TSQueryCapture *left = array_get(left_captures, i); + TSQueryCapture *right = array_get(right_captures, j); if (left->node.id == right->node.id && left->index == right->index) { i++; j++; @@ -3269,7 +3462,7 @@ static void ts_query_cursor__add_state( TSQueryCursor *self, const PatternEntry *pattern ) { - QueryStep *step = &self->query->steps.contents[pattern->step_index]; + QueryStep *step = array_get(&self->query->steps, pattern->step_index); uint32_t start_depth = self->depth - step->depth; // Keep the states array in ascending order of start_depth and pattern_index, @@ -3293,7 +3486,7 @@ static void ts_query_cursor__add_state( // need to execute in order to keep the states ordered by pattern_index. uint32_t index = self->states.size; while (index > 0) { - QueryState *prev_state = &self->states.contents[index - 1]; + QueryState *prev_state = array_get(&self->states, index - 1); if (prev_state->start_depth < start_depth) break; if (prev_state->start_depth == start_depth) { // Avoid inserting an unnecessary duplicate state, which would be @@ -3357,7 +3550,7 @@ static CaptureList *ts_query_cursor__prepare_to_capture( " abandon state. index:%u, pattern:%u, offset:%u.\n", state_index, pattern_index, byte_offset ); - QueryState *other_state = &self->states.contents[state_index]; + QueryState *other_state = array_get(&self->states, state_index); state->capture_list_id = other_state->capture_list_id; other_state->capture_list_id = NONE; other_state->dead = true; @@ -3427,8 +3620,8 @@ static QueryState *ts_query_cursor__copy_state( } array_insert(&self->states, state_index + 1, copy); - *state_ref = &self->states.contents[state_index]; - return &self->states.contents[state_index + 1]; + *state_ref = array_get(&self->states, state_index); + return array_get(&self->states, state_index + 1); } static inline bool ts_query_cursor__should_descend( @@ -3443,8 +3636,8 @@ static inline bool ts_query_cursor__should_descend( // If there are in-progress matches whose remaining steps occur // deeper in the tree, then descend. for (unsigned i = 0; i < self->states.size; i++) { - QueryState *state = &self->states.contents[i];; - QueryStep *next_step = &self->query->steps.contents[state->step_index]; + QueryState *state = array_get(&self->states, i); + QueryStep *next_step = array_get(&self->query->steps, state->step_index); if ( next_step->depth != PATTERN_DONE_MARKER && state->start_depth + next_step->depth > self->depth @@ -3485,6 +3678,31 @@ static inline bool ts_query_cursor__should_descend( return false; } +bool range_intersects(const TSRange *a, const TSRange *b) { + bool is_empty = a->start_byte == a->end_byte; + return ( + ( + a->end_byte > b->start_byte || + (is_empty && a->end_byte == b->start_byte) + ) && + ( + point_gt(a->end_point, b->start_point) || + (is_empty && point_eq(a->end_point, b->start_point)) + ) && + a->start_byte < b->end_byte && + point_lt(a->start_point, b->end_point) + ); +} + +bool range_within(const TSRange *a, const TSRange *b) { + return ( + a->start_byte >= b->start_byte && + point_gte(a->start_point, b->start_point) && + a->end_byte <= b->end_byte && + point_lte(a->end_point, b->end_point) + ); +} + // Walk the tree, processing patterns until at least one pattern finishes, // If one or more patterns finish, return `true` and store their states in the // `finished_states` array. Multiple patterns can finish on the same node. If @@ -3505,7 +3723,7 @@ static inline bool ts_query_cursor__advance( } } - if (++self->operation_count == OP_COUNT_PER_QUERY_TIMEOUT_CHECK) { + if (++self->operation_count == OP_COUNT_PER_QUERY_CALLBACK_CHECK) { self->operation_count = 0; } @@ -3518,7 +3736,6 @@ static inline bool ts_query_cursor__advance( ( self->operation_count == 0 && ( - (!clock_is_null(self->end_clock) && clock_is_gt(clock_now(), self->end_clock)) || (self->query_options && self->query_options->progress_callback && self->query_options->progress_callback(&self->query_state)) ) ) @@ -3538,8 +3755,8 @@ static inline bool ts_query_cursor__advance( // After leaving a node, remove any states that cannot make further progress. uint32_t deleted_count = 0; for (unsigned i = 0, n = self->states.size; i < n; i++) { - QueryState *state = &self->states.contents[i]; - QueryStep *step = &self->query->steps.contents[state->step_index]; + QueryState *state = array_get(&self->states, i); + QueryStep *step = array_get(&self->query->steps, state->step_index); // If a state completed its pattern inside of this node, but was deferred from finishing // in order to search for longer matches, mark it as finished. @@ -3572,7 +3789,7 @@ static inline bool ts_query_cursor__advance( } else if (deleted_count > 0) { - self->states.contents[i - deleted_count] = *state; + *array_get(&self->states, i - deleted_count) = *state; } } self->states.size -= deleted_count; @@ -3606,41 +3823,34 @@ static inline bool ts_query_cursor__advance( // Enter a new node. else { - // Get the properties of the current node. TSNode node = ts_tree_cursor_current_node(&self->cursor); TSNode parent_node = ts_tree_cursor_parent_node(&self->cursor); - uint32_t start_byte = ts_node_start_byte(node); - uint32_t end_byte = ts_node_end_byte(node); - TSPoint start_point = ts_node_start_point(node); - TSPoint end_point = ts_node_end_point(node); - bool is_empty = start_byte == end_byte; + bool parent_intersects_range = + ts_node_is_null(parent_node) || + range_intersects(&(TSRange) { + .start_point = ts_node_start_point(parent_node), + .end_point = ts_node_end_point(parent_node), + .start_byte = ts_node_start_byte(parent_node), + .end_byte = ts_node_end_byte(parent_node), + }, &self->included_range); + TSRange node_range = (TSRange) { + .start_point = ts_node_start_point(node), + .end_point = ts_node_end_point(node), + .start_byte = ts_node_start_byte(node), + .end_byte = ts_node_end_byte(node), + }; + bool node_intersects_range = + parent_intersects_range && range_intersects(&node_range, &self->included_range); + bool node_intersects_containing_range = + range_intersects(&node_range, &self->containing_range); + bool node_within_containing_range = + range_within(&node_range, &self->containing_range); - bool parent_precedes_range = !ts_node_is_null(parent_node) && ( - ts_node_end_byte(parent_node) <= self->start_byte || - point_lte(ts_node_end_point(parent_node), self->start_point) - ); - bool parent_follows_range = !ts_node_is_null(parent_node) && ( - ts_node_start_byte(parent_node) >= self->end_byte || - point_gte(ts_node_start_point(parent_node), self->end_point) - ); - bool node_precedes_range = - parent_precedes_range || - end_byte < self->start_byte || - point_lt(end_point, self->start_point) || - (!is_empty && end_byte == self->start_byte) || - (!is_empty && point_eq(end_point, self->start_point)); - - bool node_follows_range = parent_follows_range || ( - start_byte >= self->end_byte || - point_gte(start_point, self->end_point) - ); - bool parent_intersects_range = !parent_precedes_range && !parent_follows_range; - bool node_intersects_range = !node_precedes_range && !node_follows_range; - - if (self->on_visible_node) { + if (node_within_containing_range && self->on_visible_node) { TSSymbol symbol = ts_node_symbol(node); bool is_named = ts_node_is_named(node); + bool is_missing = ts_node_is_missing(node); bool has_later_siblings; bool has_later_named_siblings; bool can_have_later_siblings_with_this_field; @@ -3674,11 +3884,11 @@ static inline bool ts_query_cursor__advance( // Add new states for any patterns whose root node is a wildcard. if (!node_is_error) { for (unsigned i = 0; i < self->query->wildcard_root_pattern_count; i++) { - PatternEntry *pattern = &self->query->pattern_map.contents[i]; + PatternEntry *pattern = array_get(&self->query->pattern_map, i); // If this node matches the first step of the pattern, then add a new // state at the start of this pattern. - QueryStep *step = &self->query->steps.contents[pattern->step_index]; + QueryStep *step = array_get(&self->query->steps, pattern->step_index); uint32_t start_depth = self->depth - step->depth; if ( (pattern->is_rooted ? @@ -3696,9 +3906,9 @@ static inline bool ts_query_cursor__advance( // Add new states for any patterns whose root node matches this node. unsigned i; if (ts_query__pattern_map_search(self->query, symbol, &i)) { - PatternEntry *pattern = &self->query->pattern_map.contents[i]; + PatternEntry *pattern = array_get(&self->query->pattern_map, i); - QueryStep *step = &self->query->steps.contents[pattern->step_index]; + QueryStep *step = array_get(&self->query->steps, pattern->step_index); uint32_t start_depth = self->depth - step->depth; do { // If this node matches the first step of the pattern, then add a new @@ -3716,15 +3926,15 @@ static inline bool ts_query_cursor__advance( // Advance to the next pattern whose root node matches this node. i++; if (i == self->query->pattern_map.size) break; - pattern = &self->query->pattern_map.contents[i]; - step = &self->query->steps.contents[pattern->step_index]; + pattern = array_get(&self->query->pattern_map, i); + step = array_get(&self->query->steps, pattern->step_index); } while (step->symbol == symbol); } // Update all of the in-progress states with current node. for (unsigned j = 0, copy_count = 0; j < self->states.size; j += 1 + copy_count) { - QueryState *state = &self->states.contents[j]; - QueryStep *step = &self->query->steps.contents[state->step_index]; + QueryState *state = array_get(&self->states, j); + QueryStep *step = array_get(&self->query->steps, state->step_index); state->has_in_progress_alternatives = false; copy_count = 0; @@ -3737,9 +3947,13 @@ static inline bool ts_query_cursor__advance( // pattern. bool node_does_match = false; if (step->symbol == WILDCARD_SYMBOL) { - node_does_match = !node_is_error && (is_named || !step->is_named); + if (step->is_missing) { + node_does_match = is_missing; + } else { + node_does_match = !node_is_error && (is_named || !step->is_named); + } } else { - node_does_match = symbol == step->symbol; + node_does_match = symbol == step->symbol && (!step->is_missing || is_missing); } bool later_sibling_can_match = has_later_siblings; if ((step->is_immediate && is_named) || state->seeking_immediate_match) { @@ -3769,7 +3983,7 @@ static inline bool ts_query_cursor__advance( } if (step->negated_field_list_id) { - TSFieldId *negated_field_ids = &self->query->negated_fields.contents[step->negated_field_list_id]; + TSFieldId *negated_field_ids = array_get(&self->query->negated_fields, step->negated_field_list_id); for (;;) { TSFieldId negated_field_id = *negated_field_ids; if (negated_field_id) { @@ -3864,14 +4078,28 @@ static inline bool ts_query_cursor__advance( // Advance this state to the next step of its pattern. state->step_index++; - state->seeking_immediate_match = false; LOG( " advance state. pattern:%u, step:%u\n", state->pattern_index, state->step_index ); - QueryStep *next_step = &self->query->steps.contents[state->step_index]; + QueryStep *next_step = array_get(&self->query->steps, state->step_index); + + // For a given step, if the current symbol is the wildcard symbol, `_`, and it is **not** + // named, meaning it should capture anonymous nodes, **and** the next step is immediate, + // we reuse the `seeking_immediate_match` flag to indicate that we are looking for an + // immediate match due to an unnamed wildcard symbol. + // + // The reason for this is that typically, anchors will not consider anonymous nodes, + // but we're special casing the wildcard symbol to allow for any immediate matches, + // regardless of whether they are named or not. + if (step->symbol == WILDCARD_SYMBOL && !step->is_named && next_step->is_immediate) { + state->seeking_immediate_match = true; + } else { + state->seeking_immediate_match = false; + } + if (stop_on_definite_step && next_step->root_pattern_guaranteed) did_match = true; // If this state's next step has an alternative step, then copy the state in order @@ -3879,8 +4107,8 @@ static inline bool ts_query_cursor__advance( // so this is an interactive process. unsigned end_index = j + 1; for (unsigned k = j; k < end_index; k++) { - QueryState *child_state = &self->states.contents[k]; - QueryStep *child_step = &self->query->steps.contents[child_state->step_index]; + QueryState *child_state = array_get(&self->states, k); + QueryStep *child_step = array_get(&self->query->steps, child_state->step_index); if (child_step->alternative_index != NONE) { // A "dead-end" step exists only to add a non-sequential jump into the step sequence, // via its alternative index. When a state reaches a dead-end step, it jumps straight @@ -3921,7 +4149,7 @@ static inline bool ts_query_cursor__advance( } for (unsigned j = 0; j < self->states.size; j++) { - QueryState *state = &self->states.contents[j]; + QueryState *state = array_get(&self->states, j); if (state->dead) { array_erase(&self->states, j); j--; @@ -3933,7 +4161,7 @@ static inline bool ts_query_cursor__advance( // one state has a strict subset of another state's captures. bool did_remove = false; for (unsigned k = j + 1; k < self->states.size; k++) { - QueryState *other_state = &self->states.contents[k]; + QueryState *other_state = array_get(&self->states, k); // Query states are kept in ascending order of start_depth and pattern_index. // Since the longest-match criteria is only used for deduping matches of the same @@ -3993,7 +4221,7 @@ static inline bool ts_query_cursor__advance( state->step_index, capture_list_pool_get(&self->capture_list_pool, state->capture_list_id)->size ); - QueryStep *next_step = &self->query->steps.contents[state->step_index]; + QueryStep *next_step = array_get(&self->query->steps, state->step_index); if (next_step->depth == PATTERN_DONE_MARKER) { if (state->has_in_progress_alternatives) { LOG(" defer finishing pattern %u\n", state->pattern_index); @@ -4009,7 +4237,7 @@ static inline bool ts_query_cursor__advance( } } - if (ts_query_cursor__should_descend(self, node_intersects_range)) { + if (node_intersects_containing_range && ts_query_cursor__should_descend(self, node_intersects_range)) { switch (ts_tree_cursor_goto_first_child_internal(&self->cursor)) { case TreeCursorStepVisible: self->depth++; @@ -4038,7 +4266,7 @@ bool ts_query_cursor_next_match( } } - QueryState *state = &self->finished_states.contents[0]; + QueryState *state = array_get(&self->finished_states, 0); if (state->id == UINT32_MAX) state->id = self->next_state_id++; match->id = state->id; match->pattern_index = state->pattern_index; @@ -4058,7 +4286,7 @@ void ts_query_cursor_remove_match( uint32_t match_id ) { for (unsigned i = 0; i < self->finished_states.size; i++) { - const QueryState *state = &self->finished_states.contents[i]; + const QueryState *state = array_get(&self->finished_states, i); if (state->id == match_id) { capture_list_pool_release( &self->capture_list_pool, @@ -4072,7 +4300,7 @@ void ts_query_cursor_remove_match( // Remove unfinished query states as well to prevent future // captures for a match being removed. for (unsigned i = 0; i < self->states.size; i++) { - const QueryState *state = &self->states.contents[i]; + const QueryState *state = array_get(&self->states, i); if (state->id == match_id) { capture_list_pool_release( &self->capture_list_pool, @@ -4112,7 +4340,7 @@ bool ts_query_cursor_next_capture( uint32_t first_finished_capture_byte = first_unfinished_capture_byte; uint32_t first_finished_pattern_index = first_unfinished_pattern_index; for (unsigned i = 0; i < self->finished_states.size;) { - QueryState *state = &self->finished_states.contents[i]; + QueryState *state = array_get(&self->finished_states, i); const CaptureList *captures = capture_list_pool_get( &self->capture_list_pool, state->capture_list_id @@ -4128,15 +4356,15 @@ bool ts_query_cursor_next_capture( continue; } - TSNode node = captures->contents[state->consumed_capture_count].node; + TSNode node = array_get(captures, state->consumed_capture_count)->node; bool node_precedes_range = ( - ts_node_end_byte(node) <= self->start_byte || - point_lte(ts_node_end_point(node), self->start_point) + ts_node_end_byte(node) <= self->included_range.start_byte || + point_lte(ts_node_end_point(node), self->included_range.start_point) ); bool node_follows_range = ( - ts_node_start_byte(node) >= self->end_byte || - point_gte(ts_node_start_point(node), self->end_point) + ts_node_start_byte(node) >= self->included_range.end_byte || + point_gte(ts_node_start_point(node), self->included_range.end_point) ); bool node_outside_of_range = node_precedes_range || node_follows_range; @@ -4168,7 +4396,7 @@ bool ts_query_cursor_next_capture( if (first_finished_state) { state = first_finished_state; } else if (first_unfinished_state_is_definite) { - state = &self->states.contents[first_unfinished_state_index]; + state = array_get(&self->states, first_unfinished_state_index); } else { state = NULL; } @@ -4197,7 +4425,7 @@ bool ts_query_cursor_next_capture( ); capture_list_pool_release( &self->capture_list_pool, - self->states.contents[first_unfinished_state_index].capture_list_id + array_get(&self->states, first_unfinished_state_index)->capture_list_id ); array_erase(&self->states, first_unfinished_state_index); } diff --git a/lib/src/stack.c b/lib/src/stack.c index f0d57108..91420074 100644 --- a/lib/src/stack.c +++ b/lib/src/stack.c @@ -290,8 +290,8 @@ static StackVersion ts_stack__add_version( ) { StackHead head = { .node = node, - .node_count_at_last_error = self->heads.contents[original_version].node_count_at_last_error, - .last_external_token = self->heads.contents[original_version].last_external_token, + .node_count_at_last_error = array_get(&self->heads, original_version)->node_count_at_last_error, + .last_external_token = array_get(&self->heads, original_version)->last_external_token, .status = StackStatusActive, .lookahead_when_paused = NULL_SUBTREE, }; @@ -308,8 +308,8 @@ static void ts_stack__add_slice( SubtreeArray *subtrees ) { for (uint32_t i = self->slices.size - 1; i + 1 > 0; i--) { - StackVersion version = self->slices.contents[i].version; - if (self->heads.contents[version].node == node) { + StackVersion version = array_get(&self->slices, i)->version; + if (array_get(&self->heads, version)->node == node) { StackSlice slice = {*subtrees, version}; array_insert(&self->slices, i + 1, slice); return; @@ -349,7 +349,7 @@ static StackSliceArray stack__iter( while (self->iterators.size > 0) { for (uint32_t i = 0, size = self->iterators.size; i < size; i++) { - StackIterator *iterator = &self->iterators.contents[i]; + StackIterator *iterator = array_get(&self->iterators, i); StackNode *node = iterator->node; StackAction action = callback(payload, iterator); @@ -384,11 +384,11 @@ static StackSliceArray stack__iter( StackLink link; if (j == node->link_count) { link = node->links[0]; - next_iterator = &self->iterators.contents[i]; + next_iterator = array_get(&self->iterators, i); } else { if (self->iterators.size >= MAX_ITERATOR_COUNT) continue; link = node->links[j]; - StackIterator current_iterator = self->iterators.contents[i]; + StackIterator current_iterator = *array_get(&self->iterators, i); array_push(&self->iterators, current_iterator); next_iterator = array_back(&self->iterators); ts_subtree_array_copy(next_iterator->subtrees, &next_iterator->subtrees); @@ -444,12 +444,12 @@ void ts_stack_delete(Stack *self) { array_delete(&self->iterators); stack_node_release(self->base_node, &self->node_pool, self->subtree_pool); for (uint32_t i = 0; i < self->heads.size; i++) { - stack_head_delete(&self->heads.contents[i], &self->node_pool, self->subtree_pool); + stack_head_delete(array_get(&self->heads, i), &self->node_pool, self->subtree_pool); } array_clear(&self->heads); if (self->node_pool.contents) { for (uint32_t i = 0; i < self->node_pool.size; i++) - ts_free(self->node_pool.contents[i]); + ts_free(*array_get(&self->node_pool, i)); array_delete(&self->node_pool); } array_delete(&self->heads); @@ -460,6 +460,17 @@ uint32_t ts_stack_version_count(const Stack *self) { return self->heads.size; } +uint32_t ts_stack_halted_version_count(Stack *self) { + uint32_t count = 0; + for (uint32_t i = 0; i < self->heads.size; i++) { + StackHead *head = array_get(&self->heads, i); + if (head->status == StackStatusHalted) { + count++; + } + } + return count; +} + TSStateId ts_stack_state(const Stack *self, StackVersion version) { return array_get(&self->heads, version)->node->state; } @@ -524,6 +535,7 @@ StackSliceArray ts_stack_pop_count(Stack *self, StackVersion version, uint32_t c return stack__iter(self, version, pop_count_callback, &count, (int)count); } + forceinline StackAction pop_pending_callback(void *payload, const StackIterator *iterator) { (void)payload; if (iterator->subtree_count >= 1) { @@ -540,8 +552,8 @@ forceinline StackAction pop_pending_callback(void *payload, const StackIterator StackSliceArray ts_stack_pop_pending(Stack *self, StackVersion version) { StackSliceArray pop = stack__iter(self, version, pop_pending_callback, NULL, 0); if (pop.size > 0) { - ts_stack_renumber_version(self, pop.contents[0].version, version); - pop.contents[0].version = version; + ts_stack_renumber_version(self, array_get(&pop, 0)->version, version); + array_get(&pop, 0)->version = version; } return pop; } @@ -549,7 +561,7 @@ StackSliceArray ts_stack_pop_pending(Stack *self, StackVersion version) { forceinline StackAction pop_error_callback(void *payload, const StackIterator *iterator) { if (iterator->subtrees.size > 0) { bool *found_error = payload; - if (!*found_error && ts_subtree_is_error(iterator->subtrees.contents[0])) { + if (!*found_error && ts_subtree_is_error(*array_get(&iterator->subtrees, 0))) { *found_error = true; return StackActionPop | StackActionStop; } else { @@ -568,8 +580,8 @@ SubtreeArray ts_stack_pop_error(Stack *self, StackVersion version) { StackSliceArray pop = stack__iter(self, version, pop_error_callback, &found_error, 1); if (pop.size > 0) { ts_assert(pop.size == 1); - ts_stack_renumber_version(self, pop.contents[0].version, version); - return pop.contents[0].subtrees; + ts_stack_renumber_version(self, array_get(&pop, 0)->version, version); + return array_get(&pop, 0)->subtrees; } break; } @@ -597,7 +609,7 @@ forceinline StackAction summarize_stack_callback(void *payload, const StackItera unsigned depth = iterator->subtree_count; if (depth > session->max_depth) return StackActionStop; for (unsigned i = session->summary->size - 1; i + 1 > 0; i--) { - StackSummaryEntry entry = session->summary->contents[i]; + StackSummaryEntry entry = *array_get(session->summary, i); if (entry.depth < depth) break; if (entry.depth == depth && entry.state == state) return StackActionNone; } @@ -616,7 +628,7 @@ void ts_stack_record_summary(Stack *self, StackVersion version, unsigned max_dep }; array_init(session.summary); stack__iter(self, version, summarize_stack_callback, &session, -1); - StackHead *head = &self->heads.contents[version]; + StackHead *head = array_get(&self->heads, version); if (head->summary) { array_delete(head->summary); ts_free(head->summary); @@ -665,8 +677,8 @@ void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2) { if (v1 == v2) return; ts_assert(v2 < v1); ts_assert((uint32_t)v1 < self->heads.size); - StackHead *source_head = &self->heads.contents[v1]; - StackHead *target_head = &self->heads.contents[v2]; + StackHead *source_head = array_get(&self->heads, v1); + StackHead *target_head = array_get(&self->heads, v2); if (target_head->summary && !source_head->summary) { source_head->summary = target_head->summary; target_head->summary = NULL; @@ -677,14 +689,15 @@ void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2) { } void ts_stack_swap_versions(Stack *self, StackVersion v1, StackVersion v2) { - StackHead temporary_head = self->heads.contents[v1]; - self->heads.contents[v1] = self->heads.contents[v2]; - self->heads.contents[v2] = temporary_head; + StackHead temporary_head = *array_get(&self->heads, v1); + *array_get(&self->heads, v1) = *array_get(&self->heads, v2); + *array_get(&self->heads, v2) = temporary_head; } StackVersion ts_stack_copy_version(Stack *self, StackVersion version) { ts_assert(version < self->heads.size); - array_push(&self->heads, self->heads.contents[version]); + StackHead version_head = *array_get(&self->heads, version); + array_push(&self->heads, version_head); StackHead *head = array_back(&self->heads); stack_node_retain(head->node); if (head->last_external_token.ptr) ts_subtree_retain(head->last_external_token); @@ -694,8 +707,8 @@ StackVersion ts_stack_copy_version(Stack *self, StackVersion version) { bool ts_stack_merge(Stack *self, StackVersion version1, StackVersion version2) { if (!ts_stack_can_merge(self, version1, version2)) return false; - StackHead *head1 = &self->heads.contents[version1]; - StackHead *head2 = &self->heads.contents[version2]; + StackHead *head1 = array_get(&self->heads, version1); + StackHead *head2 = array_get(&self->heads, version2); for (uint32_t i = 0; i < head2->node->link_count; i++) { stack_node_add_link(head1->node, head2->node->links[i], self->subtree_pool); } @@ -707,8 +720,8 @@ bool ts_stack_merge(Stack *self, StackVersion version1, StackVersion version2) { } bool ts_stack_can_merge(Stack *self, StackVersion version1, StackVersion version2) { - StackHead *head1 = &self->heads.contents[version1]; - StackHead *head2 = &self->heads.contents[version2]; + StackHead *head1 = array_get(&self->heads, version1); + StackHead *head2 = array_get(&self->heads, version2); return head1->status == StackStatusActive && head2->status == StackStatusActive && @@ -753,7 +766,7 @@ Subtree ts_stack_resume(Stack *self, StackVersion version) { void ts_stack_clear(Stack *self) { stack_node_retain(self->base_node); for (uint32_t i = 0; i < self->heads.size; i++) { - stack_head_delete(&self->heads.contents[i], &self->node_pool, self->subtree_pool); + stack_head_delete(array_get(&self->heads, i), &self->node_pool, self->subtree_pool); } array_clear(&self->heads); array_push(&self->heads, ((StackHead) { @@ -776,7 +789,7 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f) array_clear(&self->iterators); for (uint32_t i = 0; i < self->heads.size; i++) { - StackHead *head = &self->heads.contents[i]; + StackHead *head = array_get(&self->heads, i); if (head->status == StackStatusHalted) continue; fprintf(f, "node_head_%u [shape=none, label=\"\"]\n", i); @@ -794,7 +807,7 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f) if (head->summary) { fprintf(f, "\nsummary:"); - for (uint32_t j = 0; j < head->summary->size; j++) fprintf(f, " %u", head->summary->contents[j].state); + for (uint32_t j = 0; j < head->summary->size; j++) fprintf(f, " %u", array_get(head->summary, j)->state); } if (head->last_external_token.ptr) { @@ -815,11 +828,11 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f) all_iterators_done = true; for (uint32_t i = 0; i < self->iterators.size; i++) { - StackIterator iterator = self->iterators.contents[i]; + StackIterator iterator = *array_get(&self->iterators, i); StackNode *node = iterator.node; for (uint32_t j = 0; j < visited_nodes.size; j++) { - if (visited_nodes.contents[j] == node) { + if (*array_get(&visited_nodes, j) == node) { node = NULL; break; } @@ -878,7 +891,7 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f) StackIterator *next_iterator; if (j == 0) { - next_iterator = &self->iterators.contents[i]; + next_iterator = array_get(&self->iterators, i); } else { array_push(&self->iterators, iterator); next_iterator = array_back(&self->iterators); diff --git a/lib/src/stack.h b/lib/src/stack.h index ac32234f..2619f1e8 100644 --- a/lib/src/stack.h +++ b/lib/src/stack.h @@ -36,6 +36,9 @@ void ts_stack_delete(Stack *self); // Get the stack's current number of versions. uint32_t ts_stack_version_count(const Stack *self); +// Get the stack's current number of halted versions. +uint32_t ts_stack_halted_version_count(Stack *self); + // Get the state at the top of the given version of the stack. If the stack is // empty, this returns the initial state, 0. TSStateId ts_stack_state(const Stack *self, StackVersion version); diff --git a/lib/src/subtree.c b/lib/src/subtree.c index 25b8bf08..97d55c86 100644 --- a/lib/src/subtree.c +++ b/lib/src/subtree.c @@ -73,14 +73,14 @@ void ts_subtree_array_copy(SubtreeArray self, SubtreeArray *dest) { dest->contents = ts_calloc(self.capacity, sizeof(Subtree)); memcpy(dest->contents, self.contents, self.size * sizeof(Subtree)); for (uint32_t i = 0; i < self.size; i++) { - ts_subtree_retain(dest->contents[i]); + ts_subtree_retain(*array_get(dest, i)); } } } void ts_subtree_array_clear(SubtreePool *pool, SubtreeArray *self) { for (uint32_t i = 0; i < self->size; i++) { - ts_subtree_release(pool, self->contents[i]); + ts_subtree_release(pool, *array_get(self, i)); } array_clear(self); } @@ -96,7 +96,7 @@ void ts_subtree_array_remove_trailing_extras( ) { array_clear(destination); while (self->size > 0) { - Subtree last = self->contents[self->size - 1]; + Subtree last = *array_get(self, self->size - 1); if (ts_subtree_extra(last)) { self->size--; array_push(destination, last); @@ -110,9 +110,9 @@ void ts_subtree_array_remove_trailing_extras( void ts_subtree_array_reverse(SubtreeArray *self) { for (uint32_t i = 0, limit = self->size / 2; i < limit; i++) { size_t reverse_index = self->size - 1 - i; - Subtree swap = self->contents[i]; - self->contents[i] = self->contents[reverse_index]; - self->contents[reverse_index] = swap; + Subtree swap = *array_get(self, i); + *array_get(self, i) = *array_get(self, reverse_index); + *array_get(self, reverse_index) = swap; } } @@ -127,7 +127,7 @@ SubtreePool ts_subtree_pool_new(uint32_t capacity) { void ts_subtree_pool_delete(SubtreePool *self) { if (self->free_trees.contents) { for (unsigned i = 0; i < self->free_trees.size; i++) { - ts_free(self->free_trees.contents[i].ptr); + ts_free(array_get(&self->free_trees, i)->ptr); } array_delete(&self->free_trees); } @@ -157,6 +157,7 @@ static inline bool ts_subtree_can_inline(Length padding, Length size, uint32_t l padding.bytes < TS_MAX_INLINE_TREE_LENGTH && padding.extent.row < 16 && padding.extent.column < TS_MAX_INLINE_TREE_LENGTH && + size.bytes < TS_MAX_INLINE_TREE_LENGTH && size.extent.row == 0 && size.extent.column < TS_MAX_INLINE_TREE_LENGTH && lookahead_bytes < 16; @@ -288,7 +289,7 @@ MutableSubtree ts_subtree_make_mut(SubtreePool *pool, Subtree self) { return result; } -static void ts_subtree__compress( +void ts_subtree_compress( MutableSubtree self, unsigned count, const TSLanguage *language, @@ -334,38 +335,6 @@ static void ts_subtree__compress( } } -void ts_subtree_balance(Subtree self, SubtreePool *pool, const TSLanguage *language) { - array_clear(&pool->tree_stack); - - if (ts_subtree_child_count(self) > 0 && self.ptr->ref_count == 1) { - array_push(&pool->tree_stack, ts_subtree_to_mut_unsafe(self)); - } - - while (pool->tree_stack.size > 0) { - MutableSubtree tree = array_pop(&pool->tree_stack); - - if (tree.ptr->repeat_depth > 0) { - Subtree child1 = ts_subtree_children(tree)[0]; - Subtree child2 = ts_subtree_children(tree)[tree.ptr->child_count - 1]; - long repeat_delta = (long)ts_subtree_repeat_depth(child1) - (long)ts_subtree_repeat_depth(child2); - if (repeat_delta > 0) { - unsigned n = (unsigned)repeat_delta; - for (unsigned i = n / 2; i > 0; i /= 2) { - ts_subtree__compress(tree, i, language, &pool->tree_stack); - n -= i; - } - } - } - - for (uint32_t i = 0; i < tree.ptr->child_count; i++) { - Subtree child = ts_subtree_children(tree)[i]; - if (ts_subtree_child_count(child) > 0 && child.ptr->ref_count == 1) { - array_push(&pool->tree_stack, ts_subtree_to_mut_unsafe(child)); - } - } - } -} - // Assign all of the node's properties that depend on its children. void ts_subtree_summarize_children( MutableSubtree self, @@ -438,7 +407,12 @@ void ts_subtree_summarize_children( self.ptr->dynamic_precedence += ts_subtree_dynamic_precedence(child); self.ptr->visible_descendant_count += ts_subtree_visible_descendant_count(child); - if (alias_sequence && alias_sequence[structural_index] != 0 && !ts_subtree_extra(child)) { + if ( + !ts_subtree_extra(child) && + ts_subtree_symbol(child) != 0 && + alias_sequence && + alias_sequence[structural_index] != 0 + ) { self.ptr->visible_descendant_count++; self.ptr->visible_child_count++; if (ts_language_symbol_metadata(language, alias_sequence[structural_index]).named) { @@ -700,12 +674,6 @@ Subtree ts_subtree_edit(Subtree self, const TSInputEdit *input_edit, SubtreePool padding = edit.new_end; } - // If the edit is a pure insertion right at the start of the subtree, - // shift the subtree over according to the insertion. - else if (edit.start.bytes == padding.bytes && is_pure_insertion) { - padding = edit.new_end; - } - // If the edit is within this subtree, resize the subtree to reflect the edit. else if ( edit.start.bytes < total_size.bytes || @@ -767,7 +735,7 @@ Subtree ts_subtree_edit(Subtree self, const TSInputEdit *input_edit, SubtreePool // Keep editing child nodes until a node is reached that starts after the edit. // Also, if this node's validity depends on its column position, then continue - // invaliditing child nodes until reaching a line break. + // invalidating child nodes until reaching a line break. if (( (child_left.bytes > edit.old_end.bytes) || (child_left.bytes == edit.old_end.bytes && child_size.bytes > 0 && i > 0) diff --git a/lib/src/subtree.h b/lib/src/subtree.h index e00334c1..ffc5fb7a 100644 --- a/lib/src/subtree.h +++ b/lib/src/subtree.h @@ -220,8 +220,8 @@ void ts_subtree_retain(Subtree self); void ts_subtree_release(SubtreePool *pool, Subtree self); int ts_subtree_compare(Subtree left, Subtree right, SubtreePool *pool); void ts_subtree_set_symbol(MutableSubtree *self, TSSymbol symbol, const TSLanguage *language); +void ts_subtree_compress(MutableSubtree self, unsigned count, const TSLanguage *language, MutableSubtreeArray *stack); void ts_subtree_summarize_children(MutableSubtree self, const TSLanguage *language); -void ts_subtree_balance(Subtree self, SubtreePool *pool, const TSLanguage *language); Subtree ts_subtree_edit(Subtree self, const TSInputEdit *edit, SubtreePool *pool); char *ts_subtree_string(Subtree self, TSSymbol alias_symbol, bool alias_is_named, const TSLanguage *language, bool include_all); void ts_subtree_print_dot_graph(Subtree self, const TSLanguage *language, FILE *f); diff --git a/lib/src/tree.c b/lib/src/tree.c index bb451180..6747fd6d 100644 --- a/lib/src/tree.c +++ b/lib/src/tree.c @@ -54,37 +54,7 @@ const TSLanguage *ts_tree_language(const TSTree *self) { void ts_tree_edit(TSTree *self, const TSInputEdit *edit) { for (unsigned i = 0; i < self->included_range_count; i++) { - TSRange *range = &self->included_ranges[i]; - if (range->end_byte >= edit->old_end_byte) { - if (range->end_byte != UINT32_MAX) { - range->end_byte = edit->new_end_byte + (range->end_byte - edit->old_end_byte); - range->end_point = point_add( - edit->new_end_point, - point_sub(range->end_point, edit->old_end_point) - ); - if (range->end_byte < edit->new_end_byte) { - range->end_byte = UINT32_MAX; - range->end_point = POINT_MAX; - } - } - } else if (range->end_byte > edit->start_byte) { - range->end_byte = edit->start_byte; - range->end_point = edit->start_point; - } - if (range->start_byte >= edit->old_end_byte) { - range->start_byte = edit->new_end_byte + (range->start_byte - edit->old_end_byte); - range->start_point = point_add( - edit->new_end_point, - point_sub(range->start_point, edit->old_end_point) - ); - if (range->start_byte < edit->new_end_byte) { - range->start_byte = UINT32_MAX; - range->start_point = POINT_MAX; - } - } else if (range->start_byte > edit->start_byte) { - range->start_byte = edit->start_byte; - range->start_point = edit->start_point; - } + ts_range_edit(&self->included_ranges[i], edit); } SubtreePool pool = ts_subtree_pool_new(0); @@ -146,7 +116,7 @@ void ts_tree_print_dot_graph(const TSTree *self, int fd) { fclose(file); } -#elif !defined(__wasi__) // WASI doesn't support dup +#elif !defined(__wasm__) // Wasm doesn't support dup #include diff --git a/lib/src/tree_cursor.c b/lib/src/tree_cursor.c index 888e7781..70ef5e39 100644 --- a/lib/src/tree_cursor.c +++ b/lib/src/tree_cursor.c @@ -16,11 +16,11 @@ typedef struct { // CursorChildIterator static inline bool ts_tree_cursor_is_entry_visible(const TreeCursor *self, uint32_t index) { - TreeCursorEntry *entry = &self->stack.contents[index]; + TreeCursorEntry *entry = array_get(&self->stack, index); if (index == 0 || ts_subtree_visible(*entry->subtree)) { return true; } else if (!ts_subtree_extra(*entry->subtree)) { - TreeCursorEntry *parent_entry = &self->stack.contents[index - 1]; + TreeCursorEntry *parent_entry = array_get(&self->stack, index - 1); return ts_language_alias_at( self->tree->language, parent_entry->subtree->ptr->production_id, @@ -129,14 +129,17 @@ static inline bool ts_tree_cursor_child_iterator_previous( }; *visible = ts_subtree_visible(*child); bool extra = ts_subtree_extra(*child); - if (!extra && self->alias_sequence) { - *visible |= self->alias_sequence[self->structural_child_index]; - self->structural_child_index--; - } self->position = length_backtrack(self->position, ts_subtree_padding(*child)); self->child_index--; + if (!extra && self->alias_sequence) { + *visible |= self->alias_sequence[self->structural_child_index]; + if (self->structural_child_index > 0) { + self->structural_child_index--; + } + } + // unsigned can underflow so compare it to child_count if (self->child_index < self->parent.ptr->child_count) { Subtree previous_child = ts_subtree_children(self->parent)[self->child_index]; @@ -304,8 +307,9 @@ int64_t ts_tree_cursor_goto_first_child_for_point(TSTreeCursor *self, TSPoint go } TreeCursorStep ts_tree_cursor_goto_sibling_internal( - TSTreeCursor *_self, - bool (*advance)(CursorChildIterator *, TreeCursorEntry *, bool *)) { + TSTreeCursor *_self, + bool (*advance)(CursorChildIterator *, TreeCursorEntry *, bool *) +) { TreeCursor *self = (TreeCursor *)_self; uint32_t initial_size = self->stack.size; @@ -370,7 +374,7 @@ TreeCursorStep ts_tree_cursor_goto_previous_sibling_internal(TSTreeCursor *_self return step; // restore position from the parent node - const TreeCursorEntry *parent = &self->stack.contents[self->stack.size - 2]; + const TreeCursorEntry *parent = array_get(&self->stack, self->stack.size - 2); Length position = parent->position; uint32_t child_index = array_back(&self->stack)->child_index; const Subtree *children = ts_subtree_children((*(parent->subtree))); @@ -421,7 +425,7 @@ void ts_tree_cursor_goto_descendant( // Ascend to the lowest ancestor that contains the goal node. for (;;) { uint32_t i = self->stack.size - 1; - TreeCursorEntry *entry = &self->stack.contents[i]; + TreeCursorEntry *entry = array_get(&self->stack, i); uint32_t next_descendant_index = entry->descendant_index + (ts_tree_cursor_is_entry_visible(self, i) ? 1 : 0) + @@ -475,7 +479,7 @@ TSNode ts_tree_cursor_current_node(const TSTreeCursor *_self) { bool is_extra = ts_subtree_extra(*last_entry->subtree); TSSymbol alias_symbol = is_extra ? 0 : self->root_alias_symbol; if (self->stack.size > 1 && !is_extra) { - TreeCursorEntry *parent_entry = &self->stack.contents[self->stack.size - 2]; + TreeCursorEntry *parent_entry = array_get(&self->stack, self->stack.size - 2); alias_symbol = ts_language_alias_at( self->tree->language, parent_entry->subtree->ptr->production_id, @@ -512,8 +516,8 @@ void ts_tree_cursor_current_status( // Walk up the tree, visiting the current node and its invisible ancestors, // because fields can refer to nodes through invisible *wrapper* nodes, for (unsigned i = self->stack.size - 1; i > 0; i--) { - TreeCursorEntry *entry = &self->stack.contents[i]; - TreeCursorEntry *parent_entry = &self->stack.contents[i - 1]; + TreeCursorEntry *entry = array_get(&self->stack, i); + TreeCursorEntry *parent_entry = array_get(&self->stack, i - 1); const TSSymbol *alias_sequence = ts_language_alias_sequence( self->tree->language, @@ -626,11 +630,11 @@ uint32_t ts_tree_cursor_current_depth(const TSTreeCursor *_self) { TSNode ts_tree_cursor_parent_node(const TSTreeCursor *_self) { const TreeCursor *self = (const TreeCursor *)_self; for (int i = (int)self->stack.size - 2; i >= 0; i--) { - TreeCursorEntry *entry = &self->stack.contents[i]; + TreeCursorEntry *entry = array_get(&self->stack, i); bool is_visible = true; TSSymbol alias_symbol = 0; if (i > 0) { - TreeCursorEntry *parent_entry = &self->stack.contents[i - 1]; + TreeCursorEntry *parent_entry = array_get(&self->stack, i - 1); alias_symbol = ts_language_alias_at( self->tree->language, parent_entry->subtree->ptr->production_id, @@ -655,8 +659,8 @@ TSFieldId ts_tree_cursor_current_field_id(const TSTreeCursor *_self) { // Walk up the tree, visiting the current node and its invisible ancestors. for (unsigned i = self->stack.size - 1; i > 0; i--) { - TreeCursorEntry *entry = &self->stack.contents[i]; - TreeCursorEntry *parent_entry = &self->stack.contents[i - 1]; + TreeCursorEntry *entry = array_get(&self->stack, i); + TreeCursorEntry *parent_entry = array_get(&self->stack, i - 1); // Stop walking up when another visible node is found. if ( diff --git a/lib/src/wasm/wasm-stdlib.h b/lib/src/wasm/wasm-stdlib.h index c1f3bc08..082ef4c2 100644 --- a/lib/src/wasm/wasm-stdlib.h +++ b/lib/src/wasm/wasm-stdlib.h @@ -1,964 +1,942 @@ unsigned char STDLIB_WASM[] = { - 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x01, 0x1e, 0x06, 0x60, + 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x01, 0x1a, 0x05, 0x60, + 0x01, 0x7f, 0x01, 0x7f, 0x60, 0x03, 0x7f, 0x7f, 0x7f, 0x01, 0x7f, 0x60, 0x02, 0x7f, 0x7f, 0x01, 0x7f, 0x60, 0x01, 0x7f, 0x00, 0x60, 0x00, 0x00, - 0x60, 0x01, 0x7f, 0x01, 0x7f, 0x60, 0x00, 0x01, 0x7f, 0x60, 0x03, 0x7f, - 0x7f, 0x7f, 0x01, 0x7f, 0x02, 0x9e, 0x01, 0x05, 0x03, 0x65, 0x6e, 0x76, - 0x06, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x02, 0x00, 0x02, 0x03, 0x65, - 0x6e, 0x76, 0x19, 0x5f, 0x5f, 0x69, 0x6e, 0x64, 0x69, 0x72, 0x65, 0x63, - 0x74, 0x5f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, - 0x61, 0x62, 0x6c, 0x65, 0x01, 0x70, 0x00, 0x01, 0x16, 0x77, 0x61, 0x73, - 0x69, 0x5f, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x70, - 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x31, 0x08, 0x61, 0x72, 0x67, 0x73, - 0x5f, 0x67, 0x65, 0x74, 0x00, 0x00, 0x16, 0x77, 0x61, 0x73, 0x69, 0x5f, - 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x70, 0x72, 0x65, - 0x76, 0x69, 0x65, 0x77, 0x31, 0x0e, 0x61, 0x72, 0x67, 0x73, 0x5f, 0x73, - 0x69, 0x7a, 0x65, 0x73, 0x5f, 0x67, 0x65, 0x74, 0x00, 0x00, 0x16, 0x77, - 0x61, 0x73, 0x69, 0x5f, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, - 0x5f, 0x70, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x31, 0x09, 0x70, 0x72, - 0x6f, 0x63, 0x5f, 0x65, 0x78, 0x69, 0x74, 0x00, 0x01, 0x03, 0x2a, 0x29, - 0x02, 0x00, 0x02, 0x02, 0x01, 0x03, 0x01, 0x00, 0x00, 0x01, 0x04, 0x00, - 0x00, 0x01, 0x02, 0x02, 0x05, 0x05, 0x03, 0x03, 0x05, 0x05, 0x00, 0x03, - 0x00, 0x03, 0x05, 0x03, 0x05, 0x03, 0x03, 0x03, 0x03, 0x05, 0x05, 0x05, - 0x03, 0x03, 0x00, 0x03, 0x03, 0x06, 0x0d, 0x02, 0x7f, 0x01, 0x41, 0x80, - 0x80, 0x04, 0x0b, 0x7f, 0x00, 0x41, 0x00, 0x0b, 0x07, 0xad, 0x02, 0x1c, - 0x11, 0x5f, 0x5f, 0x77, 0x61, 0x73, 0x6d, 0x5f, 0x63, 0x61, 0x6c, 0x6c, - 0x5f, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x00, 0x03, 0x0f, 0x5f, 0x5f, 0x73, - 0x74, 0x61, 0x63, 0x6b, 0x5f, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x65, 0x72, - 0x03, 0x00, 0x06, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x00, 0x06, 0x0a, - 0x72, 0x65, 0x73, 0x65, 0x74, 0x5f, 0x68, 0x65, 0x61, 0x70, 0x00, 0x07, - 0x06, 0x6d, 0x61, 0x6c, 0x6c, 0x6f, 0x63, 0x00, 0x08, 0x04, 0x66, 0x72, - 0x65, 0x65, 0x00, 0x09, 0x06, 0x63, 0x61, 0x6c, 0x6c, 0x6f, 0x63, 0x00, - 0x0a, 0x06, 0x6d, 0x65, 0x6d, 0x73, 0x65, 0x74, 0x00, 0x14, 0x07, 0x72, - 0x65, 0x61, 0x6c, 0x6c, 0x6f, 0x63, 0x00, 0x0b, 0x06, 0x6d, 0x65, 0x6d, - 0x63, 0x70, 0x79, 0x00, 0x13, 0x06, 0x73, 0x74, 0x72, 0x6c, 0x65, 0x6e, - 0x00, 0x15, 0x08, 0x69, 0x73, 0x77, 0x61, 0x6c, 0x6e, 0x75, 0x6d, 0x00, - 0x2b, 0x08, 0x69, 0x73, 0x77, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x00, 0x16, - 0x08, 0x69, 0x73, 0x77, 0x62, 0x6c, 0x61, 0x6e, 0x6b, 0x00, 0x22, 0x08, - 0x69, 0x73, 0x77, 0x64, 0x69, 0x67, 0x69, 0x74, 0x00, 0x23, 0x08, 0x69, - 0x73, 0x77, 0x6c, 0x6f, 0x77, 0x65, 0x72, 0x00, 0x20, 0x08, 0x69, 0x73, - 0x77, 0x73, 0x70, 0x61, 0x63, 0x65, 0x00, 0x2a, 0x08, 0x69, 0x73, 0x77, - 0x75, 0x70, 0x70, 0x65, 0x72, 0x00, 0x1e, 0x09, 0x69, 0x73, 0x77, 0x78, - 0x64, 0x69, 0x67, 0x69, 0x74, 0x00, 0x27, 0x08, 0x74, 0x6f, 0x77, 0x6c, - 0x6f, 0x77, 0x65, 0x72, 0x00, 0x1a, 0x08, 0x74, 0x6f, 0x77, 0x75, 0x70, - 0x70, 0x65, 0x72, 0x00, 0x1c, 0x06, 0x6d, 0x65, 0x6d, 0x63, 0x68, 0x72, - 0x00, 0x18, 0x06, 0x6d, 0x65, 0x6d, 0x63, 0x6d, 0x70, 0x00, 0x17, 0x07, - 0x6d, 0x65, 0x6d, 0x6d, 0x6f, 0x76, 0x65, 0x00, 0x1f, 0x06, 0x73, 0x74, - 0x72, 0x63, 0x6d, 0x70, 0x00, 0x19, 0x07, 0x73, 0x74, 0x72, 0x6e, 0x63, - 0x61, 0x74, 0x00, 0x24, 0x07, 0x73, 0x74, 0x72, 0x6e, 0x63, 0x6d, 0x70, - 0x00, 0x1d, 0x07, 0x73, 0x74, 0x72, 0x6e, 0x63, 0x70, 0x79, 0x00, 0x26, - 0x08, 0x01, 0x05, 0x0a, 0xe8, 0x2b, 0x29, 0x02, 0x00, 0x0b, 0x03, 0x00, - 0x00, 0x0b, 0x0d, 0x00, 0x41, 0xe8, 0xc2, 0x04, 0x41, 0x00, 0x41, 0x10, - 0xfc, 0x0b, 0x00, 0x0b, 0x52, 0x01, 0x01, 0x7f, 0x02, 0x40, 0x02, 0x40, - 0x23, 0x81, 0x80, 0x80, 0x80, 0x00, 0x41, 0xe8, 0xc2, 0x84, 0x80, 0x00, - 0x6a, 0x28, 0x02, 0x00, 0x0d, 0x00, 0x23, 0x81, 0x80, 0x80, 0x80, 0x00, - 0x41, 0xe8, 0xc2, 0x84, 0x80, 0x00, 0x6a, 0x41, 0x01, 0x36, 0x02, 0x00, - 0x10, 0x83, 0x80, 0x80, 0x80, 0x00, 0x10, 0x8d, 0x80, 0x80, 0x80, 0x00, - 0x21, 0x00, 0x10, 0x92, 0x80, 0x80, 0x80, 0x00, 0x20, 0x00, 0x0d, 0x01, - 0x0f, 0x0b, 0x00, 0x00, 0x0b, 0x20, 0x00, 0x10, 0x90, 0x80, 0x80, 0x80, - 0x00, 0x00, 0x0b, 0x37, 0x01, 0x01, 0x7f, 0x23, 0x81, 0x80, 0x80, 0x80, - 0x00, 0x22, 0x01, 0x41, 0xf0, 0xc2, 0x84, 0x80, 0x00, 0x6a, 0x20, 0x00, - 0x36, 0x02, 0x00, 0x20, 0x01, 0x41, 0xec, 0xc2, 0x84, 0x80, 0x00, 0x6a, - 0x20, 0x00, 0x36, 0x02, 0x00, 0x20, 0x01, 0x41, 0xf4, 0xc2, 0x84, 0x80, - 0x00, 0x6a, 0x3f, 0x00, 0x41, 0x10, 0x74, 0x36, 0x02, 0x00, 0x0b, 0xb4, - 0x01, 0x01, 0x03, 0x7f, 0x02, 0x40, 0x02, 0x40, 0x23, 0x81, 0x80, 0x80, - 0x80, 0x00, 0x22, 0x01, 0x41, 0xf4, 0xc2, 0x84, 0x80, 0x00, 0x6a, 0x28, - 0x02, 0x00, 0x20, 0x01, 0x41, 0xf0, 0xc2, 0x84, 0x80, 0x00, 0x6a, 0x28, - 0x02, 0x00, 0x22, 0x01, 0x20, 0x00, 0x6a, 0x41, 0x07, 0x6a, 0x41, 0x7c, - 0x71, 0x22, 0x02, 0x4f, 0x0d, 0x00, 0x41, 0x00, 0x21, 0x01, 0x20, 0x02, - 0x23, 0x81, 0x80, 0x80, 0x80, 0x00, 0x41, 0xec, 0xc2, 0x84, 0x80, 0x00, - 0x6a, 0x28, 0x02, 0x00, 0x6b, 0x41, 0x80, 0x80, 0x80, 0x02, 0x4a, 0x0d, - 0x01, 0x20, 0x00, 0x41, 0x7f, 0x6a, 0x41, 0x10, 0x76, 0x41, 0x01, 0x6a, - 0x40, 0x00, 0x41, 0x7f, 0x46, 0x0d, 0x01, 0x3f, 0x00, 0x21, 0x01, 0x23, - 0x81, 0x80, 0x80, 0x80, 0x00, 0x22, 0x03, 0x41, 0xf4, 0xc2, 0x84, 0x80, - 0x00, 0x6a, 0x20, 0x01, 0x41, 0x10, 0x74, 0x36, 0x02, 0x00, 0x20, 0x03, - 0x41, 0xf0, 0xc2, 0x84, 0x80, 0x00, 0x6a, 0x28, 0x02, 0x00, 0x21, 0x01, - 0x0b, 0x20, 0x01, 0x20, 0x00, 0x36, 0x02, 0x00, 0x23, 0x81, 0x80, 0x80, - 0x80, 0x00, 0x41, 0xf0, 0xc2, 0x84, 0x80, 0x00, 0x6a, 0x20, 0x02, 0x36, - 0x02, 0x00, 0x20, 0x01, 0x41, 0x04, 0x6a, 0x21, 0x01, 0x0b, 0x20, 0x01, - 0x0b, 0x48, 0x01, 0x02, 0x7f, 0x02, 0x40, 0x20, 0x00, 0x45, 0x0d, 0x00, - 0x20, 0x00, 0x41, 0x7c, 0x6a, 0x22, 0x01, 0x28, 0x02, 0x00, 0x21, 0x02, - 0x23, 0x81, 0x80, 0x80, 0x80, 0x00, 0x41, 0xf0, 0xc2, 0x84, 0x80, 0x00, - 0x6a, 0x28, 0x02, 0x00, 0x20, 0x00, 0x20, 0x02, 0x6a, 0x41, 0x03, 0x6a, - 0x41, 0x7c, 0x71, 0x47, 0x0d, 0x00, 0x23, 0x81, 0x80, 0x80, 0x80, 0x00, - 0x41, 0xf0, 0xc2, 0x84, 0x80, 0x00, 0x6a, 0x20, 0x01, 0x36, 0x02, 0x00, - 0x0b, 0x0b, 0x19, 0x00, 0x20, 0x01, 0x20, 0x00, 0x6c, 0x22, 0x00, 0x10, - 0x88, 0x80, 0x80, 0x80, 0x00, 0x41, 0x00, 0x20, 0x00, 0x10, 0x94, 0x80, - 0x80, 0x80, 0x00, 0x0b, 0x6b, 0x01, 0x02, 0x7f, 0x02, 0x40, 0x20, 0x00, - 0x45, 0x0d, 0x00, 0x20, 0x00, 0x41, 0x7c, 0x6a, 0x22, 0x02, 0x28, 0x02, - 0x00, 0x21, 0x03, 0x02, 0x40, 0x23, 0x81, 0x80, 0x80, 0x80, 0x00, 0x41, - 0xf0, 0xc2, 0x84, 0x80, 0x00, 0x6a, 0x28, 0x02, 0x00, 0x20, 0x00, 0x20, - 0x03, 0x6a, 0x41, 0x03, 0x6a, 0x41, 0x7c, 0x71, 0x47, 0x0d, 0x00, 0x23, - 0x81, 0x80, 0x80, 0x80, 0x00, 0x41, 0xf0, 0xc2, 0x84, 0x80, 0x00, 0x6a, - 0x20, 0x02, 0x36, 0x02, 0x00, 0x0c, 0x01, 0x0b, 0x20, 0x01, 0x10, 0x88, - 0x80, 0x80, 0x80, 0x00, 0x20, 0x00, 0x20, 0x02, 0x28, 0x02, 0x00, 0x10, - 0x93, 0x80, 0x80, 0x80, 0x00, 0x0f, 0x0b, 0x20, 0x01, 0x10, 0x88, 0x80, - 0x80, 0x80, 0x00, 0x0b, 0x0b, 0x00, 0x20, 0x00, 0x10, 0x90, 0x80, 0x80, - 0x80, 0x00, 0x00, 0x0b, 0xd5, 0x01, 0x01, 0x03, 0x7f, 0x23, 0x80, 0x80, - 0x80, 0x80, 0x00, 0x41, 0x10, 0x6b, 0x22, 0x00, 0x24, 0x80, 0x80, 0x80, - 0x80, 0x00, 0x02, 0x40, 0x02, 0x40, 0x02, 0x40, 0x02, 0x40, 0x02, 0x40, - 0x20, 0x00, 0x41, 0x08, 0x6a, 0x20, 0x00, 0x41, 0x0c, 0x6a, 0x10, 0x8f, - 0x80, 0x80, 0x80, 0x00, 0x0d, 0x00, 0x20, 0x00, 0x28, 0x02, 0x08, 0x41, - 0x01, 0x6a, 0x22, 0x01, 0x45, 0x0d, 0x01, 0x20, 0x00, 0x28, 0x02, 0x0c, - 0x10, 0x88, 0x80, 0x80, 0x80, 0x00, 0x22, 0x02, 0x45, 0x0d, 0x02, 0x20, - 0x01, 0x41, 0x04, 0x10, 0x8a, 0x80, 0x80, 0x80, 0x00, 0x22, 0x01, 0x45, - 0x0d, 0x03, 0x20, 0x01, 0x20, 0x02, 0x10, 0x8e, 0x80, 0x80, 0x80, 0x00, - 0x0d, 0x04, 0x20, 0x00, 0x28, 0x02, 0x08, 0x20, 0x01, 0x10, 0x84, 0x80, - 0x80, 0x80, 0x00, 0x21, 0x01, 0x20, 0x00, 0x41, 0x10, 0x6a, 0x24, 0x80, - 0x80, 0x80, 0x80, 0x00, 0x20, 0x01, 0x0f, 0x0b, 0x41, 0xc7, 0x00, 0x10, - 0x8c, 0x80, 0x80, 0x80, 0x00, 0x00, 0x0b, 0x41, 0xc6, 0x00, 0x10, 0x8c, - 0x80, 0x80, 0x80, 0x00, 0x00, 0x0b, 0x41, 0xc6, 0x00, 0x10, 0x8c, 0x80, - 0x80, 0x80, 0x00, 0x00, 0x0b, 0x20, 0x02, 0x10, 0x89, 0x80, 0x80, 0x80, - 0x00, 0x41, 0xc6, 0x00, 0x10, 0x8c, 0x80, 0x80, 0x80, 0x00, 0x00, 0x0b, - 0x20, 0x02, 0x10, 0x89, 0x80, 0x80, 0x80, 0x00, 0x20, 0x01, 0x10, 0x89, - 0x80, 0x80, 0x80, 0x00, 0x41, 0xc7, 0x00, 0x10, 0x8c, 0x80, 0x80, 0x80, - 0x00, 0x00, 0x0b, 0x11, 0x00, 0x20, 0x00, 0x20, 0x01, 0x10, 0x80, 0x80, - 0x80, 0x80, 0x00, 0x41, 0xff, 0xff, 0x03, 0x71, 0x0b, 0x11, 0x00, 0x20, - 0x00, 0x20, 0x01, 0x10, 0x81, 0x80, 0x80, 0x80, 0x00, 0x41, 0xff, 0xff, - 0x03, 0x71, 0x0b, 0x0b, 0x00, 0x20, 0x00, 0x10, 0x82, 0x80, 0x80, 0x80, - 0x00, 0x00, 0x0b, 0x02, 0x00, 0x0b, 0x0e, 0x00, 0x10, 0x91, 0x80, 0x80, - 0x80, 0x00, 0x10, 0x91, 0x80, 0x80, 0x80, 0x00, 0x0b, 0xe6, 0x07, 0x01, - 0x04, 0x7f, 0x02, 0x40, 0x02, 0x40, 0x02, 0x40, 0x20, 0x02, 0x41, 0x20, - 0x4b, 0x0d, 0x00, 0x20, 0x01, 0x41, 0x03, 0x71, 0x45, 0x0d, 0x01, 0x20, - 0x02, 0x45, 0x0d, 0x01, 0x20, 0x00, 0x20, 0x01, 0x2d, 0x00, 0x00, 0x3a, - 0x00, 0x00, 0x20, 0x02, 0x41, 0x7f, 0x6a, 0x21, 0x03, 0x20, 0x00, 0x41, - 0x01, 0x6a, 0x21, 0x04, 0x20, 0x01, 0x41, 0x01, 0x6a, 0x22, 0x05, 0x41, - 0x03, 0x71, 0x45, 0x0d, 0x02, 0x20, 0x03, 0x45, 0x0d, 0x02, 0x20, 0x00, - 0x20, 0x01, 0x2d, 0x00, 0x01, 0x3a, 0x00, 0x01, 0x20, 0x02, 0x41, 0x7e, - 0x6a, 0x21, 0x03, 0x20, 0x00, 0x41, 0x02, 0x6a, 0x21, 0x04, 0x20, 0x01, - 0x41, 0x02, 0x6a, 0x22, 0x05, 0x41, 0x03, 0x71, 0x45, 0x0d, 0x02, 0x20, - 0x03, 0x45, 0x0d, 0x02, 0x20, 0x00, 0x20, 0x01, 0x2d, 0x00, 0x02, 0x3a, - 0x00, 0x02, 0x20, 0x02, 0x41, 0x7d, 0x6a, 0x21, 0x03, 0x20, 0x00, 0x41, - 0x03, 0x6a, 0x21, 0x04, 0x20, 0x01, 0x41, 0x03, 0x6a, 0x22, 0x05, 0x41, - 0x03, 0x71, 0x45, 0x0d, 0x02, 0x20, 0x03, 0x45, 0x0d, 0x02, 0x20, 0x00, - 0x20, 0x01, 0x2d, 0x00, 0x03, 0x3a, 0x00, 0x03, 0x20, 0x02, 0x41, 0x7c, - 0x6a, 0x21, 0x03, 0x20, 0x00, 0x41, 0x04, 0x6a, 0x21, 0x04, 0x20, 0x01, - 0x41, 0x04, 0x6a, 0x21, 0x05, 0x0c, 0x02, 0x0b, 0x20, 0x00, 0x20, 0x01, - 0x20, 0x02, 0xfc, 0x0a, 0x00, 0x00, 0x20, 0x00, 0x0f, 0x0b, 0x20, 0x02, - 0x21, 0x03, 0x20, 0x00, 0x21, 0x04, 0x20, 0x01, 0x21, 0x05, 0x0b, 0x02, - 0x40, 0x02, 0x40, 0x20, 0x04, 0x41, 0x03, 0x71, 0x22, 0x02, 0x0d, 0x00, - 0x02, 0x40, 0x02, 0x40, 0x20, 0x03, 0x41, 0x10, 0x4f, 0x0d, 0x00, 0x20, - 0x03, 0x21, 0x02, 0x0c, 0x01, 0x0b, 0x02, 0x40, 0x20, 0x03, 0x41, 0x70, - 0x6a, 0x22, 0x02, 0x41, 0x10, 0x71, 0x0d, 0x00, 0x20, 0x04, 0x20, 0x05, - 0x29, 0x02, 0x00, 0x37, 0x02, 0x00, 0x20, 0x04, 0x20, 0x05, 0x29, 0x02, - 0x08, 0x37, 0x02, 0x08, 0x20, 0x04, 0x41, 0x10, 0x6a, 0x21, 0x04, 0x20, - 0x05, 0x41, 0x10, 0x6a, 0x21, 0x05, 0x20, 0x02, 0x21, 0x03, 0x0b, 0x20, - 0x02, 0x41, 0x10, 0x49, 0x0d, 0x00, 0x20, 0x03, 0x21, 0x02, 0x03, 0x40, - 0x20, 0x04, 0x20, 0x05, 0x29, 0x02, 0x00, 0x37, 0x02, 0x00, 0x20, 0x04, - 0x20, 0x05, 0x29, 0x02, 0x08, 0x37, 0x02, 0x08, 0x20, 0x04, 0x20, 0x05, - 0x29, 0x02, 0x10, 0x37, 0x02, 0x10, 0x20, 0x04, 0x20, 0x05, 0x29, 0x02, - 0x18, 0x37, 0x02, 0x18, 0x20, 0x04, 0x41, 0x20, 0x6a, 0x21, 0x04, 0x20, - 0x05, 0x41, 0x20, 0x6a, 0x21, 0x05, 0x20, 0x02, 0x41, 0x60, 0x6a, 0x22, - 0x02, 0x41, 0x0f, 0x4b, 0x0d, 0x00, 0x0b, 0x0b, 0x02, 0x40, 0x20, 0x02, - 0x41, 0x08, 0x49, 0x0d, 0x00, 0x20, 0x04, 0x20, 0x05, 0x29, 0x02, 0x00, - 0x37, 0x02, 0x00, 0x20, 0x05, 0x41, 0x08, 0x6a, 0x21, 0x05, 0x20, 0x04, - 0x41, 0x08, 0x6a, 0x21, 0x04, 0x0b, 0x02, 0x40, 0x20, 0x02, 0x41, 0x04, - 0x71, 0x45, 0x0d, 0x00, 0x20, 0x04, 0x20, 0x05, 0x28, 0x02, 0x00, 0x36, - 0x02, 0x00, 0x20, 0x05, 0x41, 0x04, 0x6a, 0x21, 0x05, 0x20, 0x04, 0x41, - 0x04, 0x6a, 0x21, 0x04, 0x0b, 0x02, 0x40, 0x20, 0x02, 0x41, 0x02, 0x71, - 0x45, 0x0d, 0x00, 0x20, 0x04, 0x20, 0x05, 0x2f, 0x00, 0x00, 0x3b, 0x00, - 0x00, 0x20, 0x04, 0x41, 0x02, 0x6a, 0x21, 0x04, 0x20, 0x05, 0x41, 0x02, - 0x6a, 0x21, 0x05, 0x0b, 0x20, 0x02, 0x41, 0x01, 0x71, 0x45, 0x0d, 0x01, - 0x20, 0x04, 0x20, 0x05, 0x2d, 0x00, 0x00, 0x3a, 0x00, 0x00, 0x20, 0x00, - 0x0f, 0x0b, 0x02, 0x40, 0x02, 0x40, 0x02, 0x40, 0x02, 0x40, 0x02, 0x40, - 0x20, 0x03, 0x41, 0x20, 0x49, 0x0d, 0x00, 0x02, 0x40, 0x02, 0x40, 0x20, - 0x02, 0x41, 0x7f, 0x6a, 0x0e, 0x03, 0x03, 0x00, 0x01, 0x07, 0x0b, 0x20, - 0x04, 0x20, 0x05, 0x28, 0x02, 0x00, 0x3b, 0x00, 0x00, 0x20, 0x04, 0x20, - 0x05, 0x41, 0x02, 0x6a, 0x28, 0x01, 0x00, 0x36, 0x02, 0x02, 0x20, 0x04, - 0x20, 0x05, 0x41, 0x06, 0x6a, 0x29, 0x01, 0x00, 0x37, 0x02, 0x06, 0x20, - 0x04, 0x41, 0x12, 0x6a, 0x21, 0x02, 0x20, 0x05, 0x41, 0x12, 0x6a, 0x21, - 0x01, 0x41, 0x0e, 0x21, 0x06, 0x20, 0x05, 0x41, 0x0e, 0x6a, 0x28, 0x01, - 0x00, 0x21, 0x05, 0x41, 0x0e, 0x21, 0x03, 0x0c, 0x03, 0x0b, 0x20, 0x04, - 0x20, 0x05, 0x28, 0x02, 0x00, 0x3a, 0x00, 0x00, 0x20, 0x04, 0x20, 0x05, - 0x41, 0x01, 0x6a, 0x28, 0x00, 0x00, 0x36, 0x02, 0x01, 0x20, 0x04, 0x20, - 0x05, 0x41, 0x05, 0x6a, 0x29, 0x00, 0x00, 0x37, 0x02, 0x05, 0x20, 0x04, - 0x41, 0x11, 0x6a, 0x21, 0x02, 0x20, 0x05, 0x41, 0x11, 0x6a, 0x21, 0x01, - 0x41, 0x0d, 0x21, 0x06, 0x20, 0x05, 0x41, 0x0d, 0x6a, 0x28, 0x00, 0x00, - 0x21, 0x05, 0x41, 0x0f, 0x21, 0x03, 0x0c, 0x02, 0x0b, 0x02, 0x40, 0x02, - 0x40, 0x20, 0x03, 0x41, 0x10, 0x4f, 0x0d, 0x00, 0x20, 0x04, 0x21, 0x02, - 0x20, 0x05, 0x21, 0x01, 0x0c, 0x01, 0x0b, 0x20, 0x04, 0x20, 0x05, 0x2d, - 0x00, 0x00, 0x3a, 0x00, 0x00, 0x20, 0x04, 0x20, 0x05, 0x28, 0x00, 0x01, - 0x36, 0x00, 0x01, 0x20, 0x04, 0x20, 0x05, 0x29, 0x00, 0x05, 0x37, 0x00, - 0x05, 0x20, 0x04, 0x20, 0x05, 0x2f, 0x00, 0x0d, 0x3b, 0x00, 0x0d, 0x20, - 0x04, 0x20, 0x05, 0x2d, 0x00, 0x0f, 0x3a, 0x00, 0x0f, 0x20, 0x04, 0x41, - 0x10, 0x6a, 0x21, 0x02, 0x20, 0x05, 0x41, 0x10, 0x6a, 0x21, 0x01, 0x0b, - 0x20, 0x03, 0x41, 0x08, 0x71, 0x0d, 0x02, 0x0c, 0x03, 0x0b, 0x20, 0x04, - 0x20, 0x05, 0x28, 0x02, 0x00, 0x22, 0x02, 0x3a, 0x00, 0x00, 0x20, 0x04, - 0x20, 0x02, 0x41, 0x10, 0x76, 0x3a, 0x00, 0x02, 0x20, 0x04, 0x20, 0x02, - 0x41, 0x08, 0x76, 0x3a, 0x00, 0x01, 0x20, 0x04, 0x20, 0x05, 0x41, 0x03, - 0x6a, 0x28, 0x00, 0x00, 0x36, 0x02, 0x03, 0x20, 0x04, 0x20, 0x05, 0x41, - 0x07, 0x6a, 0x29, 0x00, 0x00, 0x37, 0x02, 0x07, 0x20, 0x04, 0x41, 0x13, - 0x6a, 0x21, 0x02, 0x20, 0x05, 0x41, 0x13, 0x6a, 0x21, 0x01, 0x41, 0x0f, - 0x21, 0x06, 0x20, 0x05, 0x41, 0x0f, 0x6a, 0x28, 0x00, 0x00, 0x21, 0x05, - 0x41, 0x0d, 0x21, 0x03, 0x0b, 0x20, 0x04, 0x20, 0x06, 0x6a, 0x20, 0x05, - 0x36, 0x02, 0x00, 0x0b, 0x20, 0x02, 0x20, 0x01, 0x29, 0x00, 0x00, 0x37, - 0x00, 0x00, 0x20, 0x02, 0x41, 0x08, 0x6a, 0x21, 0x02, 0x20, 0x01, 0x41, - 0x08, 0x6a, 0x21, 0x01, 0x0b, 0x02, 0x40, 0x20, 0x03, 0x41, 0x04, 0x71, - 0x45, 0x0d, 0x00, 0x20, 0x02, 0x20, 0x01, 0x28, 0x00, 0x00, 0x36, 0x00, - 0x00, 0x20, 0x02, 0x41, 0x04, 0x6a, 0x21, 0x02, 0x20, 0x01, 0x41, 0x04, - 0x6a, 0x21, 0x01, 0x0b, 0x02, 0x40, 0x20, 0x03, 0x41, 0x02, 0x71, 0x45, - 0x0d, 0x00, 0x20, 0x02, 0x20, 0x01, 0x2f, 0x00, 0x00, 0x3b, 0x00, 0x00, - 0x20, 0x02, 0x41, 0x02, 0x6a, 0x21, 0x02, 0x20, 0x01, 0x41, 0x02, 0x6a, - 0x21, 0x01, 0x0b, 0x20, 0x03, 0x41, 0x01, 0x71, 0x45, 0x0d, 0x00, 0x20, - 0x02, 0x20, 0x01, 0x2d, 0x00, 0x00, 0x3a, 0x00, 0x00, 0x0b, 0x20, 0x00, - 0x0b, 0x88, 0x03, 0x02, 0x03, 0x7f, 0x01, 0x7e, 0x02, 0x40, 0x20, 0x02, - 0x41, 0x21, 0x49, 0x0d, 0x00, 0x20, 0x00, 0x20, 0x01, 0x20, 0x02, 0xfc, - 0x0b, 0x00, 0x20, 0x00, 0x0f, 0x0b, 0x02, 0x40, 0x20, 0x02, 0x45, 0x0d, - 0x00, 0x20, 0x00, 0x20, 0x01, 0x3a, 0x00, 0x00, 0x20, 0x02, 0x20, 0x00, - 0x6a, 0x22, 0x03, 0x41, 0x7f, 0x6a, 0x20, 0x01, 0x3a, 0x00, 0x00, 0x20, - 0x02, 0x41, 0x03, 0x49, 0x0d, 0x00, 0x20, 0x00, 0x20, 0x01, 0x3a, 0x00, - 0x02, 0x20, 0x00, 0x20, 0x01, 0x3a, 0x00, 0x01, 0x20, 0x03, 0x41, 0x7d, - 0x6a, 0x20, 0x01, 0x3a, 0x00, 0x00, 0x20, 0x03, 0x41, 0x7e, 0x6a, 0x20, - 0x01, 0x3a, 0x00, 0x00, 0x20, 0x02, 0x41, 0x07, 0x49, 0x0d, 0x00, 0x20, - 0x00, 0x20, 0x01, 0x3a, 0x00, 0x03, 0x20, 0x03, 0x41, 0x7c, 0x6a, 0x20, - 0x01, 0x3a, 0x00, 0x00, 0x20, 0x02, 0x41, 0x09, 0x49, 0x0d, 0x00, 0x20, - 0x00, 0x41, 0x00, 0x20, 0x00, 0x6b, 0x41, 0x03, 0x71, 0x22, 0x04, 0x6a, - 0x22, 0x05, 0x20, 0x01, 0x41, 0xff, 0x01, 0x71, 0x41, 0x81, 0x82, 0x84, - 0x08, 0x6c, 0x22, 0x03, 0x36, 0x02, 0x00, 0x20, 0x05, 0x20, 0x02, 0x20, - 0x04, 0x6b, 0x41, 0x7c, 0x71, 0x22, 0x01, 0x6a, 0x22, 0x02, 0x41, 0x7c, - 0x6a, 0x20, 0x03, 0x36, 0x02, 0x00, 0x20, 0x01, 0x41, 0x09, 0x49, 0x0d, - 0x00, 0x20, 0x05, 0x20, 0x03, 0x36, 0x02, 0x08, 0x20, 0x05, 0x20, 0x03, - 0x36, 0x02, 0x04, 0x20, 0x02, 0x41, 0x78, 0x6a, 0x20, 0x03, 0x36, 0x02, - 0x00, 0x20, 0x02, 0x41, 0x74, 0x6a, 0x20, 0x03, 0x36, 0x02, 0x00, 0x20, - 0x01, 0x41, 0x19, 0x49, 0x0d, 0x00, 0x20, 0x05, 0x20, 0x03, 0x36, 0x02, - 0x18, 0x20, 0x05, 0x20, 0x03, 0x36, 0x02, 0x14, 0x20, 0x05, 0x20, 0x03, - 0x36, 0x02, 0x10, 0x20, 0x05, 0x20, 0x03, 0x36, 0x02, 0x0c, 0x20, 0x02, - 0x41, 0x70, 0x6a, 0x20, 0x03, 0x36, 0x02, 0x00, 0x20, 0x02, 0x41, 0x6c, - 0x6a, 0x20, 0x03, 0x36, 0x02, 0x00, 0x20, 0x02, 0x41, 0x68, 0x6a, 0x20, - 0x03, 0x36, 0x02, 0x00, 0x20, 0x02, 0x41, 0x64, 0x6a, 0x20, 0x03, 0x36, - 0x02, 0x00, 0x20, 0x01, 0x20, 0x05, 0x41, 0x04, 0x71, 0x41, 0x18, 0x72, - 0x22, 0x02, 0x6b, 0x22, 0x01, 0x41, 0x20, 0x49, 0x0d, 0x00, 0x20, 0x03, - 0xad, 0x42, 0x81, 0x80, 0x80, 0x80, 0x10, 0x7e, 0x21, 0x06, 0x20, 0x05, - 0x20, 0x02, 0x6a, 0x21, 0x02, 0x03, 0x40, 0x20, 0x02, 0x20, 0x06, 0x37, - 0x03, 0x18, 0x20, 0x02, 0x20, 0x06, 0x37, 0x03, 0x10, 0x20, 0x02, 0x20, - 0x06, 0x37, 0x03, 0x08, 0x20, 0x02, 0x20, 0x06, 0x37, 0x03, 0x00, 0x20, - 0x02, 0x41, 0x20, 0x6a, 0x21, 0x02, 0x20, 0x01, 0x41, 0x60, 0x6a, 0x22, - 0x01, 0x41, 0x1f, 0x4b, 0x0d, 0x00, 0x0b, 0x0b, 0x20, 0x00, 0x0b, 0xcc, - 0x01, 0x01, 0x03, 0x7f, 0x20, 0x00, 0x21, 0x01, 0x02, 0x40, 0x02, 0x40, - 0x20, 0x00, 0x41, 0x03, 0x71, 0x45, 0x0d, 0x00, 0x02, 0x40, 0x20, 0x00, - 0x2d, 0x00, 0x00, 0x0d, 0x00, 0x20, 0x00, 0x20, 0x00, 0x6b, 0x0f, 0x0b, - 0x20, 0x00, 0x41, 0x01, 0x6a, 0x22, 0x01, 0x41, 0x03, 0x71, 0x45, 0x0d, - 0x00, 0x20, 0x01, 0x2d, 0x00, 0x00, 0x45, 0x0d, 0x01, 0x20, 0x00, 0x41, - 0x02, 0x6a, 0x22, 0x01, 0x41, 0x03, 0x71, 0x45, 0x0d, 0x00, 0x20, 0x01, - 0x2d, 0x00, 0x00, 0x45, 0x0d, 0x01, 0x20, 0x00, 0x41, 0x03, 0x6a, 0x22, - 0x01, 0x41, 0x03, 0x71, 0x45, 0x0d, 0x00, 0x20, 0x01, 0x2d, 0x00, 0x00, - 0x45, 0x0d, 0x01, 0x20, 0x00, 0x41, 0x04, 0x6a, 0x22, 0x01, 0x41, 0x03, - 0x71, 0x0d, 0x01, 0x0b, 0x20, 0x01, 0x41, 0x7c, 0x6a, 0x21, 0x02, 0x20, - 0x01, 0x41, 0x7b, 0x6a, 0x21, 0x01, 0x03, 0x40, 0x20, 0x01, 0x41, 0x04, - 0x6a, 0x21, 0x01, 0x20, 0x02, 0x41, 0x04, 0x6a, 0x22, 0x02, 0x28, 0x02, - 0x00, 0x22, 0x03, 0x41, 0x7f, 0x73, 0x20, 0x03, 0x41, 0xff, 0xfd, 0xfb, - 0x77, 0x6a, 0x71, 0x41, 0x80, 0x81, 0x82, 0x84, 0x78, 0x71, 0x45, 0x0d, - 0x00, 0x0b, 0x03, 0x40, 0x20, 0x01, 0x41, 0x01, 0x6a, 0x21, 0x01, 0x20, - 0x02, 0x2d, 0x00, 0x00, 0x21, 0x03, 0x20, 0x02, 0x41, 0x01, 0x6a, 0x21, - 0x02, 0x20, 0x03, 0x0d, 0x00, 0x0b, 0x0b, 0x20, 0x01, 0x20, 0x00, 0x6b, - 0x0b, 0x44, 0x00, 0x02, 0x40, 0x20, 0x00, 0x41, 0xff, 0xff, 0x07, 0x4b, - 0x0d, 0x00, 0x20, 0x00, 0x41, 0x08, 0x76, 0x41, 0x80, 0x80, 0x84, 0x80, - 0x00, 0x6a, 0x2d, 0x00, 0x00, 0x41, 0x05, 0x74, 0x20, 0x00, 0x41, 0x03, - 0x76, 0x41, 0x1f, 0x71, 0x72, 0x41, 0x80, 0x80, 0x84, 0x80, 0x00, 0x6a, - 0x2d, 0x00, 0x00, 0x20, 0x00, 0x41, 0x07, 0x71, 0x76, 0x41, 0x01, 0x71, - 0x0f, 0x0b, 0x20, 0x00, 0x41, 0xfe, 0xff, 0x0b, 0x49, 0x0b, 0x49, 0x01, - 0x03, 0x7f, 0x41, 0x00, 0x21, 0x03, 0x02, 0x40, 0x20, 0x02, 0x45, 0x0d, - 0x00, 0x02, 0x40, 0x03, 0x40, 0x20, 0x00, 0x2d, 0x00, 0x00, 0x22, 0x04, - 0x20, 0x01, 0x2d, 0x00, 0x00, 0x22, 0x05, 0x47, 0x0d, 0x01, 0x20, 0x01, - 0x41, 0x01, 0x6a, 0x21, 0x01, 0x20, 0x00, 0x41, 0x01, 0x6a, 0x21, 0x00, - 0x20, 0x02, 0x41, 0x7f, 0x6a, 0x22, 0x02, 0x0d, 0x00, 0x0c, 0x02, 0x0b, - 0x0b, 0x20, 0x04, 0x20, 0x05, 0x6b, 0x21, 0x03, 0x0b, 0x20, 0x03, 0x0b, - 0xf2, 0x02, 0x01, 0x03, 0x7f, 0x20, 0x02, 0x41, 0x00, 0x47, 0x21, 0x03, - 0x02, 0x40, 0x02, 0x40, 0x02, 0x40, 0x02, 0x40, 0x20, 0x00, 0x41, 0x03, - 0x71, 0x45, 0x0d, 0x00, 0x20, 0x02, 0x45, 0x0d, 0x00, 0x02, 0x40, 0x20, - 0x00, 0x2d, 0x00, 0x00, 0x20, 0x01, 0x41, 0xff, 0x01, 0x71, 0x47, 0x0d, - 0x00, 0x20, 0x00, 0x21, 0x04, 0x20, 0x02, 0x21, 0x05, 0x0c, 0x03, 0x0b, - 0x20, 0x02, 0x41, 0x7f, 0x6a, 0x22, 0x05, 0x41, 0x00, 0x47, 0x21, 0x03, - 0x20, 0x00, 0x41, 0x01, 0x6a, 0x22, 0x04, 0x41, 0x03, 0x71, 0x45, 0x0d, - 0x01, 0x20, 0x05, 0x45, 0x0d, 0x01, 0x20, 0x04, 0x2d, 0x00, 0x00, 0x20, - 0x01, 0x41, 0xff, 0x01, 0x71, 0x46, 0x0d, 0x02, 0x20, 0x02, 0x41, 0x7e, - 0x6a, 0x22, 0x05, 0x41, 0x00, 0x47, 0x21, 0x03, 0x20, 0x00, 0x41, 0x02, - 0x6a, 0x22, 0x04, 0x41, 0x03, 0x71, 0x45, 0x0d, 0x01, 0x20, 0x05, 0x45, - 0x0d, 0x01, 0x20, 0x04, 0x2d, 0x00, 0x00, 0x20, 0x01, 0x41, 0xff, 0x01, - 0x71, 0x46, 0x0d, 0x02, 0x20, 0x02, 0x41, 0x7d, 0x6a, 0x22, 0x05, 0x41, - 0x00, 0x47, 0x21, 0x03, 0x20, 0x00, 0x41, 0x03, 0x6a, 0x22, 0x04, 0x41, - 0x03, 0x71, 0x45, 0x0d, 0x01, 0x20, 0x05, 0x45, 0x0d, 0x01, 0x20, 0x04, - 0x2d, 0x00, 0x00, 0x20, 0x01, 0x41, 0xff, 0x01, 0x71, 0x46, 0x0d, 0x02, - 0x20, 0x00, 0x41, 0x04, 0x6a, 0x21, 0x04, 0x20, 0x02, 0x41, 0x7c, 0x6a, - 0x22, 0x05, 0x41, 0x00, 0x47, 0x21, 0x03, 0x0c, 0x01, 0x0b, 0x20, 0x02, - 0x21, 0x05, 0x20, 0x00, 0x21, 0x04, 0x0b, 0x20, 0x03, 0x45, 0x0d, 0x01, - 0x02, 0x40, 0x20, 0x04, 0x2d, 0x00, 0x00, 0x20, 0x01, 0x41, 0xff, 0x01, - 0x71, 0x46, 0x0d, 0x00, 0x20, 0x05, 0x41, 0x04, 0x49, 0x0d, 0x00, 0x20, - 0x01, 0x41, 0xff, 0x01, 0x71, 0x41, 0x81, 0x82, 0x84, 0x08, 0x6c, 0x21, - 0x00, 0x03, 0x40, 0x20, 0x04, 0x28, 0x02, 0x00, 0x20, 0x00, 0x73, 0x22, - 0x02, 0x41, 0x7f, 0x73, 0x20, 0x02, 0x41, 0xff, 0xfd, 0xfb, 0x77, 0x6a, - 0x71, 0x41, 0x80, 0x81, 0x82, 0x84, 0x78, 0x71, 0x0d, 0x02, 0x20, 0x04, - 0x41, 0x04, 0x6a, 0x21, 0x04, 0x20, 0x05, 0x41, 0x7c, 0x6a, 0x22, 0x05, - 0x41, 0x03, 0x4b, 0x0d, 0x00, 0x0b, 0x0b, 0x20, 0x05, 0x45, 0x0d, 0x01, - 0x0b, 0x20, 0x01, 0x41, 0xff, 0x01, 0x71, 0x21, 0x02, 0x03, 0x40, 0x02, - 0x40, 0x20, 0x04, 0x2d, 0x00, 0x00, 0x20, 0x02, 0x47, 0x0d, 0x00, 0x20, - 0x04, 0x0f, 0x0b, 0x20, 0x04, 0x41, 0x01, 0x6a, 0x21, 0x04, 0x20, 0x05, - 0x41, 0x7f, 0x6a, 0x22, 0x05, 0x0d, 0x00, 0x0b, 0x0b, 0x41, 0x00, 0x0b, - 0x67, 0x01, 0x02, 0x7f, 0x20, 0x01, 0x2d, 0x00, 0x00, 0x21, 0x02, 0x02, - 0x40, 0x20, 0x00, 0x2d, 0x00, 0x00, 0x22, 0x03, 0x45, 0x0d, 0x00, 0x20, - 0x03, 0x20, 0x02, 0x41, 0xff, 0x01, 0x71, 0x47, 0x0d, 0x00, 0x20, 0x00, - 0x41, 0x01, 0x6a, 0x21, 0x00, 0x20, 0x01, 0x41, 0x01, 0x6a, 0x21, 0x01, - 0x03, 0x40, 0x20, 0x01, 0x2d, 0x00, 0x00, 0x21, 0x02, 0x20, 0x00, 0x2d, - 0x00, 0x00, 0x22, 0x03, 0x45, 0x0d, 0x01, 0x20, 0x00, 0x41, 0x01, 0x6a, - 0x21, 0x00, 0x20, 0x01, 0x41, 0x01, 0x6a, 0x21, 0x01, 0x20, 0x03, 0x20, - 0x02, 0x41, 0xff, 0x01, 0x71, 0x46, 0x0d, 0x00, 0x0b, 0x0b, 0x20, 0x03, - 0x20, 0x02, 0x41, 0xff, 0x01, 0x71, 0x6b, 0x0b, 0x0c, 0x00, 0x20, 0x00, - 0x41, 0x00, 0x10, 0x9b, 0x80, 0x80, 0x80, 0x00, 0x0b, 0xbc, 0x02, 0x01, - 0x06, 0x7f, 0x02, 0x40, 0x20, 0x00, 0x41, 0xff, 0xff, 0x07, 0x4b, 0x0d, - 0x00, 0x20, 0x00, 0x20, 0x00, 0x41, 0xff, 0x01, 0x71, 0x22, 0x02, 0x41, - 0x03, 0x6e, 0x22, 0x03, 0x41, 0x03, 0x6c, 0x6b, 0x41, 0xff, 0x01, 0x71, - 0x41, 0x02, 0x74, 0x41, 0xc0, 0x9e, 0x84, 0x80, 0x00, 0x6a, 0x28, 0x02, - 0x00, 0x20, 0x00, 0x41, 0x08, 0x76, 0x22, 0x04, 0x41, 0xa0, 0xa9, 0x84, - 0x80, 0x00, 0x6a, 0x2d, 0x00, 0x00, 0x41, 0xd6, 0x00, 0x6c, 0x20, 0x03, - 0x6a, 0x41, 0xa0, 0xa9, 0x84, 0x80, 0x00, 0x6a, 0x2d, 0x00, 0x00, 0x6c, - 0x41, 0x0b, 0x76, 0x41, 0x06, 0x70, 0x20, 0x04, 0x41, 0x90, 0xbe, 0x84, - 0x80, 0x00, 0x6a, 0x2d, 0x00, 0x00, 0x6a, 0x41, 0x02, 0x74, 0x41, 0xd0, - 0x9e, 0x84, 0x80, 0x00, 0x6a, 0x28, 0x02, 0x00, 0x22, 0x03, 0x41, 0x08, - 0x75, 0x21, 0x04, 0x02, 0x40, 0x20, 0x03, 0x41, 0xff, 0x01, 0x71, 0x22, - 0x03, 0x41, 0x01, 0x4b, 0x0d, 0x00, 0x20, 0x04, 0x41, 0x00, 0x20, 0x03, - 0x20, 0x01, 0x73, 0x6b, 0x71, 0x20, 0x00, 0x6a, 0x0f, 0x0b, 0x20, 0x04, - 0x41, 0xff, 0x01, 0x71, 0x22, 0x03, 0x45, 0x0d, 0x00, 0x20, 0x04, 0x41, - 0x08, 0x76, 0x21, 0x04, 0x03, 0x40, 0x02, 0x40, 0x20, 0x02, 0x20, 0x03, - 0x41, 0x01, 0x76, 0x22, 0x05, 0x20, 0x04, 0x6a, 0x22, 0x06, 0x41, 0x01, - 0x74, 0x41, 0x90, 0xa6, 0x84, 0x80, 0x00, 0x6a, 0x2d, 0x00, 0x00, 0x22, - 0x07, 0x47, 0x0d, 0x00, 0x02, 0x40, 0x20, 0x06, 0x41, 0x01, 0x74, 0x41, - 0x91, 0xa6, 0x84, 0x80, 0x00, 0x6a, 0x2d, 0x00, 0x00, 0x41, 0x02, 0x74, - 0x41, 0xd0, 0x9e, 0x84, 0x80, 0x00, 0x6a, 0x28, 0x02, 0x00, 0x22, 0x03, - 0x41, 0xff, 0x01, 0x71, 0x22, 0x04, 0x41, 0x01, 0x4b, 0x0d, 0x00, 0x20, - 0x03, 0x41, 0x08, 0x75, 0x41, 0x00, 0x20, 0x04, 0x20, 0x01, 0x73, 0x6b, - 0x71, 0x20, 0x00, 0x6a, 0x0f, 0x0b, 0x41, 0x7f, 0x41, 0x01, 0x20, 0x01, - 0x1b, 0x20, 0x00, 0x6a, 0x0f, 0x0b, 0x20, 0x04, 0x20, 0x06, 0x20, 0x02, - 0x20, 0x07, 0x49, 0x22, 0x07, 0x1b, 0x21, 0x04, 0x20, 0x05, 0x20, 0x03, - 0x20, 0x05, 0x6b, 0x20, 0x07, 0x1b, 0x22, 0x03, 0x0d, 0x00, 0x0b, 0x0b, - 0x20, 0x00, 0x0b, 0x0c, 0x00, 0x20, 0x00, 0x41, 0x01, 0x10, 0x9b, 0x80, - 0x80, 0x80, 0x00, 0x0b, 0x7b, 0x01, 0x02, 0x7f, 0x02, 0x40, 0x20, 0x02, - 0x0d, 0x00, 0x41, 0x00, 0x0f, 0x0b, 0x02, 0x40, 0x02, 0x40, 0x20, 0x00, - 0x2d, 0x00, 0x00, 0x22, 0x03, 0x45, 0x0d, 0x00, 0x20, 0x00, 0x41, 0x01, - 0x6a, 0x21, 0x00, 0x20, 0x02, 0x41, 0x7f, 0x6a, 0x21, 0x02, 0x03, 0x40, - 0x20, 0x03, 0x41, 0xff, 0x01, 0x71, 0x20, 0x01, 0x2d, 0x00, 0x00, 0x22, - 0x04, 0x47, 0x0d, 0x02, 0x20, 0x04, 0x45, 0x0d, 0x02, 0x20, 0x02, 0x41, - 0x00, 0x46, 0x0d, 0x02, 0x20, 0x02, 0x41, 0x7f, 0x6a, 0x21, 0x02, 0x20, - 0x01, 0x41, 0x01, 0x6a, 0x21, 0x01, 0x20, 0x00, 0x2d, 0x00, 0x00, 0x21, - 0x03, 0x20, 0x00, 0x41, 0x01, 0x6a, 0x21, 0x00, 0x20, 0x03, 0x0d, 0x00, - 0x0b, 0x0b, 0x41, 0x00, 0x21, 0x03, 0x0b, 0x20, 0x03, 0x41, 0xff, 0x01, - 0x71, 0x20, 0x01, 0x2d, 0x00, 0x00, 0x6b, 0x0b, 0x0d, 0x00, 0x20, 0x00, - 0x10, 0x9a, 0x80, 0x80, 0x80, 0x00, 0x20, 0x00, 0x47, 0x0b, 0xbf, 0x09, - 0x01, 0x04, 0x7f, 0x02, 0x40, 0x02, 0x40, 0x02, 0x40, 0x20, 0x02, 0x41, - 0x21, 0x4f, 0x0d, 0x00, 0x20, 0x00, 0x20, 0x01, 0x46, 0x0d, 0x02, 0x20, - 0x01, 0x20, 0x00, 0x20, 0x02, 0x6a, 0x22, 0x03, 0x6b, 0x41, 0x00, 0x20, - 0x02, 0x41, 0x01, 0x74, 0x6b, 0x4b, 0x0d, 0x01, 0x0b, 0x20, 0x00, 0x20, - 0x01, 0x20, 0x02, 0xfc, 0x0a, 0x00, 0x00, 0x0c, 0x01, 0x0b, 0x20, 0x01, - 0x20, 0x00, 0x73, 0x41, 0x03, 0x71, 0x21, 0x04, 0x02, 0x40, 0x02, 0x40, - 0x02, 0x40, 0x20, 0x00, 0x20, 0x01, 0x4f, 0x0d, 0x00, 0x02, 0x40, 0x20, - 0x04, 0x45, 0x0d, 0x00, 0x20, 0x02, 0x21, 0x05, 0x20, 0x00, 0x21, 0x03, - 0x0c, 0x03, 0x0b, 0x02, 0x40, 0x20, 0x00, 0x41, 0x03, 0x71, 0x0d, 0x00, - 0x20, 0x02, 0x21, 0x05, 0x20, 0x00, 0x21, 0x03, 0x0c, 0x02, 0x0b, 0x20, - 0x02, 0x45, 0x0d, 0x03, 0x20, 0x00, 0x20, 0x01, 0x2d, 0x00, 0x00, 0x3a, - 0x00, 0x00, 0x20, 0x02, 0x41, 0x7f, 0x6a, 0x21, 0x05, 0x02, 0x40, 0x20, - 0x00, 0x41, 0x01, 0x6a, 0x22, 0x03, 0x41, 0x03, 0x71, 0x0d, 0x00, 0x20, - 0x01, 0x41, 0x01, 0x6a, 0x21, 0x01, 0x0c, 0x02, 0x0b, 0x20, 0x05, 0x45, - 0x0d, 0x03, 0x20, 0x00, 0x20, 0x01, 0x2d, 0x00, 0x01, 0x3a, 0x00, 0x01, - 0x20, 0x02, 0x41, 0x7e, 0x6a, 0x21, 0x05, 0x02, 0x40, 0x20, 0x00, 0x41, - 0x02, 0x6a, 0x22, 0x03, 0x41, 0x03, 0x71, 0x0d, 0x00, 0x20, 0x01, 0x41, - 0x02, 0x6a, 0x21, 0x01, 0x0c, 0x02, 0x0b, 0x20, 0x05, 0x45, 0x0d, 0x03, - 0x20, 0x00, 0x20, 0x01, 0x2d, 0x00, 0x02, 0x3a, 0x00, 0x02, 0x20, 0x02, - 0x41, 0x7d, 0x6a, 0x21, 0x05, 0x02, 0x40, 0x20, 0x00, 0x41, 0x03, 0x6a, - 0x22, 0x03, 0x41, 0x03, 0x71, 0x0d, 0x00, 0x20, 0x01, 0x41, 0x03, 0x6a, - 0x21, 0x01, 0x0c, 0x02, 0x0b, 0x20, 0x05, 0x45, 0x0d, 0x03, 0x20, 0x00, - 0x20, 0x01, 0x2d, 0x00, 0x03, 0x3a, 0x00, 0x03, 0x20, 0x00, 0x41, 0x04, - 0x6a, 0x21, 0x03, 0x20, 0x01, 0x41, 0x04, 0x6a, 0x21, 0x01, 0x20, 0x02, - 0x41, 0x7c, 0x6a, 0x21, 0x05, 0x0c, 0x01, 0x0b, 0x02, 0x40, 0x20, 0x04, - 0x0d, 0x00, 0x02, 0x40, 0x20, 0x03, 0x41, 0x03, 0x71, 0x45, 0x0d, 0x00, - 0x20, 0x02, 0x45, 0x0d, 0x04, 0x20, 0x00, 0x20, 0x02, 0x41, 0x7f, 0x6a, - 0x22, 0x03, 0x6a, 0x22, 0x04, 0x20, 0x01, 0x20, 0x03, 0x6a, 0x2d, 0x00, - 0x00, 0x3a, 0x00, 0x00, 0x02, 0x40, 0x20, 0x04, 0x41, 0x03, 0x71, 0x0d, - 0x00, 0x20, 0x03, 0x21, 0x02, 0x0c, 0x01, 0x0b, 0x20, 0x03, 0x45, 0x0d, - 0x04, 0x20, 0x00, 0x20, 0x02, 0x41, 0x7e, 0x6a, 0x22, 0x03, 0x6a, 0x22, - 0x04, 0x20, 0x01, 0x20, 0x03, 0x6a, 0x2d, 0x00, 0x00, 0x3a, 0x00, 0x00, - 0x02, 0x40, 0x20, 0x04, 0x41, 0x03, 0x71, 0x0d, 0x00, 0x20, 0x03, 0x21, + 0x02, 0x7c, 0x04, 0x16, 0x77, 0x61, 0x73, 0x69, 0x5f, 0x73, 0x6e, 0x61, + 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x70, 0x72, 0x65, 0x76, 0x69, 0x65, + 0x77, 0x31, 0x08, 0x61, 0x72, 0x67, 0x73, 0x5f, 0x67, 0x65, 0x74, 0x00, + 0x02, 0x16, 0x77, 0x61, 0x73, 0x69, 0x5f, 0x73, 0x6e, 0x61, 0x70, 0x73, + 0x68, 0x6f, 0x74, 0x5f, 0x70, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x31, + 0x0e, 0x61, 0x72, 0x67, 0x73, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x73, 0x5f, + 0x67, 0x65, 0x74, 0x00, 0x02, 0x16, 0x77, 0x61, 0x73, 0x69, 0x5f, 0x73, + 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x70, 0x72, 0x65, 0x76, + 0x69, 0x65, 0x77, 0x31, 0x09, 0x70, 0x72, 0x6f, 0x63, 0x5f, 0x65, 0x78, + 0x69, 0x74, 0x00, 0x03, 0x03, 0x65, 0x6e, 0x76, 0x06, 0x6d, 0x65, 0x6d, + 0x6f, 0x72, 0x79, 0x02, 0x00, 0x02, 0x03, 0x1f, 0x1e, 0x04, 0x04, 0x04, + 0x03, 0x00, 0x03, 0x02, 0x02, 0x03, 0x00, 0x00, 0x01, 0x01, 0x02, 0x00, + 0x02, 0x00, 0x01, 0x00, 0x01, 0x01, 0x00, 0x01, 0x00, 0x00, 0x01, 0x01, + 0x00, 0x00, 0x00, 0x06, 0x08, 0x01, 0x7f, 0x01, 0x41, 0x80, 0x80, 0x04, + 0x0b, 0x07, 0xad, 0x02, 0x1c, 0x11, 0x5f, 0x5f, 0x77, 0x61, 0x73, 0x6d, + 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x5f, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x00, + 0x03, 0x0f, 0x5f, 0x5f, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x5f, 0x70, 0x6f, + 0x69, 0x6e, 0x74, 0x65, 0x72, 0x03, 0x00, 0x06, 0x5f, 0x73, 0x74, 0x61, + 0x72, 0x74, 0x00, 0x05, 0x0a, 0x72, 0x65, 0x73, 0x65, 0x74, 0x5f, 0x68, + 0x65, 0x61, 0x70, 0x00, 0x06, 0x06, 0x6d, 0x61, 0x6c, 0x6c, 0x6f, 0x63, + 0x00, 0x07, 0x04, 0x66, 0x72, 0x65, 0x65, 0x00, 0x08, 0x06, 0x63, 0x61, + 0x6c, 0x6c, 0x6f, 0x63, 0x00, 0x09, 0x07, 0x72, 0x65, 0x61, 0x6c, 0x6c, + 0x6f, 0x63, 0x00, 0x0a, 0x06, 0x73, 0x74, 0x72, 0x6c, 0x65, 0x6e, 0x00, + 0x0c, 0x08, 0x69, 0x73, 0x77, 0x61, 0x6c, 0x6e, 0x75, 0x6d, 0x00, 0x20, + 0x08, 0x69, 0x73, 0x77, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x00, 0x0d, 0x08, + 0x69, 0x73, 0x77, 0x62, 0x6c, 0x61, 0x6e, 0x6b, 0x00, 0x1a, 0x08, 0x69, + 0x73, 0x77, 0x64, 0x69, 0x67, 0x69, 0x74, 0x00, 0x1b, 0x08, 0x69, 0x73, + 0x77, 0x6c, 0x6f, 0x77, 0x65, 0x72, 0x00, 0x18, 0x08, 0x69, 0x73, 0x77, + 0x73, 0x70, 0x61, 0x63, 0x65, 0x00, 0x1f, 0x08, 0x69, 0x73, 0x77, 0x75, + 0x70, 0x70, 0x65, 0x72, 0x00, 0x15, 0x09, 0x69, 0x73, 0x77, 0x78, 0x64, + 0x69, 0x67, 0x69, 0x74, 0x00, 0x1e, 0x08, 0x74, 0x6f, 0x77, 0x6c, 0x6f, + 0x77, 0x65, 0x72, 0x00, 0x11, 0x08, 0x74, 0x6f, 0x77, 0x75, 0x70, 0x70, + 0x65, 0x72, 0x00, 0x13, 0x06, 0x6d, 0x65, 0x6d, 0x63, 0x68, 0x72, 0x00, + 0x0f, 0x06, 0x6d, 0x65, 0x6d, 0x63, 0x6d, 0x70, 0x00, 0x0e, 0x06, 0x6d, + 0x65, 0x6d, 0x63, 0x70, 0x79, 0x00, 0x19, 0x07, 0x6d, 0x65, 0x6d, 0x6d, + 0x6f, 0x76, 0x65, 0x00, 0x16, 0x06, 0x6d, 0x65, 0x6d, 0x73, 0x65, 0x74, + 0x00, 0x17, 0x06, 0x73, 0x74, 0x72, 0x63, 0x6d, 0x70, 0x00, 0x10, 0x07, + 0x73, 0x74, 0x72, 0x6e, 0x63, 0x61, 0x74, 0x00, 0x1c, 0x07, 0x73, 0x74, + 0x72, 0x6e, 0x63, 0x6d, 0x70, 0x00, 0x14, 0x07, 0x73, 0x74, 0x72, 0x6e, + 0x63, 0x70, 0x79, 0x00, 0x1d, 0x08, 0x01, 0x04, 0x0c, 0x01, 0x02, 0x0a, + 0x8d, 0x2a, 0x1e, 0x02, 0x00, 0x0b, 0x0e, 0x00, 0x41, 0xec, 0xc2, 0x04, + 0x41, 0x00, 0x41, 0x84, 0x01, 0xfc, 0x0b, 0x00, 0x0b, 0xfd, 0x01, 0x01, + 0x03, 0x7f, 0x41, 0xec, 0xc2, 0x04, 0x28, 0x02, 0x00, 0x45, 0x04, 0x40, + 0x41, 0xec, 0xc2, 0x04, 0x41, 0x01, 0x36, 0x02, 0x00, 0x41, 0x84, 0xc3, + 0x04, 0x41, 0x84, 0xc3, 0x04, 0x36, 0x02, 0x00, 0x41, 0xbc, 0xc3, 0x04, + 0x41, 0x00, 0x36, 0x02, 0x00, 0x41, 0xb8, 0xc3, 0x04, 0x41, 0x80, 0x80, + 0x04, 0x36, 0x02, 0x00, 0x41, 0xb4, 0xc3, 0x04, 0x41, 0x80, 0x80, 0x04, + 0x36, 0x02, 0x00, 0x41, 0x8c, 0xc3, 0x04, 0x41, 0x84, 0xc3, 0x04, 0x36, + 0x02, 0x00, 0x41, 0x88, 0xc3, 0x04, 0x41, 0x84, 0xc3, 0x04, 0x36, 0x02, + 0x00, 0x41, 0x90, 0xc3, 0x04, 0x41, 0x80, 0xc3, 0x04, 0x28, 0x02, 0x00, + 0x36, 0x02, 0x00, 0x41, 0xe8, 0xc2, 0x04, 0x41, 0x80, 0x80, 0x04, 0x36, + 0x02, 0x00, 0x23, 0x00, 0x41, 0x10, 0x6b, 0x22, 0x00, 0x24, 0x00, 0x02, + 0x40, 0x02, 0x40, 0x02, 0x40, 0x02, 0x40, 0x20, 0x00, 0x41, 0x08, 0x6a, + 0x20, 0x00, 0x41, 0x0c, 0x6a, 0x10, 0x01, 0x41, 0xff, 0xff, 0x03, 0x71, + 0x45, 0x04, 0x40, 0x20, 0x00, 0x28, 0x02, 0x08, 0x41, 0x01, 0x6a, 0x22, + 0x01, 0x45, 0x0d, 0x01, 0x20, 0x00, 0x28, 0x02, 0x0c, 0x10, 0x07, 0x22, + 0x02, 0x45, 0x0d, 0x02, 0x20, 0x01, 0x41, 0x04, 0x10, 0x09, 0x22, 0x01, + 0x45, 0x0d, 0x03, 0x20, 0x01, 0x20, 0x02, 0x10, 0x00, 0x41, 0xff, 0xff, + 0x03, 0x71, 0x0d, 0x04, 0x20, 0x00, 0x28, 0x02, 0x08, 0x00, 0x0b, 0x41, + 0xc7, 0x00, 0x10, 0x0b, 0x00, 0x0b, 0x41, 0xc6, 0x00, 0x10, 0x0b, 0x00, + 0x0b, 0x41, 0xc6, 0x00, 0x10, 0x0b, 0x00, 0x0b, 0x20, 0x02, 0x10, 0x08, + 0x41, 0xc6, 0x00, 0x10, 0x0b, 0x00, 0x0b, 0x20, 0x02, 0x10, 0x08, 0x20, + 0x01, 0x10, 0x08, 0x41, 0xc7, 0x00, 0x10, 0x0b, 0x00, 0x0b, 0x00, 0x0b, + 0x35, 0x01, 0x01, 0x7f, 0x41, 0xf4, 0xc2, 0x04, 0x20, 0x00, 0x36, 0x02, + 0x00, 0x41, 0xf0, 0xc2, 0x04, 0x20, 0x00, 0x36, 0x02, 0x00, 0x3f, 0x00, + 0x21, 0x00, 0x20, 0x01, 0x41, 0xfc, 0xc2, 0x04, 0x6a, 0x41, 0x00, 0x36, + 0x02, 0x00, 0x20, 0x01, 0x41, 0xf8, 0xc2, 0x04, 0x6a, 0x20, 0x00, 0x41, + 0x10, 0x74, 0x36, 0x02, 0x00, 0x0b, 0xde, 0x01, 0x01, 0x04, 0x7f, 0x02, + 0x40, 0x20, 0x00, 0x45, 0x0d, 0x00, 0x02, 0x40, 0x41, 0xfc, 0xc2, 0x04, + 0x28, 0x02, 0x00, 0x22, 0x01, 0x45, 0x0d, 0x00, 0x02, 0x40, 0x20, 0x00, + 0x20, 0x01, 0x28, 0x02, 0x00, 0x4d, 0x04, 0x40, 0x20, 0x01, 0x21, 0x02, + 0x0c, 0x01, 0x0b, 0x03, 0x40, 0x20, 0x01, 0x28, 0x02, 0x04, 0x22, 0x02, + 0x45, 0x0d, 0x02, 0x20, 0x01, 0x21, 0x03, 0x20, 0x02, 0x22, 0x01, 0x28, + 0x02, 0x00, 0x20, 0x00, 0x49, 0x0d, 0x00, 0x0b, 0x0b, 0x20, 0x02, 0x28, + 0x02, 0x04, 0x21, 0x00, 0x02, 0x40, 0x20, 0x03, 0x45, 0x04, 0x40, 0x41, + 0xfc, 0xc2, 0x04, 0x20, 0x00, 0x36, 0x02, 0x00, 0x0c, 0x01, 0x0b, 0x20, + 0x03, 0x20, 0x00, 0x36, 0x02, 0x04, 0x0b, 0x20, 0x02, 0x41, 0x08, 0x6a, + 0x0f, 0x0b, 0x41, 0xf4, 0xc2, 0x04, 0x28, 0x02, 0x00, 0x22, 0x03, 0x41, + 0x08, 0x6a, 0x22, 0x01, 0x20, 0x00, 0x6a, 0x41, 0x03, 0x6a, 0x41, 0x7c, + 0x71, 0x22, 0x02, 0x41, 0xf8, 0xc2, 0x04, 0x28, 0x02, 0x00, 0x4b, 0x04, + 0x40, 0x20, 0x02, 0x41, 0xf0, 0xc2, 0x04, 0x28, 0x02, 0x00, 0x6b, 0x41, + 0x80, 0x80, 0x80, 0x02, 0x4a, 0x0d, 0x01, 0x20, 0x00, 0x41, 0x01, 0x6b, + 0x41, 0x10, 0x76, 0x41, 0x01, 0x6a, 0x40, 0x00, 0x41, 0x7f, 0x46, 0x0d, + 0x01, 0x41, 0xf8, 0xc2, 0x04, 0x3f, 0x00, 0x41, 0x10, 0x74, 0x36, 0x02, + 0x00, 0x0b, 0x20, 0x03, 0x20, 0x00, 0x36, 0x02, 0x00, 0x41, 0xf4, 0xc2, + 0x04, 0x20, 0x02, 0x36, 0x02, 0x00, 0x20, 0x01, 0x21, 0x04, 0x0b, 0x20, + 0x04, 0x0b, 0x41, 0x01, 0x02, 0x7f, 0x20, 0x00, 0x04, 0x40, 0x41, 0xf4, + 0xc2, 0x04, 0x22, 0x01, 0x28, 0x02, 0x00, 0x20, 0x00, 0x41, 0x08, 0x6b, + 0x22, 0x02, 0x28, 0x02, 0x00, 0x20, 0x00, 0x6a, 0x41, 0x03, 0x6a, 0x41, + 0x7c, 0x71, 0x47, 0x04, 0x40, 0x20, 0x00, 0x41, 0x04, 0x6b, 0x41, 0xfc, + 0xc2, 0x04, 0x22, 0x01, 0x28, 0x02, 0x00, 0x36, 0x02, 0x00, 0x0b, 0x20, + 0x01, 0x20, 0x02, 0x36, 0x02, 0x00, 0x0b, 0x0b, 0x1d, 0x00, 0x20, 0x00, + 0x20, 0x01, 0x6c, 0x22, 0x00, 0x10, 0x07, 0x21, 0x01, 0x20, 0x00, 0x04, + 0x40, 0x20, 0x01, 0x41, 0x00, 0x20, 0x00, 0xfc, 0x0b, 0x00, 0x0b, 0x20, + 0x01, 0x0b, 0x56, 0x01, 0x01, 0x7f, 0x02, 0x40, 0x20, 0x00, 0x45, 0x0d, + 0x00, 0x41, 0xf4, 0xc2, 0x04, 0x28, 0x02, 0x00, 0x20, 0x00, 0x41, 0x08, + 0x6b, 0x22, 0x02, 0x28, 0x02, 0x00, 0x20, 0x00, 0x6a, 0x41, 0x03, 0x6a, + 0x41, 0x7c, 0x71, 0x46, 0x04, 0x40, 0x41, 0xf4, 0xc2, 0x04, 0x20, 0x02, + 0x36, 0x02, 0x00, 0x0c, 0x01, 0x0b, 0x20, 0x01, 0x10, 0x07, 0x21, 0x01, + 0x20, 0x02, 0x28, 0x02, 0x00, 0x22, 0x02, 0x04, 0x40, 0x20, 0x01, 0x20, + 0x00, 0x20, 0x02, 0xfc, 0x0a, 0x00, 0x00, 0x0b, 0x20, 0x01, 0x0f, 0x0b, + 0x20, 0x01, 0x10, 0x07, 0x0b, 0x07, 0x00, 0x20, 0x00, 0x10, 0x02, 0x00, + 0x0b, 0xc5, 0x01, 0x01, 0x03, 0x7f, 0x02, 0x40, 0x02, 0x40, 0x20, 0x00, + 0x22, 0x01, 0x41, 0x03, 0x71, 0x45, 0x0d, 0x00, 0x20, 0x01, 0x2d, 0x00, + 0x00, 0x45, 0x04, 0x40, 0x41, 0x00, 0x0f, 0x0b, 0x20, 0x00, 0x41, 0x01, + 0x6a, 0x22, 0x01, 0x41, 0x03, 0x71, 0x45, 0x0d, 0x00, 0x20, 0x01, 0x2d, + 0x00, 0x00, 0x45, 0x0d, 0x01, 0x20, 0x00, 0x41, 0x02, 0x6a, 0x22, 0x01, + 0x41, 0x03, 0x71, 0x45, 0x0d, 0x00, 0x20, 0x01, 0x2d, 0x00, 0x00, 0x45, + 0x0d, 0x01, 0x20, 0x00, 0x41, 0x03, 0x6a, 0x22, 0x01, 0x41, 0x03, 0x71, + 0x45, 0x0d, 0x00, 0x20, 0x01, 0x2d, 0x00, 0x00, 0x45, 0x0d, 0x01, 0x20, + 0x00, 0x41, 0x04, 0x6a, 0x22, 0x01, 0x41, 0x03, 0x71, 0x0d, 0x01, 0x0b, + 0x20, 0x01, 0x41, 0x04, 0x6b, 0x21, 0x02, 0x20, 0x01, 0x41, 0x05, 0x6b, + 0x21, 0x01, 0x03, 0x40, 0x20, 0x01, 0x41, 0x04, 0x6a, 0x21, 0x01, 0x41, + 0x80, 0x82, 0x84, 0x08, 0x20, 0x02, 0x41, 0x04, 0x6a, 0x22, 0x02, 0x28, + 0x02, 0x00, 0x22, 0x03, 0x6b, 0x20, 0x03, 0x72, 0x41, 0x80, 0x81, 0x82, + 0x84, 0x78, 0x71, 0x41, 0x80, 0x81, 0x82, 0x84, 0x78, 0x46, 0x0d, 0x00, + 0x0b, 0x03, 0x40, 0x20, 0x01, 0x41, 0x01, 0x6a, 0x21, 0x01, 0x20, 0x02, + 0x2d, 0x00, 0x00, 0x20, 0x02, 0x41, 0x01, 0x6a, 0x21, 0x02, 0x0d, 0x00, + 0x0b, 0x0b, 0x20, 0x01, 0x20, 0x00, 0x6b, 0x0b, 0x38, 0x00, 0x20, 0x00, + 0x41, 0xff, 0xff, 0x07, 0x4d, 0x04, 0x40, 0x20, 0x00, 0x41, 0x03, 0x76, + 0x41, 0x1f, 0x71, 0x20, 0x00, 0x41, 0x08, 0x76, 0x2d, 0x00, 0x80, 0x80, + 0x04, 0x41, 0x05, 0x74, 0x72, 0x2d, 0x00, 0x80, 0x80, 0x04, 0x20, 0x00, + 0x41, 0x07, 0x71, 0x76, 0x41, 0x01, 0x71, 0x0f, 0x0b, 0x20, 0x00, 0x41, + 0xfe, 0xff, 0x0b, 0x49, 0x0b, 0x43, 0x01, 0x03, 0x7f, 0x02, 0x40, 0x20, + 0x02, 0x45, 0x0d, 0x00, 0x03, 0x40, 0x20, 0x00, 0x2d, 0x00, 0x00, 0x22, + 0x04, 0x20, 0x01, 0x2d, 0x00, 0x00, 0x22, 0x05, 0x46, 0x04, 0x40, 0x20, + 0x01, 0x41, 0x01, 0x6a, 0x21, 0x01, 0x20, 0x00, 0x41, 0x01, 0x6a, 0x21, + 0x00, 0x20, 0x02, 0x41, 0x01, 0x6b, 0x22, 0x02, 0x0d, 0x01, 0x0c, 0x02, + 0x0b, 0x0b, 0x20, 0x04, 0x20, 0x05, 0x6b, 0x21, 0x03, 0x0b, 0x20, 0x03, + 0x0b, 0xe9, 0x02, 0x01, 0x03, 0x7f, 0x20, 0x02, 0x41, 0x00, 0x47, 0x21, + 0x05, 0x02, 0x40, 0x02, 0x40, 0x02, 0x40, 0x20, 0x00, 0x41, 0x03, 0x71, + 0x45, 0x20, 0x02, 0x45, 0x72, 0x45, 0x04, 0x40, 0x20, 0x00, 0x2d, 0x00, + 0x00, 0x20, 0x01, 0x41, 0xff, 0x01, 0x71, 0x46, 0x04, 0x40, 0x20, 0x00, + 0x21, 0x03, 0x20, 0x02, 0x21, 0x04, 0x0c, 0x03, 0x0b, 0x20, 0x02, 0x41, + 0x01, 0x6b, 0x22, 0x04, 0x41, 0x00, 0x47, 0x21, 0x05, 0x20, 0x00, 0x41, + 0x01, 0x6a, 0x22, 0x03, 0x41, 0x03, 0x71, 0x45, 0x20, 0x04, 0x45, 0x72, + 0x0d, 0x01, 0x20, 0x03, 0x2d, 0x00, 0x00, 0x20, 0x01, 0x41, 0xff, 0x01, + 0x71, 0x46, 0x0d, 0x02, 0x20, 0x02, 0x41, 0x02, 0x6b, 0x22, 0x04, 0x41, + 0x00, 0x47, 0x21, 0x05, 0x20, 0x00, 0x41, 0x02, 0x6a, 0x22, 0x03, 0x41, + 0x03, 0x71, 0x45, 0x20, 0x04, 0x45, 0x72, 0x0d, 0x01, 0x20, 0x03, 0x2d, + 0x00, 0x00, 0x20, 0x01, 0x41, 0xff, 0x01, 0x71, 0x46, 0x0d, 0x02, 0x20, + 0x02, 0x41, 0x03, 0x6b, 0x22, 0x04, 0x41, 0x00, 0x47, 0x21, 0x05, 0x20, + 0x00, 0x41, 0x03, 0x6a, 0x22, 0x03, 0x41, 0x03, 0x71, 0x45, 0x20, 0x04, + 0x45, 0x72, 0x0d, 0x01, 0x20, 0x03, 0x2d, 0x00, 0x00, 0x20, 0x01, 0x41, + 0xff, 0x01, 0x71, 0x46, 0x0d, 0x02, 0x20, 0x00, 0x41, 0x04, 0x6a, 0x21, + 0x03, 0x20, 0x02, 0x41, 0x04, 0x6b, 0x22, 0x04, 0x41, 0x00, 0x47, 0x21, + 0x05, 0x0c, 0x01, 0x0b, 0x20, 0x02, 0x21, 0x04, 0x20, 0x00, 0x21, 0x03, + 0x0b, 0x20, 0x05, 0x45, 0x0d, 0x01, 0x20, 0x01, 0x41, 0xff, 0x01, 0x71, + 0x22, 0x00, 0x20, 0x03, 0x2d, 0x00, 0x00, 0x46, 0x20, 0x04, 0x41, 0x04, + 0x49, 0x72, 0x45, 0x04, 0x40, 0x20, 0x00, 0x41, 0x81, 0x82, 0x84, 0x08, + 0x6c, 0x21, 0x00, 0x03, 0x40, 0x41, 0x80, 0x82, 0x84, 0x08, 0x20, 0x03, + 0x28, 0x02, 0x00, 0x20, 0x00, 0x73, 0x22, 0x02, 0x6b, 0x20, 0x02, 0x72, + 0x41, 0x80, 0x81, 0x82, 0x84, 0x78, 0x71, 0x41, 0x80, 0x81, 0x82, 0x84, + 0x78, 0x47, 0x0d, 0x02, 0x20, 0x03, 0x41, 0x04, 0x6a, 0x21, 0x03, 0x20, + 0x04, 0x41, 0x04, 0x6b, 0x22, 0x04, 0x41, 0x03, 0x4b, 0x0d, 0x00, 0x0b, + 0x0b, 0x20, 0x04, 0x45, 0x0d, 0x01, 0x0b, 0x20, 0x01, 0x41, 0xff, 0x01, + 0x71, 0x21, 0x00, 0x03, 0x40, 0x20, 0x00, 0x20, 0x03, 0x2d, 0x00, 0x00, + 0x46, 0x04, 0x40, 0x20, 0x03, 0x0f, 0x0b, 0x20, 0x03, 0x41, 0x01, 0x6a, + 0x21, 0x03, 0x20, 0x04, 0x41, 0x01, 0x6b, 0x22, 0x04, 0x0d, 0x00, 0x0b, + 0x0b, 0x41, 0x00, 0x0b, 0x58, 0x01, 0x02, 0x7f, 0x02, 0x40, 0x20, 0x00, + 0x2d, 0x00, 0x00, 0x22, 0x02, 0x45, 0x20, 0x02, 0x20, 0x01, 0x2d, 0x00, + 0x00, 0x22, 0x03, 0x47, 0x72, 0x0d, 0x00, 0x20, 0x00, 0x41, 0x01, 0x6a, + 0x21, 0x00, 0x20, 0x01, 0x41, 0x01, 0x6a, 0x21, 0x01, 0x03, 0x40, 0x20, + 0x01, 0x2d, 0x00, 0x00, 0x21, 0x03, 0x20, 0x00, 0x2d, 0x00, 0x00, 0x22, + 0x02, 0x45, 0x0d, 0x01, 0x20, 0x00, 0x41, 0x01, 0x6a, 0x21, 0x00, 0x20, + 0x01, 0x41, 0x01, 0x6a, 0x21, 0x01, 0x20, 0x02, 0x20, 0x03, 0x46, 0x0d, + 0x00, 0x0b, 0x0b, 0x20, 0x02, 0x20, 0x03, 0x6b, 0x0b, 0x08, 0x00, 0x20, + 0x00, 0x41, 0x00, 0x10, 0x12, 0x0b, 0x90, 0x02, 0x01, 0x07, 0x7f, 0x02, + 0x40, 0x20, 0x00, 0x41, 0xff, 0xff, 0x07, 0x4b, 0x0d, 0x00, 0x20, 0x00, + 0x20, 0x00, 0x41, 0xff, 0x01, 0x71, 0x22, 0x05, 0x41, 0x03, 0x6e, 0x22, + 0x02, 0x41, 0x03, 0x6c, 0x6b, 0x41, 0xff, 0x01, 0x71, 0x41, 0x02, 0x74, + 0x28, 0x02, 0xc0, 0x9e, 0x04, 0x20, 0x02, 0x20, 0x00, 0x41, 0x08, 0x76, + 0x22, 0x02, 0x2d, 0x00, 0xa0, 0xa9, 0x04, 0x41, 0xd6, 0x00, 0x6c, 0x6a, + 0x2d, 0x00, 0xa0, 0xa9, 0x04, 0x6c, 0x41, 0x0b, 0x76, 0x41, 0x06, 0x70, + 0x20, 0x02, 0x2d, 0x00, 0x90, 0xbe, 0x04, 0x6a, 0x41, 0x02, 0x74, 0x28, + 0x02, 0xd0, 0x9e, 0x04, 0x22, 0x03, 0x41, 0x08, 0x75, 0x21, 0x02, 0x20, + 0x03, 0x41, 0xff, 0x01, 0x71, 0x22, 0x03, 0x41, 0x01, 0x4d, 0x04, 0x40, + 0x20, 0x02, 0x41, 0x00, 0x20, 0x01, 0x20, 0x03, 0x73, 0x6b, 0x71, 0x20, + 0x00, 0x6a, 0x0f, 0x0b, 0x20, 0x02, 0x41, 0xff, 0x01, 0x71, 0x22, 0x03, + 0x45, 0x0d, 0x00, 0x20, 0x02, 0x41, 0x08, 0x76, 0x21, 0x02, 0x03, 0x40, + 0x20, 0x03, 0x41, 0x01, 0x76, 0x22, 0x06, 0x20, 0x02, 0x6a, 0x22, 0x04, + 0x41, 0x01, 0x74, 0x22, 0x07, 0x2d, 0x00, 0x90, 0xa6, 0x04, 0x22, 0x08, + 0x20, 0x05, 0x46, 0x04, 0x40, 0x20, 0x07, 0x41, 0x90, 0xa6, 0x04, 0x6a, + 0x2d, 0x00, 0x01, 0x41, 0x02, 0x74, 0x28, 0x02, 0xd0, 0x9e, 0x04, 0x22, + 0x02, 0x41, 0xff, 0x01, 0x71, 0x22, 0x03, 0x41, 0x01, 0x4d, 0x04, 0x40, + 0x41, 0x00, 0x20, 0x01, 0x20, 0x03, 0x73, 0x6b, 0x20, 0x02, 0x41, 0x08, + 0x75, 0x71, 0x20, 0x00, 0x6a, 0x0f, 0x0b, 0x41, 0x7f, 0x41, 0x01, 0x20, + 0x01, 0x1b, 0x20, 0x00, 0x6a, 0x0f, 0x0b, 0x20, 0x02, 0x20, 0x04, 0x20, + 0x05, 0x20, 0x08, 0x49, 0x22, 0x04, 0x1b, 0x21, 0x02, 0x20, 0x06, 0x20, + 0x03, 0x20, 0x06, 0x6b, 0x20, 0x04, 0x1b, 0x22, 0x03, 0x0d, 0x00, 0x0b, + 0x0b, 0x20, 0x00, 0x0b, 0x08, 0x00, 0x20, 0x00, 0x41, 0x01, 0x10, 0x12, + 0x0b, 0x75, 0x01, 0x02, 0x7f, 0x20, 0x02, 0x45, 0x04, 0x40, 0x41, 0x00, + 0x0f, 0x0b, 0x02, 0x40, 0x20, 0x00, 0x2d, 0x00, 0x00, 0x22, 0x03, 0x45, + 0x04, 0x40, 0x41, 0x00, 0x21, 0x03, 0x0c, 0x01, 0x0b, 0x20, 0x00, 0x41, + 0x01, 0x6a, 0x21, 0x00, 0x20, 0x02, 0x41, 0x01, 0x6b, 0x21, 0x02, 0x02, + 0x40, 0x03, 0x40, 0x20, 0x02, 0x45, 0x20, 0x03, 0x20, 0x01, 0x2d, 0x00, + 0x00, 0x22, 0x04, 0x47, 0x20, 0x04, 0x45, 0x72, 0x72, 0x0d, 0x01, 0x20, + 0x02, 0x41, 0x01, 0x6b, 0x21, 0x02, 0x20, 0x01, 0x41, 0x01, 0x6a, 0x21, + 0x01, 0x20, 0x00, 0x2d, 0x00, 0x00, 0x21, 0x03, 0x20, 0x00, 0x41, 0x01, + 0x6a, 0x21, 0x00, 0x20, 0x03, 0x0d, 0x00, 0x0b, 0x41, 0x00, 0x21, 0x03, + 0x0b, 0x0b, 0x20, 0x03, 0x20, 0x01, 0x2d, 0x00, 0x00, 0x6b, 0x0b, 0x09, + 0x00, 0x20, 0x00, 0x10, 0x11, 0x20, 0x00, 0x47, 0x0b, 0x93, 0x0a, 0x01, + 0x04, 0x7f, 0x02, 0x40, 0x02, 0x40, 0x20, 0x02, 0x41, 0x21, 0x4f, 0x04, + 0x40, 0x20, 0x02, 0x45, 0x0d, 0x01, 0x0c, 0x02, 0x0b, 0x20, 0x00, 0x20, + 0x01, 0x46, 0x0d, 0x00, 0x20, 0x01, 0x20, 0x00, 0x20, 0x02, 0x6a, 0x22, + 0x04, 0x6b, 0x41, 0x00, 0x20, 0x02, 0x41, 0x01, 0x74, 0x6b, 0x4d, 0x04, + 0x40, 0x20, 0x02, 0x45, 0x0d, 0x01, 0x0c, 0x02, 0x0b, 0x20, 0x00, 0x20, + 0x01, 0x73, 0x41, 0x03, 0x71, 0x21, 0x03, 0x02, 0x40, 0x02, 0x40, 0x20, + 0x00, 0x20, 0x01, 0x49, 0x04, 0x40, 0x20, 0x03, 0x04, 0x40, 0x20, 0x02, + 0x21, 0x04, 0x20, 0x00, 0x21, 0x03, 0x0c, 0x03, 0x0b, 0x20, 0x00, 0x41, + 0x03, 0x71, 0x45, 0x04, 0x40, 0x20, 0x02, 0x21, 0x04, 0x20, 0x00, 0x21, + 0x03, 0x0c, 0x02, 0x0b, 0x20, 0x02, 0x45, 0x0d, 0x03, 0x20, 0x00, 0x20, + 0x01, 0x2d, 0x00, 0x00, 0x3a, 0x00, 0x00, 0x20, 0x02, 0x41, 0x01, 0x6b, + 0x21, 0x04, 0x20, 0x00, 0x41, 0x01, 0x6a, 0x22, 0x03, 0x41, 0x03, 0x71, + 0x45, 0x04, 0x40, 0x20, 0x01, 0x41, 0x01, 0x6a, 0x21, 0x01, 0x0c, 0x02, + 0x0b, 0x20, 0x04, 0x45, 0x0d, 0x03, 0x20, 0x00, 0x20, 0x01, 0x2d, 0x00, + 0x01, 0x3a, 0x00, 0x01, 0x20, 0x02, 0x41, 0x02, 0x6b, 0x21, 0x04, 0x20, + 0x00, 0x41, 0x02, 0x6a, 0x22, 0x03, 0x41, 0x03, 0x71, 0x45, 0x04, 0x40, + 0x20, 0x01, 0x41, 0x02, 0x6a, 0x21, 0x01, 0x0c, 0x02, 0x0b, 0x20, 0x04, + 0x45, 0x0d, 0x03, 0x20, 0x00, 0x20, 0x01, 0x2d, 0x00, 0x02, 0x3a, 0x00, + 0x02, 0x20, 0x02, 0x41, 0x03, 0x6b, 0x21, 0x04, 0x20, 0x00, 0x41, 0x03, + 0x6a, 0x22, 0x03, 0x41, 0x03, 0x71, 0x45, 0x04, 0x40, 0x20, 0x01, 0x41, + 0x03, 0x6a, 0x21, 0x01, 0x0c, 0x02, 0x0b, 0x20, 0x04, 0x45, 0x0d, 0x03, + 0x20, 0x00, 0x20, 0x01, 0x2d, 0x00, 0x03, 0x3a, 0x00, 0x03, 0x20, 0x00, + 0x41, 0x04, 0x6a, 0x21, 0x03, 0x20, 0x01, 0x41, 0x04, 0x6a, 0x21, 0x01, + 0x20, 0x02, 0x41, 0x04, 0x6b, 0x21, 0x04, 0x0c, 0x01, 0x0b, 0x02, 0x40, + 0x20, 0x03, 0x0d, 0x00, 0x02, 0x40, 0x20, 0x04, 0x41, 0x03, 0x71, 0x45, + 0x0d, 0x00, 0x20, 0x02, 0x45, 0x0d, 0x04, 0x20, 0x00, 0x20, 0x02, 0x41, + 0x01, 0x6b, 0x22, 0x03, 0x6a, 0x22, 0x04, 0x20, 0x01, 0x20, 0x03, 0x6a, + 0x2d, 0x00, 0x00, 0x3a, 0x00, 0x00, 0x20, 0x04, 0x41, 0x03, 0x71, 0x45, + 0x04, 0x40, 0x20, 0x03, 0x21, 0x02, 0x0c, 0x01, 0x0b, 0x20, 0x03, 0x45, + 0x0d, 0x04, 0x20, 0x00, 0x20, 0x02, 0x41, 0x02, 0x6b, 0x22, 0x03, 0x6a, + 0x22, 0x04, 0x20, 0x01, 0x20, 0x03, 0x6a, 0x2d, 0x00, 0x00, 0x3a, 0x00, + 0x00, 0x20, 0x04, 0x41, 0x03, 0x71, 0x45, 0x04, 0x40, 0x20, 0x03, 0x21, 0x02, 0x0c, 0x01, 0x0b, 0x20, 0x03, 0x45, 0x0d, 0x04, 0x20, 0x00, 0x20, - 0x02, 0x41, 0x7d, 0x6a, 0x22, 0x03, 0x6a, 0x22, 0x04, 0x20, 0x01, 0x20, - 0x03, 0x6a, 0x2d, 0x00, 0x00, 0x3a, 0x00, 0x00, 0x02, 0x40, 0x20, 0x04, - 0x41, 0x03, 0x71, 0x0d, 0x00, 0x20, 0x03, 0x21, 0x02, 0x0c, 0x01, 0x0b, - 0x20, 0x03, 0x45, 0x0d, 0x04, 0x20, 0x00, 0x20, 0x02, 0x41, 0x7c, 0x6a, - 0x22, 0x02, 0x6a, 0x20, 0x01, 0x20, 0x02, 0x6a, 0x2d, 0x00, 0x00, 0x3a, - 0x00, 0x00, 0x0b, 0x20, 0x02, 0x41, 0x04, 0x49, 0x0d, 0x00, 0x02, 0x40, - 0x20, 0x02, 0x41, 0x7c, 0x6a, 0x22, 0x06, 0x41, 0x02, 0x76, 0x41, 0x01, - 0x6a, 0x41, 0x03, 0x71, 0x22, 0x03, 0x45, 0x0d, 0x00, 0x20, 0x01, 0x41, - 0x7c, 0x6a, 0x21, 0x04, 0x20, 0x00, 0x41, 0x7c, 0x6a, 0x21, 0x05, 0x03, - 0x40, 0x20, 0x05, 0x20, 0x02, 0x6a, 0x20, 0x04, 0x20, 0x02, 0x6a, 0x28, - 0x02, 0x00, 0x36, 0x02, 0x00, 0x20, 0x02, 0x41, 0x7c, 0x6a, 0x21, 0x02, - 0x20, 0x03, 0x41, 0x7f, 0x6a, 0x22, 0x03, 0x0d, 0x00, 0x0b, 0x0b, 0x20, - 0x06, 0x41, 0x0c, 0x49, 0x0d, 0x00, 0x20, 0x01, 0x41, 0x70, 0x6a, 0x21, - 0x05, 0x20, 0x00, 0x41, 0x70, 0x6a, 0x21, 0x06, 0x03, 0x40, 0x20, 0x06, - 0x20, 0x02, 0x6a, 0x22, 0x03, 0x41, 0x0c, 0x6a, 0x20, 0x05, 0x20, 0x02, - 0x6a, 0x22, 0x04, 0x41, 0x0c, 0x6a, 0x28, 0x02, 0x00, 0x36, 0x02, 0x00, - 0x20, 0x03, 0x41, 0x08, 0x6a, 0x20, 0x04, 0x41, 0x08, 0x6a, 0x28, 0x02, - 0x00, 0x36, 0x02, 0x00, 0x20, 0x03, 0x41, 0x04, 0x6a, 0x20, 0x04, 0x41, - 0x04, 0x6a, 0x28, 0x02, 0x00, 0x36, 0x02, 0x00, 0x20, 0x03, 0x20, 0x04, - 0x28, 0x02, 0x00, 0x36, 0x02, 0x00, 0x20, 0x02, 0x41, 0x70, 0x6a, 0x22, - 0x02, 0x41, 0x03, 0x4b, 0x0d, 0x00, 0x0b, 0x0b, 0x20, 0x02, 0x45, 0x0d, - 0x02, 0x20, 0x02, 0x21, 0x03, 0x02, 0x40, 0x20, 0x02, 0x41, 0x03, 0x71, - 0x22, 0x04, 0x45, 0x0d, 0x00, 0x20, 0x01, 0x41, 0x7f, 0x6a, 0x21, 0x05, - 0x20, 0x00, 0x41, 0x7f, 0x6a, 0x21, 0x06, 0x20, 0x02, 0x21, 0x03, 0x03, - 0x40, 0x20, 0x06, 0x20, 0x03, 0x6a, 0x20, 0x05, 0x20, 0x03, 0x6a, 0x2d, - 0x00, 0x00, 0x3a, 0x00, 0x00, 0x20, 0x03, 0x41, 0x7f, 0x6a, 0x21, 0x03, - 0x20, 0x04, 0x41, 0x7f, 0x6a, 0x22, 0x04, 0x0d, 0x00, 0x0b, 0x0b, 0x20, - 0x02, 0x41, 0x04, 0x49, 0x0d, 0x02, 0x20, 0x01, 0x41, 0x7c, 0x6a, 0x21, - 0x04, 0x20, 0x00, 0x41, 0x7c, 0x6a, 0x21, 0x05, 0x03, 0x40, 0x20, 0x05, - 0x20, 0x03, 0x6a, 0x22, 0x01, 0x41, 0x03, 0x6a, 0x20, 0x04, 0x20, 0x03, - 0x6a, 0x22, 0x02, 0x41, 0x03, 0x6a, 0x2d, 0x00, 0x00, 0x3a, 0x00, 0x00, - 0x20, 0x01, 0x41, 0x02, 0x6a, 0x20, 0x02, 0x41, 0x02, 0x6a, 0x2d, 0x00, - 0x00, 0x3a, 0x00, 0x00, 0x20, 0x01, 0x41, 0x01, 0x6a, 0x20, 0x02, 0x41, - 0x01, 0x6a, 0x2d, 0x00, 0x00, 0x3a, 0x00, 0x00, 0x20, 0x01, 0x20, 0x02, - 0x2d, 0x00, 0x00, 0x3a, 0x00, 0x00, 0x20, 0x03, 0x41, 0x7c, 0x6a, 0x22, - 0x03, 0x0d, 0x00, 0x0c, 0x03, 0x0b, 0x0b, 0x20, 0x05, 0x41, 0x04, 0x49, - 0x0d, 0x00, 0x02, 0x40, 0x20, 0x05, 0x41, 0x7c, 0x6a, 0x22, 0x04, 0x41, - 0x02, 0x76, 0x41, 0x01, 0x6a, 0x41, 0x07, 0x71, 0x22, 0x02, 0x45, 0x0d, - 0x00, 0x20, 0x05, 0x20, 0x02, 0x41, 0x02, 0x74, 0x6b, 0x21, 0x05, 0x03, - 0x40, 0x20, 0x03, 0x20, 0x01, 0x28, 0x02, 0x00, 0x36, 0x02, 0x00, 0x20, - 0x01, 0x41, 0x04, 0x6a, 0x21, 0x01, 0x20, 0x03, 0x41, 0x04, 0x6a, 0x21, - 0x03, 0x20, 0x02, 0x41, 0x7f, 0x6a, 0x22, 0x02, 0x0d, 0x00, 0x0b, 0x0b, - 0x20, 0x04, 0x41, 0x1c, 0x49, 0x0d, 0x00, 0x03, 0x40, 0x20, 0x03, 0x20, - 0x01, 0x28, 0x02, 0x00, 0x36, 0x02, 0x00, 0x20, 0x03, 0x20, 0x01, 0x28, - 0x02, 0x04, 0x36, 0x02, 0x04, 0x20, 0x03, 0x20, 0x01, 0x28, 0x02, 0x08, - 0x36, 0x02, 0x08, 0x20, 0x03, 0x20, 0x01, 0x28, 0x02, 0x0c, 0x36, 0x02, - 0x0c, 0x20, 0x03, 0x20, 0x01, 0x28, 0x02, 0x10, 0x36, 0x02, 0x10, 0x20, - 0x03, 0x20, 0x01, 0x28, 0x02, 0x14, 0x36, 0x02, 0x14, 0x20, 0x03, 0x20, - 0x01, 0x28, 0x02, 0x18, 0x36, 0x02, 0x18, 0x20, 0x03, 0x20, 0x01, 0x28, - 0x02, 0x1c, 0x36, 0x02, 0x1c, 0x20, 0x01, 0x41, 0x20, 0x6a, 0x21, 0x01, - 0x20, 0x03, 0x41, 0x20, 0x6a, 0x21, 0x03, 0x20, 0x05, 0x41, 0x60, 0x6a, - 0x22, 0x05, 0x41, 0x03, 0x4b, 0x0d, 0x00, 0x0b, 0x0b, 0x20, 0x05, 0x45, - 0x0d, 0x00, 0x02, 0x40, 0x02, 0x40, 0x20, 0x05, 0x41, 0x07, 0x71, 0x22, - 0x02, 0x0d, 0x00, 0x20, 0x05, 0x21, 0x04, 0x0c, 0x01, 0x0b, 0x20, 0x05, - 0x41, 0x78, 0x71, 0x21, 0x04, 0x03, 0x40, 0x20, 0x03, 0x20, 0x01, 0x2d, - 0x00, 0x00, 0x3a, 0x00, 0x00, 0x20, 0x03, 0x41, 0x01, 0x6a, 0x21, 0x03, - 0x20, 0x01, 0x41, 0x01, 0x6a, 0x21, 0x01, 0x20, 0x02, 0x41, 0x7f, 0x6a, - 0x22, 0x02, 0x0d, 0x00, 0x0b, 0x0b, 0x20, 0x05, 0x41, 0x08, 0x49, 0x0d, - 0x00, 0x03, 0x40, 0x20, 0x03, 0x20, 0x01, 0x2d, 0x00, 0x00, 0x3a, 0x00, - 0x00, 0x20, 0x03, 0x20, 0x01, 0x2d, 0x00, 0x01, 0x3a, 0x00, 0x01, 0x20, - 0x03, 0x20, 0x01, 0x2d, 0x00, 0x02, 0x3a, 0x00, 0x02, 0x20, 0x03, 0x20, - 0x01, 0x2d, 0x00, 0x03, 0x3a, 0x00, 0x03, 0x20, 0x03, 0x20, 0x01, 0x2d, - 0x00, 0x04, 0x3a, 0x00, 0x04, 0x20, 0x03, 0x20, 0x01, 0x2d, 0x00, 0x05, - 0x3a, 0x00, 0x05, 0x20, 0x03, 0x20, 0x01, 0x2d, 0x00, 0x06, 0x3a, 0x00, - 0x06, 0x20, 0x03, 0x20, 0x01, 0x2d, 0x00, 0x07, 0x3a, 0x00, 0x07, 0x20, - 0x03, 0x41, 0x08, 0x6a, 0x21, 0x03, 0x20, 0x01, 0x41, 0x08, 0x6a, 0x21, - 0x01, 0x20, 0x04, 0x41, 0x78, 0x6a, 0x22, 0x04, 0x0d, 0x00, 0x0b, 0x0b, - 0x20, 0x00, 0x0b, 0x0d, 0x00, 0x20, 0x00, 0x10, 0x9c, 0x80, 0x80, 0x80, - 0x00, 0x20, 0x00, 0x47, 0x0b, 0x0d, 0x00, 0x20, 0x00, 0x41, 0x20, 0x46, - 0x20, 0x00, 0x41, 0x09, 0x46, 0x72, 0x0b, 0x0a, 0x00, 0x20, 0x00, 0x10, - 0xa1, 0x80, 0x80, 0x80, 0x00, 0x0b, 0x0a, 0x00, 0x20, 0x00, 0x41, 0x50, - 0x6a, 0x41, 0x0a, 0x49, 0x0b, 0x4d, 0x01, 0x02, 0x7f, 0x20, 0x00, 0x20, - 0x00, 0x10, 0x95, 0x80, 0x80, 0x80, 0x00, 0x6a, 0x21, 0x03, 0x02, 0x40, - 0x20, 0x02, 0x45, 0x0d, 0x00, 0x03, 0x40, 0x20, 0x01, 0x2d, 0x00, 0x00, - 0x22, 0x04, 0x45, 0x0d, 0x01, 0x20, 0x03, 0x20, 0x04, 0x3a, 0x00, 0x00, - 0x20, 0x03, 0x41, 0x01, 0x6a, 0x21, 0x03, 0x20, 0x01, 0x41, 0x01, 0x6a, - 0x21, 0x01, 0x20, 0x02, 0x41, 0x7f, 0x6a, 0x22, 0x02, 0x0d, 0x00, 0x0b, - 0x0b, 0x20, 0x03, 0x41, 0x00, 0x3a, 0x00, 0x00, 0x20, 0x00, 0x0b, 0xef, - 0x03, 0x01, 0x04, 0x7f, 0x02, 0x40, 0x02, 0x40, 0x02, 0x40, 0x02, 0x40, - 0x02, 0x40, 0x20, 0x01, 0x20, 0x00, 0x73, 0x41, 0x03, 0x71, 0x45, 0x0d, - 0x00, 0x20, 0x00, 0x21, 0x03, 0x0c, 0x01, 0x0b, 0x20, 0x02, 0x41, 0x00, - 0x47, 0x21, 0x04, 0x02, 0x40, 0x02, 0x40, 0x20, 0x01, 0x41, 0x03, 0x71, - 0x0d, 0x00, 0x20, 0x00, 0x21, 0x03, 0x0c, 0x01, 0x0b, 0x02, 0x40, 0x20, - 0x02, 0x0d, 0x00, 0x20, 0x00, 0x21, 0x03, 0x0c, 0x01, 0x0b, 0x20, 0x00, - 0x20, 0x01, 0x2d, 0x00, 0x00, 0x22, 0x03, 0x3a, 0x00, 0x00, 0x02, 0x40, - 0x20, 0x03, 0x0d, 0x00, 0x20, 0x00, 0x21, 0x03, 0x20, 0x02, 0x21, 0x05, - 0x0c, 0x05, 0x0b, 0x20, 0x00, 0x41, 0x01, 0x6a, 0x21, 0x03, 0x20, 0x02, - 0x41, 0x7f, 0x6a, 0x22, 0x05, 0x41, 0x00, 0x47, 0x21, 0x04, 0x02, 0x40, - 0x20, 0x01, 0x41, 0x01, 0x6a, 0x22, 0x06, 0x41, 0x03, 0x71, 0x45, 0x0d, - 0x00, 0x20, 0x05, 0x45, 0x0d, 0x00, 0x20, 0x03, 0x20, 0x06, 0x2d, 0x00, - 0x00, 0x22, 0x04, 0x3a, 0x00, 0x00, 0x20, 0x04, 0x45, 0x0d, 0x05, 0x20, - 0x00, 0x41, 0x02, 0x6a, 0x21, 0x03, 0x20, 0x02, 0x41, 0x7e, 0x6a, 0x22, - 0x05, 0x41, 0x00, 0x47, 0x21, 0x04, 0x02, 0x40, 0x20, 0x01, 0x41, 0x02, - 0x6a, 0x22, 0x06, 0x41, 0x03, 0x71, 0x45, 0x0d, 0x00, 0x20, 0x05, 0x45, - 0x0d, 0x00, 0x20, 0x03, 0x20, 0x06, 0x2d, 0x00, 0x00, 0x22, 0x04, 0x3a, - 0x00, 0x00, 0x20, 0x04, 0x45, 0x0d, 0x06, 0x20, 0x00, 0x41, 0x03, 0x6a, - 0x21, 0x03, 0x20, 0x02, 0x41, 0x7d, 0x6a, 0x22, 0x05, 0x41, 0x00, 0x47, - 0x21, 0x04, 0x02, 0x40, 0x20, 0x01, 0x41, 0x03, 0x6a, 0x22, 0x06, 0x41, - 0x03, 0x71, 0x45, 0x0d, 0x00, 0x20, 0x05, 0x45, 0x0d, 0x00, 0x20, 0x03, - 0x20, 0x06, 0x2d, 0x00, 0x00, 0x22, 0x04, 0x3a, 0x00, 0x00, 0x20, 0x04, - 0x45, 0x0d, 0x07, 0x20, 0x00, 0x41, 0x04, 0x6a, 0x21, 0x03, 0x20, 0x01, - 0x41, 0x04, 0x6a, 0x21, 0x01, 0x20, 0x02, 0x41, 0x7c, 0x6a, 0x22, 0x02, - 0x41, 0x00, 0x47, 0x21, 0x04, 0x0c, 0x03, 0x0b, 0x20, 0x06, 0x21, 0x01, - 0x20, 0x05, 0x21, 0x02, 0x0c, 0x02, 0x0b, 0x20, 0x06, 0x21, 0x01, 0x20, - 0x05, 0x21, 0x02, 0x0c, 0x01, 0x0b, 0x20, 0x06, 0x21, 0x01, 0x20, 0x05, - 0x21, 0x02, 0x0b, 0x20, 0x04, 0x45, 0x0d, 0x02, 0x02, 0x40, 0x20, 0x01, - 0x2d, 0x00, 0x00, 0x0d, 0x00, 0x20, 0x02, 0x21, 0x05, 0x0c, 0x04, 0x0b, - 0x20, 0x02, 0x41, 0x04, 0x49, 0x0d, 0x00, 0x03, 0x40, 0x20, 0x01, 0x28, - 0x02, 0x00, 0x22, 0x00, 0x41, 0x7f, 0x73, 0x20, 0x00, 0x41, 0xff, 0xfd, - 0xfb, 0x77, 0x6a, 0x71, 0x41, 0x80, 0x81, 0x82, 0x84, 0x78, 0x71, 0x0d, - 0x02, 0x20, 0x03, 0x20, 0x00, 0x36, 0x02, 0x00, 0x20, 0x03, 0x41, 0x04, - 0x6a, 0x21, 0x03, 0x20, 0x01, 0x41, 0x04, 0x6a, 0x21, 0x01, 0x20, 0x02, - 0x41, 0x7c, 0x6a, 0x22, 0x02, 0x41, 0x03, 0x4b, 0x0d, 0x00, 0x0b, 0x0b, - 0x20, 0x02, 0x45, 0x0d, 0x01, 0x0b, 0x03, 0x40, 0x20, 0x03, 0x20, 0x01, - 0x2d, 0x00, 0x00, 0x22, 0x00, 0x3a, 0x00, 0x00, 0x02, 0x40, 0x20, 0x00, - 0x0d, 0x00, 0x20, 0x02, 0x21, 0x05, 0x0c, 0x03, 0x0b, 0x20, 0x03, 0x41, - 0x01, 0x6a, 0x21, 0x03, 0x20, 0x01, 0x41, 0x01, 0x6a, 0x21, 0x01, 0x20, - 0x02, 0x41, 0x7f, 0x6a, 0x22, 0x02, 0x0d, 0x00, 0x0b, 0x0b, 0x41, 0x00, - 0x21, 0x05, 0x0b, 0x20, 0x03, 0x41, 0x00, 0x20, 0x05, 0x10, 0x94, 0x80, - 0x80, 0x80, 0x00, 0x0b, 0x11, 0x00, 0x20, 0x00, 0x20, 0x01, 0x20, 0x02, - 0x10, 0xa5, 0x80, 0x80, 0x80, 0x00, 0x1a, 0x20, 0x00, 0x0b, 0x17, 0x00, - 0x20, 0x00, 0x41, 0x50, 0x6a, 0x41, 0x0a, 0x49, 0x20, 0x00, 0x41, 0x20, - 0x72, 0x41, 0x9f, 0x7f, 0x6a, 0x41, 0x06, 0x49, 0x72, 0x0b, 0x2a, 0x01, - 0x03, 0x7f, 0x41, 0x00, 0x21, 0x01, 0x03, 0x40, 0x20, 0x00, 0x20, 0x01, - 0x6a, 0x21, 0x02, 0x20, 0x01, 0x41, 0x04, 0x6a, 0x22, 0x03, 0x21, 0x01, - 0x20, 0x02, 0x28, 0x02, 0x00, 0x0d, 0x00, 0x0b, 0x20, 0x03, 0x41, 0x7c, - 0x6a, 0x41, 0x02, 0x75, 0x0b, 0x45, 0x01, 0x01, 0x7f, 0x02, 0x40, 0x20, - 0x01, 0x45, 0x0d, 0x00, 0x20, 0x00, 0x41, 0x7c, 0x6a, 0x21, 0x00, 0x02, - 0x40, 0x03, 0x40, 0x20, 0x00, 0x41, 0x04, 0x6a, 0x22, 0x00, 0x28, 0x02, - 0x00, 0x22, 0x02, 0x45, 0x0d, 0x01, 0x20, 0x02, 0x20, 0x01, 0x47, 0x0d, - 0x00, 0x0b, 0x0b, 0x20, 0x00, 0x41, 0x00, 0x20, 0x02, 0x1b, 0x0f, 0x0b, - 0x20, 0x00, 0x20, 0x00, 0x10, 0xa8, 0x80, 0x80, 0x80, 0x00, 0x41, 0x02, - 0x74, 0x6a, 0x0b, 0x1d, 0x00, 0x02, 0x40, 0x20, 0x00, 0x0d, 0x00, 0x41, - 0x00, 0x0f, 0x0b, 0x41, 0x90, 0xc2, 0x84, 0x80, 0x00, 0x20, 0x00, 0x10, - 0xa9, 0x80, 0x80, 0x80, 0x00, 0x41, 0x00, 0x47, 0x0b, 0x24, 0x01, 0x01, - 0x7f, 0x41, 0x01, 0x21, 0x01, 0x02, 0x40, 0x20, 0x00, 0x41, 0x50, 0x6a, - 0x41, 0x0a, 0x49, 0x0d, 0x00, 0x20, 0x00, 0x10, 0x96, 0x80, 0x80, 0x80, - 0x00, 0x41, 0x00, 0x47, 0x21, 0x01, 0x0b, 0x20, 0x01, 0x0b, 0x0b, 0xf1, - 0x42, 0x01, 0x00, 0x41, 0x80, 0x80, 0x04, 0x0b, 0xe8, 0x42, 0x12, 0x11, - 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, - 0x1f, 0x20, 0x21, 0x11, 0x22, 0x23, 0x24, 0x11, 0x25, 0x26, 0x27, 0x28, - 0x29, 0x2a, 0x2b, 0x2c, 0x11, 0x2d, 0x2e, 0x2f, 0x10, 0x10, 0x30, 0x10, - 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x31, 0x32, 0x33, 0x10, 0x34, 0x35, - 0x10, 0x10, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + 0x02, 0x41, 0x03, 0x6b, 0x22, 0x03, 0x6a, 0x22, 0x04, 0x20, 0x01, 0x20, + 0x03, 0x6a, 0x2d, 0x00, 0x00, 0x3a, 0x00, 0x00, 0x20, 0x04, 0x41, 0x03, + 0x71, 0x45, 0x04, 0x40, 0x20, 0x03, 0x21, 0x02, 0x0c, 0x01, 0x0b, 0x20, + 0x03, 0x45, 0x0d, 0x04, 0x20, 0x00, 0x20, 0x02, 0x41, 0x04, 0x6b, 0x22, + 0x02, 0x6a, 0x20, 0x01, 0x20, 0x02, 0x6a, 0x2d, 0x00, 0x00, 0x3a, 0x00, + 0x00, 0x0b, 0x20, 0x02, 0x41, 0x04, 0x49, 0x0d, 0x00, 0x20, 0x02, 0x41, + 0x04, 0x6b, 0x22, 0x04, 0x41, 0x0c, 0x71, 0x41, 0x0c, 0x47, 0x04, 0x40, + 0x20, 0x04, 0x41, 0x02, 0x76, 0x41, 0x01, 0x6a, 0x41, 0x03, 0x71, 0x21, + 0x03, 0x20, 0x01, 0x41, 0x04, 0x6b, 0x21, 0x05, 0x20, 0x00, 0x41, 0x04, + 0x6b, 0x21, 0x06, 0x03, 0x40, 0x20, 0x02, 0x20, 0x06, 0x6a, 0x20, 0x02, + 0x20, 0x05, 0x6a, 0x28, 0x02, 0x00, 0x36, 0x02, 0x00, 0x20, 0x02, 0x41, + 0x04, 0x6b, 0x21, 0x02, 0x20, 0x03, 0x41, 0x01, 0x6b, 0x22, 0x03, 0x0d, + 0x00, 0x0b, 0x0b, 0x20, 0x04, 0x41, 0x0c, 0x49, 0x0d, 0x00, 0x20, 0x01, + 0x41, 0x10, 0x6b, 0x21, 0x05, 0x20, 0x00, 0x41, 0x10, 0x6b, 0x21, 0x06, + 0x03, 0x40, 0x20, 0x02, 0x20, 0x06, 0x6a, 0x22, 0x03, 0x41, 0x0c, 0x6a, + 0x20, 0x02, 0x20, 0x05, 0x6a, 0x22, 0x04, 0x41, 0x0c, 0x6a, 0x28, 0x02, + 0x00, 0x36, 0x02, 0x00, 0x20, 0x03, 0x41, 0x08, 0x6a, 0x20, 0x04, 0x41, + 0x08, 0x6a, 0x28, 0x02, 0x00, 0x36, 0x02, 0x00, 0x20, 0x03, 0x41, 0x04, + 0x6a, 0x20, 0x04, 0x41, 0x04, 0x6a, 0x28, 0x02, 0x00, 0x36, 0x02, 0x00, + 0x20, 0x03, 0x20, 0x04, 0x28, 0x02, 0x00, 0x36, 0x02, 0x00, 0x20, 0x02, + 0x41, 0x10, 0x6b, 0x22, 0x02, 0x41, 0x03, 0x4b, 0x0d, 0x00, 0x0b, 0x0b, + 0x20, 0x02, 0x45, 0x0d, 0x02, 0x20, 0x02, 0x22, 0x03, 0x41, 0x03, 0x71, + 0x22, 0x05, 0x04, 0x40, 0x20, 0x01, 0x41, 0x01, 0x6b, 0x21, 0x04, 0x20, + 0x00, 0x41, 0x01, 0x6b, 0x21, 0x06, 0x03, 0x40, 0x20, 0x03, 0x20, 0x06, + 0x6a, 0x20, 0x03, 0x20, 0x04, 0x6a, 0x2d, 0x00, 0x00, 0x3a, 0x00, 0x00, + 0x20, 0x03, 0x41, 0x01, 0x6b, 0x21, 0x03, 0x20, 0x05, 0x41, 0x01, 0x6b, + 0x22, 0x05, 0x0d, 0x00, 0x0b, 0x0b, 0x20, 0x02, 0x41, 0x04, 0x49, 0x0d, + 0x02, 0x20, 0x01, 0x41, 0x04, 0x6b, 0x21, 0x04, 0x20, 0x00, 0x41, 0x04, + 0x6b, 0x21, 0x05, 0x03, 0x40, 0x20, 0x03, 0x20, 0x05, 0x6a, 0x22, 0x01, + 0x41, 0x03, 0x6a, 0x20, 0x03, 0x20, 0x04, 0x6a, 0x22, 0x02, 0x41, 0x03, + 0x6a, 0x2d, 0x00, 0x00, 0x3a, 0x00, 0x00, 0x20, 0x01, 0x41, 0x02, 0x6a, + 0x20, 0x02, 0x41, 0x02, 0x6a, 0x2d, 0x00, 0x00, 0x3a, 0x00, 0x00, 0x20, + 0x01, 0x41, 0x01, 0x6a, 0x20, 0x02, 0x41, 0x01, 0x6a, 0x2d, 0x00, 0x00, + 0x3a, 0x00, 0x00, 0x20, 0x01, 0x20, 0x02, 0x2d, 0x00, 0x00, 0x3a, 0x00, + 0x00, 0x20, 0x03, 0x41, 0x04, 0x6b, 0x22, 0x03, 0x0d, 0x00, 0x0b, 0x0c, + 0x02, 0x0b, 0x20, 0x04, 0x41, 0x04, 0x49, 0x0d, 0x00, 0x20, 0x04, 0x41, + 0x04, 0x6b, 0x22, 0x05, 0x41, 0x1c, 0x71, 0x41, 0x1c, 0x47, 0x04, 0x40, + 0x20, 0x04, 0x20, 0x05, 0x41, 0x02, 0x76, 0x41, 0x01, 0x6a, 0x41, 0x07, + 0x71, 0x22, 0x02, 0x41, 0x02, 0x74, 0x6b, 0x21, 0x04, 0x03, 0x40, 0x20, + 0x03, 0x20, 0x01, 0x28, 0x02, 0x00, 0x36, 0x02, 0x00, 0x20, 0x01, 0x41, + 0x04, 0x6a, 0x21, 0x01, 0x20, 0x03, 0x41, 0x04, 0x6a, 0x21, 0x03, 0x20, + 0x02, 0x41, 0x01, 0x6b, 0x22, 0x02, 0x0d, 0x00, 0x0b, 0x0b, 0x20, 0x05, + 0x41, 0x1c, 0x49, 0x0d, 0x00, 0x03, 0x40, 0x20, 0x03, 0x20, 0x01, 0x28, + 0x02, 0x00, 0x36, 0x02, 0x00, 0x20, 0x03, 0x41, 0x04, 0x6a, 0x20, 0x01, + 0x41, 0x04, 0x6a, 0x28, 0x02, 0x00, 0x36, 0x02, 0x00, 0x20, 0x03, 0x41, + 0x08, 0x6a, 0x20, 0x01, 0x41, 0x08, 0x6a, 0x28, 0x02, 0x00, 0x36, 0x02, + 0x00, 0x20, 0x03, 0x41, 0x0c, 0x6a, 0x20, 0x01, 0x41, 0x0c, 0x6a, 0x28, + 0x02, 0x00, 0x36, 0x02, 0x00, 0x20, 0x03, 0x41, 0x10, 0x6a, 0x20, 0x01, + 0x41, 0x10, 0x6a, 0x28, 0x02, 0x00, 0x36, 0x02, 0x00, 0x20, 0x03, 0x41, + 0x14, 0x6a, 0x20, 0x01, 0x41, 0x14, 0x6a, 0x28, 0x02, 0x00, 0x36, 0x02, + 0x00, 0x20, 0x03, 0x41, 0x18, 0x6a, 0x20, 0x01, 0x41, 0x18, 0x6a, 0x28, + 0x02, 0x00, 0x36, 0x02, 0x00, 0x20, 0x03, 0x41, 0x1c, 0x6a, 0x20, 0x01, + 0x41, 0x1c, 0x6a, 0x28, 0x02, 0x00, 0x36, 0x02, 0x00, 0x20, 0x01, 0x41, + 0x20, 0x6a, 0x21, 0x01, 0x20, 0x03, 0x41, 0x20, 0x6a, 0x21, 0x03, 0x20, + 0x04, 0x41, 0x20, 0x6b, 0x22, 0x04, 0x41, 0x03, 0x4b, 0x0d, 0x00, 0x0b, + 0x0b, 0x20, 0x04, 0x45, 0x0d, 0x00, 0x02, 0x40, 0x20, 0x04, 0x41, 0x07, + 0x71, 0x22, 0x02, 0x45, 0x04, 0x40, 0x20, 0x04, 0x21, 0x05, 0x0c, 0x01, + 0x0b, 0x20, 0x04, 0x41, 0x78, 0x71, 0x21, 0x05, 0x03, 0x40, 0x20, 0x03, + 0x20, 0x01, 0x2d, 0x00, 0x00, 0x3a, 0x00, 0x00, 0x20, 0x03, 0x41, 0x01, + 0x6a, 0x21, 0x03, 0x20, 0x01, 0x41, 0x01, 0x6a, 0x21, 0x01, 0x20, 0x02, + 0x41, 0x01, 0x6b, 0x22, 0x02, 0x0d, 0x00, 0x0b, 0x0b, 0x20, 0x04, 0x41, + 0x08, 0x49, 0x0d, 0x00, 0x03, 0x40, 0x20, 0x03, 0x20, 0x01, 0x2d, 0x00, + 0x00, 0x3a, 0x00, 0x00, 0x20, 0x03, 0x41, 0x01, 0x6a, 0x20, 0x01, 0x41, + 0x01, 0x6a, 0x2d, 0x00, 0x00, 0x3a, 0x00, 0x00, 0x20, 0x03, 0x41, 0x02, + 0x6a, 0x20, 0x01, 0x41, 0x02, 0x6a, 0x2d, 0x00, 0x00, 0x3a, 0x00, 0x00, + 0x20, 0x03, 0x41, 0x03, 0x6a, 0x20, 0x01, 0x41, 0x03, 0x6a, 0x2d, 0x00, + 0x00, 0x3a, 0x00, 0x00, 0x20, 0x03, 0x41, 0x04, 0x6a, 0x20, 0x01, 0x41, + 0x04, 0x6a, 0x2d, 0x00, 0x00, 0x3a, 0x00, 0x00, 0x20, 0x03, 0x41, 0x05, + 0x6a, 0x20, 0x01, 0x41, 0x05, 0x6a, 0x2d, 0x00, 0x00, 0x3a, 0x00, 0x00, + 0x20, 0x03, 0x41, 0x06, 0x6a, 0x20, 0x01, 0x41, 0x06, 0x6a, 0x2d, 0x00, + 0x00, 0x3a, 0x00, 0x00, 0x20, 0x03, 0x41, 0x07, 0x6a, 0x20, 0x01, 0x41, + 0x07, 0x6a, 0x2d, 0x00, 0x00, 0x3a, 0x00, 0x00, 0x20, 0x03, 0x41, 0x08, + 0x6a, 0x21, 0x03, 0x20, 0x01, 0x41, 0x08, 0x6a, 0x21, 0x01, 0x20, 0x05, + 0x41, 0x08, 0x6b, 0x22, 0x05, 0x0d, 0x00, 0x0b, 0x0b, 0x20, 0x00, 0x0f, + 0x0b, 0x20, 0x00, 0x20, 0x01, 0x20, 0x02, 0xfc, 0x0a, 0x00, 0x00, 0x20, + 0x00, 0x0b, 0x94, 0x03, 0x02, 0x03, 0x7f, 0x01, 0x7e, 0x02, 0x40, 0x20, + 0x02, 0x41, 0x21, 0x4f, 0x04, 0x40, 0x20, 0x02, 0x45, 0x0d, 0x01, 0x20, + 0x00, 0x20, 0x01, 0x20, 0x02, 0xfc, 0x0b, 0x00, 0x20, 0x00, 0x0f, 0x0b, + 0x20, 0x02, 0x45, 0x0d, 0x00, 0x20, 0x00, 0x20, 0x01, 0x3a, 0x00, 0x00, + 0x20, 0x00, 0x20, 0x02, 0x6a, 0x22, 0x03, 0x41, 0x01, 0x6b, 0x20, 0x01, + 0x3a, 0x00, 0x00, 0x20, 0x02, 0x41, 0x03, 0x49, 0x0d, 0x00, 0x20, 0x00, + 0x20, 0x01, 0x3a, 0x00, 0x02, 0x20, 0x00, 0x20, 0x01, 0x3a, 0x00, 0x01, + 0x20, 0x03, 0x41, 0x03, 0x6b, 0x20, 0x01, 0x3a, 0x00, 0x00, 0x20, 0x03, + 0x41, 0x02, 0x6b, 0x20, 0x01, 0x3a, 0x00, 0x00, 0x20, 0x02, 0x41, 0x07, + 0x49, 0x0d, 0x00, 0x20, 0x00, 0x20, 0x01, 0x3a, 0x00, 0x03, 0x20, 0x03, + 0x41, 0x04, 0x6b, 0x20, 0x01, 0x3a, 0x00, 0x00, 0x20, 0x02, 0x41, 0x09, + 0x49, 0x0d, 0x00, 0x20, 0x00, 0x41, 0x00, 0x20, 0x00, 0x6b, 0x41, 0x03, + 0x71, 0x22, 0x05, 0x6a, 0x22, 0x04, 0x20, 0x01, 0x41, 0xff, 0x01, 0x71, + 0x41, 0x81, 0x82, 0x84, 0x08, 0x6c, 0x22, 0x03, 0x36, 0x02, 0x00, 0x20, + 0x04, 0x20, 0x02, 0x20, 0x05, 0x6b, 0x41, 0x3c, 0x71, 0x22, 0x02, 0x6a, + 0x22, 0x01, 0x41, 0x04, 0x6b, 0x20, 0x03, 0x36, 0x02, 0x00, 0x20, 0x02, + 0x41, 0x09, 0x49, 0x0d, 0x00, 0x20, 0x04, 0x20, 0x03, 0x36, 0x02, 0x08, + 0x20, 0x04, 0x20, 0x03, 0x36, 0x02, 0x04, 0x20, 0x01, 0x41, 0x08, 0x6b, + 0x20, 0x03, 0x36, 0x02, 0x00, 0x20, 0x01, 0x41, 0x0c, 0x6b, 0x20, 0x03, + 0x36, 0x02, 0x00, 0x20, 0x02, 0x41, 0x19, 0x49, 0x0d, 0x00, 0x20, 0x04, + 0x20, 0x03, 0x36, 0x02, 0x18, 0x20, 0x04, 0x20, 0x03, 0x36, 0x02, 0x14, + 0x20, 0x04, 0x20, 0x03, 0x36, 0x02, 0x10, 0x20, 0x04, 0x20, 0x03, 0x36, + 0x02, 0x0c, 0x20, 0x01, 0x41, 0x10, 0x6b, 0x20, 0x03, 0x36, 0x02, 0x00, + 0x20, 0x01, 0x41, 0x14, 0x6b, 0x20, 0x03, 0x36, 0x02, 0x00, 0x20, 0x01, + 0x41, 0x18, 0x6b, 0x20, 0x03, 0x36, 0x02, 0x00, 0x20, 0x01, 0x41, 0x1c, + 0x6b, 0x20, 0x03, 0x36, 0x02, 0x00, 0x20, 0x02, 0x20, 0x04, 0x41, 0x04, + 0x71, 0x41, 0x18, 0x72, 0x22, 0x02, 0x6b, 0x22, 0x01, 0x41, 0x20, 0x49, + 0x0d, 0x00, 0x20, 0x03, 0xad, 0x42, 0x81, 0x80, 0x80, 0x80, 0x10, 0x7e, + 0x21, 0x06, 0x20, 0x02, 0x20, 0x04, 0x6a, 0x21, 0x02, 0x03, 0x40, 0x20, + 0x02, 0x20, 0x06, 0x37, 0x03, 0x00, 0x20, 0x02, 0x41, 0x18, 0x6a, 0x20, + 0x06, 0x37, 0x03, 0x00, 0x20, 0x02, 0x41, 0x10, 0x6a, 0x20, 0x06, 0x37, + 0x03, 0x00, 0x20, 0x02, 0x41, 0x08, 0x6a, 0x20, 0x06, 0x37, 0x03, 0x00, + 0x20, 0x02, 0x41, 0x20, 0x6a, 0x21, 0x02, 0x20, 0x01, 0x41, 0x20, 0x6b, + 0x22, 0x01, 0x41, 0x1f, 0x4b, 0x0d, 0x00, 0x0b, 0x0b, 0x20, 0x00, 0x0b, + 0x09, 0x00, 0x20, 0x00, 0x10, 0x13, 0x20, 0x00, 0x47, 0x0b, 0xd5, 0x07, + 0x01, 0x04, 0x7f, 0x02, 0x40, 0x02, 0x7f, 0x02, 0x40, 0x20, 0x02, 0x41, + 0x20, 0x4d, 0x04, 0x40, 0x20, 0x01, 0x41, 0x03, 0x71, 0x45, 0x20, 0x02, + 0x45, 0x72, 0x0d, 0x01, 0x20, 0x00, 0x20, 0x01, 0x2d, 0x00, 0x00, 0x3a, + 0x00, 0x00, 0x20, 0x00, 0x41, 0x01, 0x6a, 0x20, 0x01, 0x41, 0x01, 0x6a, + 0x22, 0x03, 0x41, 0x03, 0x71, 0x45, 0x20, 0x02, 0x41, 0x01, 0x6b, 0x22, + 0x05, 0x45, 0x72, 0x0d, 0x02, 0x1a, 0x20, 0x00, 0x20, 0x01, 0x2d, 0x00, + 0x01, 0x3a, 0x00, 0x01, 0x20, 0x00, 0x41, 0x02, 0x6a, 0x20, 0x01, 0x41, + 0x02, 0x6a, 0x22, 0x03, 0x41, 0x03, 0x71, 0x45, 0x20, 0x02, 0x41, 0x02, + 0x6b, 0x22, 0x05, 0x45, 0x72, 0x0d, 0x02, 0x1a, 0x20, 0x00, 0x20, 0x01, + 0x2d, 0x00, 0x02, 0x3a, 0x00, 0x02, 0x20, 0x00, 0x41, 0x03, 0x6a, 0x20, + 0x01, 0x41, 0x03, 0x6a, 0x22, 0x03, 0x41, 0x03, 0x71, 0x45, 0x20, 0x02, + 0x41, 0x03, 0x6b, 0x22, 0x05, 0x45, 0x72, 0x0d, 0x02, 0x1a, 0x20, 0x00, + 0x20, 0x01, 0x2d, 0x00, 0x03, 0x3a, 0x00, 0x03, 0x20, 0x02, 0x41, 0x04, + 0x6b, 0x21, 0x05, 0x20, 0x01, 0x41, 0x04, 0x6a, 0x21, 0x03, 0x20, 0x00, + 0x41, 0x04, 0x6a, 0x0c, 0x02, 0x0b, 0x20, 0x02, 0x45, 0x0d, 0x02, 0x20, + 0x00, 0x20, 0x01, 0x20, 0x02, 0xfc, 0x0a, 0x00, 0x00, 0x20, 0x00, 0x0f, + 0x0b, 0x20, 0x02, 0x21, 0x05, 0x20, 0x01, 0x21, 0x03, 0x20, 0x00, 0x0b, + 0x22, 0x04, 0x41, 0x03, 0x71, 0x22, 0x02, 0x45, 0x04, 0x40, 0x02, 0x40, + 0x20, 0x05, 0x41, 0x10, 0x49, 0x04, 0x40, 0x20, 0x05, 0x21, 0x02, 0x0c, + 0x01, 0x0b, 0x20, 0x05, 0x41, 0x10, 0x6b, 0x22, 0x02, 0x41, 0x10, 0x71, + 0x45, 0x04, 0x40, 0x20, 0x04, 0x20, 0x03, 0x29, 0x02, 0x00, 0x37, 0x02, + 0x00, 0x20, 0x04, 0x20, 0x03, 0x29, 0x02, 0x08, 0x37, 0x02, 0x08, 0x20, + 0x04, 0x41, 0x10, 0x6a, 0x21, 0x04, 0x20, 0x03, 0x41, 0x10, 0x6a, 0x21, + 0x03, 0x20, 0x02, 0x21, 0x05, 0x0b, 0x20, 0x02, 0x41, 0x10, 0x49, 0x0d, + 0x00, 0x20, 0x05, 0x21, 0x02, 0x03, 0x40, 0x20, 0x04, 0x20, 0x03, 0x29, + 0x02, 0x00, 0x37, 0x02, 0x00, 0x20, 0x04, 0x41, 0x08, 0x6a, 0x20, 0x03, + 0x41, 0x08, 0x6a, 0x29, 0x02, 0x00, 0x37, 0x02, 0x00, 0x20, 0x04, 0x41, + 0x10, 0x6a, 0x20, 0x03, 0x41, 0x10, 0x6a, 0x29, 0x02, 0x00, 0x37, 0x02, + 0x00, 0x20, 0x04, 0x41, 0x18, 0x6a, 0x20, 0x03, 0x41, 0x18, 0x6a, 0x29, + 0x02, 0x00, 0x37, 0x02, 0x00, 0x20, 0x04, 0x41, 0x20, 0x6a, 0x21, 0x04, + 0x20, 0x03, 0x41, 0x20, 0x6a, 0x21, 0x03, 0x20, 0x02, 0x41, 0x20, 0x6b, + 0x22, 0x02, 0x41, 0x0f, 0x4b, 0x0d, 0x00, 0x0b, 0x0b, 0x20, 0x02, 0x41, + 0x08, 0x4f, 0x04, 0x40, 0x20, 0x04, 0x20, 0x03, 0x29, 0x02, 0x00, 0x37, + 0x02, 0x00, 0x20, 0x04, 0x41, 0x08, 0x6a, 0x21, 0x04, 0x20, 0x03, 0x41, + 0x08, 0x6a, 0x21, 0x03, 0x0b, 0x20, 0x02, 0x41, 0x04, 0x71, 0x04, 0x40, + 0x20, 0x04, 0x20, 0x03, 0x28, 0x02, 0x00, 0x36, 0x02, 0x00, 0x20, 0x04, + 0x41, 0x04, 0x6a, 0x21, 0x04, 0x20, 0x03, 0x41, 0x04, 0x6a, 0x21, 0x03, + 0x0b, 0x20, 0x02, 0x41, 0x02, 0x71, 0x04, 0x40, 0x20, 0x04, 0x20, 0x03, + 0x2f, 0x00, 0x00, 0x3b, 0x00, 0x00, 0x20, 0x04, 0x41, 0x02, 0x6a, 0x21, + 0x04, 0x20, 0x03, 0x41, 0x02, 0x6a, 0x21, 0x03, 0x0b, 0x20, 0x02, 0x41, + 0x01, 0x71, 0x45, 0x0d, 0x01, 0x20, 0x04, 0x20, 0x03, 0x2d, 0x00, 0x00, + 0x3a, 0x00, 0x00, 0x20, 0x00, 0x0f, 0x0b, 0x02, 0x40, 0x02, 0x40, 0x02, + 0x7f, 0x02, 0x40, 0x20, 0x05, 0x41, 0x20, 0x4f, 0x04, 0x40, 0x20, 0x04, + 0x20, 0x03, 0x28, 0x02, 0x00, 0x22, 0x01, 0x3a, 0x00, 0x00, 0x02, 0x40, + 0x02, 0x40, 0x20, 0x02, 0x41, 0x02, 0x6b, 0x0e, 0x02, 0x00, 0x01, 0x03, + 0x0b, 0x20, 0x04, 0x20, 0x01, 0x41, 0x08, 0x76, 0x3a, 0x00, 0x01, 0x20, + 0x04, 0x20, 0x03, 0x41, 0x06, 0x6a, 0x29, 0x01, 0x00, 0x37, 0x02, 0x06, + 0x20, 0x04, 0x20, 0x03, 0x28, 0x02, 0x04, 0x41, 0x10, 0x74, 0x20, 0x01, + 0x41, 0x10, 0x76, 0x72, 0x36, 0x02, 0x02, 0x20, 0x03, 0x41, 0x12, 0x6a, + 0x21, 0x01, 0x41, 0x0e, 0x21, 0x06, 0x20, 0x03, 0x41, 0x0e, 0x6a, 0x28, + 0x01, 0x00, 0x21, 0x03, 0x41, 0x0e, 0x21, 0x05, 0x20, 0x04, 0x41, 0x12, + 0x6a, 0x0c, 0x03, 0x0b, 0x20, 0x04, 0x20, 0x03, 0x41, 0x05, 0x6a, 0x29, + 0x00, 0x00, 0x37, 0x02, 0x05, 0x20, 0x04, 0x20, 0x03, 0x28, 0x02, 0x04, + 0x41, 0x18, 0x74, 0x20, 0x01, 0x41, 0x08, 0x76, 0x72, 0x36, 0x02, 0x01, + 0x20, 0x03, 0x41, 0x11, 0x6a, 0x21, 0x01, 0x41, 0x0d, 0x21, 0x06, 0x20, + 0x03, 0x41, 0x0d, 0x6a, 0x28, 0x00, 0x00, 0x21, 0x03, 0x41, 0x0f, 0x21, + 0x05, 0x20, 0x04, 0x41, 0x11, 0x6a, 0x0c, 0x02, 0x0b, 0x02, 0x7f, 0x20, + 0x05, 0x41, 0x10, 0x49, 0x04, 0x40, 0x20, 0x04, 0x21, 0x02, 0x20, 0x03, + 0x0c, 0x01, 0x0b, 0x20, 0x04, 0x20, 0x03, 0x2d, 0x00, 0x00, 0x3a, 0x00, + 0x00, 0x20, 0x04, 0x20, 0x03, 0x28, 0x00, 0x01, 0x36, 0x00, 0x01, 0x20, + 0x04, 0x20, 0x03, 0x29, 0x00, 0x05, 0x37, 0x00, 0x05, 0x20, 0x04, 0x20, + 0x03, 0x2f, 0x00, 0x0d, 0x3b, 0x00, 0x0d, 0x20, 0x04, 0x20, 0x03, 0x2d, + 0x00, 0x0f, 0x3a, 0x00, 0x0f, 0x20, 0x04, 0x41, 0x10, 0x6a, 0x21, 0x02, + 0x20, 0x03, 0x41, 0x10, 0x6a, 0x0b, 0x21, 0x01, 0x20, 0x05, 0x41, 0x08, + 0x71, 0x0d, 0x02, 0x0c, 0x03, 0x0b, 0x20, 0x04, 0x20, 0x01, 0x41, 0x10, + 0x76, 0x3a, 0x00, 0x02, 0x20, 0x04, 0x20, 0x01, 0x41, 0x08, 0x76, 0x3a, + 0x00, 0x01, 0x20, 0x04, 0x20, 0x03, 0x41, 0x07, 0x6a, 0x29, 0x00, 0x00, + 0x37, 0x02, 0x07, 0x20, 0x04, 0x20, 0x03, 0x28, 0x02, 0x04, 0x41, 0x08, + 0x74, 0x20, 0x01, 0x41, 0x18, 0x76, 0x72, 0x36, 0x02, 0x03, 0x20, 0x03, + 0x41, 0x13, 0x6a, 0x21, 0x01, 0x41, 0x0f, 0x21, 0x06, 0x20, 0x03, 0x41, + 0x0f, 0x6a, 0x28, 0x00, 0x00, 0x21, 0x03, 0x41, 0x0d, 0x21, 0x05, 0x20, + 0x04, 0x41, 0x13, 0x6a, 0x0b, 0x21, 0x02, 0x20, 0x04, 0x20, 0x06, 0x6a, + 0x20, 0x03, 0x36, 0x02, 0x00, 0x0b, 0x20, 0x02, 0x20, 0x01, 0x29, 0x00, + 0x00, 0x37, 0x00, 0x00, 0x20, 0x02, 0x41, 0x08, 0x6a, 0x21, 0x02, 0x20, + 0x01, 0x41, 0x08, 0x6a, 0x21, 0x01, 0x0b, 0x20, 0x05, 0x41, 0x04, 0x71, + 0x04, 0x40, 0x20, 0x02, 0x20, 0x01, 0x28, 0x00, 0x00, 0x36, 0x00, 0x00, + 0x20, 0x02, 0x41, 0x04, 0x6a, 0x21, 0x02, 0x20, 0x01, 0x41, 0x04, 0x6a, + 0x21, 0x01, 0x0b, 0x20, 0x05, 0x41, 0x02, 0x71, 0x04, 0x40, 0x20, 0x02, + 0x20, 0x01, 0x2f, 0x00, 0x00, 0x3b, 0x00, 0x00, 0x20, 0x02, 0x41, 0x02, + 0x6a, 0x21, 0x02, 0x20, 0x01, 0x41, 0x02, 0x6a, 0x21, 0x01, 0x0b, 0x20, + 0x05, 0x41, 0x01, 0x71, 0x45, 0x0d, 0x00, 0x20, 0x02, 0x20, 0x01, 0x2d, + 0x00, 0x00, 0x3a, 0x00, 0x00, 0x0b, 0x20, 0x00, 0x0b, 0x0d, 0x00, 0x20, + 0x00, 0x41, 0x20, 0x46, 0x20, 0x00, 0x41, 0x09, 0x46, 0x72, 0x0b, 0x0a, + 0x00, 0x20, 0x00, 0x41, 0x30, 0x6b, 0x41, 0x0a, 0x49, 0x0b, 0x49, 0x01, + 0x02, 0x7f, 0x20, 0x00, 0x10, 0x0c, 0x20, 0x00, 0x6a, 0x21, 0x03, 0x02, + 0x40, 0x20, 0x02, 0x45, 0x0d, 0x00, 0x03, 0x40, 0x20, 0x01, 0x2d, 0x00, + 0x00, 0x22, 0x04, 0x45, 0x0d, 0x01, 0x20, 0x03, 0x20, 0x04, 0x3a, 0x00, + 0x00, 0x20, 0x03, 0x41, 0x01, 0x6a, 0x21, 0x03, 0x20, 0x01, 0x41, 0x01, + 0x6a, 0x21, 0x01, 0x20, 0x02, 0x41, 0x01, 0x6b, 0x22, 0x02, 0x0d, 0x00, + 0x0b, 0x0b, 0x20, 0x03, 0x41, 0x00, 0x3a, 0x00, 0x00, 0x20, 0x00, 0x0b, + 0xeb, 0x03, 0x01, 0x04, 0x7f, 0x02, 0x40, 0x02, 0x40, 0x02, 0x40, 0x02, + 0x40, 0x20, 0x00, 0x20, 0x01, 0x22, 0x03, 0x73, 0x41, 0x03, 0x71, 0x04, + 0x40, 0x20, 0x00, 0x21, 0x04, 0x0c, 0x01, 0x0b, 0x20, 0x02, 0x41, 0x00, + 0x47, 0x21, 0x06, 0x02, 0x40, 0x20, 0x03, 0x41, 0x03, 0x71, 0x45, 0x04, + 0x40, 0x20, 0x00, 0x21, 0x04, 0x0c, 0x01, 0x0b, 0x20, 0x02, 0x45, 0x04, + 0x40, 0x20, 0x00, 0x21, 0x04, 0x0c, 0x01, 0x0b, 0x20, 0x00, 0x20, 0x03, + 0x2d, 0x00, 0x00, 0x22, 0x01, 0x3a, 0x00, 0x00, 0x20, 0x01, 0x45, 0x04, + 0x40, 0x20, 0x00, 0x21, 0x04, 0x20, 0x02, 0x21, 0x01, 0x0c, 0x05, 0x0b, + 0x20, 0x00, 0x41, 0x01, 0x6a, 0x21, 0x04, 0x20, 0x02, 0x41, 0x01, 0x6b, + 0x22, 0x01, 0x41, 0x00, 0x47, 0x21, 0x06, 0x20, 0x03, 0x41, 0x01, 0x6a, + 0x22, 0x05, 0x41, 0x03, 0x71, 0x45, 0x20, 0x01, 0x45, 0x72, 0x45, 0x04, + 0x40, 0x20, 0x04, 0x20, 0x05, 0x2d, 0x00, 0x00, 0x22, 0x05, 0x3a, 0x00, + 0x00, 0x20, 0x05, 0x45, 0x0d, 0x05, 0x20, 0x00, 0x41, 0x02, 0x6a, 0x21, + 0x04, 0x20, 0x02, 0x41, 0x02, 0x6b, 0x22, 0x01, 0x41, 0x00, 0x47, 0x21, + 0x06, 0x20, 0x03, 0x41, 0x02, 0x6a, 0x22, 0x05, 0x41, 0x03, 0x71, 0x45, + 0x20, 0x01, 0x45, 0x72, 0x45, 0x04, 0x40, 0x20, 0x04, 0x20, 0x05, 0x2d, + 0x00, 0x00, 0x22, 0x05, 0x3a, 0x00, 0x00, 0x20, 0x05, 0x45, 0x0d, 0x06, + 0x20, 0x00, 0x41, 0x03, 0x6a, 0x21, 0x04, 0x20, 0x02, 0x41, 0x03, 0x6b, + 0x22, 0x01, 0x41, 0x00, 0x47, 0x21, 0x06, 0x20, 0x03, 0x41, 0x03, 0x6a, + 0x22, 0x05, 0x41, 0x03, 0x71, 0x45, 0x20, 0x01, 0x45, 0x72, 0x45, 0x04, + 0x40, 0x20, 0x04, 0x20, 0x05, 0x2d, 0x00, 0x00, 0x22, 0x05, 0x3a, 0x00, + 0x00, 0x20, 0x05, 0x45, 0x0d, 0x07, 0x20, 0x00, 0x41, 0x04, 0x6a, 0x21, + 0x04, 0x20, 0x03, 0x41, 0x04, 0x6a, 0x21, 0x03, 0x20, 0x02, 0x41, 0x04, + 0x6b, 0x22, 0x02, 0x41, 0x00, 0x47, 0x21, 0x06, 0x0c, 0x03, 0x0b, 0x20, + 0x05, 0x21, 0x03, 0x20, 0x01, 0x21, 0x02, 0x0c, 0x02, 0x0b, 0x20, 0x05, + 0x21, 0x03, 0x20, 0x01, 0x21, 0x02, 0x0c, 0x01, 0x0b, 0x20, 0x05, 0x21, + 0x03, 0x20, 0x01, 0x21, 0x02, 0x0b, 0x20, 0x06, 0x45, 0x0d, 0x02, 0x20, + 0x03, 0x2d, 0x00, 0x00, 0x45, 0x04, 0x40, 0x20, 0x02, 0x21, 0x01, 0x0c, + 0x04, 0x0b, 0x20, 0x02, 0x41, 0x04, 0x49, 0x0d, 0x00, 0x03, 0x40, 0x41, + 0x80, 0x82, 0x84, 0x08, 0x20, 0x03, 0x28, 0x02, 0x00, 0x22, 0x01, 0x6b, + 0x20, 0x01, 0x72, 0x41, 0x80, 0x81, 0x82, 0x84, 0x78, 0x71, 0x41, 0x80, + 0x81, 0x82, 0x84, 0x78, 0x47, 0x0d, 0x02, 0x20, 0x04, 0x20, 0x01, 0x36, + 0x02, 0x00, 0x20, 0x04, 0x41, 0x04, 0x6a, 0x21, 0x04, 0x20, 0x03, 0x41, + 0x04, 0x6a, 0x21, 0x03, 0x20, 0x02, 0x41, 0x04, 0x6b, 0x22, 0x02, 0x41, + 0x03, 0x4b, 0x0d, 0x00, 0x0b, 0x0b, 0x20, 0x02, 0x45, 0x0d, 0x01, 0x0b, + 0x03, 0x40, 0x20, 0x04, 0x20, 0x03, 0x2d, 0x00, 0x00, 0x22, 0x01, 0x3a, + 0x00, 0x00, 0x20, 0x01, 0x45, 0x04, 0x40, 0x20, 0x02, 0x21, 0x01, 0x0c, + 0x03, 0x0b, 0x20, 0x04, 0x41, 0x01, 0x6a, 0x21, 0x04, 0x20, 0x03, 0x41, + 0x01, 0x6a, 0x21, 0x03, 0x20, 0x02, 0x41, 0x01, 0x6b, 0x22, 0x02, 0x0d, + 0x00, 0x0b, 0x0b, 0x41, 0x00, 0x21, 0x01, 0x0b, 0x20, 0x01, 0x04, 0x40, + 0x20, 0x04, 0x41, 0x00, 0x20, 0x01, 0xfc, 0x0b, 0x00, 0x0b, 0x20, 0x00, + 0x0b, 0x17, 0x00, 0x20, 0x00, 0x41, 0x30, 0x6b, 0x41, 0x0a, 0x49, 0x20, + 0x00, 0x41, 0x20, 0x72, 0x41, 0xe1, 0x00, 0x6b, 0x41, 0x06, 0x49, 0x72, + 0x0b, 0x67, 0x01, 0x02, 0x7f, 0x20, 0x00, 0x45, 0x04, 0x40, 0x41, 0x00, + 0x0f, 0x0b, 0x02, 0x7f, 0x20, 0x00, 0x04, 0x40, 0x41, 0x8c, 0xc2, 0x04, + 0x21, 0x01, 0x03, 0x40, 0x20, 0x01, 0x41, 0x04, 0x6a, 0x22, 0x01, 0x28, + 0x02, 0x00, 0x22, 0x02, 0x41, 0x00, 0x20, 0x00, 0x20, 0x02, 0x47, 0x1b, + 0x0d, 0x00, 0x0b, 0x20, 0x01, 0x41, 0x00, 0x20, 0x02, 0x1b, 0x0c, 0x01, + 0x0b, 0x41, 0x00, 0x21, 0x00, 0x03, 0x40, 0x20, 0x00, 0x41, 0x90, 0xc2, + 0x04, 0x6a, 0x20, 0x00, 0x41, 0x04, 0x6a, 0x21, 0x00, 0x28, 0x02, 0x00, + 0x0d, 0x00, 0x0b, 0x20, 0x00, 0x41, 0x04, 0x6b, 0x41, 0x7c, 0x71, 0x41, + 0x90, 0xc2, 0x04, 0x6a, 0x0b, 0x41, 0x00, 0x47, 0x0b, 0x1d, 0x01, 0x01, + 0x7f, 0x41, 0x01, 0x21, 0x01, 0x20, 0x00, 0x41, 0x30, 0x6b, 0x41, 0x0a, + 0x4f, 0x04, 0x7f, 0x20, 0x00, 0x10, 0x0d, 0x41, 0x00, 0x47, 0x05, 0x20, + 0x01, 0x0b, 0x0b, 0x0b, 0xfc, 0x42, 0x02, 0x00, 0x41, 0x80, 0x80, 0x04, + 0x0b, 0xe8, 0x42, 0x12, 0x11, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, + 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x11, 0x22, 0x23, 0x24, + 0x11, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x11, 0x2d, 0x2e, + 0x2f, 0x10, 0x10, 0x30, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x31, + 0x32, 0x33, 0x10, 0x34, 0x35, 0x10, 0x10, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, - 0x11, 0x11, 0x11, 0x36, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x36, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, - 0x11, 0x37, 0x11, 0x11, 0x11, 0x11, 0x38, 0x11, 0x39, 0x3a, 0x3b, 0x3c, - 0x3d, 0x3e, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x37, 0x11, 0x11, 0x11, 0x11, 0x38, + 0x11, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, - 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x3f, 0x10, 0x10, + 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + 0x11, 0x11, 0x3f, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, - 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x11, 0x40, 0x41, 0x11, 0x42, - 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x11, 0x4b, 0x4c, 0x4d, - 0x4e, 0x4f, 0x50, 0x51, 0x10, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, - 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x10, 0x5e, 0x5f, 0x60, 0x10, 0x11, 0x11, - 0x11, 0x61, 0x62, 0x63, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, - 0x10, 0x10, 0x11, 0x11, 0x11, 0x11, 0x64, 0x10, 0x10, 0x10, 0x10, 0x10, - 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x11, 0x11, - 0x65, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, + 0x11, 0x40, 0x41, 0x11, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, + 0x4a, 0x11, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x10, 0x52, 0x53, + 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x10, 0x5e, + 0x5f, 0x60, 0x10, 0x11, 0x11, 0x11, 0x61, 0x62, 0x63, 0x10, 0x10, 0x10, + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x11, 0x11, 0x11, 0x11, 0x64, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, - 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x11, 0x11, - 0x66, 0x67, 0x10, 0x10, 0x68, 0x69, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + 0x10, 0x10, 0x10, 0x11, 0x11, 0x65, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, + 0x10, 0x10, 0x10, 0x11, 0x11, 0x66, 0x67, 0x10, 0x10, 0x68, 0x69, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, - 0x11, 0x11, 0x11, 0x11, 0x11, 0x6a, 0x11, 0x11, 0x6b, 0x10, 0x10, 0x10, + 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x6a, 0x11, + 0x11, 0x6b, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, - 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x11, 0x6c, - 0x6d, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x6e, 0x10, + 0x10, 0x10, 0x10, 0x11, 0x6c, 0x6d, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, + 0x10, 0x10, 0x10, 0x6e, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, - 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x6f, 0x70, - 0x71, 0x72, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x73, 0x74, - 0x75, 0x10, 0x10, 0x10, 0x10, 0x10, 0x76, 0x77, 0x10, 0x10, 0x10, 0x10, - 0x78, 0x10, 0x10, 0x79, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, - 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x10, 0x10, 0x10, 0x6f, 0x70, 0x71, 0x72, 0x10, 0x10, 0x10, 0x10, 0x10, + 0x10, 0x10, 0x10, 0x73, 0x74, 0x75, 0x10, 0x10, 0x10, 0x10, 0x10, 0x76, + 0x77, 0x10, 0x10, 0x10, 0x10, 0x78, 0x10, 0x10, 0x79, 0x10, 0x10, 0x10, + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfe, 0xff, 0xff, 0x07, 0xfe, 0xff, - 0xff, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x20, 0x04, 0xff, 0xff, - 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc3, 0xff, 0x03, 0x00, 0x1f, 0x50, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xdf, 0xbc, 0x40, 0xd7, 0xff, 0xff, 0xfb, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xbf, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfe, + 0xff, 0xff, 0x07, 0xfe, 0xff, 0xff, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x20, 0x04, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0x03, 0xfc, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc3, + 0xff, 0x03, 0x00, 0x1f, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdf, 0xbc, 0x40, + 0xd7, 0xff, 0xff, 0xfb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xbf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x03, 0xfc, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xfe, 0xff, 0xff, 0xff, 0x7f, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xbf, 0xb6, 0x00, 0xff, 0xff, 0xff, 0x87, - 0x07, 0x00, 0x00, 0x00, 0xff, 0x07, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xfe, 0xff, 0xc3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xef, 0x1f, 0xfe, 0xe1, 0xff, 0x9f, 0x00, 0x00, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0xe0, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x03, 0x00, 0xff, 0xff, - 0xff, 0xff, 0xff, 0x07, 0x30, 0x04, 0xff, 0xff, 0xff, 0xfc, 0xff, 0x1f, - 0x00, 0x00, 0xff, 0xff, 0xff, 0x01, 0xff, 0x07, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0xdf, 0x3f, 0x00, 0x00, 0xf0, 0xff, 0xf8, 0x03, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xef, 0xff, 0xdf, - 0xe1, 0xff, 0xcf, 0xff, 0xfe, 0xff, 0xef, 0x9f, 0xf9, 0xff, 0xff, 0xfd, - 0xc5, 0xe3, 0x9f, 0x59, 0x80, 0xb0, 0xcf, 0xff, 0x03, 0x10, 0xee, 0x87, - 0xf9, 0xff, 0xff, 0xfd, 0x6d, 0xc3, 0x87, 0x19, 0x02, 0x5e, 0xc0, 0xff, - 0x3f, 0x00, 0xee, 0xbf, 0xfb, 0xff, 0xff, 0xfd, 0xed, 0xe3, 0xbf, 0x1b, - 0x01, 0x00, 0xcf, 0xff, 0x00, 0x1e, 0xee, 0x9f, 0xf9, 0xff, 0xff, 0xfd, - 0xed, 0xe3, 0x9f, 0x19, 0xc0, 0xb0, 0xcf, 0xff, 0x02, 0x00, 0xec, 0xc7, - 0x3d, 0xd6, 0x18, 0xc7, 0xff, 0xc3, 0xc7, 0x1d, 0x81, 0x00, 0xc0, 0xff, - 0x00, 0x00, 0xef, 0xdf, 0xfd, 0xff, 0xff, 0xfd, 0xff, 0xe3, 0xdf, 0x1d, - 0x60, 0x07, 0xcf, 0xff, 0x00, 0x00, 0xef, 0xdf, 0xfd, 0xff, 0xff, 0xfd, - 0xef, 0xe3, 0xdf, 0x1d, 0x60, 0x40, 0xcf, 0xff, 0x06, 0x00, 0xef, 0xdf, - 0xfd, 0xff, 0xff, 0xff, 0xff, 0xe7, 0xdf, 0x5d, 0xf0, 0x80, 0xcf, 0xff, - 0x00, 0xfc, 0xec, 0xff, 0x7f, 0xfc, 0xff, 0xff, 0xfb, 0x2f, 0x7f, 0x80, - 0x5f, 0xff, 0xc0, 0xff, 0x0c, 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0xff, 0x07, 0x3f, 0x20, 0xff, 0x03, 0x00, 0x00, 0x00, 0x00, 0xd6, 0xf7, - 0xff, 0xff, 0xaf, 0xff, 0xff, 0x3b, 0x5f, 0x20, 0xff, 0xf3, 0x00, 0x00, - 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xff, 0x03, 0x00, 0x00, 0xff, 0xfe, - 0xff, 0xff, 0xff, 0x1f, 0xfe, 0xff, 0x03, 0xff, 0xff, 0xfe, 0xff, 0xff, - 0xff, 0x1f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0x7f, 0xf9, 0xff, 0x03, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xbf, 0x20, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xf7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0x3d, 0x7f, 0x3d, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3d, - 0xff, 0xff, 0xff, 0xff, 0x3d, 0x7f, 0x3d, 0xff, 0x7f, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0x3d, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0x07, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f, 0x3f, 0xfe, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xff, 0x7f, 0x02, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x00, 0xff, 0xbf, 0xb6, + 0x00, 0xff, 0xff, 0xff, 0x87, 0x07, 0x00, 0x00, 0x00, 0xff, 0x07, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xc3, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xef, 0x1f, 0xfe, + 0xe1, 0xff, 0x9f, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, + 0xe0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0x03, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0x07, 0x30, 0x04, 0xff, + 0xff, 0xff, 0xfc, 0xff, 0x1f, 0x00, 0x00, 0xff, 0xff, 0xff, 0x01, 0xff, + 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xdf, 0x3f, 0x00, + 0x00, 0xf0, 0xff, 0xf8, 0x03, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xef, 0xff, 0xdf, 0xe1, 0xff, 0xcf, 0xff, 0xfe, 0xff, 0xef, + 0x9f, 0xf9, 0xff, 0xff, 0xfd, 0xc5, 0xe3, 0x9f, 0x59, 0x80, 0xb0, 0xcf, + 0xff, 0x03, 0x10, 0xee, 0x87, 0xf9, 0xff, 0xff, 0xfd, 0x6d, 0xc3, 0x87, + 0x19, 0x02, 0x5e, 0xc0, 0xff, 0x3f, 0x00, 0xee, 0xbf, 0xfb, 0xff, 0xff, + 0xfd, 0xed, 0xe3, 0xbf, 0x1b, 0x01, 0x00, 0xcf, 0xff, 0x00, 0x1e, 0xee, + 0x9f, 0xf9, 0xff, 0xff, 0xfd, 0xed, 0xe3, 0x9f, 0x19, 0xc0, 0xb0, 0xcf, + 0xff, 0x02, 0x00, 0xec, 0xc7, 0x3d, 0xd6, 0x18, 0xc7, 0xff, 0xc3, 0xc7, + 0x1d, 0x81, 0x00, 0xc0, 0xff, 0x00, 0x00, 0xef, 0xdf, 0xfd, 0xff, 0xff, + 0xfd, 0xff, 0xe3, 0xdf, 0x1d, 0x60, 0x07, 0xcf, 0xff, 0x00, 0x00, 0xef, + 0xdf, 0xfd, 0xff, 0xff, 0xfd, 0xef, 0xe3, 0xdf, 0x1d, 0x60, 0x40, 0xcf, + 0xff, 0x06, 0x00, 0xef, 0xdf, 0xfd, 0xff, 0xff, 0xff, 0xff, 0xe7, 0xdf, + 0x5d, 0xf0, 0x80, 0xcf, 0xff, 0x00, 0xfc, 0xec, 0xff, 0x7f, 0xfc, 0xff, + 0xff, 0xfb, 0x2f, 0x7f, 0x80, 0x5f, 0xff, 0xc0, 0xff, 0x0c, 0x00, 0xfe, + 0xff, 0xff, 0xff, 0xff, 0x7f, 0xff, 0x07, 0x3f, 0x20, 0xff, 0x03, 0x00, + 0x00, 0x00, 0x00, 0xd6, 0xf7, 0xff, 0xff, 0xaf, 0xff, 0xff, 0x3b, 0x5f, + 0x20, 0xff, 0xf3, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xff, + 0x03, 0x00, 0x00, 0xff, 0xfe, 0xff, 0xff, 0xff, 0x1f, 0xfe, 0xff, 0x03, + 0xff, 0xff, 0xfe, 0xff, 0xff, 0xff, 0x1f, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0xf9, 0xff, + 0x03, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f, 0xff, + 0xff, 0xff, 0xff, 0xbf, 0x20, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf7, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3d, 0x7f, 0x3d, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x3d, 0xff, 0xff, 0xff, 0xff, 0x3d, 0x7f, 0x3d, + 0xff, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3d, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x07, 0x00, 0x00, 0x00, 0x00, 0xff, + 0xff, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0x3f, 0x3f, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x9f, 0xff, 0xff, 0xfe, 0xff, - 0xff, 0x07, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc7, - 0xff, 0x01, 0xff, 0xdf, 0x0f, 0x00, 0xff, 0xff, 0x0f, 0x00, 0xff, 0xff, - 0x0f, 0x00, 0xff, 0xdf, 0x0d, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xcf, 0xff, 0xff, 0x01, 0x80, 0x10, 0xff, 0x03, 0x00, 0x00, 0x00, 0x00, - 0xff, 0x03, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0x07, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0x3f, 0x00, 0xff, 0xff, 0xff, 0x7f, 0xff, 0x0f, - 0xff, 0x01, 0xc0, 0xff, 0xff, 0xff, 0xff, 0x3f, 0x1f, 0x00, 0xff, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0xff, 0xff, 0xff, 0x03, 0xff, 0x03, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0xff, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0x7f, 0xfe, 0xff, 0x1f, 0x00, 0xff, 0x03, 0xff, 0x03, 0x80, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xef, 0xff, 0xef, 0x0f, 0xff, 0x03, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf3, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xbf, 0xff, 0x03, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x7f, 0x00, 0xff, 0xe3, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f, 0xff, 0x01, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xe7, 0x00, 0x00, 0x00, 0x00, 0x00, 0xde, - 0x6f, 0x04, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x9f, 0xff, 0xff, 0xfe, 0xff, 0xff, 0x07, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xc7, 0xff, 0x01, 0xff, 0xdf, 0x0f, 0x00, 0xff, + 0xff, 0x0f, 0x00, 0xff, 0xff, 0x0f, 0x00, 0xff, 0xdf, 0x0d, 0x00, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xcf, 0xff, 0xff, 0x01, 0x80, 0x10, 0xff, + 0x03, 0x00, 0x00, 0x00, 0x00, 0xff, 0x03, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x07, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f, 0x00, 0xff, + 0xff, 0xff, 0x7f, 0xff, 0x0f, 0xff, 0x01, 0xc0, 0xff, 0xff, 0xff, 0xff, + 0x3f, 0x1f, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0f, 0xff, 0xff, 0xff, + 0x03, 0xff, 0x03, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x0f, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0xfe, 0xff, 0x1f, 0x00, 0xff, + 0x03, 0xff, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xef, 0xff, 0xef, + 0x0f, 0xff, 0x03, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xf3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xbf, 0xff, 0x03, 0x00, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0xff, 0xe3, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x3f, 0xff, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe7, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xde, 0x6f, 0x04, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x80, 0xff, 0x1f, 0x00, 0xff, 0xff, - 0x3f, 0x3f, 0xff, 0xff, 0xff, 0xff, 0x3f, 0x3f, 0xff, 0xaa, 0xff, 0xff, - 0xff, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xdf, 0x5f, 0xdc, 0x1f, - 0xcf, 0x0f, 0xff, 0x1f, 0xdc, 0x1f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x80, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x80, + 0xff, 0x1f, 0x00, 0xff, 0xff, 0x3f, 0x3f, 0xff, 0xff, 0xff, 0xff, 0x3f, + 0x3f, 0xff, 0xaa, 0xff, 0xff, 0xff, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xdf, 0x5f, 0xdc, 0x1f, 0xcf, 0x0f, 0xff, 0x1f, 0xdc, 0x1f, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x02, 0x80, 0x00, 0x00, 0xff, 0x1f, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x84, 0xfc, 0x2f, 0x3e, 0x50, + 0xbd, 0xff, 0xf3, 0xe0, 0x43, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xc0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x03, 0x00, 0x00, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0x1f, 0x78, 0x0c, 0x00, 0xff, 0xff, 0xff, 0xff, 0xbf, + 0x20, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0xff, + 0xff, 0x7f, 0x00, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0xff, + 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, + 0x00, 0x00, 0x00, 0xfe, 0x03, 0x3e, 0x1f, 0xfe, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0xe0, 0xfe, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf7, 0xe0, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x7f, 0x00, 0x00, 0xff, 0xff, 0xff, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x1f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x3f, 0xff, 0x1f, 0xff, 0xff, 0xff, 0x0f, 0x00, 0x00, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x7f, 0xf0, 0x8f, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, + 0x00, 0x80, 0xff, 0xfc, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xf9, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7c, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0xff, 0xbf, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x00, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x2f, 0x00, 0xff, 0x03, 0x00, + 0x00, 0xfc, 0xe8, 0xff, 0xff, 0xff, 0xff, 0xff, 0x07, 0xff, 0xff, 0xff, + 0xff, 0x07, 0x00, 0xff, 0xff, 0xff, 0x1f, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xf7, 0xff, 0x00, 0x80, 0xff, 0x03, 0xff, 0xff, 0xff, 0x7f, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0xff, 0x3f, 0xff, 0x03, 0xff, + 0xff, 0x7f, 0xfc, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x05, + 0x00, 0x00, 0x38, 0xff, 0xff, 0x3c, 0x00, 0x7e, 0x7e, 0x7e, 0x00, 0x7f, + 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf7, 0xff, 0x00, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x07, 0xff, 0x03, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x00, 0xff, 0xff, 0x7f, 0xf8, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0f, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x03, 0x00, 0x00, 0x00, 0x00, 0x7f, 0x00, 0xf8, 0xe0, 0xff, + 0xfd, 0x7f, 0x5f, 0xdb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x03, 0x00, 0x00, 0x00, 0xf8, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f, 0x00, + 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0x0f, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xdf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x1f, 0x00, 0x00, 0xff, 0x03, 0xfe, + 0xff, 0xff, 0x07, 0xfe, 0xff, 0xff, 0x07, 0xc0, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0xfc, 0xfc, 0xfc, 0x1c, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xef, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xb7, 0xff, + 0x3f, 0xff, 0x3f, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x07, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x1f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x84, 0xfc, 0x2f, 0x3e, 0x50, 0xbd, 0xff, 0xf3, 0xe0, 0x43, - 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0x03, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x1f, 0x78, - 0x0c, 0x00, 0xff, 0xff, 0xff, 0xff, 0xbf, 0x20, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0xff, 0xff, 0x7f, 0x00, 0x7f, 0x7f, - 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x00, 0x00, 0x00, 0xfe, 0x03, - 0x3e, 0x1f, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x7f, 0xe0, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xf7, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0xff, 0xff, - 0xff, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, + 0xff, 0xff, 0x1f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0xe0, 0xff, 0xff, 0xff, + 0x07, 0xff, 0xff, 0xff, 0xff, 0xff, 0x07, 0xff, 0xff, 0xff, 0x3f, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0xff, 0x3e, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f, 0xff, 0x03, 0xff, 0xff, 0xff, + 0xff, 0x0f, 0xff, 0xff, 0xff, 0xff, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0xff, + 0xff, 0x3f, 0x00, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, + 0xfd, 0xff, 0xff, 0xff, 0xff, 0xbf, 0x91, 0xff, 0xff, 0x3f, 0x00, 0xff, + 0xff, 0x7f, 0x00, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xff, 0x37, 0x00, 0xff, 0xff, 0x3f, 0x00, 0xff, + 0xff, 0xff, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x6f, 0xf0, 0xef, 0xfe, 0xff, 0xff, 0x3f, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x1f, 0xff, 0xff, 0xff, 0x1f, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xfe, 0xff, 0xff, 0x1f, 0x00, 0x00, 0x00, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f, 0x00, 0xff, 0xff, 0x3f, 0x00, 0xff, + 0xff, 0x07, 0x00, 0xff, 0xff, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x07, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0x07, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0xff, 0x03, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, + 0xff, 0xff, 0x1f, 0x80, 0x00, 0xff, 0xff, 0x3f, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xff, 0x7f, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0x3f, 0x00, 0x00, 0x00, 0xc0, 0xff, 0x00, 0x00, 0xfc, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff, 0xff, + 0x01, 0xff, 0x03, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc7, 0xff, 0x70, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x47, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0x1e, 0x00, 0xff, 0x17, 0x00, 0x00, 0x00, 0x00, 0xff, + 0xff, 0xfb, 0xff, 0xff, 0xff, 0x9f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x7f, 0xbd, 0xff, 0xbf, 0xff, 0x01, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x01, 0xff, 0x03, 0xef, 0x9f, 0xf9, 0xff, 0xff, + 0xfd, 0xed, 0xe3, 0x9f, 0x19, 0x81, 0xe0, 0x0f, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xbb, + 0x07, 0xff, 0x83, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xb3, 0x00, 0xff, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f, 0x7f, 0x00, + 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x7f, 0x11, 0x00, 0xff, 0x03, 0x00, 0x00, 0x00, 0x00, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f, 0x01, 0xff, 0x03, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xe7, 0xff, 0x07, 0xff, 0x03, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x03, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xfc, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, 0x1a, + 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe7, 0x7f, 0x00, + 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x20, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01, 0xff, + 0xfd, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x7f, 0x01, 0x00, 0xff, 0x03, 0x00, + 0x00, 0xfc, 0xff, 0xff, 0xff, 0xfc, 0xff, 0xff, 0xfe, 0x7f, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xfb, 0xff, 0xff, 0xff, + 0xff, 0x7f, 0xb4, 0xcb, 0x00, 0xff, 0x03, 0xbf, 0xfd, 0xff, 0xff, 0xff, + 0x7f, 0x7b, 0x01, 0xff, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7f, 0x00, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01, 0xff, + 0xff, 0xff, 0x7f, 0xff, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x3f, 0x00, 0x00, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x0f, 0x00, 0xff, 0x03, 0xf8, + 0xff, 0xff, 0xe0, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x87, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0xff, 0xff, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x1f, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f, 0xff, 0x1f, - 0xff, 0xff, 0xff, 0x0f, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0xf0, 0x8f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x80, 0xff, 0xfc, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf9, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x80, 0xff, 0xbf, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0x0f, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0x2f, 0x00, 0xff, 0x03, 0x00, 0x00, 0xfc, 0xe8, 0xff, 0xff, - 0xff, 0xff, 0xff, 0x07, 0xff, 0xff, 0xff, 0xff, 0x07, 0x00, 0xff, 0xff, - 0xff, 0x1f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf7, 0xff, 0x00, 0x80, - 0xff, 0x03, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x7f, 0x00, 0xff, 0x3f, 0xff, 0x03, 0xff, 0xff, 0x7f, 0xfc, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x05, 0x00, 0x00, 0x38, 0xff, 0xff, - 0x3c, 0x00, 0x7e, 0x7e, 0x7e, 0x00, 0x7f, 0x7f, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xf7, 0xff, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x07, 0xff, 0x03, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x00, 0xff, 0xff, 0x7f, 0xf8, - 0xff, 0xff, 0xff, 0xff, 0xff, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x7f, 0x00, 0xf8, 0xe0, 0xff, 0xfd, 0x7f, 0x5f, 0xdb, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x03, 0x00, 0x00, 0x00, 0xf8, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xfc, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0x0f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdf, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0x1f, 0x00, 0x00, 0xff, 0x03, 0xfe, 0xff, 0xff, 0x07, 0xfe, 0xff, - 0xff, 0x07, 0xc0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0x7f, 0xfc, 0xfc, 0xfc, 0x1c, 0x00, 0x00, 0x00, 0x00, 0xff, 0xef, - 0xff, 0xff, 0x7f, 0xff, 0xff, 0xb7, 0xff, 0x3f, 0xff, 0x3f, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x1f, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x1f, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, - 0xff, 0xff, 0x00, 0xe0, 0xff, 0xff, 0xff, 0x07, 0xff, 0xff, 0xff, 0xff, - 0xff, 0x07, 0xff, 0xff, 0xff, 0x3f, 0xff, 0xff, 0xff, 0xff, 0x0f, 0xff, - 0x3e, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0x3f, 0xff, 0x03, 0xff, 0xff, 0xff, 0xff, 0x0f, 0xff, 0xff, 0xff, - 0xff, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0x0f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0xff, 0xff, 0x3f, 0x00, 0xff, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0xfd, 0xff, 0xff, 0xff, 0xff, - 0xbf, 0x91, 0xff, 0xff, 0x3f, 0x00, 0xff, 0xff, 0x7f, 0x00, 0xff, 0xff, - 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, - 0x37, 0x00, 0xff, 0xff, 0x3f, 0x00, 0xff, 0xff, 0xff, 0x03, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6f, 0xf0, - 0xef, 0xfe, 0xff, 0xff, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, - 0xff, 0x1f, 0xff, 0xff, 0xff, 0x1f, 0x00, 0x00, 0x00, 0x00, 0xff, 0xfe, - 0xff, 0xff, 0x1f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x3f, 0x00, 0xff, 0xff, 0x3f, 0x00, 0xff, 0xff, 0x07, 0x00, 0xff, 0xff, - 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x07, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x07, 0x00, 0xff, 0xff, - 0xff, 0xff, 0xff, 0x00, 0xff, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x1f, 0x80, 0x00, - 0xff, 0xff, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, - 0x7f, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f, 0x00, - 0x00, 0x00, 0xc0, 0xff, 0x00, 0x00, 0xfc, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0x01, 0x00, 0x00, 0xff, 0xff, 0xff, 0x01, 0xff, 0x03, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xc7, 0xff, 0x70, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x47, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x1e, 0x00, - 0xff, 0x17, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xfb, 0xff, 0xff, 0xff, - 0x9f, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xbd, - 0xff, 0xbf, 0xff, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01, - 0xff, 0x03, 0xef, 0x9f, 0xf9, 0xff, 0xff, 0xfd, 0xed, 0xe3, 0x9f, 0x19, - 0x81, 0xe0, 0x0f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xbb, 0x07, 0xff, 0x83, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xb3, 0x00, - 0xff, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0x3f, 0x7f, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x11, 0x00, - 0xff, 0x03, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x3f, 0x01, 0xff, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, - 0xff, 0xe7, 0xff, 0x07, 0xff, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x03, - 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xfc, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, 0x1a, 0x00, 0x00, 0x00, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xe7, 0x7f, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0x20, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0x01, 0xff, 0xfd, 0xff, 0xff, 0xff, 0xff, - 0x7f, 0x7f, 0x01, 0x00, 0xff, 0x03, 0x00, 0x00, 0xfc, 0xff, 0xff, 0xff, - 0xfc, 0xff, 0xff, 0xfe, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x7f, 0xfb, 0xff, 0xff, 0xff, 0xff, 0x7f, 0xb4, 0xcb, 0x00, - 0xff, 0x03, 0xbf, 0xfd, 0xff, 0xff, 0xff, 0x7f, 0x7b, 0x01, 0xff, 0x03, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0x7f, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0x01, 0xff, 0xff, 0xff, 0x7f, 0xff, 0x03, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xff, 0xff, 0xff, 0x3f, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x00, 0x0f, 0x00, 0xff, 0x03, 0xf8, 0xff, 0xff, 0xe0, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x87, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0x80, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x07, 0x00, 0xff, 0xff, - 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x00, 0xf0, 0x00, + 0xff, 0x07, 0x00, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x07, 0x00, 0xf0, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0x07, 0xff, 0x1f, 0xff, 0x01, 0xff, 0x43, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xdf, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xdf, 0x64, 0xde, 0xff, 0xeb, 0xef, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xbf, 0xe7, 0xdf, 0xdf, 0xff, 0xff, - 0xff, 0x7b, 0x5f, 0xfc, 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x07, 0xff, 0x1f, 0xff, + 0x01, 0xff, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xdf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xdf, 0x64, + 0xde, 0xff, 0xeb, 0xef, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xbf, + 0xe7, 0xdf, 0xdf, 0xff, 0xff, 0xff, 0x7b, 0x5f, 0xfc, 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f, 0xff, - 0xff, 0xff, 0xfd, 0xff, 0xff, 0xf7, 0xff, 0xff, 0xff, 0xf7, 0xff, 0xff, - 0xdf, 0xff, 0xff, 0xff, 0xdf, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, - 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, 0xf7, 0xcf, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xf9, 0xdb, 0x07, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0x1f, 0x80, 0x3f, 0xff, 0x43, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0xff, 0x03, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x1f, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x8f, 0x08, - 0xff, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xef, 0xff, - 0xff, 0xff, 0x96, 0xfe, 0xf7, 0x0a, 0x84, 0xea, 0x96, 0xaa, 0x96, 0xf7, - 0xf7, 0x5e, 0xff, 0xfb, 0xff, 0x0f, 0xee, 0xfb, 0xff, 0x0f, 0x00, 0x00, + 0xff, 0xff, 0xff, 0x3f, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, 0xf7, 0xff, + 0xff, 0xff, 0xf7, 0xff, 0xff, 0xdf, 0xff, 0xff, 0xff, 0xdf, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, 0xff, + 0xfd, 0xff, 0xff, 0xf7, 0xcf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xf9, 0xdb, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xff, 0xff, 0xff, 0x03, 0xff, 0xff, 0xff, 0x03, 0xff, 0xff, 0xff, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x1f, 0x80, 0x3f, 0xff, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x56, 0x01, 0x00, 0x00, 0x39, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x20, - 0x00, 0x00, 0x00, 0xe0, 0xff, 0xff, 0x00, 0xbf, 0x1d, 0x00, 0x00, 0xe7, - 0x02, 0x00, 0x00, 0x79, 0x00, 0x00, 0x02, 0x24, 0x00, 0x00, 0x01, 0x01, - 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, - 0x00, 0x00, 0x00, 0xfe, 0xff, 0xff, 0x01, 0x39, 0xff, 0xff, 0x00, 0x18, - 0xff, 0xff, 0x01, 0x87, 0xff, 0xff, 0x00, 0xd4, 0xfe, 0xff, 0x00, 0xc3, - 0x00, 0x00, 0x01, 0xd2, 0x00, 0x00, 0x01, 0xce, 0x00, 0x00, 0x01, 0xcd, - 0x00, 0x00, 0x01, 0x4f, 0x00, 0x00, 0x01, 0xca, 0x00, 0x00, 0x01, 0xcb, - 0x00, 0x00, 0x01, 0xcf, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x01, 0xd3, - 0x00, 0x00, 0x01, 0xd1, 0x00, 0x00, 0x00, 0xa3, 0x00, 0x00, 0x01, 0xd5, - 0x00, 0x00, 0x00, 0x82, 0x00, 0x00, 0x01, 0xd6, 0x00, 0x00, 0x01, 0xda, - 0x00, 0x00, 0x01, 0xd9, 0x00, 0x00, 0x01, 0xdb, 0x00, 0x00, 0x00, 0x38, - 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0xb1, 0xff, 0xff, 0x01, 0x9f, - 0xff, 0xff, 0x01, 0xc8, 0xff, 0xff, 0x02, 0x28, 0x24, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x00, 0x33, - 0xff, 0xff, 0x00, 0x26, 0xff, 0xff, 0x01, 0x7e, 0xff, 0xff, 0x01, 0x2b, - 0x2a, 0x00, 0x01, 0x5d, 0xff, 0xff, 0x01, 0x28, 0x2a, 0x00, 0x00, 0x3f, - 0x2a, 0x00, 0x01, 0x3d, 0xff, 0xff, 0x01, 0x45, 0x00, 0x00, 0x01, 0x47, - 0x00, 0x00, 0x00, 0x1f, 0x2a, 0x00, 0x00, 0x1c, 0x2a, 0x00, 0x00, 0x1e, - 0x2a, 0x00, 0x00, 0x2e, 0xff, 0xff, 0x00, 0x32, 0xff, 0xff, 0x00, 0x36, - 0xff, 0xff, 0x00, 0x35, 0xff, 0xff, 0x00, 0x4f, 0xa5, 0x00, 0x00, 0x4b, - 0xa5, 0x00, 0x00, 0x31, 0xff, 0xff, 0x00, 0x28, 0xa5, 0x00, 0x00, 0x44, - 0xa5, 0x00, 0x00, 0x2f, 0xff, 0xff, 0x00, 0x2d, 0xff, 0xff, 0x00, 0xf7, - 0x29, 0x00, 0x00, 0x41, 0xa5, 0x00, 0x00, 0xfd, 0x29, 0x00, 0x00, 0x2b, - 0xff, 0xff, 0x00, 0x2a, 0xff, 0xff, 0x00, 0xe7, 0x29, 0x00, 0x00, 0x43, - 0xa5, 0x00, 0x00, 0x2a, 0xa5, 0x00, 0x00, 0xbb, 0xff, 0xff, 0x00, 0x27, - 0xff, 0xff, 0x00, 0xb9, 0xff, 0xff, 0x00, 0x25, 0xff, 0xff, 0x00, 0x15, - 0xa5, 0x00, 0x00, 0x12, 0xa5, 0x00, 0x02, 0x24, 0x4c, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x01, 0x20, 0x00, 0x00, 0x00, 0xe0, 0xff, 0xff, 0x01, 0x01, - 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x00, 0x54, 0x00, 0x00, 0x01, 0x74, - 0x00, 0x00, 0x01, 0x26, 0x00, 0x00, 0x01, 0x25, 0x00, 0x00, 0x01, 0x40, - 0x00, 0x00, 0x01, 0x3f, 0x00, 0x00, 0x00, 0xda, 0xff, 0xff, 0x00, 0xdb, - 0xff, 0xff, 0x00, 0xe1, 0xff, 0xff, 0x00, 0xc0, 0xff, 0xff, 0x00, 0xc1, - 0xff, 0xff, 0x01, 0x08, 0x00, 0x00, 0x00, 0xc2, 0xff, 0xff, 0x00, 0xc7, - 0xff, 0xff, 0x00, 0xd1, 0xff, 0xff, 0x00, 0xca, 0xff, 0xff, 0x00, 0xf8, - 0xff, 0xff, 0x00, 0xaa, 0xff, 0xff, 0x00, 0xb0, 0xff, 0xff, 0x00, 0x07, - 0x00, 0x00, 0x00, 0x8c, 0xff, 0xff, 0x01, 0xc4, 0xff, 0xff, 0x00, 0xa0, - 0xff, 0xff, 0x01, 0xf9, 0xff, 0xff, 0x02, 0x1a, 0x70, 0x00, 0x01, 0x01, - 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x01, 0x20, 0x00, 0x00, 0x00, 0xe0, - 0xff, 0xff, 0x01, 0x50, 0x00, 0x00, 0x01, 0x0f, 0x00, 0x00, 0x00, 0xf1, - 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x01, 0x30, 0x00, 0x00, 0x00, 0xd0, - 0xff, 0xff, 0x01, 0x01, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x00, 0x00, - 0x00, 0x00, 0x00, 0xc0, 0x0b, 0x00, 0x01, 0x60, 0x1c, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x01, 0xd0, 0x97, 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 0xf8, - 0xff, 0xff, 0x02, 0x05, 0x8a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x40, - 0xf4, 0xff, 0x00, 0x9e, 0xe7, 0xff, 0x00, 0xc2, 0x89, 0x00, 0x00, 0xdb, - 0xe7, 0xff, 0x00, 0x92, 0xe7, 0xff, 0x00, 0x93, 0xe7, 0xff, 0x00, 0x9c, - 0xe7, 0xff, 0x00, 0x9d, 0xe7, 0xff, 0x00, 0xa4, 0xe7, 0xff, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x38, 0x8a, 0x00, 0x00, 0x04, 0x8a, 0x00, 0x00, 0xe6, - 0x0e, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x00, 0x00, - 0x00, 0x00, 0x00, 0xc5, 0xff, 0xff, 0x01, 0x41, 0xe2, 0xff, 0x02, 0x1d, - 0x8f, 0x00, 0x00, 0x08, 0x00, 0x00, 0x01, 0xf8, 0xff, 0xff, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x56, 0x00, 0x00, 0x01, 0xaa, 0xff, 0xff, 0x00, 0x4a, - 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x70, - 0x00, 0x00, 0x00, 0x7e, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x01, 0xb6, - 0xff, 0xff, 0x01, 0xf7, 0xff, 0xff, 0x00, 0xdb, 0xe3, 0xff, 0x01, 0x9c, - 0xff, 0xff, 0x01, 0x90, 0xff, 0xff, 0x01, 0x80, 0xff, 0xff, 0x01, 0x82, - 0xff, 0xff, 0x02, 0x05, 0xac, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x10, - 0x00, 0x00, 0x00, 0xf0, 0xff, 0xff, 0x01, 0x1c, 0x00, 0x00, 0x01, 0x01, - 0x00, 0x00, 0x01, 0xa3, 0xe2, 0xff, 0x01, 0x41, 0xdf, 0xff, 0x01, 0xba, - 0xdf, 0xff, 0x00, 0xe4, 0xff, 0xff, 0x02, 0x0b, 0xb1, 0x00, 0x01, 0x01, - 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x01, 0x30, 0x00, 0x00, 0x00, 0xd0, - 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x01, 0x09, 0xd6, 0xff, 0x01, 0x1a, - 0xf1, 0xff, 0x01, 0x19, 0xd6, 0xff, 0x00, 0xd5, 0xd5, 0xff, 0x00, 0xd8, - 0xd5, 0xff, 0x01, 0xe4, 0xd5, 0xff, 0x01, 0x03, 0xd6, 0xff, 0x01, 0xe1, - 0xd5, 0xff, 0x01, 0xe2, 0xd5, 0xff, 0x01, 0xc1, 0xd5, 0xff, 0x00, 0x00, - 0x00, 0x00, 0x00, 0xa0, 0xe3, 0xff, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, - 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x02, 0x0c, 0xbc, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x01, 0xbc, - 0x5a, 0xff, 0x01, 0xa0, 0x03, 0x00, 0x01, 0xfc, 0x75, 0xff, 0x01, 0xd8, - 0x5a, 0xff, 0x00, 0x30, 0x00, 0x00, 0x01, 0xb1, 0x5a, 0xff, 0x01, 0xb5, - 0x5a, 0xff, 0x01, 0xbf, 0x5a, 0xff, 0x01, 0xee, 0x5a, 0xff, 0x01, 0xd6, - 0x5a, 0xff, 0x01, 0xeb, 0x5a, 0xff, 0x01, 0xd0, 0xff, 0xff, 0x01, 0xbd, - 0x5a, 0xff, 0x01, 0xc8, 0x75, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, - 0x68, 0xff, 0x00, 0x60, 0xfc, 0xff, 0x00, 0x00, 0x00, 0x00, 0x01, 0x20, - 0x00, 0x00, 0x00, 0xe0, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x01, 0x28, - 0x00, 0x00, 0x00, 0xd8, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x01, 0x40, - 0x00, 0x00, 0x00, 0xc0, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x01, 0x20, - 0x00, 0x00, 0x00, 0xe0, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x01, 0x20, - 0x00, 0x00, 0x00, 0xe0, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x01, 0x22, - 0x00, 0x00, 0x00, 0xde, 0xff, 0xff, 0x30, 0x0c, 0x31, 0x0d, 0x78, 0x0e, - 0x7f, 0x0f, 0x80, 0x10, 0x81, 0x11, 0x86, 0x12, 0x89, 0x13, 0x8a, 0x13, - 0x8e, 0x14, 0x8f, 0x15, 0x90, 0x16, 0x93, 0x13, 0x94, 0x17, 0x95, 0x18, - 0x96, 0x19, 0x97, 0x1a, 0x9a, 0x1b, 0x9c, 0x19, 0x9d, 0x1c, 0x9e, 0x1d, - 0x9f, 0x1e, 0xa6, 0x1f, 0xa9, 0x1f, 0xae, 0x1f, 0xb1, 0x20, 0xb2, 0x20, - 0xb7, 0x21, 0xbf, 0x22, 0xc5, 0x23, 0xc8, 0x23, 0xcb, 0x23, 0xdd, 0x24, - 0xf2, 0x23, 0xf6, 0x25, 0xf7, 0x26, 0x20, 0x2d, 0x3a, 0x2e, 0x3d, 0x2f, - 0x3e, 0x30, 0x3f, 0x31, 0x40, 0x31, 0x43, 0x32, 0x44, 0x33, 0x45, 0x34, - 0x50, 0x35, 0x51, 0x36, 0x52, 0x37, 0x53, 0x38, 0x54, 0x39, 0x59, 0x3a, - 0x5b, 0x3b, 0x5c, 0x3c, 0x61, 0x3d, 0x63, 0x3e, 0x65, 0x3f, 0x66, 0x40, - 0x68, 0x41, 0x69, 0x42, 0x6a, 0x40, 0x6b, 0x43, 0x6c, 0x44, 0x6f, 0x42, - 0x71, 0x45, 0x72, 0x46, 0x75, 0x47, 0x7d, 0x48, 0x82, 0x49, 0x87, 0x4a, - 0x89, 0x4b, 0x8a, 0x4c, 0x8b, 0x4c, 0x8c, 0x4d, 0x92, 0x4e, 0x9d, 0x4f, - 0x9e, 0x50, 0x45, 0x57, 0x7b, 0x1d, 0x7c, 0x1d, 0x7d, 0x1d, 0x7f, 0x58, - 0x86, 0x59, 0x88, 0x5a, 0x89, 0x5a, 0x8a, 0x5a, 0x8c, 0x5b, 0x8e, 0x5c, - 0x8f, 0x5c, 0xac, 0x5d, 0xad, 0x5e, 0xae, 0x5e, 0xaf, 0x5e, 0xc2, 0x5f, - 0xcc, 0x60, 0xcd, 0x61, 0xce, 0x61, 0xcf, 0x62, 0xd0, 0x63, 0xd1, 0x64, - 0xd5, 0x65, 0xd6, 0x66, 0xd7, 0x67, 0xf0, 0x68, 0xf1, 0x69, 0xf2, 0x6a, - 0xf3, 0x6b, 0xf4, 0x6c, 0xf5, 0x6d, 0xf9, 0x6e, 0xfd, 0x2d, 0xfe, 0x2d, - 0xff, 0x2d, 0x50, 0x69, 0x51, 0x69, 0x52, 0x69, 0x53, 0x69, 0x54, 0x69, - 0x55, 0x69, 0x56, 0x69, 0x57, 0x69, 0x58, 0x69, 0x59, 0x69, 0x5a, 0x69, - 0x5b, 0x69, 0x5c, 0x69, 0x5d, 0x69, 0x5e, 0x69, 0x5f, 0x69, 0x82, 0x00, - 0x83, 0x00, 0x84, 0x00, 0x85, 0x00, 0x86, 0x00, 0x87, 0x00, 0x88, 0x00, - 0x89, 0x00, 0xc0, 0x75, 0xcf, 0x76, 0x80, 0x89, 0x81, 0x8a, 0x82, 0x8b, - 0x85, 0x8c, 0x86, 0x8d, 0x70, 0x9d, 0x71, 0x9d, 0x76, 0x9e, 0x77, 0x9e, - 0x78, 0x9f, 0x79, 0x9f, 0x7a, 0xa0, 0x7b, 0xa0, 0x7c, 0xa1, 0x7d, 0xa1, - 0xb3, 0xa2, 0xba, 0xa3, 0xbb, 0xa3, 0xbc, 0xa4, 0xbe, 0xa5, 0xc3, 0xa2, - 0xcc, 0xa4, 0xda, 0xa6, 0xdb, 0xa6, 0xe5, 0x6a, 0xea, 0xa7, 0xeb, 0xa7, - 0xec, 0x6e, 0xf3, 0xa2, 0xf8, 0xa8, 0xf9, 0xa8, 0xfa, 0xa9, 0xfb, 0xa9, - 0xfc, 0xa4, 0x26, 0xb0, 0x2a, 0xb1, 0x2b, 0xb2, 0x4e, 0xb3, 0x84, 0x08, - 0x62, 0xba, 0x63, 0xbb, 0x64, 0xbc, 0x65, 0xbd, 0x66, 0xbe, 0x6d, 0xbf, - 0x6e, 0xc0, 0x6f, 0xc1, 0x70, 0xc2, 0x7e, 0xc3, 0x7f, 0xc3, 0x7d, 0xcf, - 0x8d, 0xd0, 0x94, 0xd1, 0xab, 0xd2, 0xac, 0xd3, 0xad, 0xd4, 0xb0, 0xd5, - 0xb1, 0xd6, 0xb2, 0xd7, 0xc4, 0xd8, 0xc5, 0xd9, 0xc6, 0xda, 0x07, 0x08, - 0x09, 0x0a, 0x0b, 0x0c, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, - 0x06, 0x06, 0x0d, 0x06, 0x06, 0x0e, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, - 0x06, 0x06, 0x0f, 0x10, 0x11, 0x12, 0x06, 0x13, 0x06, 0x06, 0x06, 0x06, - 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x14, 0x15, 0x06, 0x06, 0x06, 0x06, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0f, 0xff, 0x03, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x1f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0x8f, 0x08, 0xff, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xef, 0xff, 0xff, 0xff, 0x96, 0xfe, 0xf7, 0x0a, 0x84, + 0xea, 0x96, 0xaa, 0x96, 0xf7, 0xf7, 0x5e, 0xff, 0xfb, 0xff, 0x0f, 0xee, + 0xfb, 0xff, 0x0f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x03, 0xff, 0xff, 0xff, + 0x03, 0xff, 0xff, 0xff, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x56, + 0x01, 0x00, 0x00, 0x39, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x20, 0x00, 0x00, 0x00, 0xe0, 0xff, 0xff, 0x00, + 0xbf, 0x1d, 0x00, 0x00, 0xe7, 0x02, 0x00, 0x00, 0x79, 0x00, 0x00, 0x02, + 0x24, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x02, 0x00, 0x00, 0x00, 0xfe, 0xff, 0xff, 0x01, + 0x39, 0xff, 0xff, 0x00, 0x18, 0xff, 0xff, 0x01, 0x87, 0xff, 0xff, 0x00, + 0xd4, 0xfe, 0xff, 0x00, 0xc3, 0x00, 0x00, 0x01, 0xd2, 0x00, 0x00, 0x01, + 0xce, 0x00, 0x00, 0x01, 0xcd, 0x00, 0x00, 0x01, 0x4f, 0x00, 0x00, 0x01, + 0xca, 0x00, 0x00, 0x01, 0xcb, 0x00, 0x00, 0x01, 0xcf, 0x00, 0x00, 0x00, + 0x61, 0x00, 0x00, 0x01, 0xd3, 0x00, 0x00, 0x01, 0xd1, 0x00, 0x00, 0x00, + 0xa3, 0x00, 0x00, 0x01, 0xd5, 0x00, 0x00, 0x00, 0x82, 0x00, 0x00, 0x01, + 0xd6, 0x00, 0x00, 0x01, 0xda, 0x00, 0x00, 0x01, 0xd9, 0x00, 0x00, 0x01, + 0xdb, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, + 0xb1, 0xff, 0xff, 0x01, 0x9f, 0xff, 0xff, 0x01, 0xc8, 0xff, 0xff, 0x02, + 0x28, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0x00, 0x33, 0xff, 0xff, 0x00, 0x26, 0xff, 0xff, 0x01, + 0x7e, 0xff, 0xff, 0x01, 0x2b, 0x2a, 0x00, 0x01, 0x5d, 0xff, 0xff, 0x01, + 0x28, 0x2a, 0x00, 0x00, 0x3f, 0x2a, 0x00, 0x01, 0x3d, 0xff, 0xff, 0x01, + 0x45, 0x00, 0x00, 0x01, 0x47, 0x00, 0x00, 0x00, 0x1f, 0x2a, 0x00, 0x00, + 0x1c, 0x2a, 0x00, 0x00, 0x1e, 0x2a, 0x00, 0x00, 0x2e, 0xff, 0xff, 0x00, + 0x32, 0xff, 0xff, 0x00, 0x36, 0xff, 0xff, 0x00, 0x35, 0xff, 0xff, 0x00, + 0x4f, 0xa5, 0x00, 0x00, 0x4b, 0xa5, 0x00, 0x00, 0x31, 0xff, 0xff, 0x00, + 0x28, 0xa5, 0x00, 0x00, 0x44, 0xa5, 0x00, 0x00, 0x2f, 0xff, 0xff, 0x00, + 0x2d, 0xff, 0xff, 0x00, 0xf7, 0x29, 0x00, 0x00, 0x41, 0xa5, 0x00, 0x00, + 0xfd, 0x29, 0x00, 0x00, 0x2b, 0xff, 0xff, 0x00, 0x2a, 0xff, 0xff, 0x00, + 0xe7, 0x29, 0x00, 0x00, 0x43, 0xa5, 0x00, 0x00, 0x2a, 0xa5, 0x00, 0x00, + 0xbb, 0xff, 0xff, 0x00, 0x27, 0xff, 0xff, 0x00, 0xb9, 0xff, 0xff, 0x00, + 0x25, 0xff, 0xff, 0x00, 0x15, 0xa5, 0x00, 0x00, 0x12, 0xa5, 0x00, 0x02, + 0x24, 0x4c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x20, 0x00, 0x00, 0x00, + 0xe0, 0xff, 0xff, 0x01, 0x01, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x00, + 0x54, 0x00, 0x00, 0x01, 0x74, 0x00, 0x00, 0x01, 0x26, 0x00, 0x00, 0x01, + 0x25, 0x00, 0x00, 0x01, 0x40, 0x00, 0x00, 0x01, 0x3f, 0x00, 0x00, 0x00, + 0xda, 0xff, 0xff, 0x00, 0xdb, 0xff, 0xff, 0x00, 0xe1, 0xff, 0xff, 0x00, + 0xc0, 0xff, 0xff, 0x00, 0xc1, 0xff, 0xff, 0x01, 0x08, 0x00, 0x00, 0x00, + 0xc2, 0xff, 0xff, 0x00, 0xc7, 0xff, 0xff, 0x00, 0xd1, 0xff, 0xff, 0x00, + 0xca, 0xff, 0xff, 0x00, 0xf8, 0xff, 0xff, 0x00, 0xaa, 0xff, 0xff, 0x00, + 0xb0, 0xff, 0xff, 0x00, 0x07, 0x00, 0x00, 0x00, 0x8c, 0xff, 0xff, 0x01, + 0xc4, 0xff, 0xff, 0x00, 0xa0, 0xff, 0xff, 0x01, 0xf9, 0xff, 0xff, 0x02, + 0x1a, 0x70, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x01, + 0x20, 0x00, 0x00, 0x00, 0xe0, 0xff, 0xff, 0x01, 0x50, 0x00, 0x00, 0x01, + 0x0f, 0x00, 0x00, 0x00, 0xf1, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x30, 0x00, 0x00, 0x00, 0xd0, 0xff, 0xff, 0x01, 0x01, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x0b, 0x00, 0x01, + 0x60, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xd0, 0x97, 0x00, 0x01, + 0x08, 0x00, 0x00, 0x00, 0xf8, 0xff, 0xff, 0x02, 0x05, 0x8a, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x40, 0xf4, 0xff, 0x00, 0x9e, 0xe7, 0xff, 0x00, + 0xc2, 0x89, 0x00, 0x00, 0xdb, 0xe7, 0xff, 0x00, 0x92, 0xe7, 0xff, 0x00, + 0x93, 0xe7, 0xff, 0x00, 0x9c, 0xe7, 0xff, 0x00, 0x9d, 0xe7, 0xff, 0x00, + 0xa4, 0xe7, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x8a, 0x00, 0x00, + 0x04, 0x8a, 0x00, 0x00, 0xe6, 0x0e, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc5, 0xff, 0xff, 0x01, + 0x41, 0xe2, 0xff, 0x02, 0x1d, 0x8f, 0x00, 0x00, 0x08, 0x00, 0x00, 0x01, + 0xf8, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x56, 0x00, 0x00, 0x01, + 0xaa, 0xff, 0xff, 0x00, 0x4a, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, + 0x80, 0x00, 0x00, 0x00, 0x70, 0x00, 0x00, 0x00, 0x7e, 0x00, 0x00, 0x00, + 0x09, 0x00, 0x00, 0x01, 0xb6, 0xff, 0xff, 0x01, 0xf7, 0xff, 0xff, 0x00, + 0xdb, 0xe3, 0xff, 0x01, 0x9c, 0xff, 0xff, 0x01, 0x90, 0xff, 0xff, 0x01, + 0x80, 0xff, 0xff, 0x01, 0x82, 0xff, 0xff, 0x02, 0x05, 0xac, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x10, 0x00, 0x00, 0x00, 0xf0, 0xff, 0xff, 0x01, + 0x1c, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0xa3, 0xe2, 0xff, 0x01, + 0x41, 0xdf, 0xff, 0x01, 0xba, 0xdf, 0xff, 0x00, 0xe4, 0xff, 0xff, 0x02, + 0x0b, 0xb1, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x01, + 0x30, 0x00, 0x00, 0x00, 0xd0, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x09, 0xd6, 0xff, 0x01, 0x1a, 0xf1, 0xff, 0x01, 0x19, 0xd6, 0xff, 0x00, + 0xd5, 0xd5, 0xff, 0x00, 0xd8, 0xd5, 0xff, 0x01, 0xe4, 0xd5, 0xff, 0x01, + 0x03, 0xd6, 0xff, 0x01, 0xe1, 0xd5, 0xff, 0x01, 0xe2, 0xd5, 0xff, 0x01, + 0xc1, 0xd5, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0xe3, 0xff, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x02, + 0x0c, 0xbc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0x01, 0xbc, 0x5a, 0xff, 0x01, 0xa0, 0x03, 0x00, 0x01, + 0xfc, 0x75, 0xff, 0x01, 0xd8, 0x5a, 0xff, 0x00, 0x30, 0x00, 0x00, 0x01, + 0xb1, 0x5a, 0xff, 0x01, 0xb5, 0x5a, 0xff, 0x01, 0xbf, 0x5a, 0xff, 0x01, + 0xee, 0x5a, 0xff, 0x01, 0xd6, 0x5a, 0xff, 0x01, 0xeb, 0x5a, 0xff, 0x01, + 0xd0, 0xff, 0xff, 0x01, 0xbd, 0x5a, 0xff, 0x01, 0xc8, 0x75, 0xff, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x30, 0x68, 0xff, 0x00, 0x60, 0xfc, 0xff, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x20, 0x00, 0x00, 0x00, 0xe0, 0xff, 0xff, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x28, 0x00, 0x00, 0x00, 0xd8, 0xff, 0xff, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x40, 0x00, 0x00, 0x00, 0xc0, 0xff, 0xff, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x20, 0x00, 0x00, 0x00, 0xe0, 0xff, 0xff, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x20, 0x00, 0x00, 0x00, 0xe0, 0xff, 0xff, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x22, 0x00, 0x00, 0x00, 0xde, 0xff, 0xff, 0x30, + 0x0c, 0x31, 0x0d, 0x78, 0x0e, 0x7f, 0x0f, 0x80, 0x10, 0x81, 0x11, 0x86, + 0x12, 0x89, 0x13, 0x8a, 0x13, 0x8e, 0x14, 0x8f, 0x15, 0x90, 0x16, 0x93, + 0x13, 0x94, 0x17, 0x95, 0x18, 0x96, 0x19, 0x97, 0x1a, 0x9a, 0x1b, 0x9c, + 0x19, 0x9d, 0x1c, 0x9e, 0x1d, 0x9f, 0x1e, 0xa6, 0x1f, 0xa9, 0x1f, 0xae, + 0x1f, 0xb1, 0x20, 0xb2, 0x20, 0xb7, 0x21, 0xbf, 0x22, 0xc5, 0x23, 0xc8, + 0x23, 0xcb, 0x23, 0xdd, 0x24, 0xf2, 0x23, 0xf6, 0x25, 0xf7, 0x26, 0x20, + 0x2d, 0x3a, 0x2e, 0x3d, 0x2f, 0x3e, 0x30, 0x3f, 0x31, 0x40, 0x31, 0x43, + 0x32, 0x44, 0x33, 0x45, 0x34, 0x50, 0x35, 0x51, 0x36, 0x52, 0x37, 0x53, + 0x38, 0x54, 0x39, 0x59, 0x3a, 0x5b, 0x3b, 0x5c, 0x3c, 0x61, 0x3d, 0x63, + 0x3e, 0x65, 0x3f, 0x66, 0x40, 0x68, 0x41, 0x69, 0x42, 0x6a, 0x40, 0x6b, + 0x43, 0x6c, 0x44, 0x6f, 0x42, 0x71, 0x45, 0x72, 0x46, 0x75, 0x47, 0x7d, + 0x48, 0x82, 0x49, 0x87, 0x4a, 0x89, 0x4b, 0x8a, 0x4c, 0x8b, 0x4c, 0x8c, + 0x4d, 0x92, 0x4e, 0x9d, 0x4f, 0x9e, 0x50, 0x45, 0x57, 0x7b, 0x1d, 0x7c, + 0x1d, 0x7d, 0x1d, 0x7f, 0x58, 0x86, 0x59, 0x88, 0x5a, 0x89, 0x5a, 0x8a, + 0x5a, 0x8c, 0x5b, 0x8e, 0x5c, 0x8f, 0x5c, 0xac, 0x5d, 0xad, 0x5e, 0xae, + 0x5e, 0xaf, 0x5e, 0xc2, 0x5f, 0xcc, 0x60, 0xcd, 0x61, 0xce, 0x61, 0xcf, + 0x62, 0xd0, 0x63, 0xd1, 0x64, 0xd5, 0x65, 0xd6, 0x66, 0xd7, 0x67, 0xf0, + 0x68, 0xf1, 0x69, 0xf2, 0x6a, 0xf3, 0x6b, 0xf4, 0x6c, 0xf5, 0x6d, 0xf9, + 0x6e, 0xfd, 0x2d, 0xfe, 0x2d, 0xff, 0x2d, 0x50, 0x69, 0x51, 0x69, 0x52, + 0x69, 0x53, 0x69, 0x54, 0x69, 0x55, 0x69, 0x56, 0x69, 0x57, 0x69, 0x58, + 0x69, 0x59, 0x69, 0x5a, 0x69, 0x5b, 0x69, 0x5c, 0x69, 0x5d, 0x69, 0x5e, + 0x69, 0x5f, 0x69, 0x82, 0x00, 0x83, 0x00, 0x84, 0x00, 0x85, 0x00, 0x86, + 0x00, 0x87, 0x00, 0x88, 0x00, 0x89, 0x00, 0xc0, 0x75, 0xcf, 0x76, 0x80, + 0x89, 0x81, 0x8a, 0x82, 0x8b, 0x85, 0x8c, 0x86, 0x8d, 0x70, 0x9d, 0x71, + 0x9d, 0x76, 0x9e, 0x77, 0x9e, 0x78, 0x9f, 0x79, 0x9f, 0x7a, 0xa0, 0x7b, + 0xa0, 0x7c, 0xa1, 0x7d, 0xa1, 0xb3, 0xa2, 0xba, 0xa3, 0xbb, 0xa3, 0xbc, + 0xa4, 0xbe, 0xa5, 0xc3, 0xa2, 0xcc, 0xa4, 0xda, 0xa6, 0xdb, 0xa6, 0xe5, + 0x6a, 0xea, 0xa7, 0xeb, 0xa7, 0xec, 0x6e, 0xf3, 0xa2, 0xf8, 0xa8, 0xf9, + 0xa8, 0xfa, 0xa9, 0xfb, 0xa9, 0xfc, 0xa4, 0x26, 0xb0, 0x2a, 0xb1, 0x2b, + 0xb2, 0x4e, 0xb3, 0x84, 0x08, 0x62, 0xba, 0x63, 0xbb, 0x64, 0xbc, 0x65, + 0xbd, 0x66, 0xbe, 0x6d, 0xbf, 0x6e, 0xc0, 0x6f, 0xc1, 0x70, 0xc2, 0x7e, + 0xc3, 0x7f, 0xc3, 0x7d, 0xcf, 0x8d, 0xd0, 0x94, 0xd1, 0xab, 0xd2, 0xac, + 0xd3, 0xad, 0xd4, 0xb0, 0xd5, 0xb1, 0xd6, 0xb2, 0xd7, 0xc4, 0xd8, 0xc5, + 0xd9, 0xc6, 0xda, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x0d, 0x06, 0x06, 0x0e, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x0f, 0x10, 0x11, 0x12, 0x06, + 0x13, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x14, + 0x15, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, @@ -968,24 +946,23 @@ unsigned char STDLIB_WASM[] = { 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, - 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x16, 0x17, 0x06, 0x06, - 0x06, 0x18, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x16, 0x17, 0x06, 0x06, 0x06, 0x18, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, - 0x06, 0x19, 0x06, 0x06, 0x06, 0x06, 0x1a, 0x06, 0x06, 0x06, 0x06, 0x06, - 0x06, 0x06, 0x1b, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, - 0x06, 0x06, 0x1c, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x19, 0x06, 0x06, 0x06, 0x06, 0x1a, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x1b, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x1c, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, - 0x06, 0x06, 0x06, 0x06, 0x1d, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x1d, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, @@ -995,9 +972,9 @@ unsigned char STDLIB_WASM[] = { 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, - 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x1e, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, - 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1e, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -1006,297 +983,254 @@ unsigned char STDLIB_WASM[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0x2b, 0x2b, - 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x01, 0x00, 0x54, 0x56, 0x56, 0x56, - 0x56, 0x56, 0x56, 0x56, 0x56, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x18, 0x00, 0x00, 0x00, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x07, - 0x2b, 0x2b, 0x5b, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x4a, 0x56, - 0x56, 0x05, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, - 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x24, 0x50, 0x79, 0x31, 0x50, 0x31, - 0x50, 0x31, 0x38, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, - 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x4e, 0x31, 0x02, 0x4e, 0x0d, 0x0d, - 0x4e, 0x03, 0x4e, 0x00, 0x24, 0x6e, 0x00, 0x4e, 0x31, 0x26, 0x6e, 0x51, - 0x4e, 0x24, 0x50, 0x4e, 0x39, 0x14, 0x81, 0x1b, 0x1d, 0x1d, 0x53, 0x31, - 0x50, 0x31, 0x50, 0x0d, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x1b, 0x53, - 0x24, 0x50, 0x31, 0x02, 0x5c, 0x7b, 0x5c, 0x7b, 0x5c, 0x7b, 0x5c, 0x7b, - 0x5c, 0x7b, 0x14, 0x79, 0x5c, 0x7b, 0x5c, 0x7b, 0x5c, 0x2d, 0x2b, 0x49, - 0x03, 0x48, 0x03, 0x78, 0x5c, 0x7b, 0x14, 0x00, 0x96, 0x0a, 0x01, 0x2b, - 0x28, 0x06, 0x06, 0x00, 0x2a, 0x06, 0x2a, 0x2a, 0x2b, 0x07, 0xbb, 0xb5, - 0x2b, 0x1e, 0x00, 0x2b, 0x07, 0x2b, 0x2b, 0x2b, 0x01, 0x2b, 0x2b, 0x2b, + 0x00, 0x00, 0x24, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x01, + 0x00, 0x54, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x2b, 0x2b, 0x2b, + 0x2b, 0x2b, 0x2b, 0x2b, 0x07, 0x2b, 0x2b, 0x5b, 0x56, 0x56, 0x56, 0x56, + 0x56, 0x56, 0x56, 0x4a, 0x56, 0x56, 0x05, 0x31, 0x50, 0x31, 0x50, 0x31, + 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x24, + 0x50, 0x79, 0x31, 0x50, 0x31, 0x50, 0x31, 0x38, 0x50, 0x31, 0x50, 0x31, + 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x4e, + 0x31, 0x02, 0x4e, 0x0d, 0x0d, 0x4e, 0x03, 0x4e, 0x00, 0x24, 0x6e, 0x00, + 0x4e, 0x31, 0x26, 0x6e, 0x51, 0x4e, 0x24, 0x50, 0x4e, 0x39, 0x14, 0x81, + 0x1b, 0x1d, 0x1d, 0x53, 0x31, 0x50, 0x31, 0x50, 0x0d, 0x31, 0x50, 0x31, + 0x50, 0x31, 0x50, 0x1b, 0x53, 0x24, 0x50, 0x31, 0x02, 0x5c, 0x7b, 0x5c, + 0x7b, 0x5c, 0x7b, 0x5c, 0x7b, 0x5c, 0x7b, 0x14, 0x79, 0x5c, 0x7b, 0x5c, + 0x7b, 0x5c, 0x2d, 0x2b, 0x49, 0x03, 0x48, 0x03, 0x78, 0x5c, 0x7b, 0x14, + 0x00, 0x96, 0x0a, 0x01, 0x2b, 0x28, 0x06, 0x06, 0x00, 0x2a, 0x06, 0x2a, + 0x2a, 0x2b, 0x07, 0xbb, 0xb5, 0x2b, 0x1e, 0x00, 0x2b, 0x07, 0x2b, 0x2b, + 0x2b, 0x01, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, + 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, + 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x01, 0x2b, + 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, + 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2a, 0x2b, + 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, + 0xcd, 0x46, 0xcd, 0x2b, 0x00, 0x25, 0x2b, 0x07, 0x01, 0x06, 0x01, 0x55, + 0x56, 0x56, 0x56, 0x56, 0x56, 0x55, 0x56, 0x56, 0x02, 0x24, 0x81, 0x81, + 0x81, 0x81, 0x81, 0x15, 0x81, 0x81, 0x81, 0x00, 0x00, 0x2b, 0x00, 0xb2, + 0xd1, 0xb2, 0xd1, 0xb2, 0xd1, 0xb2, 0xd1, 0x00, 0x00, 0xcd, 0xcc, 0x01, + 0x00, 0xd7, 0xd7, 0xd7, 0xd7, 0xd7, 0x83, 0x81, 0x81, 0x81, 0x81, 0x81, + 0x81, 0x81, 0x81, 0x81, 0x81, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, + 0xac, 0xac, 0xac, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x31, 0x50, 0x31, + 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x02, 0x00, 0x00, 0x31, + 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, + 0x50, 0x31, 0x50, 0x31, 0x50, 0x4e, 0x31, 0x50, 0x31, 0x50, 0x4e, 0x31, + 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, + 0x50, 0x31, 0x02, 0x87, 0xa6, 0x87, 0xa6, 0x87, 0xa6, 0x87, 0xa6, 0x87, + 0xa6, 0x87, 0xa6, 0x87, 0xa6, 0x87, 0xa6, 0x2a, 0x2b, 0x2b, 0x2b, 0x2b, + 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x00, 0x00, 0x00, 0x54, + 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x54, 0x56, + 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x0c, + 0x00, 0x0c, 0x2a, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, + 0x2b, 0x2b, 0x2b, 0x2b, 0x07, 0x2a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x2a, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, + 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, + 0x2b, 0x2b, 0x2b, 0x56, 0x56, 0x6c, 0x81, 0x15, 0x00, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, - 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x01, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, - 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2a, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, - 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0xcd, 0x46, 0xcd, 0x2b, 0x00, - 0x25, 0x2b, 0x07, 0x01, 0x06, 0x01, 0x55, 0x56, 0x56, 0x56, 0x56, 0x56, - 0x55, 0x56, 0x56, 0x02, 0x24, 0x81, 0x81, 0x81, 0x81, 0x81, 0x15, 0x81, - 0x81, 0x81, 0x00, 0x00, 0x2b, 0x00, 0xb2, 0xd1, 0xb2, 0xd1, 0xb2, 0xd1, - 0xb2, 0xd1, 0x00, 0x00, 0xcd, 0xcc, 0x01, 0x00, 0xd7, 0xd7, 0xd7, 0xd7, - 0xd7, 0x83, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, - 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, 0x1c, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, - 0x31, 0x50, 0x31, 0x02, 0x00, 0x00, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, - 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, - 0x4e, 0x31, 0x50, 0x31, 0x50, 0x4e, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, - 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x02, 0x87, 0xa6, - 0x87, 0xa6, 0x87, 0xa6, 0x87, 0xa6, 0x87, 0xa6, 0x87, 0xa6, 0x87, 0xa6, - 0x87, 0xa6, 0x2a, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, - 0x2b, 0x2b, 0x2b, 0x00, 0x00, 0x00, 0x54, 0x56, 0x56, 0x56, 0x56, 0x56, + 0x2b, 0x2b, 0x2b, 0x07, 0x6c, 0x03, 0x41, 0x2b, 0x2b, 0x56, 0x56, 0x56, + 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x2c, + 0x56, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, + 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x0c, 0x6c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, + 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, + 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, + 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, + 0x25, 0x06, 0x25, 0x56, 0x7a, 0x9e, 0x26, 0x06, 0x25, 0x06, 0x25, 0x06, + 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, + 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, + 0x25, 0x06, 0x01, 0x2b, 0x2b, 0x4f, 0x56, 0x56, 0x2c, 0x2b, 0x7f, 0x56, + 0x56, 0x39, 0x2b, 0x2b, 0x55, 0x56, 0x56, 0x2b, 0x2b, 0x4f, 0x56, 0x56, + 0x2c, 0x2b, 0x7f, 0x56, 0x56, 0x81, 0x37, 0x75, 0x5b, 0x7b, 0x5c, 0x2b, + 0x2b, 0x4f, 0x56, 0x56, 0x02, 0xac, 0x04, 0x00, 0x00, 0x39, 0x2b, 0x2b, + 0x55, 0x56, 0x56, 0x2b, 0x2b, 0x4f, 0x56, 0x56, 0x2c, 0x2b, 0x2b, 0x56, + 0x56, 0x32, 0x13, 0x81, 0x57, 0x00, 0x6f, 0x81, 0x7e, 0xc9, 0xd7, 0x7e, + 0x2d, 0x81, 0x81, 0x0e, 0x7e, 0x39, 0x7f, 0x6f, 0x57, 0x00, 0x81, 0x81, + 0x7e, 0x15, 0x00, 0x7e, 0x03, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, + 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x07, 0x2b, 0x24, 0x2b, 0x97, 0x2b, 0x2b, + 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2a, 0x2b, 0x2b, 0x2b, 0x2b, + 0x2b, 0x56, 0x56, 0x56, 0x56, 0x56, 0x80, 0x81, 0x81, 0x81, 0x81, 0x39, + 0xbb, 0x2a, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, + 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, + 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, + 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x01, 0x81, 0x81, 0x81, 0x81, 0x81, + 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0xc9, 0xac, + 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, + 0xac, 0xac, 0xd0, 0x0d, 0x00, 0x4e, 0x31, 0x02, 0xb4, 0xc1, 0xc1, 0xd7, + 0xd7, 0x24, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, + 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, + 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0xd7, + 0xd7, 0x53, 0xc1, 0x47, 0xd4, 0xd7, 0xd7, 0xd7, 0x05, 0x2b, 0x2b, 0x2b, + 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x07, 0x01, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x31, 0x50, 0x31, + 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x0d, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, + 0x50, 0x31, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, + 0x79, 0x5c, 0x7b, 0x5c, 0x7b, 0x4f, 0x7b, 0x5c, 0x7b, 0x5c, 0x7b, 0x5c, + 0x7b, 0x5c, 0x7b, 0x5c, 0x7b, 0x5c, 0x7b, 0x5c, 0x7b, 0x5c, 0x7b, 0x5c, + 0x7b, 0x5c, 0x2d, 0x2b, 0x2b, 0x79, 0x14, 0x5c, 0x7b, 0x5c, 0x2d, 0x79, + 0x2a, 0x5c, 0x27, 0x5c, 0x7b, 0x5c, 0x7b, 0x5c, 0x7b, 0xa4, 0x00, 0x0a, + 0xb4, 0x5c, 0x7b, 0x5c, 0x7b, 0x4f, 0x03, 0x2a, 0x2b, 0x2b, 0x2b, 0x2b, + 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, + 0x2b, 0x2b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x2a, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, + 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, + 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, + 0x07, 0x00, 0x48, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, + 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x55, 0x56, 0x56, 0x56, + 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x0e, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, + 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x07, 0x00, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x24, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, + 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x07, 0x00, 0x00, 0x00, + 0x00, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, + 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x54, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, - 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x0c, 0x00, 0x0c, 0x2a, 0x2b, 0x2b, - 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x07, - 0x2a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x2a, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, + 0x2b, 0x2b, 0x2b, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, + 0x56, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a, 0x2b, + 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x56, 0x56, 0x56, + 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a, 0x2b, 0x2b, 0x2b, 0x2b, - 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, - 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x56, 0x56, - 0x6c, 0x81, 0x15, 0x00, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, - 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, - 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, - 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x07, 0x6c, - 0x03, 0x41, 0x2b, 0x2b, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, - 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x2c, 0x56, 0x2b, 0x2b, 0x2b, 0x2b, - 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, - 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, + 0x2b, 0x2b, 0x55, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, + 0x56, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x6c, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x25, 0x06, 0x25, - 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, - 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, - 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, - 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x56, 0x7a, - 0x9e, 0x26, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, - 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, - 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x25, 0x06, 0x01, 0x2b, 0x2b, - 0x4f, 0x56, 0x56, 0x2c, 0x2b, 0x7f, 0x56, 0x56, 0x39, 0x2b, 0x2b, 0x55, - 0x56, 0x56, 0x2b, 0x2b, 0x4f, 0x56, 0x56, 0x2c, 0x2b, 0x7f, 0x56, 0x56, - 0x81, 0x37, 0x75, 0x5b, 0x7b, 0x5c, 0x2b, 0x2b, 0x4f, 0x56, 0x56, 0x02, - 0xac, 0x04, 0x00, 0x00, 0x39, 0x2b, 0x2b, 0x55, 0x56, 0x56, 0x2b, 0x2b, - 0x4f, 0x56, 0x56, 0x2c, 0x2b, 0x2b, 0x56, 0x56, 0x32, 0x13, 0x81, 0x57, - 0x00, 0x6f, 0x81, 0x7e, 0xc9, 0xd7, 0x7e, 0x2d, 0x81, 0x81, 0x0e, 0x7e, - 0x39, 0x7f, 0x6f, 0x57, 0x00, 0x81, 0x81, 0x7e, 0x15, 0x00, 0x7e, 0x03, - 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, - 0x07, 0x2b, 0x24, 0x2b, 0x97, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, - 0x2b, 0x2b, 0x2a, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x56, 0x56, 0x56, 0x56, - 0x56, 0x80, 0x81, 0x81, 0x81, 0x81, 0x39, 0xbb, 0x2a, 0x2b, 0x2b, 0x2b, - 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, - 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, - 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, - 0x2b, 0x01, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, - 0x81, 0x81, 0x81, 0x81, 0x81, 0xc9, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, - 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, 0xd0, 0x0d, 0x00, - 0x4e, 0x31, 0x02, 0xb4, 0xc1, 0xc1, 0xd7, 0xd7, 0x24, 0x50, 0x31, 0x50, - 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, - 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, - 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0xd7, 0xd7, 0x53, 0xc1, 0x47, 0xd4, - 0xd7, 0xd7, 0xd7, 0x05, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, - 0x2b, 0x2b, 0x2b, 0x2b, 0x07, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x06, 0x27, 0x51, 0x6f, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x7c, 0x00, 0x00, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x83, 0x8e, 0x92, 0x97, 0x00, 0xaa, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb4, 0xc4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x4e, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, - 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x0d, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x24, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x31, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2b, 0x2b, 0x2b, 0x2b, - 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x79, 0x5c, 0x7b, 0x5c, 0x7b, - 0x4f, 0x7b, 0x5c, 0x7b, 0x5c, 0x7b, 0x5c, 0x7b, 0x5c, 0x7b, 0x5c, 0x7b, - 0x5c, 0x7b, 0x5c, 0x7b, 0x5c, 0x7b, 0x5c, 0x7b, 0x5c, 0x2d, 0x2b, 0x2b, - 0x79, 0x14, 0x5c, 0x7b, 0x5c, 0x2d, 0x79, 0x2a, 0x5c, 0x27, 0x5c, 0x7b, - 0x5c, 0x7b, 0x5c, 0x7b, 0xa4, 0x00, 0x0a, 0xb4, 0x5c, 0x7b, 0x5c, 0x7b, - 0x4f, 0x03, 0x2a, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, - 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a, - 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, - 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, - 0x2b, 0x2b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc6, 0xc9, 0x00, + 0x00, 0x00, 0xdb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2b, - 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x07, 0x00, 0x48, 0x56, 0x56, - 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, - 0x2b, 0x2b, 0x2b, 0x55, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, - 0x56, 0x56, 0x56, 0x56, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xde, 0x00, 0x00, 0x00, 0x00, 0xe1, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xe4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xe7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x24, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, - 0x07, 0x00, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, - 0x56, 0x56, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0x2b, - 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, - 0x2b, 0x2b, 0x2b, 0x07, 0x00, 0x00, 0x00, 0x00, 0x56, 0x56, 0x56, 0x56, - 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, - 0x56, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a, - 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x56, 0x56, - 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x0e, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xea, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, - 0x2b, 0x2b, 0x2b, 0x2b, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, - 0x56, 0x56, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2b, 0x2b, - 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x2b, 0x55, 0x56, 0x56, - 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x56, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x27, 0x51, 0x6f, 0x77, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0x00, - 0x00, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x83, 0x8e, - 0x92, 0x97, 0x00, 0xaa, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xb4, 0xc4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xed, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xc6, 0xc9, 0x00, 0x00, 0x00, 0xdb, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xde, 0x00, 0x00, - 0x00, 0x00, 0xe1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe4, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe7, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xea, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0xed, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x0a, 0x00, - 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x0c, 0x00, - 0x00, 0x00, 0x85, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x01, 0x20, - 0x00, 0x00, 0x02, 0x20, 0x00, 0x00, 0x03, 0x20, 0x00, 0x00, 0x04, 0x20, - 0x00, 0x00, 0x05, 0x20, 0x00, 0x00, 0x06, 0x20, 0x00, 0x00, 0x08, 0x20, - 0x00, 0x00, 0x09, 0x20, 0x00, 0x00, 0x0a, 0x20, 0x00, 0x00, 0x28, 0x20, - 0x00, 0x00, 0x29, 0x20, 0x00, 0x00, 0x5f, 0x20, 0x00, 0x00, 0x00, 0x30, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x92, 0x05, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x01, 0xc9, 0x04, 0x2c, 0x00, 0x2a, 0x5f, 0x5f, 0x69, 0x6d, - 0x70, 0x6f, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x77, 0x61, 0x73, 0x69, 0x5f, - 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x70, 0x72, 0x65, - 0x76, 0x69, 0x65, 0x77, 0x31, 0x5f, 0x61, 0x72, 0x67, 0x73, 0x5f, 0x67, - 0x65, 0x74, 0x01, 0x30, 0x5f, 0x5f, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, - 0x65, 0x64, 0x5f, 0x77, 0x61, 0x73, 0x69, 0x5f, 0x73, 0x6e, 0x61, 0x70, - 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x70, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, - 0x31, 0x5f, 0x61, 0x72, 0x67, 0x73, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x73, - 0x5f, 0x67, 0x65, 0x74, 0x02, 0x2b, 0x5f, 0x5f, 0x69, 0x6d, 0x70, 0x6f, - 0x72, 0x74, 0x65, 0x64, 0x5f, 0x77, 0x61, 0x73, 0x69, 0x5f, 0x73, 0x6e, - 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x70, 0x72, 0x65, 0x76, 0x69, - 0x65, 0x77, 0x31, 0x5f, 0x70, 0x72, 0x6f, 0x63, 0x5f, 0x65, 0x78, 0x69, - 0x74, 0x03, 0x11, 0x5f, 0x5f, 0x77, 0x61, 0x73, 0x6d, 0x5f, 0x63, 0x61, - 0x6c, 0x6c, 0x5f, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x04, 0x13, 0x75, 0x6e, - 0x64, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x5f, 0x77, 0x65, 0x61, 0x6b, - 0x3a, 0x6d, 0x61, 0x69, 0x6e, 0x05, 0x12, 0x5f, 0x5f, 0x77, 0x61, 0x73, - 0x6d, 0x5f, 0x69, 0x6e, 0x69, 0x74, 0x5f, 0x6d, 0x65, 0x6d, 0x6f, 0x72, - 0x79, 0x06, 0x06, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x07, 0x0a, 0x72, - 0x65, 0x73, 0x65, 0x74, 0x5f, 0x68, 0x65, 0x61, 0x70, 0x08, 0x06, 0x6d, - 0x61, 0x6c, 0x6c, 0x6f, 0x63, 0x09, 0x04, 0x66, 0x72, 0x65, 0x65, 0x0a, - 0x06, 0x63, 0x61, 0x6c, 0x6c, 0x6f, 0x63, 0x0b, 0x07, 0x72, 0x65, 0x61, - 0x6c, 0x6c, 0x6f, 0x63, 0x0c, 0x05, 0x5f, 0x45, 0x78, 0x69, 0x74, 0x0d, - 0x0b, 0x5f, 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x76, 0x6f, 0x69, 0x64, - 0x0e, 0x0f, 0x5f, 0x5f, 0x77, 0x61, 0x73, 0x69, 0x5f, 0x61, 0x72, 0x67, - 0x73, 0x5f, 0x67, 0x65, 0x74, 0x0f, 0x15, 0x5f, 0x5f, 0x77, 0x61, 0x73, - 0x69, 0x5f, 0x61, 0x72, 0x67, 0x73, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x73, - 0x5f, 0x67, 0x65, 0x74, 0x10, 0x10, 0x5f, 0x5f, 0x77, 0x61, 0x73, 0x69, - 0x5f, 0x70, 0x72, 0x6f, 0x63, 0x5f, 0x65, 0x78, 0x69, 0x74, 0x11, 0x05, - 0x64, 0x75, 0x6d, 0x6d, 0x79, 0x12, 0x11, 0x5f, 0x5f, 0x77, 0x61, 0x73, - 0x6d, 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x5f, 0x64, 0x74, 0x6f, 0x72, 0x73, - 0x13, 0x06, 0x6d, 0x65, 0x6d, 0x63, 0x70, 0x79, 0x14, 0x06, 0x6d, 0x65, - 0x6d, 0x73, 0x65, 0x74, 0x15, 0x06, 0x73, 0x74, 0x72, 0x6c, 0x65, 0x6e, - 0x16, 0x08, 0x69, 0x73, 0x77, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x17, 0x06, - 0x6d, 0x65, 0x6d, 0x63, 0x6d, 0x70, 0x18, 0x06, 0x6d, 0x65, 0x6d, 0x63, - 0x68, 0x72, 0x19, 0x06, 0x73, 0x74, 0x72, 0x63, 0x6d, 0x70, 0x1a, 0x08, - 0x74, 0x6f, 0x77, 0x6c, 0x6f, 0x77, 0x65, 0x72, 0x1b, 0x07, 0x63, 0x61, - 0x73, 0x65, 0x6d, 0x61, 0x70, 0x1c, 0x08, 0x74, 0x6f, 0x77, 0x75, 0x70, - 0x70, 0x65, 0x72, 0x1d, 0x07, 0x73, 0x74, 0x72, 0x6e, 0x63, 0x6d, 0x70, - 0x1e, 0x08, 0x69, 0x73, 0x77, 0x75, 0x70, 0x70, 0x65, 0x72, 0x1f, 0x07, - 0x6d, 0x65, 0x6d, 0x6d, 0x6f, 0x76, 0x65, 0x20, 0x08, 0x69, 0x73, 0x77, - 0x6c, 0x6f, 0x77, 0x65, 0x72, 0x21, 0x07, 0x69, 0x73, 0x62, 0x6c, 0x61, - 0x6e, 0x6b, 0x22, 0x08, 0x69, 0x73, 0x77, 0x62, 0x6c, 0x61, 0x6e, 0x6b, - 0x23, 0x08, 0x69, 0x73, 0x77, 0x64, 0x69, 0x67, 0x69, 0x74, 0x24, 0x07, - 0x73, 0x74, 0x72, 0x6e, 0x63, 0x61, 0x74, 0x25, 0x09, 0x5f, 0x5f, 0x73, - 0x74, 0x70, 0x6e, 0x63, 0x70, 0x79, 0x26, 0x07, 0x73, 0x74, 0x72, 0x6e, - 0x63, 0x70, 0x79, 0x27, 0x09, 0x69, 0x73, 0x77, 0x78, 0x64, 0x69, 0x67, - 0x69, 0x74, 0x28, 0x06, 0x77, 0x63, 0x73, 0x6c, 0x65, 0x6e, 0x29, 0x06, - 0x77, 0x63, 0x73, 0x63, 0x68, 0x72, 0x2a, 0x08, 0x69, 0x73, 0x77, 0x73, - 0x70, 0x61, 0x63, 0x65, 0x2b, 0x08, 0x69, 0x73, 0x77, 0x61, 0x6c, 0x6e, - 0x75, 0x6d, 0x07, 0x33, 0x02, 0x00, 0x0f, 0x5f, 0x5f, 0x73, 0x74, 0x61, - 0x63, 0x6b, 0x5f, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x01, 0x1f, - 0x47, 0x4f, 0x54, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x69, 0x6e, 0x74, - 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x5f, 0x5f, 0x6d, 0x65, 0x6d, 0x6f, - 0x72, 0x79, 0x5f, 0x62, 0x61, 0x73, 0x65, 0x09, 0x0a, 0x01, 0x00, 0x07, - 0x2e, 0x72, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x00, 0x76, 0x09, 0x70, 0x72, - 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x73, 0x01, 0x0c, 0x70, 0x72, 0x6f, - 0x63, 0x65, 0x73, 0x73, 0x65, 0x64, 0x2d, 0x62, 0x79, 0x01, 0x05, 0x63, - 0x6c, 0x61, 0x6e, 0x67, 0x56, 0x31, 0x37, 0x2e, 0x30, 0x2e, 0x36, 0x20, - 0x28, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6c, 0x6c, 0x76, 0x6d, - 0x2f, 0x6c, 0x6c, 0x76, 0x6d, 0x2d, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, - 0x74, 0x20, 0x36, 0x30, 0x30, 0x39, 0x37, 0x30, 0x38, 0x62, 0x34, 0x33, - 0x36, 0x37, 0x31, 0x37, 0x31, 0x63, 0x63, 0x64, 0x62, 0x66, 0x34, 0x62, - 0x35, 0x39, 0x30, 0x35, 0x63, 0x62, 0x36, 0x61, 0x38, 0x30, 0x33, 0x37, - 0x35, 0x33, 0x66, 0x65, 0x31, 0x38, 0x29, 0x00, 0x39, 0x0f, 0x74, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x5f, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x73, 0x03, 0x2b, 0x0b, 0x62, 0x75, 0x6c, 0x6b, 0x2d, 0x6d, 0x65, 0x6d, - 0x6f, 0x72, 0x79, 0x2b, 0x0f, 0x6d, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, - 0x2d, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x73, 0x2b, 0x08, 0x73, 0x69, - 0x67, 0x6e, 0x2d, 0x65, 0x78, 0x74 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x09, + 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x0b, + 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x85, 0x00, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x01, 0x20, 0x00, 0x00, 0x02, 0x20, 0x00, 0x00, 0x03, + 0x20, 0x00, 0x00, 0x04, 0x20, 0x00, 0x00, 0x05, 0x20, 0x00, 0x00, 0x06, + 0x20, 0x00, 0x00, 0x08, 0x20, 0x00, 0x00, 0x09, 0x20, 0x00, 0x00, 0x0a, + 0x20, 0x00, 0x00, 0x28, 0x20, 0x00, 0x00, 0x29, 0x20, 0x00, 0x00, 0x5f, + 0x20, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x41, 0xe8, 0xc2, 0x04, 0x0b, 0x04, 0x00, 0x00, 0x02, 0x00, 0x00, 0x8e, + 0x01, 0x09, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x73, 0x02, + 0x08, 0x6c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, 0x01, 0x03, 0x43, + 0x31, 0x31, 0x00, 0x0c, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, + 0x64, 0x2d, 0x62, 0x79, 0x01, 0x05, 0x63, 0x6c, 0x61, 0x6e, 0x67, 0x5f, + 0x32, 0x31, 0x2e, 0x31, 0x2e, 0x34, 0x2d, 0x77, 0x61, 0x73, 0x69, 0x2d, + 0x73, 0x64, 0x6b, 0x20, 0x28, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, + 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, + 0x6c, 0x6c, 0x76, 0x6d, 0x2f, 0x6c, 0x6c, 0x76, 0x6d, 0x2d, 0x70, 0x72, + 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x20, 0x32, 0x32, 0x32, 0x66, 0x63, 0x31, + 0x31, 0x66, 0x32, 0x62, 0x38, 0x66, 0x32, 0x35, 0x66, 0x36, 0x61, 0x30, + 0x66, 0x34, 0x39, 0x37, 0x36, 0x32, 0x37, 0x32, 0x65, 0x66, 0x31, 0x62, + 0x62, 0x37, 0x62, 0x66, 0x34, 0x39, 0x35, 0x32, 0x31, 0x64, 0x29, 0x00, + 0xa4, 0x01, 0x0f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x66, 0x65, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x09, 0x2b, 0x0f, 0x6d, 0x75, 0x74, + 0x61, 0x62, 0x6c, 0x65, 0x2d, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x73, + 0x2b, 0x13, 0x6e, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x70, 0x70, 0x69, 0x6e, + 0x67, 0x2d, 0x66, 0x70, 0x74, 0x6f, 0x69, 0x6e, 0x74, 0x2b, 0x0b, 0x62, + 0x75, 0x6c, 0x6b, 0x2d, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x2b, 0x08, + 0x73, 0x69, 0x67, 0x6e, 0x2d, 0x65, 0x78, 0x74, 0x2b, 0x0f, 0x72, 0x65, + 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2d, 0x74, 0x79, 0x70, 0x65, + 0x73, 0x2b, 0x0a, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x2b, 0x0e, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x65, 0x64, 0x2d, + 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x2b, 0x0f, 0x62, 0x75, 0x6c, 0x6b, 0x2d, + 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x2d, 0x6f, 0x70, 0x74, 0x2b, 0x16, + 0x63, 0x61, 0x6c, 0x6c, 0x2d, 0x69, 0x6e, 0x64, 0x69, 0x72, 0x65, 0x63, + 0x74, 0x2d, 0x6f, 0x76, 0x65, 0x72, 0x6c, 0x6f, 0x6e, 0x67 }; -unsigned int STDLIB_WASM_LEN = 15582; +unsigned int STDLIB_WASM_LEN = 14794; diff --git a/lib/src/wasm_store.c b/lib/src/wasm_store.c index 98cc5420..cd94f6a0 100644 --- a/lib/src/wasm_store.c +++ b/lib/src/wasm_store.c @@ -32,7 +32,7 @@ const char *STDLIB_SYMBOLS[] = { #include "./stdlib-symbols.txt" }; -// The contents of the `dylink.0` custom section of a wasm module, +// The contents of the `dylink.0` custom section of a Wasm module, // as specified by the current WebAssembly dynamic linking ABI proposal. typedef struct { uint32_t memory_size; @@ -43,15 +43,15 @@ typedef struct { // WasmLanguageId - A pointer used to identify a language. This language id is // reference-counted, so that its ownership can be shared between the language -// itself and the instances of the language that are held in wasm stores. +// itself and the instances of the language that are held in Wasm stores. typedef struct { volatile uint32_t ref_count; volatile uint32_t is_language_deleted; } WasmLanguageId; -// LanguageWasmModule - Additional data associated with a wasm-backed +// LanguageWasmModule - Additional data associated with a Wasm-backed // `TSLanguage`. This data is read-only and does not reference a particular -// wasm store, so it can be shared by all users of a `TSLanguage`. A pointer to +// Wasm store, so it can be shared by all users of a `TSLanguage`. A pointer to // this is stored on the language itself. typedef struct { volatile uint32_t ref_count; @@ -64,7 +64,7 @@ typedef struct { } LanguageWasmModule; // LanguageWasmInstance - Additional data associated with an instantiation of -// a `TSLanguage` in a particular wasm store. The wasm store holds one of +// a `TSLanguage` in a particular Wasm store. The Wasm store holds one of // these structs for each language that it has instantiated. typedef struct { WasmLanguageId *language_id; @@ -91,7 +91,7 @@ typedef struct { uint32_t args_sizes_get; } BuiltinFunctionIndices; -// TSWasmStore - A struct that allows a given `Parser` to use wasm-backed +// TSWasmStore - A struct that allows a given `Parser` to use Wasm-backed // languages. This struct is mutable, and can only be used by one parser at a // time. struct TSWasmStore { @@ -115,9 +115,9 @@ struct TSWasmStore { typedef Array(char) StringData; // LanguageInWasmMemory - The memory layout of a `TSLanguage` when compiled to -// wasm32. This is used to copy static language data out of the wasm memory. +// wasm32. This is used to copy static language data out of the Wasm memory. typedef struct { - uint32_t version; + uint32_t abi_version; uint32_t symbol_count; uint32_t alias_count; uint32_t token_count; @@ -153,10 +153,18 @@ typedef struct { int32_t deserialize; } external_scanner; int32_t primary_state_ids; + int32_t name; + int32_t reserved_words; + uint16_t max_reserved_word_set_size; + uint32_t supertype_count; + int32_t supertype_symbols; + int32_t supertype_map_slices; + int32_t supertype_map_entries; + TSLanguageMetadata metadata; } LanguageInWasmMemory; // LexerInWasmMemory - The memory layout of a `TSLexer` when compiled to wasm32. -// This is used to copy mutable lexing state in and out of the wasm memory. +// This is used to copy mutable lexing state in and out of the Wasm memory. typedef struct { int32_t lookahead; TSSymbol result_symbol; @@ -244,7 +252,7 @@ static bool wasm_dylink_info__parse( } /******************************************* - * Native callbacks exposed to wasm modules + * Native callbacks exposed to Wasm modules *******************************************/ static wasm_trap_t *callback__abort( @@ -253,7 +261,7 @@ static bool wasm_dylink_info__parse( wasmtime_val_raw_t *args_and_results, size_t args_and_results_len ) { - return wasmtime_trap_new("wasm module called abort", 24); + return wasmtime_trap_new("Wasm module called abort", 24); } static wasm_trap_t *callback__debug_message( @@ -414,6 +422,17 @@ static void *copy_strings( return result; } +static void *copy_string( + const uint8_t *data, + int32_t address +) { + const char *string = (const char *)&data[address]; + size_t len = strlen(string); + char *result = ts_malloc(len + 1); + memcpy(result, string, len + 1); + return result; +} + static bool name_eq(const wasm_name_t *name, const char *string) { return strncmp(string, name->data, name->size) == 0; } @@ -635,7 +654,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) { }, }; - // Create all of the wasm functions. + // Create all of the Wasm functions. unsigned builtin_definitions_len = array_len(builtin_definitions); unsigned lexer_definitions_len = array_len(lexer_definitions); for (unsigned i = 0; i < builtin_definitions_len; i++) { @@ -660,7 +679,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) { wasm_error->kind = TSWasmErrorKindCompile; format( &wasm_error->message, - "failed to compile wasm stdlib: %.*s", + "failed to compile Wasm stdlib: %.*s", (int)message.size, message.data ); goto error; @@ -683,7 +702,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) { wasm_error->kind = TSWasmErrorKindCompile; format( &wasm_error->message, - "wasm stdlib is missing the 'memory' import" + "Wasm stdlib is missing the 'memory' import" ); goto error; } @@ -699,7 +718,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) { wasm_error->kind = TSWasmErrorKindAllocate; format( &wasm_error->message, - "failed to allocate wasm memory: %.*s", + "failed to allocate Wasm memory: %.*s", (int)message.size, message.data ); goto error; @@ -718,7 +737,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) { wasm_error->kind = TSWasmErrorKindAllocate; format( &wasm_error->message, - "failed to allocate wasm table: %.*s", + "failed to allocate Wasm table: %.*s", (int)message.size, message.data ); goto error; @@ -735,6 +754,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) { wasmtime_val_t stack_pointer_value = WASM_I32_VAL(0); wasmtime_global_t stack_pointer_global; error = wasmtime_global_new(context, var_i32_type, &stack_pointer_value, &stack_pointer_global); + wasm_globaltype_delete(var_i32_type); ts_assert(!error); *self = (TSWasmStore) { @@ -760,7 +780,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) { wasm_error->kind = TSWasmErrorKindInstantiate; format( &wasm_error->message, - "unexpected import in wasm stdlib: %.*s\n", + "unexpected import in Wasm stdlib: %.*s\n", (int)import_name->size, import_name->data ); goto error; @@ -777,7 +797,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) { wasm_error->kind = TSWasmErrorKindInstantiate; format( &wasm_error->message, - "failed to instantiate wasm stdlib module: %.*s", + "failed to instantiate Wasm stdlib module: %.*s", (int)message.size, message.data ); goto error; @@ -787,7 +807,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) { wasm_error->kind = TSWasmErrorKindInstantiate; format( &wasm_error->message, - "trapped when instantiating wasm stdlib module: %.*s", + "trapped when instantiating Wasm stdlib module: %.*s", (int)message.size, message.data ); goto error; @@ -848,7 +868,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) { wasm_error->kind = TSWasmErrorKindInstantiate; format( &wasm_error->message, - "missing malloc reset function in wasm stdlib" + "missing malloc reset function in Wasm stdlib" ); goto error; } @@ -858,7 +878,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) { wasm_error->kind = TSWasmErrorKindInstantiate; format( &wasm_error->message, - "missing exported symbol in wasm stdlib: %s", + "missing exported symbol in Wasm stdlib: %s", STDLIB_SYMBOLS[i] ); goto error; @@ -877,7 +897,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) { wasm_error->kind = TSWasmErrorKindAllocate; format( &wasm_error->message, - "failed to grow wasm table to initial size: %.*s", + "failed to grow Wasm table to initial size: %.*s", (int)message.size, message.data ); goto error; @@ -927,7 +947,7 @@ void ts_wasm_store_delete(TSWasmStore *self) { wasmtime_store_delete(self->store); wasm_engine_delete(self->engine); for (unsigned i = 0; i < self->language_instances.size; i++) { - LanguageWasmInstance *instance = &self->language_instances.contents[i]; + LanguageWasmInstance *instance = array_get(&self->language_instances, i); language_id_delete(instance->language_id); } array_delete(&self->language_instances); @@ -937,7 +957,7 @@ void ts_wasm_store_delete(TSWasmStore *self) { size_t ts_wasm_store_language_count(const TSWasmStore *self) { size_t result = 0; for (unsigned i = 0; i < self->language_instances.size; i++) { - const WasmLanguageId *id = self->language_instances.contents[i].language_id; + const WasmLanguageId *id = array_get(&self->language_instances, i)->language_id; if (!id->is_language_deleted) { result++; } @@ -1044,7 +1064,7 @@ static bool ts_wasm_store__instantiate( wasmtime_error_message(error, &message); format( error_message, - "error instantiating wasm module: %.*s\n", + "error instantiating Wasm module: %.*s\n", (int)message.size, message.data ); goto error; @@ -1053,7 +1073,7 @@ static bool ts_wasm_store__instantiate( wasm_trap_message(trap, &message); format( error_message, - "trap when instantiating wasm module: %.*s\n", + "trap when instantiating Wasm module: %.*s\n", (int)message.size, message.data ); goto error; @@ -1163,17 +1183,17 @@ const TSLanguage *ts_wasm_store_load_language( if (!wasm_dylink_info__parse((const unsigned char *)wasm, wasm_len, &dylink_info)) { wasm_error->kind = TSWasmErrorKindParse; - format(&wasm_error->message, "failed to parse dylink section of wasm module"); + format(&wasm_error->message, "failed to parse dylink section of Wasm module"); goto error; } - // Compile the wasm code. + // Compile the Wasm code. error = wasmtime_module_new(self->engine, (const uint8_t *)wasm, wasm_len, &module); if (error) { wasm_message_t message; wasmtime_error_message(error, &message); wasm_error->kind = TSWasmErrorKindCompile; - format(&wasm_error->message, "error compiling wasm module: %.*s", (int)message.size, message.data); + format(&wasm_error->message, "error compiling Wasm module: %.*s", (int)message.size, message.data); wasm_byte_vec_delete(&message); goto error; } @@ -1194,32 +1214,39 @@ const TSLanguage *ts_wasm_store_load_language( goto error; } - // Copy all of the static data out of the language object in wasm memory, + // Copy all of the static data out of the language object in Wasm memory, // constructing a native language object. LanguageInWasmMemory wasm_language; wasmtime_context_t *context = wasmtime_store_context(self->store); const uint8_t *memory = wasmtime_memory_data(context, &self->memory); memcpy(&wasm_language, &memory[language_address], sizeof(LanguageInWasmMemory)); + bool has_supertypes = + wasm_language.abi_version > LANGUAGE_VERSION_WITH_RESERVED_WORDS && + wasm_language.supertype_count > 0; + int32_t addresses[] = { - wasm_language.alias_map, - wasm_language.alias_sequences, - wasm_language.field_map_entries, - wasm_language.field_map_slices, - wasm_language.field_names, - wasm_language.keyword_lex_fn, - wasm_language.lex_fn, - wasm_language.lex_modes, - wasm_language.parse_actions, wasm_language.parse_table, - wasm_language.primary_state_ids, - wasm_language.primary_state_ids, - wasm_language.public_symbol_map, wasm_language.small_parse_table, wasm_language.small_parse_table_map, - wasm_language.symbol_metadata, - wasm_language.symbol_metadata, + wasm_language.parse_actions, wasm_language.symbol_names, + wasm_language.field_names, + wasm_language.field_map_slices, + wasm_language.field_map_entries, + wasm_language.symbol_metadata, + wasm_language.public_symbol_map, + wasm_language.alias_map, + wasm_language.alias_sequences, + wasm_language.lex_modes, + wasm_language.lex_fn, + wasm_language.keyword_lex_fn, + wasm_language.primary_state_ids, + wasm_language.name, + wasm_language.reserved_words, + has_supertypes ? wasm_language.supertype_symbols : 0, + has_supertypes ? wasm_language.supertype_map_entries : 0, + has_supertypes ? wasm_language.supertype_map_slices : 0, wasm_language.external_token_count > 0 ? wasm_language.external_scanner.states : 0, wasm_language.external_token_count > 0 ? wasm_language.external_scanner.symbol_map : 0, wasm_language.external_token_count > 0 ? wasm_language.external_scanner.create : 0, @@ -1237,7 +1264,7 @@ const TSLanguage *ts_wasm_store_load_language( StringData field_name_buffer = array_new(); *language = (TSLanguage) { - .version = wasm_language.version, + .abi_version = wasm_language.abi_version, .symbol_count = wasm_language.symbol_count, .alias_count = wasm_language.alias_count, .token_count = wasm_language.token_count, @@ -1246,8 +1273,10 @@ const TSLanguage *ts_wasm_store_load_language( .large_state_count = wasm_language.large_state_count, .production_id_count = wasm_language.production_id_count, .field_count = wasm_language.field_count, + .supertype_count = wasm_language.supertype_count, .max_alias_sequence_length = wasm_language.max_alias_sequence_length, .keyword_capture_token = wasm_language.keyword_capture_token, + .metadata = wasm_language.metadata, .parse_table = copy( &memory[wasm_language.parse_table], wasm_language.large_state_count * wasm_language.symbol_count * sizeof(uint16_t) @@ -1274,21 +1303,21 @@ const TSLanguage *ts_wasm_store_load_language( ), .lex_modes = copy( &memory[wasm_language.lex_modes], - wasm_language.state_count * sizeof(TSLexMode) + wasm_language.state_count * sizeof(TSLexerMode) ), }; if (language->field_count > 0 && language->production_id_count > 0) { language->field_map_slices = copy( &memory[wasm_language.field_map_slices], - wasm_language.production_id_count * sizeof(TSFieldMapSlice) + wasm_language.production_id_count * sizeof(TSMapSlice) ); // Determine the number of field map entries by finding the greatest index // in any of the slices. uint32_t field_map_entry_count = 0; for (uint32_t i = 0; i < wasm_language.production_id_count; i++) { - TSFieldMapSlice slice = language->field_map_slices[i]; + TSMapSlice slice = language->field_map_slices[i]; uint32_t slice_end = slice.index + slice.length; if (slice_end > field_map_entry_count) { field_map_entry_count = slice_end; @@ -1307,6 +1336,37 @@ const TSLanguage *ts_wasm_store_load_language( ); } + if (has_supertypes) { + language->supertype_symbols = copy( + &memory[wasm_language.supertype_symbols], + wasm_language.supertype_count * sizeof(TSSymbol) + ); + + // Determine the number of supertype map slices by finding the greatest + // supertype ID. + int largest_supertype = 0; + for (unsigned i = 0; i < language->supertype_count; i++) { + TSSymbol supertype = language->supertype_symbols[i]; + if (supertype > largest_supertype) { + largest_supertype = supertype; + } + } + + language->supertype_map_slices = copy( + &memory[wasm_language.supertype_map_slices], + (largest_supertype + 1) * sizeof(TSMapSlice) + ); + + TSSymbol last_supertype = language->supertype_symbols[language->supertype_count - 1]; + TSMapSlice last_slice = language->supertype_map_slices[last_supertype]; + uint32_t supertype_map_entry_count = last_slice.index + last_slice.length; + + language->supertype_map_entries = copy( + &memory[wasm_language.supertype_map_entries], + supertype_map_entry_count * sizeof(char *) + ); + } + if (language->max_alias_sequence_length > 0 && language->production_id_count > 0) { // The alias map contains symbols, alias counts, and aliases, terminated by a null symbol. int32_t alias_map_size = 0; @@ -1317,11 +1377,12 @@ const TSLanguage *ts_wasm_store_load_language( if (symbol == 0) break; uint16_t value_count; memcpy(&value_count, &memory[wasm_language.alias_map + alias_map_size], sizeof(value_count)); + alias_map_size += sizeof(uint16_t); alias_map_size += value_count * sizeof(TSSymbol); } language->alias_map = copy( &memory[wasm_language.alias_map], - alias_map_size * sizeof(TSSymbol) + alias_map_size ); language->alias_sequences = copy( &memory[wasm_language.alias_sequences], @@ -1343,13 +1404,22 @@ const TSLanguage *ts_wasm_store_load_language( ); } - if (language->version >= LANGUAGE_VERSION_WITH_PRIMARY_STATES) { + if (language->abi_version >= LANGUAGE_VERSION_WITH_PRIMARY_STATES) { language->primary_state_ids = copy( &memory[wasm_language.primary_state_ids], wasm_language.state_count * sizeof(TSStateId) ); } + if (language->abi_version >= LANGUAGE_VERSION_WITH_RESERVED_WORDS) { + language->name = copy_string(memory, wasm_language.name); + language->reserved_words = copy( + &memory[wasm_language.reserved_words], + wasm_language.max_reserved_word_set_size * sizeof(TSSymbol) + ); + language->max_reserved_word_set_size = wasm_language.max_reserved_word_set_size; + } + if (language->external_token_count > 0) { language->external_scanner.symbol_map = copy( &memory[wasm_language.external_scanner.symbol_map], @@ -1374,15 +1444,15 @@ const TSLanguage *ts_wasm_store_load_language( .ref_count = 1, }; - // The lex functions are not used for wasm languages. Use those two fields - // to mark this language as WASM-based and to store the language's - // WASM-specific data. + // The lex functions are not used for Wasm languages. Use those two fields + // to mark this language as Wasm-based and to store the language's + // Wasm-specific data. language->lex_fn = ts_wasm_store__sentinel_lex_fn; language->keyword_lex_fn = (bool (*)(TSLexer *, TSStateId))language_module; // Clear out any instances of languages that have been deleted. for (unsigned i = 0; i < self->language_instances.size; i++) { - WasmLanguageId *id = self->language_instances.contents[i].language_id; + WasmLanguageId *id = array_get(&self->language_instances, i)->language_id; if (id->is_language_deleted) { language_id_delete(id); array_erase(&self->language_instances, i); @@ -1423,7 +1493,7 @@ bool ts_wasm_store_add_language( // instances of languages that have been deleted. bool exists = false; for (unsigned i = 0; i < self->language_instances.size; i++) { - WasmLanguageId *id = self->language_instances.contents[i].language_id; + WasmLanguageId *id = array_get(&self->language_instances, i)->language_id; if (id->is_language_deleted) { language_id_delete(id); array_erase(&self->language_instances, i); @@ -1494,7 +1564,7 @@ bool ts_wasm_store_start(TSWasmStore *self, TSLexer *lexer, const TSLanguage *la uint32_t instance_index; if (!ts_wasm_store_add_language(self, language, &instance_index)) return false; self->current_lexer = lexer; - self->current_instance = &self->language_instances.contents[instance_index]; + self->current_instance = array_get(&self->language_instances, instance_index); self->has_error = false; ts_wasm_store_reset_heap(self); return true; @@ -1527,7 +1597,7 @@ static void ts_wasm_store__call( // wasmtime_error_message(error, &message); // fprintf( // stderr, - // "error in wasm module: %.*s\n", + // "error in Wasm module: %.*s\n", // (int)message.size, message.data // ); wasmtime_error_delete(error); @@ -1537,7 +1607,7 @@ static void ts_wasm_store__call( // wasm_trap_message(trap, &message); // fprintf( // stderr, - // "trap in wasm module: %.*s\n", + // "trap in Wasm module: %.*s\n", // (int)message.size, message.data // ); wasm_trap_delete(trap); @@ -1545,13 +1615,22 @@ static void ts_wasm_store__call( } } +// The data fields of TSLexer, without the function pointers. +// +// This portion of the struct needs to be copied in and out +// of Wasm memory before and after calling a scan function. +typedef struct { + int32_t lookahead; + TSSymbol result_symbol; +} TSLexerDataPrefix; + static bool ts_wasm_store__call_lex_function(TSWasmStore *self, unsigned function_index, TSStateId state) { wasmtime_context_t *context = wasmtime_store_context(self->store); uint8_t *memory_data = wasmtime_memory_data(context, &self->memory); memcpy( &memory_data[self->lexer_address], - &self->current_lexer->lookahead, - sizeof(self->current_lexer->lookahead) + self->current_lexer, + sizeof(TSLexerDataPrefix) ); wasmtime_val_raw_t args[2] = { @@ -1563,9 +1642,9 @@ static bool ts_wasm_store__call_lex_function(TSWasmStore *self, unsigned functio bool result = args[0].i32; memcpy( - &self->current_lexer->lookahead, + self->current_lexer, &memory_data[self->lexer_address], - sizeof(self->current_lexer->lookahead) + sizeof(self->current_lexer->result_symbol) + sizeof(TSLexerDataPrefix) ); return result; } @@ -1610,8 +1689,8 @@ bool ts_wasm_store_call_scanner_scan( memcpy( &memory_data[self->lexer_address], - &self->current_lexer->lookahead, - sizeof(self->current_lexer->lookahead) + self->current_lexer, + sizeof(TSLexerDataPrefix) ); uint32_t valid_tokens_address = @@ -1626,9 +1705,9 @@ bool ts_wasm_store_call_scanner_scan( if (self->has_error) return false; memcpy( - &self->current_lexer->lookahead, + self->current_lexer, &memory_data[self->lexer_address], - sizeof(self->current_lexer->lookahead) + sizeof(self->current_lexer->result_symbol) + sizeof(TSLexerDataPrefix) ); return args[0].i32; } @@ -1713,8 +1792,8 @@ void ts_wasm_language_release(const TSLanguage *self) { LanguageWasmModule *module = ts_language__wasm_module(self); ts_assert(module->ref_count > 0); if (atomic_dec(&module->ref_count) == 0) { - // Update the language id to reflect that the language is deleted. This allows any wasm stores - // that hold wasm instances for this language to delete those instances. + // Update the language id to reflect that the language is deleted. This allows any Wasm stores + // that hold Wasm instances for this language to delete those instances. atomic_inc(&module->language_id->is_language_deleted); language_id_delete(module->language_id); @@ -1729,8 +1808,13 @@ void ts_wasm_language_release(const TSLanguage *self) { ts_free((void *)self->external_scanner.symbol_map); ts_free((void *)self->field_map_entries); ts_free((void *)self->field_map_slices); + ts_free((void *)self->supertype_symbols); + ts_free((void *)self->supertype_map_entries); + ts_free((void *)self->supertype_map_slices); ts_free((void *)self->field_names); ts_free((void *)self->lex_modes); + ts_free((void *)self->name); + ts_free((void *)self->reserved_words); ts_free((void *)self->parse_actions); ts_free((void *)self->parse_table); ts_free((void *)self->primary_state_ids); @@ -1751,8 +1835,8 @@ void ts_wasm_language_release(const TSLanguage *self) { #else -// If the WASM feature is not enabled, define dummy versions of all of the -// wasm-related functions. +// If the Wasm feature is not enabled, define dummy versions of all of the +// Wasm-related functions. void ts_wasm_store_delete(TSWasmStore *self) { (void)self; diff --git a/rustfmt.toml b/rustfmt.toml deleted file mode 100644 index 4fd08b8e..00000000 --- a/rustfmt.toml +++ /dev/null @@ -1,6 +0,0 @@ -comment_width = 100 -format_code_in_doc_comments = true -format_macro_matchers = true -imports_granularity = "Crate" -group_imports = "StdExternalCrate" -wrap_comments = true diff --git a/test/fixtures/fixtures.json b/test/fixtures/fixtures.json new file mode 100644 index 00000000..c1e2b167 --- /dev/null +++ b/test/fixtures/fixtures.json @@ -0,0 +1,17 @@ +[ + ["bash","v0.25.0"], + ["c","v0.24.1"], + ["cpp","v0.23.4"], + ["embedded-template","v0.25.0"], + ["go","v0.25.0"], + ["html","v0.23.2"], + ["java","v0.23.5"], + ["javascript","v0.25.0"], + ["jsdoc","v0.23.2"], + ["json","v0.24.8"], + ["php","v0.24.2"], + ["python","v0.23.6"], + ["ruby","v0.23.1"], + ["rust","v0.24.0"], + ["typescript","v0.23.2"] +] \ No newline at end of file diff --git a/test/fixtures/test_grammars/aliased_inlined_rules/grammar.js b/test/fixtures/test_grammars/aliased_inlined_rules/grammar.js index 2f1091e7..9d41fef2 100644 --- a/test/fixtures/test_grammars/aliased_inlined_rules/grammar.js +++ b/test/fixtures/test_grammars/aliased_inlined_rules/grammar.js @@ -2,7 +2,7 @@ // shows that you can alias a rule that would otherwise be anonymous, and it will then appear as a // named node. -module.exports = grammar({ +export default grammar({ name: 'aliased_inlined_rules', extras: $ => [/\s/], diff --git a/test/fixtures/test_grammars/aliased_rules/grammar.js b/test/fixtures/test_grammars/aliased_rules/grammar.js index a615a90d..df721d5b 100644 --- a/test/fixtures/test_grammars/aliased_rules/grammar.js +++ b/test/fixtures/test_grammars/aliased_rules/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: 'aliased_rules', extras: $ => [ diff --git a/test/fixtures/test_grammars/aliased_token_rules/grammar.js b/test/fixtures/test_grammars/aliased_token_rules/grammar.js index 704a2a34..2a6f4be3 100644 --- a/test/fixtures/test_grammars/aliased_token_rules/grammar.js +++ b/test/fixtures/test_grammars/aliased_token_rules/grammar.js @@ -1,7 +1,7 @@ // This grammar shows that `ALIAS` rules can be applied directly to `TOKEN` and `IMMEDIATE_TOKEN` // rules. -module.exports = grammar({ +export default grammar({ name: 'aliased_token_rules', extras: $ => [/\s/], diff --git a/test/fixtures/test_grammars/aliased_unit_reductions/grammar.js b/test/fixtures/test_grammars/aliased_unit_reductions/grammar.js index 9b39de28..77186d07 100644 --- a/test/fixtures/test_grammars/aliased_unit_reductions/grammar.js +++ b/test/fixtures/test_grammars/aliased_unit_reductions/grammar.js @@ -5,7 +5,7 @@ // their parent rule. In that situation, eliminating the invisible node could cause the alias to be // incorrectly applied to its child. -module.exports = grammar({ +export default grammar({ name: 'aliased_unit_reductions', extras: $ => [/\s/], diff --git a/test/fixtures/test_grammars/aliases_in_root/corpus.txt b/test/fixtures/test_grammars/aliases_in_root/corpus.txt new file mode 100644 index 00000000..ed78852b --- /dev/null +++ b/test/fixtures/test_grammars/aliases_in_root/corpus.txt @@ -0,0 +1,13 @@ +====================================== +Aliases within the root node +====================================== + +# this is a comment +foo foo + +--- + +(document + (comment) + (bar) + (foo)) diff --git a/test/fixtures/test_grammars/aliases_in_root/grammar.js b/test/fixtures/test_grammars/aliases_in_root/grammar.js new file mode 100644 index 00000000..9d46af2f --- /dev/null +++ b/test/fixtures/test_grammars/aliases_in_root/grammar.js @@ -0,0 +1,19 @@ +export default grammar({ + name: 'aliases_in_root', + + extras: $ => [ + /\s/, + $.comment, + ], + + rules: { + document: $ => seq( + alias($.foo, $.bar), + $.foo, + ), + + foo: $ => "foo", + + comment: $ => /#.*/ + } +}); diff --git a/test/fixtures/test_grammars/anonymous_error/corpus.txt b/test/fixtures/test_grammars/anonymous_error/corpus.txt new file mode 100644 index 00000000..f1dd3d34 --- /dev/null +++ b/test/fixtures/test_grammars/anonymous_error/corpus.txt @@ -0,0 +1,9 @@ +====================== +A simple error literal +====================== + +ERROR + +--- + +(document) diff --git a/test/fixtures/test_grammars/anonymous_error/grammar.js b/test/fixtures/test_grammars/anonymous_error/grammar.js new file mode 100644 index 00000000..72870612 --- /dev/null +++ b/test/fixtures/test_grammars/anonymous_error/grammar.js @@ -0,0 +1,6 @@ +export default grammar({ + name: 'anonymous_error', + rules: { + document: $ => repeat(choice('ok', 'ERROR')), + } +}); diff --git a/test/fixtures/test_grammars/anonymous_tokens_with_escaped_chars/corpus.txt b/test/fixtures/test_grammars/anonymous_tokens_with_escaped_chars/corpus.txt index 749264c6..c69d0484 100644 --- a/test/fixtures/test_grammars/anonymous_tokens_with_escaped_chars/corpus.txt +++ b/test/fixtures/test_grammars/anonymous_tokens_with_escaped_chars/corpus.txt @@ -19,7 +19,7 @@ anonymous tokens defined with LF escape sequence anonymous tokens defined with CR escape sequence ================================================= - + --- (first_rule) diff --git a/test/fixtures/test_grammars/anonymous_tokens_with_escaped_chars/grammar.js b/test/fixtures/test_grammars/anonymous_tokens_with_escaped_chars/grammar.js index 3e7e294b..cffe374d 100644 --- a/test/fixtures/test_grammars/anonymous_tokens_with_escaped_chars/grammar.js +++ b/test/fixtures/test_grammars/anonymous_tokens_with_escaped_chars/grammar.js @@ -4,7 +4,7 @@ // characters. This grammar tests that this escaping works. The test is basically that the generated // parser compiles successfully. -module.exports = grammar({ +export default grammar({ name: "anonymous_tokens_with_escaped_chars", rules: { first_rule: $ => choice( diff --git a/test/fixtures/test_grammars/associativity_left/grammar.js b/test/fixtures/test_grammars/associativity_left/grammar.js index 6dbc4671..3b0b8e54 100644 --- a/test/fixtures/test_grammars/associativity_left/grammar.js +++ b/test/fixtures/test_grammars/associativity_left/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: 'associativity_left', rules: { diff --git a/test/fixtures/test_grammars/associativity_missing/grammar.js b/test/fixtures/test_grammars/associativity_missing/grammar.js index 9c1ed980..c540d132 100644 --- a/test/fixtures/test_grammars/associativity_missing/grammar.js +++ b/test/fixtures/test_grammars/associativity_missing/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: 'associativity_missing', rules: { diff --git a/test/fixtures/test_grammars/associativity_right/grammar.js b/test/fixtures/test_grammars/associativity_right/grammar.js index 69bfd065..a45a02fd 100644 --- a/test/fixtures/test_grammars/associativity_right/grammar.js +++ b/test/fixtures/test_grammars/associativity_right/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: 'associativity_right', rules: { diff --git a/test/fixtures/test_grammars/conflict_in_repeat_rule/grammar.js b/test/fixtures/test_grammars/conflict_in_repeat_rule/grammar.js index c23e8a7c..e3d4b2d8 100644 --- a/test/fixtures/test_grammars/conflict_in_repeat_rule/grammar.js +++ b/test/fixtures/test_grammars/conflict_in_repeat_rule/grammar.js @@ -2,7 +2,7 @@ // parser generator in order to implement repetition. There is no way of referring to these rules in // the grammar DSL, so these conflicts must be resolved by referring to their parent rules. -module.exports = grammar({ +export default grammar({ name: 'conflict_in_repeat_rule', rules: { diff --git a/test/fixtures/test_grammars/conflict_in_repeat_rule_after_external_token/grammar.js b/test/fixtures/test_grammars/conflict_in_repeat_rule_after_external_token/grammar.js index 27364b22..85145f7f 100644 --- a/test/fixtures/test_grammars/conflict_in_repeat_rule_after_external_token/grammar.js +++ b/test/fixtures/test_grammars/conflict_in_repeat_rule_after_external_token/grammar.js @@ -2,7 +2,7 @@ // after an external token is consumed. This tests that the logic for determining the repeat rule's // "parent" rule works in the presence of external tokens. -module.exports = grammar({ +export default grammar({ name: 'conflict_in_repeat_rule_after_external_token', externals: $ => [ @@ -29,4 +29,4 @@ module.exports = grammar({ identifier: $ => /[a-z]+/ } -}); \ No newline at end of file +}); diff --git a/test/fixtures/test_grammars/conflicting_precedence/grammar.js b/test/fixtures/test_grammars/conflicting_precedence/grammar.js index 8092f8e9..98b41def 100644 --- a/test/fixtures/test_grammars/conflicting_precedence/grammar.js +++ b/test/fixtures/test_grammars/conflicting_precedence/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: 'conflicting_precedence', rules: { diff --git a/test/fixtures/test_grammars/depends_on_column/grammar.js b/test/fixtures/test_grammars/depends_on_column/grammar.js index 6f74810e..95646d86 100644 --- a/test/fixtures/test_grammars/depends_on_column/grammar.js +++ b/test/fixtures/test_grammars/depends_on_column/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: "depends_on_column", rules: { x_is_at: ($) => seq(/[ \r\n]*/, choice($.odd_column, $.even_column), "x"), diff --git a/test/fixtures/test_grammars/dynamic_precedence/grammar.js b/test/fixtures/test_grammars/dynamic_precedence/grammar.js index 321f1139..e4ad3a8e 100644 --- a/test/fixtures/test_grammars/dynamic_precedence/grammar.js +++ b/test/fixtures/test_grammars/dynamic_precedence/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: 'dynamic_precedence', extras: $ => [/\s/], diff --git a/test/fixtures/test_grammars/epsilon_external_extra_tokens/corpus.txt b/test/fixtures/test_grammars/epsilon_external_extra_tokens/corpus.txt new file mode 100644 index 00000000..776db2ec --- /dev/null +++ b/test/fixtures/test_grammars/epsilon_external_extra_tokens/corpus.txt @@ -0,0 +1,9 @@ +========================== +A document +========================== + +a b + +--- + +(document) diff --git a/test/fixtures/test_grammars/epsilon_external_extra_tokens/grammar.js b/test/fixtures/test_grammars/epsilon_external_extra_tokens/grammar.js new file mode 100644 index 00000000..aefa4e0f --- /dev/null +++ b/test/fixtures/test_grammars/epsilon_external_extra_tokens/grammar.js @@ -0,0 +1,11 @@ +export default grammar({ + name: 'epsilon_external_extra_tokens', + + extras: $ => [/\s/, $.comment], + + externals: $ => [$.comment], + + rules: { + document: $ => seq('a', 'b'), + } +}); diff --git a/test/fixtures/test_grammars/epsilon_external_extra_tokens/scanner.c b/test/fixtures/test_grammars/epsilon_external_extra_tokens/scanner.c new file mode 100644 index 00000000..c8949d1d --- /dev/null +++ b/test/fixtures/test_grammars/epsilon_external_extra_tokens/scanner.c @@ -0,0 +1,33 @@ +#include "tree_sitter/parser.h" + +enum TokenType { + COMMENT +}; + +void *tree_sitter_epsilon_external_extra_tokens_external_scanner_create(void) { + return NULL; +} + +bool tree_sitter_epsilon_external_extra_tokens_external_scanner_scan( + void *payload, + TSLexer *lexer, + const bool *valid_symbols +) { + lexer->result_symbol = COMMENT; + return true; +} + +unsigned tree_sitter_epsilon_external_extra_tokens_external_scanner_serialize( + void *payload, + char *buffer +) { + return 0; +} + +void tree_sitter_epsilon_external_extra_tokens_external_scanner_deserialize( + void *payload, + const char *buffer, + unsigned length +) {} + +void tree_sitter_epsilon_external_extra_tokens_external_scanner_destroy(void *payload) {} diff --git a/test/fixtures/test_grammars/epsilon_external_tokens/grammar.js b/test/fixtures/test_grammars/epsilon_external_tokens/grammar.js index 27deef47..14163d80 100644 --- a/test/fixtures/test_grammars/epsilon_external_tokens/grammar.js +++ b/test/fixtures/test_grammars/epsilon_external_tokens/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: 'epsilon_external_tokens', extras: $ => [/\s/], diff --git a/test/fixtures/test_grammars/epsilon_rules/grammar.js b/test/fixtures/test_grammars/epsilon_rules/grammar.js index 8cba729b..159ccd9f 100644 --- a/test/fixtures/test_grammars/epsilon_rules/grammar.js +++ b/test/fixtures/test_grammars/epsilon_rules/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: 'epsilon_rules', rules: { diff --git a/test/fixtures/test_grammars/external_and_internal_anonymous_tokens/grammar.js b/test/fixtures/test_grammars/external_and_internal_anonymous_tokens/grammar.js index e289f5ad..6e74b9d3 100644 --- a/test/fixtures/test_grammars/external_and_internal_anonymous_tokens/grammar.js +++ b/test/fixtures/test_grammars/external_and_internal_anonymous_tokens/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: 'external_and_internal_anonymous_tokens', externals: $ => [ diff --git a/test/fixtures/test_grammars/external_and_internal_tokens/grammar.js b/test/fixtures/test_grammars/external_and_internal_tokens/grammar.js index 6b70c98a..370a5723 100644 --- a/test/fixtures/test_grammars/external_and_internal_tokens/grammar.js +++ b/test/fixtures/test_grammars/external_and_internal_tokens/grammar.js @@ -2,7 +2,7 @@ // validity of an *internal* token. This is done by including the names of that internal token // (`line_break`) in the grammar's `externals` field. -module.exports = grammar({ +export default grammar({ name: 'external_and_internal_tokens', externals: $ => [ diff --git a/test/fixtures/test_grammars/external_extra_tokens/grammar.js b/test/fixtures/test_grammars/external_extra_tokens/grammar.js index a5390c8a..c64ecdc1 100644 --- a/test/fixtures/test_grammars/external_extra_tokens/grammar.js +++ b/test/fixtures/test_grammars/external_extra_tokens/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: "external_extra_tokens", externals: $ => [ diff --git a/test/fixtures/test_grammars/external_tokens/grammar.js b/test/fixtures/test_grammars/external_tokens/grammar.js index 457eee94..c4ff6cb7 100644 --- a/test/fixtures/test_grammars/external_tokens/grammar.js +++ b/test/fixtures/test_grammars/external_tokens/grammar.js @@ -2,7 +2,7 @@ // that track the nesting depth of parentheses, similar to Ruby's percent // string literals. -module.exports = grammar({ +export default grammar({ name: "external_tokens", externals: $ => [ diff --git a/test/fixtures/test_grammars/external_tokens/scanner.c b/test/fixtures/test_grammars/external_tokens/scanner.c index 6d80827e..da40c485 100644 --- a/test/fixtures/test_grammars/external_tokens/scanner.c +++ b/test/fixtures/test_grammars/external_tokens/scanner.c @@ -30,7 +30,7 @@ void tree_sitter_external_tokens_external_scanner_destroy(void *payload) { unsigned tree_sitter_external_tokens_external_scanner_serialize( void *payload, char *buffer -) { return true; } +) { return 0; } void tree_sitter_external_tokens_external_scanner_deserialize( void *payload, diff --git a/test/fixtures/test_grammars/external_unicode_column_alignment/grammar.js b/test/fixtures/test_grammars/external_unicode_column_alignment/grammar.js index 3016b31d..e578920c 100644 --- a/test/fixtures/test_grammars/external_unicode_column_alignment/grammar.js +++ b/test/fixtures/test_grammars/external_unicode_column_alignment/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: "external_unicode_column_alignment", externals: $ => [ diff --git a/test/fixtures/test_grammars/extra_non_terminals/corpus.txt b/test/fixtures/test_grammars/extra_non_terminals/corpus.txt index 52b7d864..b58fa68b 100644 --- a/test/fixtures/test_grammars/extra_non_terminals/corpus.txt +++ b/test/fixtures/test_grammars/extra_non_terminals/corpus.txt @@ -12,11 +12,12 @@ a b c d Extras ============== -a (one) b (two) (three) c d +a (one) b (two) (three) c d // e --- (module - (comment) - (comment) - (comment)) + (comment (paren_comment)) + (comment (paren_comment)) + (comment (paren_comment)) + (comment (line_comment))) diff --git a/test/fixtures/test_grammars/extra_non_terminals/grammar.js b/test/fixtures/test_grammars/extra_non_terminals/grammar.js index d13cd68a..116cb0a6 100644 --- a/test/fixtures/test_grammars/extra_non_terminals/grammar.js +++ b/test/fixtures/test_grammars/extra_non_terminals/grammar.js @@ -1,6 +1,6 @@ // This grammar has an "extra" rule, `comment`, that is a non-terminal. -module.exports = grammar({ +export default grammar({ name: "extra_non_terminals", extras: $ => [ @@ -9,7 +9,12 @@ module.exports = grammar({ ], rules: { - module: $ => seq('a', 'b', 'c', 'd'), - comment: $ => seq('(', repeat(/[a-z]+/), ')'), + module: _ => seq('a', 'b', 'c', 'd'), + + comment: $ => choice($.paren_comment, $.line_comment), + + paren_comment: _ => token(seq('(', repeat(/[a-z]+/), ')')), + + line_comment: _ => token(seq('//', /.*/)), } }) diff --git a/test/fixtures/test_grammars/extra_non_terminals_with_shared_rules/grammar.js b/test/fixtures/test_grammars/extra_non_terminals_with_shared_rules/grammar.js index 28539871..993e89b4 100644 --- a/test/fixtures/test_grammars/extra_non_terminals_with_shared_rules/grammar.js +++ b/test/fixtures/test_grammars/extra_non_terminals_with_shared_rules/grammar.js @@ -1,7 +1,7 @@ // This grammar has a non-terminal extra rule `macro_statement` that contains // child rules that are also used elsewhere in the grammar. -module.exports = grammar({ +export default grammar({ name: "extra_non_terminals_with_shared_rules", extras: $ => [/\s+/, $.macro_statement], diff --git a/test/fixtures/test_grammars/get_col_eof/grammar.js b/test/fixtures/test_grammars/get_col_eof/grammar.js index 3b70db2f..a2cf6ef4 100644 --- a/test/fixtures/test_grammars/get_col_eof/grammar.js +++ b/test/fixtures/test_grammars/get_col_eof/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: "get_col_eof", externals: $ => [ diff --git a/test/fixtures/test_grammars/get_col_should_hang_not_crash/grammar.js b/test/fixtures/test_grammars/get_col_should_hang_not_crash/grammar.js index 83d57d2c..d45ea967 100644 --- a/test/fixtures/test_grammars/get_col_should_hang_not_crash/grammar.js +++ b/test/fixtures/test_grammars/get_col_should_hang_not_crash/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: 'get_col_should_hang_not_crash', externals: $ => [ diff --git a/test/fixtures/test_grammars/immediate_tokens/grammar.js b/test/fixtures/test_grammars/immediate_tokens/grammar.js index 7505a811..223de3d3 100644 --- a/test/fixtures/test_grammars/immediate_tokens/grammar.js +++ b/test/fixtures/test_grammars/immediate_tokens/grammar.js @@ -3,7 +3,7 @@ // When there are *no* leading `extras`, an immediate token is preferred over a normal token which // would otherwise match. -module.exports = grammar({ +export default grammar({ name: "immediate_tokens", extras: $ => [/\s/], diff --git a/test/fixtures/test_grammars/indirect_recursion_in_transitions/expected_error.txt b/test/fixtures/test_grammars/indirect_recursion_in_transitions/expected_error.txt new file mode 100644 index 00000000..4f244a6c --- /dev/null +++ b/test/fixtures/test_grammars/indirect_recursion_in_transitions/expected_error.txt @@ -0,0 +1 @@ +Grammar contains an indirectly recursive rule: type_expression -> _expression -> identifier_expression -> type_expression \ No newline at end of file diff --git a/test/fixtures/test_grammars/indirect_recursion_in_transitions/grammar.js b/test/fixtures/test_grammars/indirect_recursion_in_transitions/grammar.js new file mode 100644 index 00000000..23ce2b24 --- /dev/null +++ b/test/fixtures/test_grammars/indirect_recursion_in_transitions/grammar.js @@ -0,0 +1,16 @@ +export default grammar({ + name: 'indirect_recursive_in_single_symbol_transitions', + rules: { + source_file: $ => repeat($._statement), + + _statement: $ => seq($.initialization_part, $.type_expression), + + type_expression: $ => choice('int', $._expression), + + initialization_part: $ => seq('=', $._expression), + + _expression: $ => choice($.identifier_expression, $.type_expression), + + identifier_expression: $ => choice(/[a-zA-Z_][a-zA-Z0-9_]*/, $.type_expression), + } +}); diff --git a/test/fixtures/test_grammars/inline_rules/grammar.js b/test/fixtures/test_grammars/inline_rules/grammar.js index 4477097d..c8f3275c 100644 --- a/test/fixtures/test_grammars/inline_rules/grammar.js +++ b/test/fixtures/test_grammars/inline_rules/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: "inline_rules", extras: $ => [/\s/], diff --git a/test/fixtures/test_grammars/inlined_aliased_rules/grammar.js b/test/fixtures/test_grammars/inlined_aliased_rules/grammar.js index 8578659b..84612330 100644 --- a/test/fixtures/test_grammars/inlined_aliased_rules/grammar.js +++ b/test/fixtures/test_grammars/inlined_aliased_rules/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: "inlined_aliased_rules", extras: $ => [/\s/], diff --git a/test/fixtures/test_grammars/inverted_external_token/grammar.js b/test/fixtures/test_grammars/inverted_external_token/grammar.js index 3530d0db..e1d78b89 100644 --- a/test/fixtures/test_grammars/inverted_external_token/grammar.js +++ b/test/fixtures/test_grammars/inverted_external_token/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: "inverted_external_token", externals: $ => [$.line_break], diff --git a/test/fixtures/test_grammars/invisible_start_rule/grammar.js b/test/fixtures/test_grammars/invisible_start_rule/grammar.js index 4afa4d66..d9fdf6f5 100644 --- a/test/fixtures/test_grammars/invisible_start_rule/grammar.js +++ b/test/fixtures/test_grammars/invisible_start_rule/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: "invisible_start_rule", rules: { _value: $ => choice($.a, $.b), diff --git a/test/fixtures/test_grammars/lexical_conflicts_due_to_state_merging/grammar.js b/test/fixtures/test_grammars/lexical_conflicts_due_to_state_merging/grammar.js index 4868dc81..2dc3639b 100644 --- a/test/fixtures/test_grammars/lexical_conflicts_due_to_state_merging/grammar.js +++ b/test/fixtures/test_grammars/lexical_conflicts_due_to_state_merging/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: 'lexical_conflicts_due_to_state_merging', rules: { diff --git a/test/fixtures/test_grammars/named_precedences/grammar.js b/test/fixtures/test_grammars/named_precedences/grammar.js index 2132385b..52c767c5 100644 --- a/test/fixtures/test_grammars/named_precedences/grammar.js +++ b/test/fixtures/test_grammars/named_precedences/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: 'named_precedences', conflicts: $ => [ diff --git a/test/fixtures/test_grammars/named_rule_aliased_as_anonymous/grammar.js b/test/fixtures/test_grammars/named_rule_aliased_as_anonymous/grammar.js index 3f30de56..6bc56822 100644 --- a/test/fixtures/test_grammars/named_rule_aliased_as_anonymous/grammar.js +++ b/test/fixtures/test_grammars/named_rule_aliased_as_anonymous/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: 'named_rule_aliased_as_anonymous', rules: { diff --git a/test/fixtures/test_grammars/nested_inlined_rules/grammar.js b/test/fixtures/test_grammars/nested_inlined_rules/grammar.js index 7aaf601b..3ca4d264 100644 --- a/test/fixtures/test_grammars/nested_inlined_rules/grammar.js +++ b/test/fixtures/test_grammars/nested_inlined_rules/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: 'nested_inlined_rules', inline: $ => [ diff --git a/test/fixtures/test_grammars/next_sibling_from_zwt/grammar.js b/test/fixtures/test_grammars/next_sibling_from_zwt/grammar.js index 857b94ad..f9a31c44 100644 --- a/test/fixtures/test_grammars/next_sibling_from_zwt/grammar.js +++ b/test/fixtures/test_grammars/next_sibling_from_zwt/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: "next_sibling_from_zwt", extras: $ => [ /\s|\\\r?\n/, @@ -19,4 +19,3 @@ module.exports = grammar({ ), } }); - diff --git a/test/fixtures/test_grammars/partially_resolved_conflict/grammar.js b/test/fixtures/test_grammars/partially_resolved_conflict/grammar.js index cd0d1d65..23b11167 100644 --- a/test/fixtures/test_grammars/partially_resolved_conflict/grammar.js +++ b/test/fixtures/test_grammars/partially_resolved_conflict/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: 'partially_resolved_conflict', rules: { diff --git a/test/fixtures/test_grammars/precedence_on_single_child_missing/grammar.js b/test/fixtures/test_grammars/precedence_on_single_child_missing/grammar.js index fbdb450f..0c9c1b45 100644 --- a/test/fixtures/test_grammars/precedence_on_single_child_missing/grammar.js +++ b/test/fixtures/test_grammars/precedence_on_single_child_missing/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: 'precedence_on_single_child_missing', rules: { diff --git a/test/fixtures/test_grammars/precedence_on_single_child_negative/grammar.js b/test/fixtures/test_grammars/precedence_on_single_child_negative/grammar.js index 798075db..6d9ea114 100644 --- a/test/fixtures/test_grammars/precedence_on_single_child_negative/grammar.js +++ b/test/fixtures/test_grammars/precedence_on_single_child_negative/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: 'precedence_on_single_child_negative', rules: { diff --git a/test/fixtures/test_grammars/precedence_on_single_child_positive/grammar.js b/test/fixtures/test_grammars/precedence_on_single_child_positive/grammar.js index d2e57c30..0492f0ac 100644 --- a/test/fixtures/test_grammars/precedence_on_single_child_positive/grammar.js +++ b/test/fixtures/test_grammars/precedence_on_single_child_positive/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: 'precedence_on_single_child_positive', rules: { diff --git a/test/fixtures/test_grammars/precedence_on_subsequence/grammar.js b/test/fixtures/test_grammars/precedence_on_subsequence/grammar.js index 3a5bdefb..f6caad5e 100644 --- a/test/fixtures/test_grammars/precedence_on_subsequence/grammar.js +++ b/test/fixtures/test_grammars/precedence_on_subsequence/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: 'precedence_on_subsequence', rules: { diff --git a/test/fixtures/test_grammars/precedence_on_token/grammar.js b/test/fixtures/test_grammars/precedence_on_token/grammar.js index e56f2d81..67a0b9bf 100644 --- a/test/fixtures/test_grammars/precedence_on_token/grammar.js +++ b/test/fixtures/test_grammars/precedence_on_token/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: 'precedence_on_token', extras: $ => [ diff --git a/test/fixtures/test_grammars/readme_grammar/grammar.js b/test/fixtures/test_grammars/readme_grammar/grammar.js index 9f3ce6dd..0547df4c 100644 --- a/test/fixtures/test_grammars/readme_grammar/grammar.js +++ b/test/fixtures/test_grammars/readme_grammar/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: 'readme_grammar', // Things that can appear anywhere in the language, like comments @@ -31,6 +31,6 @@ module.exports = grammar({ comment: _ => /#.*/, - variable: _ => /[a-zA-Z]\w*/, + variable: _ => new RustRegex('(?i:[a-z])\\w*'), }, }); diff --git a/test/fixtures/test_grammars/reserved_words/corpus.txt b/test/fixtures/test_grammars/reserved_words/corpus.txt new file mode 100644 index 00000000..607efe40 --- /dev/null +++ b/test/fixtures/test_grammars/reserved_words/corpus.txt @@ -0,0 +1,101 @@ +============== +Valid Code +============== + +if (a) { + var b = { + c: d, + e: f, + }; + while (g) { + h(); + } +} + +--- + +(program + (if_statement + (parenthesized_expression (identifier)) + (block + (var_declaration + (identifier) + (object + (pair (identifier) (identifier)) + (pair (identifier) (identifier)))) + (while_statement + (parenthesized_expression (identifier)) + (block (expression_statement (call_expression (identifier)))))))) + +================================================ +Error detected at globally-reserved word +================================================ + +var a = + +if (something) { + c(); +} + +--- + +(program + (ERROR (identifier)) + (if_statement + (parenthesized_expression (identifier)) + (block + (expression_statement (call_expression (identifier)))))) + +================================================ +Object keys that are reserved in other contexts +================================================ + +var x = { + if: a, + while: b, +}; + +--- + +(program + (var_declaration + (identifier) + (object + (pair (identifier) (identifier)) + (pair (identifier) (identifier))))) + +================================================ +Error detected at context-specific reserved word +================================================ + +var x = { +var y = z; + +--- + +(program + (ERROR (identifier)) + + ; Important - var declaration is still recognized, + ; because in this example grammar, `var` is a keyword + ; even within object literals. + (var_declaration + (identifier) + (identifier))) + +============================================= +Other tokens that overlap with keyword tokens +============================================= + +var a = /reserved-words-should-not-affect-this/; +var d = /if/; + +--- + +(program + (var_declaration + (identifier) + (regex (regex_pattern))) + (var_declaration + (identifier) + (regex (regex_pattern)))) diff --git a/test/fixtures/test_grammars/reserved_words/grammar.js b/test/fixtures/test_grammars/reserved_words/grammar.js new file mode 100644 index 00000000..74c88eea --- /dev/null +++ b/test/fixtures/test_grammars/reserved_words/grammar.js @@ -0,0 +1,67 @@ +const RESERVED_NAMES = ["if", "while", "var"]; +const RESERVED_PROPERTY_NAMES = ["var"]; + +export default grammar({ + name: "reserved_words", + + reserved: { + global: $ => RESERVED_NAMES, + property: $ => RESERVED_PROPERTY_NAMES, + }, + + word: $ => $.identifier, + + rules: { + program: $ => repeat($._statement), + + block: $ => seq("{", repeat($._statement), "}"), + + _statement: $ => choice( + $.var_declaration, + $.if_statement, + $.while_statement, + $.expression_statement, + ), + + var_declaration: $ => seq("var", $.identifier, "=", $._expression, ";"), + + if_statement: $ => seq("if", $.parenthesized_expression, $.block), + + while_statement: $ => seq("while", $.parenthesized_expression, $.block), + + expression_statement: $ => seq($._expression, ";"), + + _expression: $ => choice( + $.identifier, + $.parenthesized_expression, + $.call_expression, + $.member_expression, + $.object, + $.regex, + ), + + parenthesized_expression: $ => seq("(", $._expression, ")"), + + member_expression: $ => seq($._expression, ".", $.identifier), + + call_expression: $ => seq($._expression, "(", repeat(seq($._expression, ",")), ")"), + + object: $ => seq("{", repeat(seq(choice($.pair, $.getter), ",")), "}"), + + regex: $ => seq('/', $.regex_pattern, '/'), + + regex_pattern: $ => token(prec(-1, /[^/\n]+/)), + + pair: $ => seq(reserved('property', $.identifier), ":", $._expression), + + getter: $ => seq( + "get", + reserved('property', $.identifier), + "(", + ")", + $.block, + ), + + identifier: $ => /[a-z_]\w*/, + }, +}); diff --git a/test/fixtures/test_grammars/start_rule_is_blank/grammar.js b/test/fixtures/test_grammars/start_rule_is_blank/grammar.js index b38e0de0..6fa28ae5 100644 --- a/test/fixtures/test_grammars/start_rule_is_blank/grammar.js +++ b/test/fixtures/test_grammars/start_rule_is_blank/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: 'start_rule_is_blank', rules: { diff --git a/test/fixtures/test_grammars/start_rule_is_token/grammar.js b/test/fixtures/test_grammars/start_rule_is_token/grammar.js index f00433ea..05742687 100644 --- a/test/fixtures/test_grammars/start_rule_is_token/grammar.js +++ b/test/fixtures/test_grammars/start_rule_is_token/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: 'start_rule_is_token', rules: { diff --git a/test/fixtures/test_grammars/unicode_classes/grammar.js b/test/fixtures/test_grammars/unicode_classes/grammar.js index 25dcf13d..2aafdea4 100644 --- a/test/fixtures/test_grammars/unicode_classes/grammar.js +++ b/test/fixtures/test_grammars/unicode_classes/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: 'unicode_classes', rules: { diff --git a/test/fixtures/test_grammars/unused_rules/grammar.js b/test/fixtures/test_grammars/unused_rules/grammar.js index 462243c8..78020419 100644 --- a/test/fixtures/test_grammars/unused_rules/grammar.js +++ b/test/fixtures/test_grammars/unused_rules/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: 'unused_rules', rules: { diff --git a/test/fixtures/test_grammars/uses_current_column/grammar.js b/test/fixtures/test_grammars/uses_current_column/grammar.js index 795ad597..1e93f06b 100644 --- a/test/fixtures/test_grammars/uses_current_column/grammar.js +++ b/test/fixtures/test_grammars/uses_current_column/grammar.js @@ -1,4 +1,4 @@ -module.exports = grammar({ +export default grammar({ name: 'uses_current_column', externals: $ => [ diff --git a/xtask/src/build_wasm.rs b/xtask/src/build_wasm.rs deleted file mode 100644 index ee3c7d5b..00000000 --- a/xtask/src/build_wasm.rs +++ /dev/null @@ -1,228 +0,0 @@ -use std::{ - ffi::{OsStr, OsString}, - fmt::Write, - fs, - process::Command, -}; - -use anyhow::{anyhow, Result}; - -use crate::{bail_on_err, BuildWasm, EMSCRIPTEN_TAG}; - -#[derive(PartialEq, Eq)] -enum EmccSource { - Native, - Docker, - Podman, -} - -pub fn run_wasm(args: &BuildWasm) -> Result<()> { - let mut emscripten_flags = vec!["-O3", "--minify", "0"]; - - if args.debug { - emscripten_flags.extend(["-s", "ASSERTIONS=1", "-s", "SAFE_HEAP=1", "-O0", "-g"]); - } - - if args.verbose { - emscripten_flags.extend(["-s", "VERBOSE=1", "-v"]); - } - - let emcc_name = if cfg!(windows) { "emcc.bat" } else { "emcc" }; - - // Order of preference: emscripten > docker > podman > error - let source = if !args.docker && Command::new(emcc_name).output().is_ok() { - EmccSource::Native - } else if Command::new("docker") - .arg("info") - .output() - .is_ok_and(|out| out.status.success()) - { - EmccSource::Docker - } else if Command::new("podman") - .arg("--version") - .output() - .is_ok_and(|out| out.status.success()) - { - EmccSource::Podman - } else { - return Err(anyhow!( - "You must have either emcc, docker, or podman on your PATH to run this command" - )); - }; - - let mut command = match source { - EmccSource::Native => Command::new(emcc_name), - EmccSource::Docker | EmccSource::Podman => { - let mut command = match source { - EmccSource::Docker => Command::new("docker"), - EmccSource::Podman => Command::new("podman"), - _ => unreachable!(), - }; - command.args(["run", "--rm"]); - - // Mount the root directory as a volume, which is the repo root - let mut volume_string = OsString::from(std::env::current_dir().unwrap()); - volume_string.push(":/src:Z"); - command.args([OsStr::new("--volume"), &volume_string]); - - // In case `docker` is an alias to `podman`, ensure that podman - // mounts the current directory as writable by the container - // user which has the same uid as the host user. Setting the - // podman-specific variable is more reliable than attempting to - // detect whether `docker` is an alias for `podman`. - // see https://docs.podman.io/en/latest/markdown/podman-run.1.html#userns-mode - command.env("PODMAN_USERNS", "keep-id"); - - // Get the current user id so that files created in the docker container will have - // the same owner. - #[cfg(unix)] - { - #[link(name = "c")] - extern "C" { - fn getuid() -> u32; - } - // don't need to set user for podman since PODMAN_USERNS=keep-id is already set - if source == EmccSource::Docker { - let user_id = unsafe { getuid() }; - command.args(["--user", &user_id.to_string()]); - } - }; - - // Run `emcc` in a container using the `emscripten-slim` image - command.args([EMSCRIPTEN_TAG, "emcc"]); - command - } - }; - - fs::create_dir_all("target/scratch").unwrap(); - - let exported_functions = concat!( - include_str!("../../lib/src/wasm/stdlib-symbols.txt"), - include_str!("../../lib/binding_web/exports.txt") - ) - .replace('"', "") - .lines() - .fold(String::new(), |mut output, line| { - let _ = write!(output, "_{line}"); - output - }) - .trim_end_matches(',') - .to_string(); - - let exported_functions = format!("EXPORTED_FUNCTIONS={exported_functions}"); - let exported_runtime_methods = "EXPORTED_RUNTIME_METHODS=stringToUTF16,AsciiToString"; - - emscripten_flags.extend([ - "-s", - "WASM=1", - "-s", - "INITIAL_MEMORY=33554432", - "-s", - "ALLOW_MEMORY_GROWTH=1", - "-s", - "SUPPORT_BIG_ENDIAN=1", - "-s", - "MAIN_MODULE=2", - "-s", - "FILESYSTEM=0", - "-s", - "NODEJS_CATCH_EXIT=0", - "-s", - "NODEJS_CATCH_REJECTION=0", - "-s", - &exported_functions, - "-s", - exported_runtime_methods, - "-fno-exceptions", - "-std=c11", - "-D", - "fprintf(...)=", - "-D", - "NDEBUG=", - "-D", - "_POSIX_C_SOURCE=200112L", - "-D", - "_DEFAULT_SOURCE=", - "-I", - "lib/src", - "-I", - "lib/include", - "--js-library", - "lib/binding_web/imports.js", - "--pre-js", - "lib/binding_web/prefix.js", - "--post-js", - "lib/binding_web/binding.js", - "--post-js", - "lib/binding_web/suffix.js", - "lib/src/lib.c", - "lib/binding_web/binding.c", - "-o", - "target/scratch/tree-sitter.js", - ]); - - bail_on_err( - &command.args(emscripten_flags).spawn()?.wait_with_output()?, - "Failed to compile the Tree-sitter WASM library", - )?; - - fs::rename( - "target/scratch/tree-sitter.js", - "lib/binding_web/tree-sitter.js", - )?; - - fs::rename( - "target/scratch/tree-sitter.wasm", - "lib/binding_web/tree-sitter.wasm", - )?; - - Ok(()) -} - -pub fn run_wasm_stdlib() -> Result<()> { - let export_flags = include_str!("../../lib/src/wasm/stdlib-symbols.txt") - .lines() - .map(|line| format!("-Wl,--export={}", &line[1..line.len() - 1])) - .collect::>(); - - let mut command = Command::new("target/wasi-sdk-21.0/bin/clang-17"); - - let output = command - .args([ - "-o", - "stdlib.wasm", - "-Os", - "-fPIC", - "-Wl,--no-entry", - "-Wl,--stack-first", - "-Wl,-z", - "-Wl,stack-size=65536", - "-Wl,--import-undefined", - "-Wl,--import-memory", - "-Wl,--import-table", - "-Wl,--strip-debug", - "-Wl,--export=reset_heap", - "-Wl,--export=__wasm_call_ctors", - "-Wl,--export=__stack_pointer", - ]) - .args(export_flags) - .arg("lib/src/wasm/stdlib.c") - .output()?; - - bail_on_err(&output, "Failed to compile the Tree-sitter WASM stdlib")?; - - let xxd = Command::new("xxd") - .args(["-C", "-i", "stdlib.wasm"]) - .output()?; - - bail_on_err( - &xxd, - "Failed to run xxd on the compiled Tree-sitter WASM stdlib", - )?; - - fs::write("lib/src/wasm/wasm-stdlib.h", xxd.stdout)?; - - fs::rename("stdlib.wasm", "target/stdlib.wasm")?; - - Ok(()) -} diff --git a/xtask/src/bump.rs b/xtask/src/bump.rs deleted file mode 100644 index 671793cf..00000000 --- a/xtask/src/bump.rs +++ /dev/null @@ -1,318 +0,0 @@ -use std::{cmp::Ordering, path::Path}; - -use anyhow::{anyhow, Result}; -use git2::{DiffOptions, Repository}; -use indoc::indoc; -use semver::{BuildMetadata, Prerelease, Version}; -use toml::Value; - -use crate::BumpVersion; - -pub fn get_latest_tag(repo: &Repository) -> Result { - let mut tags = repo - .tag_names(None)? - .into_iter() - .filter_map(|tag| tag.map(String::from)) - .filter_map(|tag| Version::parse(tag.strip_prefix('v').unwrap_or(&tag)).ok()) - .collect::>(); - - tags.sort_by( - |a, b| match (a.pre != Prerelease::EMPTY, b.pre != Prerelease::EMPTY) { - (true, true) | (false, false) => a.cmp(b), - (true, false) => Ordering::Less, - (false, true) => Ordering::Greater, - }, - ); - - tags.last() - .map(std::string::ToString::to_string) - .ok_or_else(|| anyhow!("No tags found")) -} - -pub fn run(args: BumpVersion) -> Result<()> { - let repo = Repository::open(".")?; - let latest_tag = get_latest_tag(&repo)?; - let current_version = Version::parse(&latest_tag)?; - let latest_tag_sha = repo.revparse_single(&format!("v{latest_tag}"))?.id(); - - let workspace_toml_version = Version::parse(&fetch_workspace_version()?)?; - - if current_version.major != workspace_toml_version.major - && current_version.minor != workspace_toml_version.minor - { - eprintln!( - indoc! {" - Seems like the workspace Cargo.toml ({}) version does not match up with the latest git tag ({}). - Please ensure you don't change that yourself, this subcommand will handle this for you. - "}, - workspace_toml_version, latest_tag - ); - return Ok(()); - } - - let mut revwalk = repo.revwalk()?; - revwalk.push_range(format!("{latest_tag_sha}..HEAD").as_str())?; - let mut diff_options = DiffOptions::new(); - - let mut should_increment_patch = false; - let mut should_increment_minor = false; - - for oid in revwalk { - let oid = oid?; - let commit = repo.find_commit(oid)?; - let message = commit.message().unwrap(); - let message = message.trim(); - - let diff = { - let parent = commit.parent(0).unwrap(); - let parent_tree = parent.tree().unwrap(); - let commit_tree = commit.tree().unwrap(); - repo.diff_tree_to_tree( - Some(&parent_tree), - Some(&commit_tree), - Some(&mut diff_options), - )? - }; - - let mut source_code_changed = false; - diff.foreach( - &mut |delta, _| { - let path = delta.new_file().path().unwrap().to_str().unwrap(); - if path.ends_with("rs") || path.ends_with("js") || path.ends_with('c') { - source_code_changed = true; - } - true - }, - None, - None, - None, - )?; - - if source_code_changed { - should_increment_patch = true; - - let Some((prefix, _)) = message.split_once(':') else { - continue; - }; - - let convention = if prefix.contains('(') { - prefix.split_once('(').unwrap().0 - } else { - prefix - }; - - if ["feat", "feat!"].contains(&convention) || prefix.ends_with('!') { - should_increment_minor = true; - } - } - } - - let next_version = if let Some(version) = args.version { - version - } else { - let mut next_version = current_version.clone(); - if should_increment_minor { - next_version.minor += 1; - next_version.patch = 0; - next_version.pre = Prerelease::EMPTY; - next_version.build = BuildMetadata::EMPTY; - } else if should_increment_patch { - next_version.patch += 1; - next_version.pre = Prerelease::EMPTY; - next_version.build = BuildMetadata::EMPTY; - } else { - return Err(anyhow!(format!( - "No source code changed since {current_version}" - ))); - } - next_version - }; - - println!("Bumping from {current_version} to {next_version}"); - update_crates(¤t_version, &next_version)?; - update_makefile(&next_version)?; - update_cmake(&next_version)?; - update_npm(&next_version)?; - update_zig(&next_version)?; - tag_next_version(&repo, &next_version)?; - - Ok(()) -} - -fn tag_next_version(repo: &Repository, next_version: &Version) -> Result<()> { - // first add the manifests - - let mut index = repo.index()?; - - for file in [ - "Cargo.toml", - "Cargo.lock", - "cli/Cargo.toml", - "cli/config/Cargo.toml", - "cli/loader/Cargo.toml", - "lib/Cargo.toml", - "highlight/Cargo.toml", - "tags/Cargo.toml", - "cli/npm/package.json", - "lib/binding_web/package.json", - "Makefile", - "lib/CMakeLists.txt", - "build.zig.zon", - ] { - index.add_path(Path::new(file))?; - } - - index.write()?; - - let tree_id = index.write_tree()?; - let tree = repo.find_tree(tree_id)?; - let signature = repo.signature()?; - let parent_commit = repo.revparse_single("HEAD")?.peel_to_commit()?; - - let commit_id = repo.commit( - Some("HEAD"), - &signature, - &signature, - &format!("{next_version}"), - &tree, - &[&parent_commit], - )?; - - let tag = repo.tag( - &format!("v{next_version}"), - &repo.find_object(commit_id, None)?, - &signature, - &format!("v{next_version}"), - false, - )?; - - println!("Tagged commit {commit_id} with tag {tag}"); - - Ok(()) -} - -fn update_makefile(next_version: &Version) -> Result<()> { - let makefile = std::fs::read_to_string("Makefile")?; - let makefile = makefile - .lines() - .map(|line| { - if line.starts_with("VERSION") { - format!("VERSION := {next_version}") - } else { - line.to_string() - } - }) - .collect::>() - .join("\n") - + "\n"; - - std::fs::write("Makefile", makefile)?; - - Ok(()) -} - -fn update_cmake(next_version: &Version) -> Result<()> { - let cmake = std::fs::read_to_string("lib/CMakeLists.txt")?; - let cmake = cmake - .lines() - .map(|line| { - if line.contains(" VERSION") { - let start_quote = line.find('"').unwrap(); - let end_quote = line.rfind('"').unwrap(); - format!( - "{}{next_version}{}", - &line[..start_quote + 1], - &line[end_quote..] - ) - } else { - line.to_string() - } - }) - .collect::>() - .join("\n") - + "\n"; - - std::fs::write("lib/CMakeLists.txt", cmake)?; - - Ok(()) -} - -fn update_crates(current_version: &Version, next_version: &Version) -> Result<()> { - let mut cmd = std::process::Command::new("cargo"); - cmd.arg("workspaces").arg("version"); - - if next_version.minor > current_version.minor { - cmd.arg("minor"); - } else { - cmd.arg("patch"); - } - - cmd.arg("--no-git-commit") - .arg("--yes") - .arg("--force") - .arg("tree-sitter{,-cli,-config,-generate,-loader,-highlight,-tags}") - .arg("--ignore-changes") - .arg("lib/language/*"); - - let status = cmd.status()?; - - if !status.success() { - return Err(anyhow!("Failed to update crates")); - } - - Ok(()) -} - -fn update_npm(next_version: &Version) -> Result<()> { - for path in ["lib/binding_web/package.json", "cli/npm/package.json"] { - let package_json = - serde_json::from_str::(&std::fs::read_to_string(path)?)?; - - let mut package_json = package_json - .as_object() - .ok_or_else(|| anyhow!("Invalid package.json"))? - .clone(); - package_json.insert( - "version".to_string(), - serde_json::Value::String(next_version.to_string()), - ); - - let package_json = serde_json::to_string_pretty(&package_json)? + "\n"; - - std::fs::write(path, package_json)?; - } - - Ok(()) -} - -fn update_zig(next_version: &Version) -> Result<()> { - let zig = std::fs::read_to_string("build.zig.zon")?; - - let zig = zig - .lines() - .map(|line| { - if line.starts_with(" .version") { - format!(" .version = \"{next_version}\",") - } else { - line.to_string() - } - }) - .collect::>() - .join("\n") - + "\n"; - - std::fs::write("build.zig.zon", zig)?; - - Ok(()) -} - -/// read Cargo.toml and get the version -fn fetch_workspace_version() -> Result { - let cargo_toml = toml::from_str::(&std::fs::read_to_string("Cargo.toml")?)?; - - Ok(cargo_toml["workspace"]["package"]["version"] - .as_str() - .unwrap() - .trim_matches('"') - .to_string()) -} diff --git a/xtask/src/fetch.rs b/xtask/src/fetch.rs deleted file mode 100644 index f48373db..00000000 --- a/xtask/src/fetch.rs +++ /dev/null @@ -1,119 +0,0 @@ -use std::{path::Path, process::Command}; - -use anyhow::Result; - -use crate::{bail_on_err, EMSCRIPTEN_VERSION}; - -pub fn run_fixtures() -> Result<()> { - let grammars_dir = Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .join("test") - .join("fixtures") - .join("grammars"); - - [ - ("bash", "master"), - ("c", "master"), - ("cpp", "master"), - ("embedded-template", "master"), - ("go", "master"), - ("html", "master"), - ("java", "master"), - ("javascript", "master"), - ("jsdoc", "master"), - ("json", "master"), - ("php", "master"), - ("python", "master"), - ("ruby", "master"), - ("rust", "master"), - ("typescript", "master"), - ] - .iter() - .try_for_each(|(grammar, r#ref)| { - let grammar_dir = grammars_dir.join(grammar); - let grammar_url = format!("https://github.com/tree-sitter/tree-sitter-{grammar}"); - - println!("Updating the {grammar} grammar..."); - - if !grammar_dir.exists() { - let mut command = Command::new("git"); - command.args([ - "clone", - "--depth", - "1", - &grammar_url, - &grammar_dir.to_string_lossy(), - ]); - bail_on_err( - &command.spawn()?.wait_with_output()?, - "Failed to clone the {grammar} grammar", - )?; - } - - std::env::set_current_dir(&grammar_dir)?; - - let mut command = Command::new("git"); - command.args(["fetch", "origin", r#ref, "--depth", "1"]); - bail_on_err( - &command.spawn()?.wait_with_output()?, - "Failed to fetch the {grammar} grammar", - )?; - - let mut command = Command::new("git"); - command.args(["reset", "--hard", "FETCH_HEAD"]); - bail_on_err( - &command.spawn()?.wait_with_output()?, - "Failed to reset the {grammar} grammar", - )?; - - Ok(()) - }) -} - -pub fn run_emscripten() -> Result<()> { - let emscripten_dir = Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .join("target") - .join("emsdk"); - - if emscripten_dir.exists() { - println!("Emscripten SDK already exists"); - return Ok(()); - } - println!("Cloning the Emscripten SDK..."); - - let mut command = Command::new("git"); - command.args([ - "clone", - "https://github.com/emscripten-core/emsdk.git", - &emscripten_dir.to_string_lossy(), - ]); - bail_on_err( - &command.spawn()?.wait_with_output()?, - "Failed to clone the Emscripten SDK", - )?; - - std::env::set_current_dir(&emscripten_dir)?; - - let emsdk = if cfg!(windows) { - "emsdk.bat" - } else { - "./emsdk" - }; - - let mut command = Command::new(emsdk); - command.args(["install", EMSCRIPTEN_VERSION]); - bail_on_err( - &command.spawn()?.wait_with_output()?, - "Failed to install Emscripten", - )?; - - let mut command = Command::new(emsdk); - command.args(["activate", EMSCRIPTEN_VERSION]); - bail_on_err( - &command.spawn()?.wait_with_output()?, - "Failed to activate Emscripten", - ) -} diff --git a/xtask/src/upgrade_wasmtime.rs b/xtask/src/upgrade_wasmtime.rs deleted file mode 100644 index 6a8f4976..00000000 --- a/xtask/src/upgrade_wasmtime.rs +++ /dev/null @@ -1,145 +0,0 @@ -use std::{path::Path, process::Command}; - -use anyhow::{Context, Result}; -use git2::Repository; -use semver::Version; - -use crate::UpgradeWasmtime; - -const WASMTIME_RELEASE_URL: &str = "https://github.com/bytecodealliance/wasmtime/releases/download"; - -fn update_cargo(version: &Version) -> Result<()> { - let file = std::fs::read_to_string("lib/Cargo.toml")?; - let mut old_lines = file.lines(); - let mut new_lines = Vec::with_capacity(old_lines.size_hint().0); - - while let Some(line) = old_lines.next() { - new_lines.push(line.to_string()); - if line == "[dependencies.wasmtime-c-api]" { - let _ = old_lines.next(); - new_lines.push(format!("version = \"{version}\"")); - } - } - - std::fs::write("lib/Cargo.toml", new_lines.join("\n") + "\n")?; - - Command::new("cargo") - .arg("update") - .status() - .map(|_| ()) - .with_context(|| "Failed to execute cargo update") -} - -fn zig_fetch(lines: &mut Vec, version: &Version, url_suffix: &str) -> Result<()> { - let url = &format!("{WASMTIME_RELEASE_URL}/v{version}/wasmtime-v{version}-{url_suffix}"); - println!(" Fetching {url}"); - lines.push(format!(" .url = \"{url}\",")); - - let output = Command::new("zig") - .arg("fetch") - .arg(url) - .output() - .with_context(|| format!("Failed to execute zig fetch {url}"))?; - - let hash = String::from_utf8_lossy(&output.stdout); - lines.push(format!(" .hash = \"{}\",", hash.trim_end())); - - Ok(()) -} - -fn update_zig(version: &Version) -> Result<()> { - let file = std::fs::read_to_string("build.zig.zon")?; - let mut old_lines = file.lines(); - let new_lines = &mut Vec::with_capacity(old_lines.size_hint().0); - - while let Some(line) = old_lines.next() { - new_lines.push(line.to_string()); - match line { - " .wasmtime_c_api_aarch64_android = .{" => { - let (_, _) = (old_lines.next(), old_lines.next()); - zig_fetch(new_lines, version, "aarch64-android-c-api.tar.xz")?; - } - " .wasmtime_c_api_aarch64_linux = .{" => { - let (_, _) = (old_lines.next(), old_lines.next()); - zig_fetch(new_lines, version, "aarch64-linux-c-api.tar.xz")?; - } - " .wasmtime_c_api_aarch64_macos = .{" => { - let (_, _) = (old_lines.next(), old_lines.next()); - zig_fetch(new_lines, version, "aarch64-macos-c-api.tar.xz")?; - } - " .wasmtime_c_api_riscv64gc_linux = .{" => { - let (_, _) = (old_lines.next(), old_lines.next()); - zig_fetch(new_lines, version, "riscv64gc-linux-c-api.tar.xz")?; - } - " .wasmtime_c_api_s390x_linux = .{" => { - let (_, _) = (old_lines.next(), old_lines.next()); - zig_fetch(new_lines, version, "s390x-linux-c-api.tar.xz")?; - } - " .wasmtime_c_api_x86_64_android = .{" => { - let (_, _) = (old_lines.next(), old_lines.next()); - zig_fetch(new_lines, version, "x86_64-android-c-api.tar.xz")?; - } - " .wasmtime_c_api_x86_64_linux = .{" => { - let (_, _) = (old_lines.next(), old_lines.next()); - zig_fetch(new_lines, version, "x86_64-linux-c-api.tar.xz")?; - } - " .wasmtime_c_api_x86_64_macos = .{" => { - let (_, _) = (old_lines.next(), old_lines.next()); - zig_fetch(new_lines, version, "x86_64-macos-c-api.tar.xz")?; - } - " .wasmtime_c_api_x86_64_mingw = .{" => { - let (_, _) = (old_lines.next(), old_lines.next()); - zig_fetch(new_lines, version, "x86_64-mingw-c-api.zip")?; - } - " .wasmtime_c_api_x86_64_musl = .{" => { - let (_, _) = (old_lines.next(), old_lines.next()); - zig_fetch(new_lines, version, "x86_64-musl-c-api.tar.xz")?; - } - " .wasmtime_c_api_x86_64_windows = .{" => { - let (_, _) = (old_lines.next(), old_lines.next()); - zig_fetch(new_lines, version, "x86_64-windows-c-api.zip")?; - } - _ => {} - } - } - - std::fs::write("build.zig.zon", new_lines.join("\n") + "\n")?; - - Ok(()) -} - -fn create_commit(repo: &Repository, version: &Version) -> Result<()> { - let mut index = repo.index()?; - index.add_path(Path::new("lib/Cargo.toml"))?; - index.add_path(Path::new("build.zig.zon"))?; - index.write()?; - - let tree_id = index.write_tree()?; - let tree = repo.find_tree(tree_id)?; - let signature = repo.signature()?; - let parent_commit = repo.revparse_single("HEAD")?.peel_to_commit()?; - - let _ = repo.commit( - Some("HEAD"), - &signature, - &signature, - &format!("build(deps): bump wasmtime-c-api to v{version}"), - &tree, - &[&parent_commit], - )?; - - Ok(()) -} - -pub fn run(args: &UpgradeWasmtime) -> Result<()> { - println!("Upgrading wasmtime for Rust"); - update_cargo(&args.version)?; - - println!("Upgrading wasmtime for Zig"); - update_zig(&args.version)?; - - let repo = Repository::open(".")?; - create_commit(&repo, &args.version)?; - - Ok(()) -}