build(deps): bump cc from 1.2.52 to 1.2.53 in the cargo group

Bumps the cargo group with 1 update: [cc](https://github.com/rust-lang/cc-rs).


Updates `cc` from 1.2.52 to 1.2.53
- [Release notes](https://github.com/rust-lang/cc-rs/releases)
- [Changelog](https://github.com/rust-lang/cc-rs/blob/main/CHANGELOG.md)
- [Commits](https://github.com/rust-lang/cc-rs/compare/cc-v1.2.52...cc-v1.2.53)

---
updated-dependencies:
- dependency-name: cc
  dependency-version: 1.2.53
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cargo
...

Signed-off-by: dependabot[bot] <support@github.com>

.editorconfig

@@ -10,6 +10,9 @@ insert_final_newline = true
 [*.rs]
 indent_size = 4
 
+[*.{zig,zon}]
+indent_size = 4
+
 [Makefile]
 indent_style = tab
 indent_size = 8

.envrc

@@ -0,0 +1 @@
+use flake

.gitattributes

@@ -3,5 +3,4 @@
 /lib/src/unicode/*.h linguist-vendored
 /lib/src/unicode/LICENSE linguist-vendored
 
-/cli/src/generate/prepare_grammar/*.json -diff
 Cargo.lock -diff

.github/FUNDING.yml

@@ -0,0 +1,15 @@
+# These are supported funding model platforms
+
+github: tree-sitter
+patreon: # Replace with a single Patreon username
+open_collective: tree-sitter # Replace with a single Open Collective username
+ko_fi: amaanq
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
+polar: # Replace with a single Polar username
+buy_me_a_coffee: # Replace with a single Buy Me a Coffee username
+thanks_dev: # Replace with a single thanks.dev username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,6 +1,6 @@
 name: Bug Report
 description: Report a problem
-labels: [bug]
+type: Bug
 body:
   - type: textarea
     attributes:
@@ -13,9 +13,11 @@ body:
     attributes:
       label: "Steps to reproduce"
       placeholder: |
+        ```sh
         git clone --depth=1 https://github.com/tree-sitter/tree-sitter-ruby
         cd tree-sitter-ruby
         tree-sitter generate
+        ```
     validations:
       required: true
 

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -1,6 +1,6 @@
 name: Feature request
 description: Request an enhancement
-labels: [enhancement]
+type: Feature
 body:
   - type: markdown
     attributes:

.github/actions/cache/action.yml

@@ -1,24 +1,25 @@
-name: 'Cache'
-description: "This action caches fixtures"
+name: Cache
+
+description: This action caches fixtures
+
 outputs:
   cache-hit:
-    description: 'Cache hit'
-    value: ${{ steps.cache_output.outputs.cache-hit }}
+    description: Cache hit
+    value: ${{ steps.cache.outputs.cache-hit }}
+
 runs:
-  using: "composite"
+  using: composite
   steps:
     - uses: actions/cache@v4
-      id: cache_fixtures
+      id: cache
       with:
         path: |
           test/fixtures/grammars
           target/release/tree-sitter-*.wasm
         key: fixtures-${{ join(matrix.*, '_') }}-${{ hashFiles(
-          'cli/src/generate/**',
-          'script/generate-fixtures*',
+          'crates/generate/src/**',
+          'lib/src/parser.h',
+          'lib/src/array.h',
+          'lib/src/alloc.h',
           'test/fixtures/grammars/*/**/src/*.c',
           '.github/actions/cache/action.yml') }}
-
-    - run: echo "cache-hit=${{ steps.cache_fixtures.outputs.cache-hit }}" >> $GITHUB_OUTPUT
-      shell: bash
-      id: cache_output

.github/cliff.toml

@@ -16,13 +16,13 @@ body = """
     {% for commit in commits%}\
         {% if not commit.scope %}\
             - {{ commit.message | upper_first }}\
-              {% if commit.github.pr_number %} (<https://github.com/{{ remote.github.owner }}/{{ remote.github.repo }}/pull/{{ commit.github.pr_number }}>){%- endif %}
+              {% if commit.remote.pr_number %} (<https://github.com/{{ remote.github.owner }}/{{ remote.github.repo }}/pull/{{ commit.remote.pr_number }}>){%- endif %}
         {% endif %}\
     {% endfor %}\
     {% for group, commits in commits | group_by(attribute="scope") %}\
         {% for commit in commits %}\
             - **{{commit.scope}}**: {{ commit.message | upper_first }}\
-                {% if commit.github.pr_number %} (<https://github.com/{{ remote.github.owner }}/{{ remote.github.repo }}/pull/{{ commit.github.pr_number }}>){%- endif %}
+                {% if commit.remote.pr_number %} (<https://github.com/{{ remote.github.owner }}/{{ remote.github.repo }}/pull/{{ commit.remote.pr_number }}>){%- endif %}
         {% endfor %}\
     {% endfor %}
 {% endfor %}

.github/dependabot.yml

@@ -4,6 +4,8 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+    cooldown:
+      default-days: 3
     commit-message:
       prefix: "build(deps)"
     labels:
@@ -12,10 +14,16 @@ updates:
     groups:
       cargo:
         patterns: ["*"]
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major", "version-update:semver-minor"]
+
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
       interval: "weekly"
+    cooldown:
+      default-days: 3
     commit-message:
       prefix: "ci"
     labels:
@@ -24,3 +32,22 @@ updates:
     groups:
       actions:
         patterns: ["*"]
+
+  - package-ecosystem: "npm"
+    versioning-strategy: increase
+    directories:
+      - "/crates/npm"
+      - "/crates/eslint"
+      - "/lib/binding_web"
+    schedule:
+      interval: "weekly"
+    cooldown:
+      default-days: 3
+    commit-message:
+      prefix: "build(deps)"
+    labels:
+      - "dependencies"
+      - "npm"
+    groups:
+      npm:
+        patterns: ["*"]

.github/scripts/close_spam.js

@@ -0,0 +1,29 @@
+module.exports = async ({ github, context }) => {
+  let target = context.payload.issue;
+  if (target) {
+    await github.rest.issues.update({
+      ...context.repo,
+      issue_number: target.number,
+      state: "closed",
+      state_reason: "not_planned",
+      title: "[spam]",
+      body: "",
+      type: null,
+    });
+  } else {
+    target = context.payload.pull_request;
+    await github.rest.pulls.update({
+      ...context.repo,
+      pull_number: target.number,
+      state: "closed",
+      title: "[spam]",
+      body: "",
+    });
+  }
+
+  await github.rest.issues.lock({
+    ...context.repo,
+    issue_number: target.number,
+    lock_reason: "spam",
+  });
+};

.github/scripts/cross.sh

@@ -1,17 +0,0 @@
-#!/bin/bash
-
-# set -x
-set -e
-
-if [ "$BUILD_CMD" != "cross" ]; then
-    echo "cross.sh - is a helper to assist only in cross compiling environments" >&2
-    echo "To use this tool set the BUILD_CMD env var to the \"cross\" value" >&2
-    exit 111
-fi
-
-if [ -z "$CROSS_IMAGE" ]; then
-    echo "The CROSS_IMAGE env var should be provided" >&2
-    exit 111
-fi
-
-docker run --rm -v /home/runner:/home/runner -w "$PWD" "$CROSS_IMAGE" "$@"

.github/scripts/make.sh

@@ -1,19 +0,0 @@
-#!/bin/bash
-
-# set -x
-set -e
-
-if [ "$BUILD_CMD" == "cross" ]; then
-    if [ -z "$CC" ]; then
-        echo "make.sh: CC is not set" >&2
-        exit 111
-    fi
-    if [ -z "$AR" ]; then
-        echo "make.sh: AR is not set" >&2
-        exit 111
-    fi
-
-    cross.sh make CC=$CC AR=$AR "$@"
-else
-    make "$@"
-fi

.github/scripts/tree-sitter.sh

@@ -1,28 +0,0 @@
-#!/bin/bash
-
-# set -x
-set -e
-
-if [ -z "$ROOT" ]; then
-    echo "The ROOT env var should be set to absolute path of a repo root folder" >&2
-    exit 111
-fi
-
-if [ -z "$TARGET" ]; then
-    echo "The TARGET env var should be equal to a \`cargo build --target <TARGET>\` command value" >&2
-    exit 111
-fi
-
-tree_sitter="$ROOT"/target/"$TARGET"/release/tree-sitter
-
-if [ "$BUILD_CMD" == "cross" ]; then
-    if [ -z "$CROSS_RUNNER" ]; then
-        echo "The CROSS_RUNNER env var should be set to a CARGO_TARGET_*_RUNNER env var value" >&2
-        echo "that is available in a docker image used by the cross tool under the hood" >&2
-        exit 111
-    fi
-
-    cross.sh $CROSS_RUNNER "$tree_sitter" "$@"
-else
-    "$tree_sitter" "$@"
-fi

.github/scripts/wasm_stdlib.js

@@ -0,0 +1,25 @@
+module.exports = async ({ github, context, core }) => {
+  if (context.eventName !== 'pull_request') return;
+
+  const prNumber = context.payload.pull_request.number;
+  const owner = context.repo.owner;
+  const repo = context.repo.repo;
+
+  const { data: files } = await github.rest.pulls.listFiles({
+    owner,
+    repo,
+    pull_number: prNumber
+  });
+
+  const changedFiles = files.map(file => file.filename);
+
+  const wasmStdLibSrc = 'crates/language/wasm/';
+  const dirChanged = changedFiles.some(file => file.startsWith(wasmStdLibSrc));
+
+  if (!dirChanged) return;
+
+  const wasmStdLibHeader = 'lib/src/wasm/wasm-stdlib.h';
+  const requiredChanged = changedFiles.includes(wasmStdLibHeader);
+
+  if (!requiredChanged) core.setFailed(`Changes detected in ${wasmStdLibSrc} but ${wasmStdLibHeader} was not modified.`);
+};

.github/workflows/backport.yml

@@ -0,0 +1,31 @@
+name: Backport Pull Request
+
+on:
+  pull_request_target:
+    types: [closed, labeled]
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  backport:
+    if: github.event.pull_request.merged
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Create app token
+        uses: actions/create-github-app-token@v2
+        id: app-token
+        with:
+          app-id: ${{ vars.BACKPORT_APP }}
+          private-key: ${{ secrets.BACKPORT_KEY }}
+
+      - name: Create backport PR
+        uses: korthout/backport-action@v4
+        with:
+          pull_title: "${pull_title}"
+          label_pattern: "^ci:backport ([^ ]+)$"
+          github_token: ${{ steps.app-token.outputs.token }}

.github/workflows/bindgen.yml

@@ -0,0 +1,30 @@
+name: Check Bindgen Output
+
+on:
+  pull_request:
+    paths:
+      - lib/include/tree_sitter/api.h
+      - lib/binding_rust/bindings.rs
+  push:
+    branches: [master]
+    paths:
+      - lib/include/tree_sitter/api.h
+      - lib/binding_rust/bindings.rs
+
+jobs:
+  check-bindgen:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Set up stable Rust toolchain
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: stable
+
+      - name: Generate bindings
+        run: cargo xtask generate-bindings
+
+      - name: Check if the bindgen output changed
+        run: git diff --exit-code lib/binding_rust/bindings.rs

.github/workflows/build.yml

@@ -1,14 +1,9 @@
 name: Build & Test
 
-env:
-  CARGO_TERM_COLOR: always
-  RUSTFLAGS: "-D warnings"
-  CROSS_DEBUG: 1
-
 on:
   workflow_call:
     inputs:
-      run_test:
+      run-test:
         default: true
         type: boolean
 
@@ -21,168 +16,296 @@ jobs:
       fail-fast: false
       matrix:
         platform:
-        - linux-arm64       #
-        - linux-arm         #
-        - linux-x64         #
-        - linux-x86         #
-        - linux-powerpc64   #
-        - windows-arm64     #
-        - windows-x64       # <-- No C library build - requires an additional adapted Makefile for `cl.exe` compiler
-        - windows-x86       #     -- // --
-        - macos-arm64       #
-        - macos-x64         #
+          - linux-arm64
+          - linux-arm
+          - linux-x64
+          - linux-x86
+          - linux-powerpc64
+          - windows-arm64
+          - windows-x64
+          - windows-x86
+          - macos-arm64
+          - macos-x64
+          - wasm32
 
         include:
-        # When adding a new `target`:
-        # 1. Define a new platform alias above
-        # 2. Add a new record to a matrix map in `cli/npm/install.js`
-        - { platform: linux-arm64       , target: aarch64-unknown-linux-gnu           , os: ubuntu-latest  , use-cross: true    }
-        - { platform: linux-arm         , target: arm-unknown-linux-gnueabi           , os: ubuntu-latest  , use-cross: true    }
-        - { platform: linux-x64         , target: x86_64-unknown-linux-gnu            , os: ubuntu-20.04   , cli_features: wasm } #2272
-        - { platform: linux-x86         , target: i686-unknown-linux-gnu              , os: ubuntu-latest  , use-cross: true    }
-        - { platform: linux-powerpc64   , target: powerpc64-unknown-linux-gnu         , os: ubuntu-latest  , use-cross: true    }
-        - { platform: windows-arm64     , target: aarch64-pc-windows-msvc             , os: windows-latest                      }
-        - { platform: windows-x64       , target: x86_64-pc-windows-msvc              , os: windows-latest , cli_features: wasm }
-        - { platform: windows-x86       , target: i686-pc-windows-msvc                , os: windows-latest                      }
-        - { platform: macos-arm64       , target: aarch64-apple-darwin                , os: macos-14       , cli_features: wasm }
-        - { platform: macos-x64         , target: x86_64-apple-darwin                 , os: macos-12       , cli_features: wasm }
+          # When adding a new `target`:
+          # 1. Define a new platform alias above
+          # 2. Add a new record to the matrix map in `crates/cli/npm/install.js`
+          - { platform: linux-arm64     , target: aarch64-unknown-linux-gnu     , os: ubuntu-24.04-arm }
+          - { platform: linux-arm       , target: armv7-unknown-linux-gnueabihf , os: ubuntu-24.04-arm }
+          - { platform: linux-x64       , target: x86_64-unknown-linux-gnu      , os: ubuntu-24.04     }
+          - { platform: linux-x86       , target: i686-unknown-linux-gnu        , os: ubuntu-24.04     }
+          - { platform: linux-powerpc64 , target: powerpc64-unknown-linux-gnu   , os: ubuntu-24.04     }
+          - { platform: windows-arm64   , target: aarch64-pc-windows-msvc       , os: windows-11-arm   }
+          - { platform: windows-x64     , target: x86_64-pc-windows-msvc        , os: windows-2025     }
+          - { platform: windows-x86     , target: i686-pc-windows-msvc          , os: windows-2025     }
+          - { platform: macos-arm64     , target: aarch64-apple-darwin          , os: macos-15         }
+          - { platform: macos-x64       , target: x86_64-apple-darwin           , os: macos-15-intel   }
+          - { platform: wasm32          , target: wasm32-unknown-unknown        , os: ubuntu-24.04     }
 
-        # Cross compilers for C library
-        - { platform: linux-arm64       , cc: aarch64-linux-gnu-gcc             , ar: aarch64-linux-gnu-ar           }
-        - { platform: linux-arm         , cc: arm-linux-gnueabi-gcc             , ar: arm-linux-gnueabi-ar           }
-        - { platform: linux-x86         , cc: i686-linux-gnu-gcc                , ar: i686-linux-gnu-ar              }
-        - { platform: linux-powerpc64   , cc: powerpc64-linux-gnu-gcc           , ar: powerpc64-linux-gnu-ar         }
+          # Extra features
+          - { platform: linux-arm64     , features: wasm }
+          - { platform: linux-x64       , features: wasm }
+          - { platform: macos-arm64     , features: wasm }
+          - { platform: macos-x64       , features: wasm }
 
-        # See #2041 tree-sitter issue
-        - { platform: windows-x64   , rust-test-threads: 1 }
-        - { platform: windows-x86   , rust-test-threads: 1 }
+          # Cross-compilation
+          - { platform: linux-arm       , cross: true }
+          - { platform: linux-x86       , cross: true }
+          - { platform: linux-powerpc64 , cross: true }
 
-        # CLI only build
-        - { platform: windows-arm64 , cli-only: true }
+          # Compile-only
+          - { platform: wasm32          , no-run: true }
 
     env:
-      BUILD_CMD: cargo
-      EXE: ${{ contains(matrix.target, 'windows') && '.exe' || '' }}
+      CARGO_TERM_COLOR: always
+      RUSTFLAGS: -D warnings
 
     defaults:
       run:
         shell: bash
 
     steps:
-    - uses: actions/checkout@v4
+    - name: Checkout repository
+      uses: actions/checkout@v6
 
-    - name: Read Emscripten version
-      run: echo "EMSCRIPTEN_VERSION=$(cat cli/loader/emscripten-version)" >> $GITHUB_ENV
+    - name: Set up cross-compilation
+      if: matrix.cross
+      run: |
+        for target in armv7-unknown-linux-gnueabihf i686-unknown-linux-gnu powerpc64-unknown-linux-gnu; do
+          camel_target=${target//-/_}; target_cc=${target/-unknown/}
+          printf 'CC_%s=%s\n' "$camel_target" "${target_cc/v7/}-gcc"
+          printf 'AR_%s=%s\n' "$camel_target" "${target_cc/v7/}-ar"
+          printf 'CARGO_TARGET_%s_LINKER=%s\n' "${camel_target^^}" "${target_cc/v7/}-gcc"
+        done >> $GITHUB_ENV
+        {
+          printf 'CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_RUNNER=qemu-arm -L /usr/arm-linux-gnueabihf\n'
+          printf 'CARGO_TARGET_POWERPC64_UNKNOWN_LINUX_GNU_RUNNER=qemu-ppc64 -L /usr/powerpc64-linux-gnu\n'
+        } >> $GITHUB_ENV
+
+    - name: Get emscripten version
+      if: contains(matrix.features, 'wasm')
+      run: printf 'EMSCRIPTEN_VERSION=%s\n' "$(<crates/loader/emscripten-version)" >> $GITHUB_ENV
 
     - name: Install Emscripten
-      if: ${{ !matrix.cli-only && !matrix.use-cross }}
+      if: contains(matrix.features, 'wasm')
       uses: mymindstorm/setup-emsdk@v14
       with:
         version: ${{ env.EMSCRIPTEN_VERSION }}
 
-    - run: rustup toolchain install stable --profile minimal
-    - run: rustup target add ${{ matrix.target }}
-    - uses: Swatinem/rust-cache@v2
-
-    - name: Install cross
-      if: ${{ matrix.use-cross }}
-      uses: taiki-e/install-action@v2
+    - name: Set up Rust
+      uses: actions-rust-lang/setup-rust-toolchain@v1
       with:
-        tool: cross
+        target: ${{ matrix.target }}
 
-    - name: Build custom cross image
-      if: ${{ matrix.use-cross && matrix.os == 'ubuntu-latest' }}
+    - name: Install cross-compilation toolchain
+      if: matrix.cross
       run: |
-        target="${{ matrix.target }}"
-        image=ghcr.io/cross-rs/$target:custom
-        echo "CROSS_IMAGE=$image"                              >> $GITHUB_ENV
-
-        echo "[target.$target]"                                >> Cross.toml
-        echo "image = \"$image\""                              >> Cross.toml
-        echo "CROSS_CONFIG=$PWD/Cross.toml"                    >> $GITHUB_ENV
-
-        echo "FROM ghcr.io/cross-rs/$target:edge"                            >> Dockerfile
-        echo "RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash -" >> Dockerfile
-        echo "RUN apt-get update && apt-get -y install nodejs"               >> Dockerfile
-        docker build -t $image .
-
-    - name: Setup env extras
-      env:
-        RUST_TEST_THREADS: ${{ matrix.rust-test-threads }}
-        USE_CROSS: ${{ matrix.use-cross }}
-        TARGET: ${{ matrix.target }}
-        CC: ${{ matrix.cc }}
-        AR: ${{ matrix.ar }}
-      run: |
-        PATH="$PWD/.github/scripts:$PATH"
-        echo "$PWD/.github/scripts" >> $GITHUB_PATH
-
-        echo "TREE_SITTER=tree-sitter.sh" >> $GITHUB_ENV
-        echo "TARGET=$TARGET" >> $GITHUB_ENV
-        echo "ROOT=$PWD" >> $GITHUB_ENV
-
-        [ -n "$RUST_TEST_THREADS" ] && \
-        echo "RUST_TEST_THREADS=$RUST_TEST_THREADS" >> $GITHUB_ENV
-
-        [ -n "$CC" ] && echo "CC=$CC" >> $GITHUB_ENV
-        [ -n "$AR" ] && echo "AR=$AR" >> $GITHUB_ENV
-
-        if [ "$USE_CROSS" == "true" ]; then
-          echo "BUILD_CMD=cross" >> $GITHUB_ENV
-          runner=$(BUILD_CMD=cross cross.sh bash -c "env | sed -nr '/^CARGO_TARGET_.*_RUNNER=/s///p'")
-          [ -n "$runner" ] && echo "CROSS_RUNNER=$runner" >> $GITHUB_ENV
+        sudo apt-get update -qy
+        if [[ $PLATFORM == linux-arm ]]; then
+          sudo apt-get install -qy {binutils,gcc}-arm-linux-gnueabihf qemu-user
+        elif [[ $PLATFORM == linux-x86 ]]; then
+          sudo apt-get install -qy {binutils,gcc}-i686-linux-gnu
+        elif [[ $PLATFORM == linux-powerpc64 ]]; then
+          sudo apt-get install -qy {binutils,gcc}-powerpc64-linux-gnu qemu-user
         fi
+      env:
+        PLATFORM: ${{ matrix.platform }}
 
-    - name: Build C library
-      if: ${{ !contains(matrix.os, 'windows') }} # Requires an additional adapted Makefile for `cl.exe` compiler
-      run: make.sh -j CFLAGS="-Werror"
+    - name: Install MinGW and Clang (Windows x64 MSYS2)
+      if: matrix.platform == 'windows-x64'
+      uses: msys2/setup-msys2@v2
+      with:
+        update: true
+        install: |
+          mingw-w64-x86_64-toolchain
+          mingw-w64-x86_64-clang
+          mingw-w64-x86_64-make
+          mingw-w64-x86_64-cmake
 
-    - name: Build wasm library
-      if: ${{ !matrix.cli-only && !matrix.use-cross }} # No sense to build on the same Github runner hosts many times
-      run: script/build-wasm
+    # TODO: Remove RUSTFLAGS="--cap-lints allow" once we use a wasmtime release that addresses
+    # the `mismatched-lifetime-syntaxes` lint
+    - name: Build wasmtime library (Windows x64 MSYS2)
+      if: contains(matrix.features, 'wasm') && matrix.platform == 'windows-x64'
+      run: |
+        mkdir -p target
+        WASMTIME_VERSION=$(cargo metadata --format-version=1 --locked --features wasm | \
+                           jq -r '.packages[] | select(.name == "wasmtime-c-api-impl") | .version')
+        curl -LSs "$WASMTIME_REPO/archive/refs/tags/v${WASMTIME_VERSION}.tar.gz" | tar xzf - -C target
+        cd target/wasmtime-${WASMTIME_VERSION}
+        cmake -S crates/c-api -B target/c-api \
+          -DCMAKE_INSTALL_PREFIX="$PWD/artifacts" \
+          -DWASMTIME_DISABLE_ALL_FEATURES=ON \
+          -DWASMTIME_FEATURE_CRANELIFT=ON \
+          -DWASMTIME_TARGET='x86_64-pc-windows-gnu'
+        cmake --build target/c-api && cmake --install target/c-api
+        printf 'CMAKE_PREFIX_PATH=%s\n' "$PWD/artifacts" >> $GITHUB_ENV
+      env:
+        WASMTIME_REPO: https://github.com/bytecodealliance/wasmtime
+        RUSTFLAGS: ${{ env.RUSTFLAGS }} --cap-lints allow
 
-    - run: $BUILD_CMD build --release --target=${{ matrix.target }} --features=${{ matrix.cli_features }}
+    - name: Build C library (Windows x64 MSYS2 CMake)
+      if: matrix.platform == 'windows-x64'
+      shell: msys2 {0}
+      run: |
+        cmake -G Ninja -S . -B build/static \
+          -DBUILD_SHARED_LIBS=OFF \
+          -DCMAKE_BUILD_TYPE=Debug \
+          -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
+          -DTREE_SITTER_FEATURE_WASM=$WASM \
+          -DCMAKE_C_COMPILER=clang
+        cmake --build build/static
 
-    - run: script/fetch-fixtures
+        cmake -G Ninja -S . -B build/shared \
+          -DBUILD_SHARED_LIBS=ON \
+          -DCMAKE_BUILD_TYPE=Debug \
+          -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
+          -DTREE_SITTER_FEATURE_WASM=$WASM \
+          -DCMAKE_C_COMPILER=clang
+        cmake --build build/shared
+        rm -rf \
+          build/{static,shared} \
+          "${CMAKE_PREFIX_PATH}/artifacts" \
+          target/wasmtime-${WASMTIME_VERSION}
+      env:
+        WASM: ${{ contains(matrix.features, 'wasm') && 'ON' || 'OFF' }}
 
-    - uses: ./.github/actions/cache
+    # TODO: Remove RUSTFLAGS="--cap-lints allow" once we use a wasmtime release that addresses
+    # the `mismatched-lifetime-syntaxes` lint
+    - name: Build wasmtime library
+      if: contains(matrix.features, 'wasm')
+      run: |
+        mkdir -p target
+        WASMTIME_VERSION=$(cargo metadata --format-version=1 --locked --features wasm | \
+                           jq -r '.packages[] | select(.name == "wasmtime-c-api-impl") | .version')
+        curl -LSs "$WASMTIME_REPO/archive/refs/tags/v${WASMTIME_VERSION}.tar.gz" | tar xzf - -C target
+        cd target/wasmtime-${WASMTIME_VERSION}
+        cmake -S crates/c-api -B target/c-api \
+          -DCMAKE_INSTALL_PREFIX="$PWD/artifacts" \
+          -DWASMTIME_DISABLE_ALL_FEATURES=ON \
+          -DWASMTIME_FEATURE_CRANELIFT=ON \
+          -DWASMTIME_TARGET='${{ matrix.target }}'
+        cmake --build target/c-api && cmake --install target/c-api
+        printf 'CMAKE_PREFIX_PATH=%s\n' "$PWD/artifacts" >> $GITHUB_ENV
+      env:
+        WASMTIME_REPO: https://github.com/bytecodealliance/wasmtime
+        RUSTFLAGS: ${{ env.RUSTFLAGS }} --cap-lints allow
+
+    - name: Build C library (make)
+      if: runner.os != 'Windows'
+      run: |
+        if [[ $PLATFORM == linux-arm ]]; then
+          CC=arm-linux-gnueabihf-gcc; AR=arm-linux-gnueabihf-ar
+        elif [[ $PLATFORM == linux-x86 ]]; then
+          CC=i686-linux-gnu-gcc; AR=i686-linux-gnu-ar
+        elif [[ $PLATFORM == linux-powerpc64 ]]; then
+          CC=powerpc64-linux-gnu-gcc; AR=powerpc64-linux-gnu-ar
+        else
+          CC=gcc; AR=ar
+        fi
+        make -j CFLAGS="$CFLAGS" CC=$CC AR=$AR
+      env:
+        PLATFORM: ${{ matrix.platform }}
+        CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types
+
+    - name: Build C library (CMake)
+      if: "!matrix.cross"
+      run: |
+        cmake -S . -B build/static \
+          -DBUILD_SHARED_LIBS=OFF \
+          -DCMAKE_BUILD_TYPE=Debug \
+          -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
+          -DTREE_SITTER_FEATURE_WASM=$WASM
+        cmake --build build/static --verbose
+
+        cmake -S . -B build/shared \
+          -DBUILD_SHARED_LIBS=ON \
+          -DCMAKE_BUILD_TYPE=Debug \
+          -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
+          -DTREE_SITTER_FEATURE_WASM=$WASM
+        cmake --build build/shared --verbose
+      env:
+        CC: ${{ contains(matrix.platform, 'linux') && 'clang' || '' }}
+        WASM: ${{ contains(matrix.features, 'wasm') && 'ON' || 'OFF' }}
+
+    - name: Build Wasm library
+      if: contains(matrix.features, 'wasm')
+      shell: bash
+      run: |
+        cd lib/binding_web
+        npm ci
+        CJS=true npm run build
+        CJS=true npm run build:debug
+        npm run build
+        npm run build:debug
+
+    - name: Check no_std builds
+      if: inputs.run-test && !matrix.no-run
+      working-directory: lib
+      shell: bash
+      run: cargo check --no-default-features --target='${{ matrix.target }}'
+
+    - name: Build target
+      run: cargo build --release --target='${{ matrix.target }}' --features='${{ matrix.features }}' $PACKAGE
+      env:
+        PACKAGE: ${{ matrix.platform == 'wasm32' && '-p tree-sitter' || '' }}
+
+    - name: Cache fixtures
       id: cache
+      if: inputs.run-test && !matrix.no-run
+      uses: ./.github/actions/cache
+
+    - name: Fetch fixtures
+      if: inputs.run-test && !matrix.no-run
+      run: cargo run -p xtask --target='${{ matrix.target }}' -- fetch-fixtures
 
     - name: Generate fixtures
-      if: ${{ !matrix.cli-only && inputs.run_test && steps.cache.outputs.cache-hit != 'true' }} # Can't natively run CLI on Github runner's host
-      run: script/generate-fixtures
+      if: inputs.run-test && !matrix.no-run && steps.cache.outputs.cache-hit != 'true'
+      run: cargo run -p xtask --target='${{ matrix.target }}' -- generate-fixtures
 
-    - name: Generate WASM fixtures
-      if: ${{ !matrix.cli-only && !matrix.use-cross && inputs.run_test && steps.cache.outputs.cache-hit != 'true' }} # See comment for the "Build wasm library" step
-      run: script/generate-fixtures-wasm
+    - name: Generate Wasm fixtures
+      if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm') && steps.cache.outputs.cache-hit != 'true'
+      run: cargo run -p xtask --target='${{ matrix.target }}' -- generate-fixtures --wasm
 
     - name: Run main tests
-      if: ${{ !matrix.cli-only && inputs.run_test }} # Can't natively run CLI on Github runner's host
-      run: $BUILD_CMD test --target=${{ matrix.target }} --features=${{ matrix.cli_features }}
+      if: inputs.run-test && !matrix.no-run
+      run: cargo test --target='${{ matrix.target }}' --features='${{ matrix.features }}'
 
-    - name: Run wasm tests
-      if: ${{ !matrix.cli-only && !matrix.use-cross && inputs.run_test }} # See comment for the "Build wasm library" step
-      run: script/test-wasm
-
-    - name: Run benchmarks
-      if: ${{ !matrix.cli-only && !matrix.use-cross && inputs.run_test }} # Cross-compiled benchmarks make no sense
-      run: $BUILD_CMD bench benchmark -p tree-sitter-cli --target=${{ matrix.target }}
+    - name: Run Wasm tests
+      if: inputs.run-test && !matrix.no-run && contains(matrix.features, 'wasm')
+      run: cargo run -p xtask --target='${{ matrix.target }}' -- test-wasm
 
     - name: Upload CLI artifact
-      uses: actions/upload-artifact@v4
+      if: "!matrix.no-run"
+      uses: actions/upload-artifact@v6
       with:
         name: tree-sitter.${{ matrix.platform }}
-        path: target/${{ matrix.target }}/release/tree-sitter${{ env.EXE }}
+        path: target/${{ matrix.target }}/release/tree-sitter${{ contains(matrix.target, 'windows') && '.exe' || '' }}
         if-no-files-found: error
         retention-days: 7
 
-    - name: Upload WASM artifacts
-      if: ${{ matrix.platform == 'linux-x64' }}
-      uses: actions/upload-artifact@v4
+    - name: Upload Wasm artifacts
+      if: matrix.platform == 'linux-x64'
+      uses: actions/upload-artifact@v6
       with:
         name: tree-sitter.wasm
         path: |
-          lib/binding_web/tree-sitter.js
-          lib/binding_web/tree-sitter.wasm
+          lib/binding_web/web-tree-sitter.js
+          lib/binding_web/web-tree-sitter.js.map
+          lib/binding_web/web-tree-sitter.cjs
+          lib/binding_web/web-tree-sitter.cjs.map
+          lib/binding_web/web-tree-sitter.wasm
+          lib/binding_web/web-tree-sitter.wasm.map
+          lib/binding_web/debug/web-tree-sitter.cjs
+          lib/binding_web/debug/web-tree-sitter.cjs.map
+          lib/binding_web/debug/web-tree-sitter.js
+          lib/binding_web/debug/web-tree-sitter.js.map
+          lib/binding_web/debug/web-tree-sitter.wasm
+          lib/binding_web/debug/web-tree-sitter.wasm.map
+          lib/binding_web/lib/*.c
+          lib/binding_web/lib/*.h
+          lib/binding_web/lib/*.ts
+          lib/binding_web/src/*.ts
         if-no-files-found: error
         retention-days: 7

.github/workflows/ci.yml

@@ -1,9 +1,21 @@
 name: CI
+
 on:
   pull_request:
+    paths-ignore:
+      - docs/**
+      - "**/README.md"
+      - CONTRIBUTING.md
+      - LICENSE
+      - cli/src/templates
   push:
-    branches:
-      - 'master'
+    branches: [master]
+    paths-ignore:
+      - docs/**
+      - "**/README.md"
+      - CONTRIBUTING.md
+      - LICENSE
+      - cli/src/templates
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -13,15 +25,25 @@ jobs:
   checks:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
-    - run: rustup toolchain install stable --profile minimal
-    - run: rustup toolchain install nightly --profile minimal
-    - run: rustup component add --toolchain nightly rustfmt
-    - uses: Swatinem/rust-cache@v2
-    - run: make lint
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Set up stable Rust toolchain
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: stable
+          components: clippy, rustfmt
+
+      - name: Lint files
+        run: |
+          make lint
+          make lint-web
 
   sanitize:
     uses: ./.github/workflows/sanitize.yml
 
   build:
     uses: ./.github/workflows/build.yml
+
+  check-wasm-stdlib:
+    uses: ./.github/workflows/wasm_stdlib.yml

.github/workflows/docs.yml

@@ -0,0 +1,50 @@
+name: Deploy Docs
+on:
+  push:
+    branches: [master]
+    paths: [docs/**]
+  workflow_dispatch:
+
+jobs:
+  deploy-docs:
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write
+      pages: write
+      id-token: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Set up Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+
+      - name: Install mdbook
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          jq_expr='.assets[] | select(.name | contains("x86_64-unknown-linux-gnu")) | .browser_download_url'
+          url=$(gh api repos/rust-lang/mdbook/releases/tags/v0.4.52 --jq "$jq_expr")
+          mkdir mdbook
+          curl -sSL "$url" | tar -xz -C mdbook
+          printf '%s/mdbook\n' "$PWD" >> "$GITHUB_PATH"
+
+      - name: Install mdbook-admonish
+        run: cargo install mdbook-admonish
+
+      - name: Build Book
+        run: mdbook build docs
+
+      - name: Setup Pages
+        uses: actions/configure-pages@v5
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v4
+        with:
+          path: docs/book
+
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

.github/workflows/nvim_ts.yml

@@ -0,0 +1,69 @@
+name: nvim-treesitter parser tests
+
+on:
+  pull_request:
+    paths:
+      - 'crates/cli/**'
+      - 'crates/config/**'
+      - 'crates/generate/**'
+      - 'crates/loader/**'
+      - '.github/workflows/nvim_ts.yml'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  check_compilation:
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        type: [generate, build]
+    name: ${{ matrix.os }} - ${{ matrix.type }}
+    runs-on: ${{ matrix.os }}
+    env:
+      NVIM: ${{ matrix.os == 'windows-latest' && 'nvim-win64\\bin\\nvim.exe' || 'nvim' }}
+      NVIM_TS_DIR: nvim-treesitter
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: actions/checkout@v6
+        with:
+          repository: nvim-treesitter/nvim-treesitter
+          path: ${{ env.NVIM_TS_DIR }}
+          ref: main
+
+      - if: runner.os != 'Windows'
+        run: echo ${{ github.workspace }}/target/release >> $GITHUB_PATH
+
+      - if: runner.os == 'Windows'
+        run: echo ${{ github.workspace }}/target/release >> $env:GITHUB_PATH
+
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+      - run: cargo build --release
+      - uses: ilammy/msvc-dev-cmd@v1
+
+      - name: Install and prepare Neovim
+        run: bash ./scripts/ci-install.sh
+        working-directory: ${{ env.NVIM_TS_DIR }}
+
+      - if: matrix.type == 'generate'
+        name: Generate and compile parsers
+        run: $NVIM -l ./scripts/install-parsers.lua --generate --max-jobs=2
+        working-directory: ${{ env.NVIM_TS_DIR }}
+        shell: bash
+
+      - if: matrix.type == 'build'
+        name: Compile parsers
+        run: $NVIM -l ./scripts/install-parsers.lua --max-jobs=10
+        working-directory: ${{ env.NVIM_TS_DIR }}
+        shell: bash
+
+      - if: "!cancelled()"
+        name: Check query files
+        run: $NVIM -l ./scripts/check-queries.lua
+        working-directory: ${{ env.NVIM_TS_DIR }}
+        shell: bash

.github/workflows/release.yml

@@ -1,4 +1,5 @@
 name: Release
+
 on:
   workflow_dispatch:
   push:
@@ -9,19 +10,22 @@ jobs:
   build:
     uses: ./.github/workflows/build.yml
     with:
-      run_test: false
+      run-test: false
 
   release:
-    name: Release
+    name: Release on GitHub
     runs-on: ubuntu-latest
     needs: build
     permissions:
+      id-token: write
+      attestations: write
       contents: write
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout repository
+        uses: actions/checkout@v6
 
       - name: Download build artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v7
         with:
           path: artifacts
 
@@ -31,9 +35,13 @@ jobs:
 
       - name: Prepare release artifacts
         run: |
-          mkdir -p target
-          mv artifacts/tree-sitter.wasm/* target/
+          mkdir -p target web
+          mv artifacts/tree-sitter.wasm/* web/
+
+          tar -czf target/web-tree-sitter.tar.gz -C web .
+
           rm -r artifacts/tree-sitter.wasm
+
           for platform in $(cd artifacts; ls | sed 's/^tree-sitter\.//'); do
             exe=$(ls artifacts/tree-sitter.$platform/tree-sitter*)
             gzip --stdout --name $exe > target/tree-sitter-$platform.gz
@@ -41,60 +49,81 @@ jobs:
           rm -rf artifacts
           ls -l target/
 
-      - name: Create release
-        uses: softprops/action-gh-release@v2
+      - name: Generate attestations
+        uses: actions/attest-build-provenance@v3
         with:
-          name: ${{ github.ref_name }}
-          tag_name: ${{ github.ref_name }}
-          fail_on_unmatched_files: true
-          files: |
+          subject-path: |
             target/tree-sitter-*.gz
-            target/tree-sitter.wasm
-            target/tree-sitter.js
+            target/web-tree-sitter.tar.gz
+
+      - name: Create release
+        run: |-
+          gh release create $GITHUB_REF_NAME \
+            target/tree-sitter-*.gz \
+            target/web-tree-sitter.tar.gz
+        env:
+          GH_TOKEN: ${{ github.token }}
 
   crates_io:
-    name: Publish CLI to Crates.io
+    name: Publish packages to Crates.io
     runs-on: ubuntu-latest
+    environment: crates
+    permissions:
+      id-token: write
+      contents: read
     needs: release
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout repository
+        uses: actions/checkout@v6
 
-      - name: Setup Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
+      - name: Set up Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+
+      - name: Set up registry token
+        id: auth
+        uses: rust-lang/crates-io-auth-action@v1
 
       - name: Publish crates to Crates.io
         uses: katyo/publish-crates@v2
         with:
-          registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+          registry-token: ${{ steps.auth.outputs.token }}
 
   npm:
-    name: Publish lib to npmjs.com
+    name: Publish packages to npmjs.com
     runs-on: ubuntu-latest
+    environment: npm
+    permissions:
+      id-token: write
+      contents: read
     needs: release
     strategy:
       fail-fast: false
       matrix:
-        directory: ["cli/npm", "lib/binding_web"]
+        directory: [crates/cli/npm, lib/binding_web]
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Set up Node
+        uses: actions/setup-node@v6
+        with:
+          node-version: 24
+          registry-url: https://registry.npmjs.org
+
+      - name: Set up Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
 
       - name: Build wasm
         if: matrix.directory == 'lib/binding_web'
-        run: ./script/build-wasm
-
-      - name: Setup Node
-        uses: actions/setup-node@v4
-        with:
-          node-version: 18
-          registry-url: "https://registry.npmjs.org"
-
-      - name: Publish lib to npmjs.com
-        env:
-          NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}
         run: |
           cd ${{ matrix.directory }}
-          npm publish
+          npm ci
+          npm run build
+          npm run build:debug
+          CJS=true npm run build
+          CJS=true npm run build:debug
+          npm run build:dts
+
+      - name: Publish to npmjs.com
+        working-directory: ${{ matrix.directory }}
+        run: npm publish

.github/workflows/response.yml

@@ -1,34 +1,47 @@
-name: no_response
+name: No response
+
 on:
   schedule:
-    - cron: '30 1 * * *' # Run every day at 01:30
+    - cron: "30 1 * * *" # Run every day at 01:30
   workflow_dispatch:
   issue_comment:
 
+permissions:
+  issues: write
+  pull-requests: write
+
 jobs:
   close:
+    name: Close issues with no response
     if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest
-    permissions:
-      issues: write
-      pull-requests: write
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/github-script@v7
+      - name: Checkout script
+        uses: actions/checkout@v6
+        with:
+          sparse-checkout: .github/scripts/close_unresponsive.js
+          sparse-checkout-cone-mode: false
+
+      - name: Run script
+        uses: actions/github-script@v8
         with:
           script: |
             const script = require('./.github/scripts/close_unresponsive.js')
             await script({github, context})
 
   remove_label:
+    name: Remove response label
     if: github.event_name == 'issue_comment'
     runs-on: ubuntu-latest
-    permissions:
-      issues: write
-      pull-requests: write
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/github-script@v7
+      - name: Checkout script
+        uses: actions/checkout@v6
+        with:
+          sparse-checkout: .github/scripts/remove_response_label.js
+          sparse-checkout-cone-mode: false
+
+      - name: Run script
+        uses: actions/github-script@v8
         with:
           script: |
             const script = require('./.github/scripts/remove_response_label.js')

.github/workflows/reviewers_remove.yml

@@ -1,16 +1,24 @@
-name: "reviewers: remove"
+name: Remove Reviewers
+
 on:
   pull_request_target:
     types: [converted_to_draft, closed]
+
+permissions:
+  pull-requests: write
+
 jobs:
   remove-reviewers:
     runs-on: ubuntu-latest
-    permissions:
-      pull-requests: write
     steps:
-      - uses: actions/checkout@v4
-      - name: 'Remove reviewers'
-        uses: actions/github-script@v7
+      - name: Checkout script
+        uses: actions/checkout@v6
+        with:
+          sparse-checkout: .github/scripts/reviewers_remove.js
+          sparse-checkout-cone-mode: false
+
+      - name: Run script
+        uses: actions/github-script@v8
         with:
           script: |
             const script = require('./.github/scripts/reviewers_remove.js')

.github/workflows/sanitize.yml

@@ -8,39 +8,44 @@ on:
   workflow_call:
 
 jobs:
-  check_undefined_behaviour:
-    name: Sanitizer checks
+  check-undefined-behaviour:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     env:
       TREE_SITTER: ${{ github.workspace }}/target/release/tree-sitter
     steps:
-    - name: Checkout source code
-      uses: actions/checkout@v4
+      - name: Checkout repository
+        uses: actions/checkout@v6
 
-    - name: Install UBSAN library
-      run: sudo apt-get update -y && sudo apt-get install -y libubsan1
+      - name: Install UBSAN library
+        run: sudo apt-get update -y && sudo apt-get install -y libubsan1
 
-    - run: rustup toolchain install stable --profile minimal
-    - uses: Swatinem/rust-cache@v2
-    - run: cargo build --release
-    - run: script/fetch-fixtures
+      - name: Set up Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
 
-    - uses: ./.github/actions/cache
-      id: cache
+      - name: Build project
+        run: cargo build --release
 
-    - if: ${{ steps.cache.outputs.cache-hit != 'true' }}
-      run: script/generate-fixtures
+      - name: Cache fixtures
+        uses: ./.github/actions/cache
+        id: cache
 
-    - name: Run main tests with undefined behaviour sanitizer (UBSAN)
-      env:
-        CFLAGS: -fsanitize=undefined
-        RUSTFLAGS: ${{ env.RUSTFLAGS }} -lubsan
-      run: cargo test -- --test-threads 1
+      - name: Fetch fixtures
+        run: cargo xtask fetch-fixtures
 
-    - name: Run main tests with address sanitizer (ASAN)
-      env:
-        ASAN_OPTIONS: verify_asan_link_order=0
-        CFLAGS: -fsanitize=address
-        RUSTFLAGS: ${{ env.RUSTFLAGS }} -lasan --cfg sanitizing
-      run: cargo test -- --test-threads 1
+      - name: Generate fixtures
+        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+        run: cargo xtask generate-fixtures
+
+      - name: Run main tests with undefined behaviour sanitizer (UBSAN)
+        run: cargo test -- --test-threads 1
+        env:
+          CFLAGS: -fsanitize=undefined
+          RUSTFLAGS: ${{ env.RUSTFLAGS }} -lubsan
+
+      - name: Run main tests with address sanitizer (ASAN)
+        run: cargo test -- --test-threads 1
+        env:
+          ASAN_OPTIONS: verify_asan_link_order=0
+          CFLAGS: -fsanitize=address
+          RUSTFLAGS: ${{ env.RUSTFLAGS }} -lasan --cfg sanitizing

.github/workflows/spam.yml

@@ -0,0 +1,29 @@
+name: Close as spam
+
+on:
+  issues:
+    types: [labeled]
+  pull_request_target:
+    types: [labeled]
+
+permissions:
+  issues: write
+  pull-requests: write
+
+jobs:
+  spam:
+    runs-on: ubuntu-latest
+    if: github.event.label.name == 'spam'
+    steps:
+      - name: Checkout script
+        uses: actions/checkout@v6
+        with:
+          sparse-checkout: .github/scripts/close_spam.js
+          sparse-checkout-cone-mode: false
+
+      - name: Run script
+        uses: actions/github-script@v8
+        with:
+          script: |
+            const script = require('./.github/scripts/close_spam.js')
+            await script({github, context})

.github/workflows/wasm_exports.yml

@@ -0,0 +1,41 @@
+name: Check Wasm Exports
+
+on:
+  pull_request:
+    paths:
+      - lib/include/tree_sitter/api.h
+      - lib/binding_web/**
+      - xtask/src/**
+  push:
+    branches: [master]
+    paths:
+      - lib/include/tree_sitter/api.h
+      - lib/binding_rust/bindings.rs
+      - CMakeLists.txt
+
+jobs:
+  check-wasm-exports:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Set up stable Rust toolchain
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: stable
+
+      - name: Install wasm-objdump
+        run: sudo apt-get update -y && sudo apt-get install -y wabt
+
+      - name: Build C library (make)
+        run: make -j CFLAGS="$CFLAGS"
+        env:
+          CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types
+
+      - name: Build Wasm Library
+        working-directory: lib/binding_web
+        run: npm ci && npm run build:debug
+
+      - name: Check Wasm exports
+        run: cargo xtask check-wasm-exports

.github/workflows/wasm_stdlib.yml

@@ -0,0 +1,19 @@
+name: Check Wasm Stdlib build
+
+on:
+  workflow_call:
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Check directory changes
+        uses: actions/github-script@v8
+        with:
+          script: |
+            const scriptPath = `${process.env.GITHUB_WORKSPACE}/.github/scripts/wasm_stdlib.js`;
+            const script = require(scriptPath);
+            return script({ github, context, core });

.gitignore

@@ -1,30 +1,40 @@
 log*.html
+.direnv
 
 .idea
 *.xcodeproj
 .vscode
 .cache
+.zig-cache
+.direnv
 
+profile*
 fuzz-results
-
-/tree-sitter.pc
+test/fuzz/out
 test/fixtures/grammars/*
 !test/fixtures/grammars/.gitkeep
-package-lock.json
+
 node_modules
 
 docs/assets/js/tree-sitter.js
 
+/tree-sitter.pc
+
 /target
 *.rs.bk
 *.a
 *.dylib
 *.so
 *.so.[0-9]*
+*.dll
 *.o
 *.obj
 *.exp
 *.lib
 *.wasm
 .swiftpm
+.build
+build
 zig-*
+
+/result

.zed/settings.json

@@ -0,0 +1,11 @@
+{
+  "lsp": {
+    "rust-analyzer": {
+      "initialization_options": {
+        "cargo": {
+          "features": "all"
+        }
+      }
+    }
+  }
+}

CHANGELOG.md

@@ -1,379 +0,0 @@
-# Changelog
-
-## [0.22.6] — 2024-05-05
-
-### Features
-
-- Improve handling of serialization buffer overflows (<https://github.com/tree-sitter/tree-sitter/pull/3318>)
-- Reverse iteration through node parents (<https://github.com/tree-sitter/tree-sitter/pull/3214>)
-- **cli**: Support `NO_COLOR` (<https://github.com/tree-sitter/tree-sitter/pull/3299>)
-- **cli**: Add test listing and allow users to parse a specific test number (<https://github.com/tree-sitter/tree-sitter/pull/3067>)
-- **grammar**: Add "inherits" field if available (<https://github.com/tree-sitter/tree-sitter/pull/3295>)
-
-### Bug Fixes
-
-- Correctly load field data from wasm languages
-- Improve error message when the `tree-sitter` field is malformed
-- Don't error out on package.json lookup errors if `--no-bindings` is passed (<https://github.com/tree-sitter/tree-sitter/pull/3323>)
-- **cli**: Keep default cc flags in build
-- **cli**: Properly account for multi-grammar repos when using docker to build a wasm parser (<https://github.com/tree-sitter/tree-sitter/pull/3337>)
-- **generate**: Don't check arbitrarily named dirs
-- **generate**: Take `AsRef<Path>` for the path parameter to avoid clones (<https://github.com/tree-sitter/tree-sitter/pull/3322>)
-- **highlight**: Correct signature of `ts_highlighter_add_language`
-- **lib**: Do not return field names for extras (<https://github.com/tree-sitter/tree-sitter/pull/3330>)
-- **lib**: Advance the lookahead end byte by 4 when there's an invalid code point (<https://github.com/tree-sitter/tree-sitter/pull/3305>)
-- **rust**: Update README example (<https://github.com/tree-sitter/tree-sitter/pull/3307>)
-- **rust**: Use unix + wasi cfg instead of not windows for fd (<https://github.com/tree-sitter/tree-sitter/pull/3304>)
-- **test**: Allow newlines in between test name and attribute
-- **wasm**: Correct `childrenFromFieldXXX` method signatures (<https://github.com/tree-sitter/tree-sitter/pull/3301>)
-- **xtask**: Always bump every crate in tandem
-- **zig**: Make usable as a zig dependency (<https://github.com/tree-sitter/tree-sitter/pull/3315>)
-
-### Documentation
-
-- Mention build command variables
-- Swap `\s` for `\\s` in query example
-- **highlight**: Typo (<https://github.com/tree-sitter/tree-sitter/pull/3290>)
-
-### Refactor
-
-- **tests**: Migrate remaining `grammar.json` tests to `grammar.js` (<https://github.com/tree-sitter/tree-sitter/pull/3325>)
-
-### Build System and CI
-
-- Add nightly rustfmt to workflow for linting (<https://github.com/tree-sitter/tree-sitter/pull/3333>)
-- Fix address sanitizer step (<https://github.com/tree-sitter/tree-sitter/pull/3188>)
-- **deps**: Bump cc from 1.0.92 to 1.0.94 in the cargo group (<https://github.com/tree-sitter/tree-sitter/pull/3298>)
-- **deps**: Bump the cargo group with 6 updates (<https://github.com/tree-sitter/tree-sitter/pull/3313>)
-- **xtask**: Bump `build.zig.zon` version when bumping versions
-
-## [0.22.5] — 2024-04-14
-
-### Bug Fixes
-
-- Avoid generating unused character set constants
-- **cli**: Test parsing on windows (<https://github.com/tree-sitter/tree-sitter/pull/3289>)
-- **rust**: Compilation on wasm32-wasi (<https://github.com/tree-sitter/tree-sitter/pull/3293>)
-
-## [0.22.4] — 2024-04-12
-
-### Bug Fixes
-
-- Fix sorting of transitions within a lex state
-- Include 2-character ranges in array-based state transitions
-
-### Build System and CI
-
-- Always bump at least the patch version in bump xtask
-
-## [0.22.3] — 2024-04-12
-
-### Features
-
-- Add strncat to wasm stdlib
-- Generate simpler code for matching large character sets (<https://github.com/tree-sitter/tree-sitter/pull/3234>)
-- When loading languages via WASM, gracefully handle memory errors and leaks in external scanners (<https://github.com/tree-sitter/tree-sitter/pull/3181>)
-
-### Bug Fixes
-
-- **bindings**: Add utf-8 flag to python & node (<https://github.com/tree-sitter/tree-sitter/pull/3278>)
-- **bindings**: Generate parser.c if missing (<https://github.com/tree-sitter/tree-sitter/pull/3277>)
-- **bindings**: Remove required platforms for swift (<https://github.com/tree-sitter/tree-sitter/pull/3264>)
-- **cli**: Fix mismatched parenthesis when accounting for `&&` (<https://github.com/tree-sitter/tree-sitter/pull/3274>)
-- **lib**: Do not consider childless nodes for ts_node_parent (<https://github.com/tree-sitter/tree-sitter/pull/3191>)
-- **lib**: Properly account for aliased root nodes and root nodes with
-  children in `ts_subtree_string` (<https://github.com/tree-sitter/tree-sitter/pull/3191>)
-- **lib**: Account for the root node of a tree cursor being an alias (<https://github.com/tree-sitter/tree-sitter/pull/3191>)
-- **lib**: Use correct format specifier in log message (<https://github.com/tree-sitter/tree-sitter/pull/3255>)
-- **parser**: Fix variadic macro (<https://github.com/tree-sitter/tree-sitter/pull/3229>)
-- render: Proper function prototypes (<https://github.com/tree-sitter/tree-sitter/pull/3277>)
-- **windows**: Add `/utf-8` flag for parsers using unicode symbols (<https://github.com/tree-sitter/tree-sitter/pull/3223>)
-- Add a semicolon after SKIP macros (<https://github.com/tree-sitter/tree-sitter/pull/3264>)
-- Add back `build-wasm` temporarily (<https://github.com/tree-sitter/tree-sitter/pull/3203>)
-- Add lifetime to matches function (<https://github.com/tree-sitter/tree-sitter/pull/3254>)
-- Default output directory for `build --wasm` should use current_dir (<https://github.com/tree-sitter/tree-sitter/pull/3203>)
-- Fix sorting of wasm stdlib symbols
-- Insert "tree-sitter" section in current directory's package.json if it exists (<https://github.com/tree-sitter/tree-sitter/pull/3224>)
-- Tie the lifetime of the cursor to the query in `QueryCursor::captures()` (<https://github.com/tree-sitter/tree-sitter/pull/3266>)
-- Wrong flag check in `build.rs`
-
-### Performance
-
-- **cli**: Reduced the compile time of generated parsers by generating C code with fewer conditionals (<https://github.com/tree-sitter/tree-sitter/pull/3234>)
-
-### Documentation
-
-- Add NGINX grammar
-
-### Refactor
-
-- **parser**: Make REDUCE macro non-variadic (<https://github.com/tree-sitter/tree-sitter/pull/3280>)
-- **js**: Misc fixes & tidying
-- **rust**: Misc fixes & tidying
-
-### Testing
-
-- Add regression test for node parent + string bug (<https://github.com/tree-sitter/tree-sitter/pull/3191>)
-- **test**: Allow colons in test names (<https://github.com/tree-sitter/tree-sitter/pull/3264>)
-
-### Build System and CI
-
-- Upgrade wasmtime
-- Update emscripten version (<https://github.com/tree-sitter/tree-sitter/pull/3272>)
-- **dependabot**: Improve PR labels (<https://github.com/tree-sitter/tree-sitter/pull/3282>)
-
-## [0.22.2] — 2024-03-17
-
-### Breaking
-
-- **cli**: Add a separate build command to compile parsers
-
-### Features
-
-- **bindings/rust**: Expose `Parser::included_ranges`
-- Lower the lib's MSRV (<https://github.com/tree-sitter/tree-sitter/pull/3169>)
-- **lib**: Implement Display for Node (<https://github.com/tree-sitter/tree-sitter/pull/3177>)
-
-### Bug Fixes
-
-- **bindings/wasm**: Fix `Parser.getIncludedRanges()` (<https://github.com/tree-sitter/tree-sitter/pull/3164>)
-- **lib**: Makefile installation on macOS (<https://github.com/tree-sitter/tree-sitter/pull/3167>)
-- **lib**: Makefile installation (<https://github.com/tree-sitter/tree-sitter/pull/3173>)
-- **lib**: Avoid possible UB of calling memset on a null ptr when 0 is passed into `array_grow_by` (<https://github.com/tree-sitter/tree-sitter/pull/3176>)
-- **lib**: Allow hiding symbols (<https://github.com/tree-sitter/tree-sitter/pull/3180>)
-
-### Documentation
-
-- Fix typo (<https://github.com/tree-sitter/tree-sitter/pull/3158>)
-- **licensfe**: Update year (<https://github.com/tree-sitter/tree-sitter/pull/3183>)
-
-### Refactor
-
-- Remove dependency on which crate (<https://github.com/tree-sitter/tree-sitter/pull/3172>)
-- Turbofish styling
-
-### Testing
-
-- Fix header writes (<https://github.com/tree-sitter/tree-sitter/pull/3174>)
-
-### Build System and CI
-
-- Simplify workflows (<https://github.com/tree-sitter/tree-sitter/pull/3002>)
-- **lib**: Allow overriding CFLAGS on the commandline (<https://github.com/tree-sitter/tree-sitter/pull/3159>)
-
-## [0.22.1] — 2024-03-10
-
-### Bug Fixes
-
-- Cli build script behavior on release
-
-## [0.22.0] — 2024-03-10
-
-### Breaking
-
-- Remove top-level `corpus` dir for tests
-  The cli will now only look in `test/corpus` for tests
-- Remove redundant escape regex & curly brace regex preprocessing (<https://github.com/tree-sitter/tree-sitter/pull/2838>)
-- **bindings**: Convert node bindings to NAPI (<https://github.com/tree-sitter/tree-sitter/pull/3077>)
-- **wasm**: Make `current*`, `is*`, and `has*` methods properties (<https://github.com/tree-sitter/tree-sitter/pull/3103>)
-- **wasm**: Keep API in-line with upstream and start aligning with node (<https://github.com/tree-sitter/tree-sitter/pull/3149>)
-
-### Features
-
-- Add xtasks to assist with bumping crates (<https://github.com/tree-sitter/tree-sitter/pull/3065>)
-- Improve language bindings (<https://github.com/tree-sitter/tree-sitter/pull/2438>)
-- Expose the allocator and array header files for external scanners (<https://github.com/tree-sitter/tree-sitter/pull/3063>)
-- Add typings for the node bindings
-- Replace `nan` with `node-addon-api` and conditionally print logs
-- **bindings**: Add more make targets
-- **bindings**: Add peerDependencies for npm
-- **bindings**: Add prebuildify to node
-- **bindings**: Remove dsl types file (<https://github.com/tree-sitter/tree-sitter/pull/3126>)
-- **node**: Type tag the language (<https://github.com/tree-sitter/tree-sitter/pull/3109>)
-- **test**: Add attributes for corpus tests
-
-### Bug Fixes
-
-- Apply some `scan-build` suggestions (unused assignment/garbage access) (<https://github.com/tree-sitter/tree-sitter/pull/3056>)
-- Wrap `||` comparison in parentheses when `&&` is used (<https://github.com/tree-sitter/tree-sitter/pull/3070>)
-- Ignore unused variables in the array macros (<https://github.com/tree-sitter/tree-sitter/pull/3083>)
-- `binding.cc` overwrite should replace `PARSER_NAME` (<https://github.com/tree-sitter/tree-sitter/pull/3116>)
-- Don't use `__declspec(dllexport)` on windows (<https://github.com/tree-sitter/tree-sitter/pull/3128>)
-- Parsers should export the language function on windows
-- Allow the regex `v` flag (<https://github.com/tree-sitter/tree-sitter/pull/3154>)
-- **assertions**: Case shouldn't matter for comment node detection
-- **bindings**: Editorconfig and setup.py fixes (<https://github.com/tree-sitter/tree-sitter/pull/3082>)
-- **bindings**: Insert `types` after `main` if it exists (<https://github.com/tree-sitter/tree-sitter/pull/3122>)
-- **bindings**: Fix template oversights (<https://github.com/tree-sitter/tree-sitter/pull/3155>)
-- **cli**: Only output the sources with `--no-bindings` (<https://github.com/tree-sitter/tree-sitter/pull/3123>)
-- **generate**: Add `.npmignore`, populate Swift's exclude list (<https://github.com/tree-sitter/tree-sitter/pull/3085>)
-- **generate**: Extern allocator functions for the template don't need to be "exported" (<https://github.com/tree-sitter/tree-sitter/pull/3132>)
-- **generate**: Camel case name in `Cargo.toml` description (<https://github.com/tree-sitter/tree-sitter/pull/3140>)
-- **lib**: Include `api.h` so `ts_set_allocator` is visible (<https://github.com/tree-sitter/tree-sitter/pull/3092>)
-
-### Documentation
-
-- Add GitHub user and PR info to the changelog
-- Add css for inline code (<https://github.com/tree-sitter/tree-sitter/pull/2844>)
-- Document test attributes
-- Add `Ohm` language parser
-- Remove duplicate `the`'s (<https://github.com/tree-sitter/tree-sitter/pull/3120>)
-- Add discord and matrix badges (<https://github.com/tree-sitter/tree-sitter/pull/3148>)
-
-### Refactor
-
-- Rename TS_REUSE_ALLOCATOR flag (<https://github.com/tree-sitter/tree-sitter/pull/3088>)
-- Remove extern/const where possible
-- **array**: Use pragma GCC in clang too
-- **bindings**: Remove npmignore (<https://github.com/tree-sitter/tree-sitter/pull/3089>)
-
-### Testing
-
-- Don't use TS_REUSE_ALLOCATOR on Darwin systems (<https://github.com/tree-sitter/tree-sitter/pull/3087>)
-- Add test case for parse stack merging with incorrect error cost bug (<https://github.com/tree-sitter/tree-sitter/pull/3098>)
-
-### Build System and CI
-
-- Improve changelog settings (<https://github.com/tree-sitter/tree-sitter/pull/3064>)
-- Unify crate versions via workspace (<https://github.com/tree-sitter/tree-sitter/pull/3074>)
-- Update `cc` to remove annoying debug output (<https://github.com/tree-sitter/tree-sitter/pull/3075>)
-- Adjust dependabot settings (<https://github.com/tree-sitter/tree-sitter/pull/3079>)
-- Use c11 everywhere
-- Add uninstall command
-- Don't skip tests on failing lint (<https://github.com/tree-sitter/tree-sitter/pull/3102>)
-- Remove unused deps, bump deps, and bump MSRV to 1.74.1 (<https://github.com/tree-sitter/tree-sitter/pull/3153>)
-- **bindings**: Metadata improvements
-- **bindings**: Make everything c11 (<https://github.com/tree-sitter/tree-sitter/pull/3099>)
-- **dependabot**: Update weekly instead of daily (<https://github.com/tree-sitter/tree-sitter/pull/3112>)
-- **deps**: Bump the cargo group with 1 update (<https://github.com/tree-sitter/tree-sitter/pull/3081>)
-- **deps**: Bump the cargo group with 1 update (<https://github.com/tree-sitter/tree-sitter/pull/3097>)
-- **deps**: Bump deps & lockfile (<https://github.com/tree-sitter/tree-sitter/pull/3060>)
-- **deps**: Bump the cargo group with 4 updates (<https://github.com/tree-sitter/tree-sitter/pull/3134>)
-- **lint**: Detect if `Cargo.lock` needs to be updated (<https://github.com/tree-sitter/tree-sitter/pull/3066>)
-- **lint**: Make lockfile check quiet (<https://github.com/tree-sitter/tree-sitter/pull/3078>)
-- **swift**: Move 'cLanguageStandard' behind 'targets' (<https://github.com/tree-sitter/tree-sitter/pull/3101>)
-
-### Other
-
-- Make Node.js language bindings context aware (<https://github.com/tree-sitter/tree-sitter/pull/2841>)
-  They don't have any dynamic global data, so all it takes is just declaring them as such
-- Fix crash when attempting to load ancient languages via wasm (<https://github.com/tree-sitter/tree-sitter/pull/3068>)
-- Use workspace dependencies for internal crates like Tree-sitter (<https://github.com/tree-sitter/tree-sitter/pull/3076>)
-- Remove vendored wasmtime headers (https://github.com/tree-sitter/tree-sitter/pull/3084)
-  When building rust binding, use wasmtime headers provided via cargo
-  by the wasmtime-c-api crate.
-- Fix invalid parse stack recursive merging with mismatched error cost (<https://github.com/tree-sitter/tree-sitter/pull/3086>)
-  Allowing this invalid merge caused an invariant to be violated
-  later on during parsing, when handling a later error.
-- Fix regression in `subtree_compare` (<https://github.com/tree-sitter/tree-sitter/pull/3111>)
-- docs: Add `Ohm` language parser (<https://github.com/tree-sitter/tree-sitter/pull/3114>)
-- Delete `binding_files.rs` (<https://github.com/tree-sitter/tree-sitter/pull/3106>)
-- **bindings**: Consistent wording (<https://github.com/tree-sitter/tree-sitter/pull/3096>)
-- **bindings**: Ignore more artifacts (<https://github.com/tree-sitter/tree-sitter/pull/3119>)
-
-## [0.21.0] — 2024-02-21
-
-### Breaking
-
-- Remove the apply-all-captures flag, make last-wins precedence the default
-
-  **NOTE**: This change might cause breakage in your grammar's highlight tests.
-  Just flip the order around of the relevant queries, and keep in mind that the
-  last query that matches will win.
-
-### Features
-
-- Use lockfiles to dedup recompilation
-- Improve error message for files with an unknown grammar path (<https://github.com/tree-sitter/tree-sitter/pull/2475>)
-- Implement first-line-regex (<https://github.com/tree-sitter/tree-sitter/pull/2479>)
-- Error out if an empty string is in the `extras` array
-- Allow specifying an external scanner's files (<https://github.com/tree-sitter/tree-sitter/pull/3031>)
-- Better error info when a scanner is missing required symbols
-- **cli**: Add an optional `grammar-path` argument for the playground (<https://github.com/tree-sitter/tree-sitter/pull/3014>)
-- **cli**: Add optional `config-path` argument (<https://github.com/tree-sitter/tree-sitter/pull/3050>)
-- **loader**: Add more commonly used default parser directories
-
-### Bug Fixes
-
-- Prettify xml output and add node position info (<https://github.com/tree-sitter/tree-sitter/pull/2970>)
-- Inherited grammar generation
-- Properly error out when the word property is an invalid rule
-- Update schema for regex flags (<https://github.com/tree-sitter/tree-sitter/pull/3006>)
-- Properly handle `Query.matches` when filtering out results (<https://github.com/tree-sitter/tree-sitter/pull/3013>)
-- Sexp format edge case with quoted closed parenthesis (<https://github.com/tree-sitter/tree-sitter/pull/3016>)
-- Always push the default files if there's no `externals`
-- Don't log NUL characters (<https://github.com/tree-sitter/tree-sitter/pull/3037>)
-- Don't throw an error if the user uses `map` in the grammar (<https://github.com/tree-sitter/tree-sitter/pull/3041>)
-- Remove redundant imports (<https://github.com/tree-sitter/tree-sitter/pull/3047>)
-- **cli**: Installation via a HTTP tunnel proxy (<https://github.com/tree-sitter/tree-sitter/pull/2824>)
-- **cli**: Don't update tests automatically if parse errors are detected (<https://github.com/tree-sitter/tree-sitter/pull/3033>)
-- **cli**: Don't use `long` for `grammar_path`
-- **test**: Allow writing updates to tests without erroneous nodes instead of denying all of them if a single error is found
-- **test**: Edge case when parsing `UNEXPECTED`/`MISSING` nodes with an indentation level greater than 0
-- **wasm**: Remove C++ mangled symbols (<https://github.com/tree-sitter/tree-sitter/pull/2971>)
-
-### Documentation
-
-- Create issue template (<https://github.com/tree-sitter/tree-sitter/pull/2978>)
-- Document regex limitations
-- Mention that `token($.foo)` is illegal
-- Explicitly mention behavior of walking outside the given "root" node for a `TSTreeCursor` (<https://github.com/tree-sitter/tree-sitter/pull/3021>)
-- Small fixes (<https://github.com/tree-sitter/tree-sitter/pull/2987>)
-- Add `Tact` language parser (<https://github.com/tree-sitter/tree-sitter/pull/3030>)
-- **web**: Provide deno usage information (<https://github.com/tree-sitter/tree-sitter/pull/2498>)
-
-### Refactor
-
-- Extract regex check into a function and lower its precedence
-- `&PathBuf` -> `&Path` (<https://github.com/tree-sitter/tree-sitter/pull/3035>)
-- Name anonymous types in api.h (<https://github.com/tree-sitter/tree-sitter/pull/1659>)
-
-### Testing
-
-- Add quotes around bash variables (<https://github.com/tree-sitter/tree-sitter/pull/3023>)
-- Update html tests
-
-### Build System and CI
-
-- Only create release for normal semver tags (<https://github.com/tree-sitter/tree-sitter/pull/2973>)
-- Add useful development targets to makefile (<https://github.com/tree-sitter/tree-sitter/pull/2979>)
-- Remove minimum glibc information in summary page (<https://github.com/tree-sitter/tree-sitter/pull/2988>)
-- Use the native m1 mac runner (<https://github.com/tree-sitter/tree-sitter/pull/2995>)
-- Add editorconfig (<https://github.com/tree-sitter/tree-sitter/pull/2998>)
-- Remove symbolic links from repository (<https://github.com/tree-sitter/tree-sitter/pull/2997>)
-- Move common Cargo.toml keys into the workspace and inherit them (<https://github.com/tree-sitter/tree-sitter/pull/3019>)
-- Remove reviewers when drafting or closing a PR (<https://github.com/tree-sitter/tree-sitter/pull/2963>)
-- Enable creating changelogs with git-cliff (<https://github.com/tree-sitter/tree-sitter/pull/3040>)
-- Cache fixtures (<https://github.com/tree-sitter/tree-sitter/pull/3038>)
-- Don't cancel jobs on master (<https://github.com/tree-sitter/tree-sitter/pull/3052>)
-- Relax caching requirements (<https://github.com/tree-sitter/tree-sitter/pull/3051>)
-- **deps**: Bump clap from 4.4.18 to 4.5.0 (<https://github.com/tree-sitter/tree-sitter/pull/3007>)
-- **deps**: Bump wasmtime from v16.0.0 to v17.0.1 (<https://github.com/tree-sitter/tree-sitter/pull/3008>)
-- **deps**: Bump wasmtime to v18.0.1 (<https://github.com/tree-sitter/tree-sitter/pull/3057>)
-- **sanitize**: Add a timeout of 60 minutes (<https://github.com/tree-sitter/tree-sitter/pull/3017>)
-- **sanitize**: Reduce timeout to 20 minutes (<https://github.com/tree-sitter/tree-sitter/pull/3054>)
-
-### Other
-
-- Document preferred language for scanner (<https://github.com/tree-sitter/tree-sitter/pull/2972>)
-- Add java and tsx to corpus tests (<https://github.com/tree-sitter/tree-sitter/pull/2992>)
-- Provide a CLI flag to open `log.html` (<https://github.com/tree-sitter/tree-sitter/pull/2996>)
-- Some more clippy lints (<https://github.com/tree-sitter/tree-sitter/pull/3010>)
-- Remove deprecated query parsing mechanism (<https://github.com/tree-sitter/tree-sitter/pull/3011>)
-- Print out full compiler arguments ran when it fails (<https://github.com/tree-sitter/tree-sitter/pull/3018>)
-- Deprecate C++ scanners (<https://github.com/tree-sitter/tree-sitter/pull/3020>)
-- Add some documentation to the playground page (<https://github.com/tree-sitter/tree-sitter/pull/1495>)
-- Update relevant rust tests (<https://github.com/tree-sitter/tree-sitter/pull/2947>)
-- Clippy lints (<https://github.com/tree-sitter/tree-sitter/pull/3032>)
-- Error out when multiple arguments are passed to `token`/`token.immediate` (<https://github.com/tree-sitter/tree-sitter/pull/3036>)
-- Tidying
-- Prefer turbofish syntax where possible (<https://github.com/tree-sitter/tree-sitter/pull/3048>)
-- Use published wasmtime crates
-- Cleaner cast
-- Update `Cargo.lock`
-- Get rid of `github_issue_test` file (<https://github.com/tree-sitter/tree-sitter/pull/3055>)
-- **cli**: Use spawn to display `emcc`'s stdout and stderr (<https://github.com/tree-sitter/tree-sitter/pull/2494>)
-- **cli**: Warn users when a query path needed for a subcommand isn't specified in a grammar's package.json
-- **generate**: Dedup and warn about duplicate or invalid rules (<https://github.com/tree-sitter/tree-sitter/pull/2994>)
-- **test**: Use different languages for async tests (<https://github.com/tree-sitter/tree-sitter/pull/2953>)
-- **wasm**: Use `SIDE_MODULE=2` to silence warning (<https://github.com/tree-sitter/tree-sitter/pull/3003>)

CMakeLists.txt

@@ -0,0 +1,95 @@
+cmake_minimum_required(VERSION 3.13)
+
+project(tree-sitter
+        VERSION "0.27.0"
+        DESCRIPTION "An incremental parsing system for programming tools"
+        HOMEPAGE_URL "https://tree-sitter.github.io/tree-sitter/"
+        LANGUAGES C)
+
+option(BUILD_SHARED_LIBS "Build using shared libraries" ON)
+option(TREE_SITTER_FEATURE_WASM "Enable the Wasm feature" OFF)
+option(AMALGAMATED "Build using an amalgamated source" OFF)
+
+if(AMALGAMATED)
+  set(TS_SOURCE_FILES "${PROJECT_SOURCE_DIR}/lib/src/lib.c")
+else()
+  file(GLOB TS_SOURCE_FILES lib/src/*.c)
+  list(REMOVE_ITEM TS_SOURCE_FILES "${PROJECT_SOURCE_DIR}/lib/src/lib.c")
+endif()
+
+add_library(tree-sitter ${TS_SOURCE_FILES})
+
+target_include_directories(tree-sitter PRIVATE lib/src lib/src/wasm PUBLIC lib/include)
+
+if(MSVC)
+  target_compile_options(tree-sitter PRIVATE
+                         /wd4018 # disable 'signed/unsigned mismatch'
+                         /wd4232 # disable 'nonstandard extension used'
+                         /wd4244 # disable 'possible loss of data'
+                         /wd4267 # disable 'possible loss of data (size_t)'
+                         /wd4701 # disable 'potentially uninitialized local variable'
+                         /we4022 # treat 'incompatible types' as an error
+                         /W4)
+else()
+  target_compile_options(tree-sitter PRIVATE
+                         -Wall -Wextra -Wshadow -Wpedantic
+                         -Werror=incompatible-pointer-types)
+endif()
+
+if(TREE_SITTER_FEATURE_WASM)
+  if(NOT DEFINED CACHE{WASMTIME_INCLUDE_DIR})
+    message(CHECK_START "Looking for wasmtime headers")
+    find_path(WASMTIME_INCLUDE_DIR wasmtime.h
+              PATHS ENV DEP_WASMTIME_C_API_INCLUDE)
+    if(NOT WASMTIME_INCLUDE_DIR)
+      unset(WASMTIME_INCLUDE_DIR CACHE)
+      message(FATAL_ERROR "Could not find wasmtime headers.\nDid you forget to set CMAKE_INCLUDE_PATH?")
+    endif()
+    message(CHECK_PASS "found")
+  endif()
+
+  if(NOT DEFINED CACHE{WASMTIME_LIBRARY})
+    message(CHECK_START "Looking for wasmtime library")
+    find_library(WASMTIME_LIBRARY wasmtime)
+    if(NOT WASMTIME_LIBRARY)
+      unset(WASMTIME_LIBRARY CACHE)
+      message(FATAL_ERROR "Could not find wasmtime library.\nDid you forget to set CMAKE_LIBRARY_PATH?")
+    endif()
+    message(CHECK_PASS "found")
+  endif()
+
+  target_compile_definitions(tree-sitter PUBLIC TREE_SITTER_FEATURE_WASM)
+  target_include_directories(tree-sitter SYSTEM PRIVATE "${WASMTIME_INCLUDE_DIR}")
+  target_link_libraries(tree-sitter PUBLIC "${WASMTIME_LIBRARY}")
+  set_property(TARGET tree-sitter PROPERTY C_STANDARD_REQUIRED ON)
+
+  if(NOT BUILD_SHARED_LIBS)
+    if(WIN32)
+      target_compile_definitions(tree-sitter PRIVATE WASM_API_EXTERN= WASI_API_EXTERN=)
+      target_link_libraries(tree-sitter INTERFACE ws2_32 advapi32 userenv ntdll shell32 ole32 bcrypt)
+    elseif(NOT APPLE)
+      target_link_libraries(tree-sitter INTERFACE pthread dl m)
+    endif()
+  endif()
+endif()
+
+set_target_properties(tree-sitter
+                      PROPERTIES
+                      C_STANDARD 11
+                      C_VISIBILITY_PRESET hidden
+                      POSITION_INDEPENDENT_CODE ON
+                      SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}"
+                      DEFINE_SYMBOL "")
+
+target_compile_definitions(tree-sitter PRIVATE _POSIX_C_SOURCE=200112L _DEFAULT_SOURCE _BSD_SOURCE _DARWIN_C_SOURCE)
+
+include(GNUInstallDirs)
+
+configure_file(lib/tree-sitter.pc.in "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter.pc" @ONLY)
+
+install(FILES lib/include/tree_sitter/api.h
+        DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/tree_sitter")
+install(FILES "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter.pc"
+        DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
+install(TARGETS tree-sitter
+        LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}")

CONTRIBUTING.md

@@ -1 +1 @@
-See [section-6-contributing.md](./docs/section-6-contributing.md)
+See [docs/src/6-contributing.md](./docs/src/6-contributing.md)

Cargo.toml

@@ -1,28 +1,86 @@
 [workspace]
-default-members = ["cli"]
+default-members = ["crates/cli"]
 members = [
-  "cli",
-  "cli/config",
-  "cli/loader",
+  "crates/cli",
+  "crates/config",
+  "crates/generate",
+  "crates/highlight",
+  "crates/loader",
+  "crates/tags",
+  "crates/xtask",
+  "crates/language",
   "lib",
-  "lib/language",
-  "tags",
-  "highlight",
-  "xtask",
 ]
 resolver = "2"
 
 [workspace.package]
-version = "0.22.6"
-authors = ["Max Brunsfeld <maxbrunsfeld@gmail.com>"]
+version = "0.27.0"
+authors = [
+  "Max Brunsfeld <maxbrunsfeld@gmail.com>",
+  "Amaan Qureshi <amaanq12@gmail.com>",
+]
 edition = "2021"
-rust-version = "1.74.1"
+rust-version = "1.85"
 homepage = "https://tree-sitter.github.io/tree-sitter"
 repository = "https://github.com/tree-sitter/tree-sitter"
 license = "MIT"
 keywords = ["incremental", "parsing"]
 categories = ["command-line-utilities", "parsing"]
 
+[workspace.lints.clippy]
+dbg_macro = "deny"
+todo = "deny"
+pedantic = { level = "warn", priority = -1 }
+nursery = { level = "warn", priority = -1 }
+cargo = { level = "warn", priority = -1 }
+
+# The lints below are a specific subset of the pedantic+nursery lints
+# that we explicitly allow in the tree-sitter codebase because they either:
+#
+# 1. Contain false positives,
+# 2. Are unnecessary, or
+# 3. Worsen the code
+
+branches_sharing_code = "allow"
+cast_lossless = "allow"
+cast_possible_truncation = "allow"
+cast_possible_wrap = "allow"
+cast_precision_loss = "allow"
+cast_sign_loss = "allow"
+checked_conversions = "allow"
+cognitive_complexity = "allow"
+collection_is_never_read = "allow"
+fallible_impl_from = "allow"
+fn_params_excessive_bools = "allow"
+inline_always = "allow"
+if_not_else = "allow"
+items_after_statements = "allow"
+match_wildcard_for_single_variants = "allow"
+missing_errors_doc = "allow"
+missing_panics_doc = "allow"
+module_name_repetitions = "allow"
+multiple_crate_versions = "allow"
+needless_for_each = "allow"
+obfuscated_if_else = "allow"
+option_if_let_else = "allow"
+or_fun_call = "allow"
+range_plus_one = "allow"
+redundant_clone = "allow"
+redundant_closure_for_method_calls = "allow"
+ref_option = "allow"
+similar_names = "allow"
+string_lit_as_bytes = "allow"
+struct_excessive_bools = "allow"
+struct_field_names = "allow"
+transmute_undefined_repr = "allow"
+too_many_lines = "allow"
+unnecessary_wraps = "allow"
+unused_self = "allow"
+used_underscore_items = "allow"
+
+[workspace.lints.rust]
+mismatched_lifetime_syntaxes = "allow"
+
 [profile.optimize]
 inherits = "release"
 strip = true         # Automatically strip symbols from the binary.
@@ -34,59 +92,72 @@ codegen-units = 1    # Maximum size reduction optimizations.
 inherits = "optimize"
 opt-level = "s"       # Optimize for size.
 
-[profile.profile]
-inherits = "optimize"
-strip = false
+[profile.release-dev]
+inherits = "release"
+lto = false
+debug = true
+debug-assertions = true
+overflow-checks = true
+incremental = true
+codegen-units = 256
 
 [workspace.dependencies]
-anstyle = "1.0.7"
-anyhow = "1.0.86"
-cc = "1.0.98"
-clap = { version = "4.5.4", features = [
+ansi_colours = "1.2.3"
+anstyle = "1.0.13"
+anyhow = "1.0.100"
+bstr = "1.12.0"
+cc = "1.2.53"
+clap = { version = "4.5.54", features = [
   "cargo",
   "derive",
   "env",
   "help",
+  "string",
   "unstable-styles",
 ] }
-ctor = "0.2.8"
-ctrlc = { version = "3.4.4", features = ["termination"] }
-dirs = "5.0.1"
-filetime = "0.2.23"
-fs4 = "0.8.3"
-git2 = "0.18.3"
-glob = "0.3.1"
+clap_complete = "4.5.65"
+clap_complete_nushell = "4.5.10"
+crc32fast = "1.5.0"
+ctor = "0.2.9"
+ctrlc = { version = "3.5.0", features = ["termination"] }
+dialoguer = { version = "0.11.0", features = ["fuzzy-select"] }
+etcetera = "0.11.0"
+fs4 = "0.12.0"
+glob = "0.3.3"
 heck = "0.5.0"
 html-escape = "0.2.13"
-indexmap = "2.2.6"
-indoc = "2.0.5"
-lazy_static = "1.4.0"
-libloading = "0.8.3"
-log = { version = "0.4.21", features = ["std"] }
-memchr = "2.7.2"
-once_cell = "1.19.0"
-pretty_assertions = "1.4.0"
+indexmap = "2.12.1"
+indoc = "2.0.6"
+libloading = "0.9.0"
+log = { version = "0.4.28", features = ["std"] }
+memchr = "2.7.6"
+once_cell = "1.21.3"
+pretty_assertions = "1.4.1"
 rand = "0.8.5"
-regex = "1.10.4"
-regex-syntax = "0.8.3"
-rustc-hash = "1.1.0"
-semver = "1.0.23"
-serde = { version = "1.0.202", features = ["derive"] }
-serde_derive = "1.0.197"
-serde_json = { version = "1.0.117", features = ["preserve_order"] }
-similar = "2.5.0"
-smallbitvec = "2.5.3"
-tempfile = "3.10.1"
-thiserror = "1.0.61"
+regex = "1.11.3"
+regex-syntax = "0.8.6"
+rustc-hash = "2.1.1"
+schemars = "1.0.5"
+semver = { version = "1.0.27", features = ["serde"] }
+serde = { version = "1.0.219", features = ["derive"] }
+serde_json = { version = "1.0.149", features = ["preserve_order"] }
+similar = "2.7.0"
+smallbitvec = "2.6.0"
+streaming-iterator = "0.1.9"
+tempfile = "3.23.0"
+thiserror = "2.0.17"
 tiny_http = "0.12.0"
-toml = "0.8.13"
-unindent = "0.2.3"
+topological-sort = "0.2.2"
+unindent = "0.2.4"
 walkdir = "2.5.0"
-wasmparser = "0.206.0"
-webbrowser = "1.0.1"
+wasmparser = "0.243.0"
+webbrowser = "1.0.5"
 
-tree-sitter = { version = "0.22.6", path = "./lib" }
-tree-sitter-loader = { version = "0.22.6", path = "./cli/loader" }
-tree-sitter-config = { version = "0.22.6", path = "./cli/config" }
-tree-sitter-highlight = { version = "0.22.6", path = "./highlight" }
-tree-sitter-tags = { version = "0.22.6", path = "./tags" }
+tree-sitter = { version = "0.27.0", path = "./lib" }
+tree-sitter-generate = { version = "0.27.0", path = "./crates/generate" }
+tree-sitter-loader = { version = "0.27.0", path = "./crates/loader" }
+tree-sitter-config = { version = "0.27.0", path = "./crates/config" }
+tree-sitter-highlight = { version = "0.27.0", path = "./crates/highlight" }
+tree-sitter-tags = { version = "0.27.0", path = "./crates/tags" }
+
+tree-sitter-language = { version = "0.1", path = "./crates/language" }

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2018-2024 Max Brunsfeld
+Copyright (c) 2018 Max Brunsfeld
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -1,9 +1,12 @@
-VERSION := 0.22.6
+VERSION := 0.27.0
+DESCRIPTION := An incremental parsing system for programming tools
+HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/
 
 # install directory layout
 PREFIX ?= /usr/local
 INCLUDEDIR ?= $(PREFIX)/include
 LIBDIR ?= $(PREFIX)/lib
+BINDIR ?= $(PREFIX)/bin
 PCLIBDIR ?= $(LIBDIR)/pkgconfig
 
 # collect sources
@@ -19,8 +22,9 @@ OBJ := $(SRC:.c=.o)
 
 # define default flags, and override to append mandatory flags
 ARFLAGS := rcs
-CFLAGS ?= -O3 -Wall -Wextra -Wshadow -pedantic
+CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types
 override CFLAGS += -std=c11 -fPIC -fvisibility=hidden
+override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE -D_BSD_SOURCE -D_DARWIN_C_SOURCE
 override CFLAGS += -Ilib/src -Ilib/src/wasm -Ilib/include
 
 # ABI versioning
@@ -28,22 +32,25 @@ SONAME_MAJOR := $(word 1,$(subst ., ,$(VERSION)))
 SONAME_MINOR := $(word 2,$(subst ., ,$(VERSION)))
 
 # OS-specific bits
-ifeq ($(OS),Windows_NT)
-	$(error "Windows is not supported")
-else ifeq ($(shell uname),Darwin)
+MACHINE := $(shell $(CC) -dumpmachine)
+
+ifneq ($(findstring darwin,$(MACHINE)),)
 	SOEXT = dylib
-	SOEXTVER_MAJOR = $(SONAME_MAJOR).dylib
-	SOEXTVER = $(SONAME_MAJOR).$(SONAME_MINOR).dylib
-	LINKSHARED += -dynamiclib -Wl,-install_name,$(LIBDIR)/libtree-sitter.$(SONAME_MAJOR).dylib
+	SOEXTVER_MAJOR = $(SONAME_MAJOR).$(SOEXT)
+	SOEXTVER = $(SONAME_MAJOR).$(SONAME_MINOR).$(SOEXT)
+	LINKSHARED += -dynamiclib -Wl,-install_name,$(LIBDIR)/libtree-sitter.$(SOEXTVER)
+else ifneq ($(findstring mingw32,$(MACHINE)),)
+	SOEXT = dll
+	LINKSHARED += -s -shared -Wl,--out-implib,libtree-sitter.dll.a
 else
 	SOEXT = so
-	SOEXTVER_MAJOR = so.$(SONAME_MAJOR)
-	SOEXTVER = so.$(SONAME_MAJOR).$(SONAME_MINOR)
-	LINKSHARED += -shared -Wl,-soname,libtree-sitter.so.$(SONAME_MAJOR)
-endif
+	SOEXTVER_MAJOR = $(SOEXT).$(SONAME_MAJOR)
+	SOEXTVER = $(SOEXT).$(SONAME_MAJOR).$(SONAME_MINOR)
+	LINKSHARED += -shared -Wl,-soname,libtree-sitter.$(SOEXTVER)
 ifneq ($(filter $(shell uname),FreeBSD NetBSD DragonFly),)
 	PCLIBDIR := $(PREFIX)/libdata/pkgconfig
 endif
+endif
 
 all: libtree-sitter.a libtree-sitter.$(SOEXT) tree-sitter.pc
 
@@ -56,24 +63,39 @@ ifneq ($(STRIP),)
 	$(STRIP) $@
 endif
 
-tree-sitter.pc: tree-sitter.pc.in
-	sed -e 's|@VERSION@|$(VERSION)|' \
-		-e 's|@LIBDIR@|$(LIBDIR)|' \
-		-e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \
-		-e 's|=$(PREFIX)|=$${prefix}|' \
-		-e 's|@PREFIX@|$(PREFIX)|' $< > $@
+ifneq ($(findstring mingw32,$(MACHINE)),)
+libtree-sitter.dll.a: libtree-sitter.$(SOEXT)
+endif
+
+tree-sitter.pc: lib/tree-sitter.pc.in
+	sed -e 's|@PROJECT_VERSION@|$(VERSION)|' \
+		-e 's|@CMAKE_INSTALL_LIBDIR@|$(LIBDIR:$(PREFIX)/%=%)|' \
+		-e 's|@CMAKE_INSTALL_INCLUDEDIR@|$(INCLUDEDIR:$(PREFIX)/%=%)|' \
+		-e 's|@PROJECT_DESCRIPTION@|$(DESCRIPTION)|' \
+		-e 's|@PROJECT_HOMEPAGE_URL@|$(HOMEPAGE_URL)|' \
+		-e 's|@CMAKE_INSTALL_PREFIX@|$(PREFIX)|' $< > $@
+
+shared: libtree-sitter.$(SOEXT)
+
+static: libtree-sitter.a
 
 clean:
-	$(RM) $(OBJ) tree-sitter.pc libtree-sitter.a libtree-sitter.$(SOEXT)
+	$(RM) $(OBJ) tree-sitter.pc libtree-sitter.a libtree-sitter.$(SOEXT) libtree-stitter.dll.a
 
 install: all
 	install -d '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter '$(DESTDIR)$(PCLIBDIR)' '$(DESTDIR)$(LIBDIR)'
 	install -m644 lib/include/tree_sitter/api.h '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/api.h
 	install -m644 tree-sitter.pc '$(DESTDIR)$(PCLIBDIR)'/tree-sitter.pc
 	install -m644 libtree-sitter.a '$(DESTDIR)$(LIBDIR)'/libtree-sitter.a
+ifneq ($(findstring mingw32,$(MACHINE)),)
+	install -d '$(DESTDIR)$(BINDIR)'
+	install -m755 libtree-sitter.dll '$(DESTDIR)$(BINDIR)'/libtree-sitter.dll
+	install -m755 libtree-sitter.dll.a '$(DESTDIR)$(LIBDIR)'/libtree-sitter.dll.a
+else
 	install -m755 libtree-sitter.$(SOEXT) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXTVER)
-	ln -sf libtree-sitter.$(SOEXTVER) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXTVER_MAJOR)
-	ln -sf libtree-sitter.$(SOEXTVER_MAJOR) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXT)
+	cd '$(DESTDIR)$(LIBDIR)' && ln -sf libtree-sitter.$(SOEXTVER) libtree-sitter.$(SOEXTVER_MAJOR)
+	cd '$(DESTDIR)$(LIBDIR)' && ln -sf libtree-sitter.$(SOEXTVER_MAJOR) libtree-sitter.$(SOEXT)
+endif
 
 uninstall:
 	$(RM) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.a \
@@ -82,31 +104,36 @@ uninstall:
 		'$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXT) \
 		'$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/api.h \
 		'$(DESTDIR)$(PCLIBDIR)'/tree-sitter.pc
+	rmdir '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter
 
-.PHONY: all install uninstall clean
+.PHONY: all shared static install uninstall clean
 
 
 ##### Dev targets #####
 
 test:
-	script/fetch-fixtures
-	script/generate-fixtures
-	script/test
+	cargo xtask fetch-fixtures
+	cargo xtask generate-fixtures
+	cargo xtask test
 
-test_wasm:
-	script/generate-fixtures-wasm
-	script/test-wasm
+test-wasm:
+	cargo xtask generate-fixtures --wasm
+	cargo xtask test-wasm
 
 lint:
 	cargo update --workspace --locked --quiet
 	cargo check --workspace --all-targets
-	cargo +nightly fmt --all --check
+	cargo fmt --all --check
 	cargo clippy --workspace --all-targets -- -D warnings
 
+lint-web:
+	npm --prefix lib/binding_web ci
+	npm --prefix lib/binding_web run lint
+
 format:
-	cargo +nightly fmt --all
+	cargo fmt --all
 
 changelog:
-	@git-cliff --config script/cliff.toml --output CHANGELOG.md --latest --github-token $(shell gh auth token)
+	@git-cliff --config .github/cliff.toml --prepend CHANGELOG.md --latest --github-token $(shell gh auth token)
 
-.PHONY: test test_wasm lint format changelog
+.PHONY: test test-wasm lint format changelog

Package.swift

@@ -15,27 +15,21 @@ let package = Package(
         .target(name: "TreeSitter",
                 path: "lib",
                 exclude: [
-                    "binding_rust",
-                    "binding_web",
-                    "node_modules",
-                    "Cargo.toml",
-                    "README.md",
-                    "src/unicode/README.md",
-                    "src/unicode/LICENSE",
-                    "src/unicode/ICU_SHA",
-                    "src/get_changed_ranges.c",
-                    "src/tree_cursor.c",
-                    "src/stack.c",
-                    "src/node.c",
-                    "src/lexer.c",
-                    "src/parser.c",
-                    "src/language.c",
-                    "src/alloc.c",
-                    "src/subtree.c",
-                    "src/tree.c",
-                    "src/query.c"
+                        "src/unicode/ICU_SHA",
+                        "src/unicode/README.md",
+                        "src/unicode/LICENSE",
+                        "src/wasm/stdlib-symbols.txt",
+                        "src/lib.c",
                 ],
-                sources: ["src/lib.c"]),
+                sources: ["src"],
+                publicHeadersPath: "include",
+                cSettings: [
+                        .headerSearchPath("src"),
+                        .define("_POSIX_C_SOURCE", to: "200112L"),
+                        .define("_DEFAULT_SOURCE"),
+                        .define("_BSD_SOURCE"),
+                        .define("_DARWIN_C_SOURCE"),
+                ]),
     ],
     cLanguageStandard: .c11
 )

README.md

@@ -14,8 +14,8 @@ Tree-sitter is a parser generator tool and an incremental parsing library. It ca
 ## Links
 - [Documentation](https://tree-sitter.github.io)
 - [Rust binding](lib/binding_rust/README.md)
-- [WASM binding](lib/binding_web/README.md)
-- [Command-line interface](cli/README.md)
+- [Wasm binding](lib/binding_web/README.md)
+- [Command-line interface](crates/cli/README.md)
 
 [discord]: https://img.shields.io/discord/1063097320771698699?logo=discord&label=discord
 [matrix]: https://img.shields.io/matrix/tree-sitter-chat%3Amatrix.org?logo=matrix&label=matrix

build.zig

@@ -1,18 +1,142 @@
 const std = @import("std");
 
-pub fn build(b: *std.Build) void {
-    var lib = b.addStaticLibrary(.{
+pub fn build(b: *std.Build) !void {
+    const target = b.standardTargetOptions(.{});
+    const optimize = b.standardOptimizeOption(.{});
+
+    const wasm = b.option(bool, "enable-wasm", "Enable Wasm support") orelse false;
+    const shared = b.option(bool, "build-shared", "Build a shared library") orelse false;
+    const amalgamated = b.option(bool, "amalgamated", "Build using an amalgamated source") orelse false;
+
+    const lib: *std.Build.Step.Compile = b.addLibrary(.{
         .name = "tree-sitter",
-        .target = b.standardTargetOptions(.{}),
-        .optimize = b.standardOptimizeOption(.{}),
+        .linkage = if (shared) .dynamic else .static,
+        .root_module = b.createModule(.{
+            .target = target,
+            .optimize = optimize,
+            .link_libc = true,
+            .pic = if (shared) true else null,
+        }),
     });
 
-    lib.linkLibC();
-    lib.addCSourceFile(.{ .file = b.path("lib/src/lib.c"), .flags = &.{"-std=c11"} });
+    if (amalgamated) {
+        lib.addCSourceFile(.{
+            .file = b.path("lib/src/lib.c"),
+            .flags = &.{"-std=c11"},
+        });
+    } else {
+        const files = try findSourceFiles(b);
+        defer b.allocator.free(files);
+        lib.addCSourceFiles(.{
+            .root = b.path("lib/src"),
+            .files = files,
+            .flags = &.{"-std=c11"},
+        });
+    }
+
     lib.addIncludePath(b.path("lib/include"));
     lib.addIncludePath(b.path("lib/src"));
+    lib.addIncludePath(b.path("lib/src/wasm"));
+
+    lib.root_module.addCMacro("_POSIX_C_SOURCE", "200112L");
+    lib.root_module.addCMacro("_DEFAULT_SOURCE", "");
+    lib.root_module.addCMacro("_BSD_SOURCE", "");
+    lib.root_module.addCMacro("_DARWIN_C_SOURCE", "");
+
+    if (wasm) {
+        if (b.lazyDependency(wasmtimeDep(target.result), .{})) |wasmtime| {
+            lib.root_module.addCMacro("TREE_SITTER_FEATURE_WASM", "");
+            lib.addSystemIncludePath(wasmtime.path("include"));
+            lib.addLibraryPath(wasmtime.path("lib"));
+            if (shared) lib.linkSystemLibrary("wasmtime");
+        }
+    }
 
     lib.installHeadersDirectory(b.path("lib/include"), ".", .{});
 
     b.installArtifact(lib);
 }
+
+/// Get the name of the wasmtime dependency for this target.
+pub fn wasmtimeDep(target: std.Target) []const u8 {
+    const arch = target.cpu.arch;
+    const os = target.os.tag;
+    const abi = target.abi;
+    return @as(?[]const u8, switch (os) {
+        .linux => switch (arch) {
+            .x86_64 => switch (abi) {
+                .gnu => "wasmtime_c_api_x86_64_linux",
+                .musl => "wasmtime_c_api_x86_64_musl",
+                .android => "wasmtime_c_api_x86_64_android",
+                else => null,
+            },
+            .aarch64 => switch (abi) {
+                .gnu => "wasmtime_c_api_aarch64_linux",
+                .musl => "wasmtime_c_api_aarch64_musl",
+                .android => "wasmtime_c_api_aarch64_android",
+                else => null,
+            },
+            .x86 => switch (abi) {
+                .gnu => "wasmtime_c_api_i686_linux",
+                else => null,
+            },
+            .arm => switch (abi) {
+                .gnueabi => "wasmtime_c_api_armv7_linux",
+                else => null,
+            },
+            .s390x => switch (abi) {
+                .gnu => "wasmtime_c_api_s390x_linux",
+                else => null,
+            },
+            .riscv64 => switch (abi) {
+                .gnu => "wasmtime_c_api_riscv64gc_linux",
+                else => null,
+            },
+            else => null,
+        },
+        .windows => switch (arch) {
+            .x86_64 => switch (abi) {
+                .gnu => "wasmtime_c_api_x86_64_mingw",
+                .msvc => "wasmtime_c_api_x86_64_windows",
+                else => null,
+            },
+            .aarch64 => switch (abi) {
+                .msvc => "wasmtime_c_api_aarch64_windows",
+                else => null,
+            },
+            .x86 => switch (abi) {
+                .msvc => "wasmtime_c_api_i686_windows",
+                else => null,
+            },
+            else => null,
+        },
+        .macos => switch (arch) {
+            .x86_64 => "wasmtime_c_api_x86_64_macos",
+            .aarch64 => "wasmtime_c_api_aarch64_macos",
+            else => null,
+        },
+        else => null,
+    }) orelse std.debug.panic(
+        "Unsupported target for wasmtime: {s}-{s}-{s}",
+        .{ @tagName(arch), @tagName(os), @tagName(abi) },
+    );
+}
+
+fn findSourceFiles(b: *std.Build) ![]const []const u8 {
+    var sources: std.ArrayListUnmanaged([]const u8) = .empty;
+
+    var dir = try b.build_root.handle.openDir("lib/src", .{ .iterate = true });
+    var iter = dir.iterate();
+    defer dir.close();
+
+    while (try iter.next()) |entry| {
+        if (entry.kind != .file) continue;
+        const file = entry.name;
+        const ext = std.fs.path.extension(file);
+        if (std.mem.eql(u8, ext, ".c") and !std.mem.eql(u8, file, "lib.c")) {
+            try sources.append(b.allocator, b.dupe(file));
+        }
+    }
+
+    return sources.toOwnedSlice(b.allocator);
+}

build.zig.zon

@@ -1,10 +1,96 @@
 .{
-    .name = "tree-sitter",
-    .version = "0.22.6",
+    .name = .tree_sitter,
+    .fingerprint = 0x841224b447ac0d4f,
+    .version = "0.27.0",
+    .minimum_zig_version = "0.14.1",
     .paths = .{
         "build.zig",
         "build.zig.zon",
         "lib/src",
         "lib/include",
+        "README.md",
+        "LICENSE",
+    },
+    .dependencies = .{
+        .wasmtime_c_api_aarch64_android = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-android-c-api.tar.xz",
+            .hash = "N-V-__8AAIfPIgdw2YnV3QyiFQ2NHdrxrXzzCdjYJyxJDOta",
+            .lazy = true,
+        },
+        .wasmtime_c_api_aarch64_linux = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-linux-c-api.tar.xz",
+            .hash = "N-V-__8AAIt97QZi7Pf7nNJ2mVY6uxA80Klyuvvtop3pLMRK",
+            .lazy = true,
+        },
+        .wasmtime_c_api_aarch64_macos = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-macos-c-api.tar.xz",
+            .hash = "N-V-__8AAAO48QQf91w9RmmUDHTja8DrXZA1n6Bmc8waW3qe",
+            .lazy = true,
+        },
+        .wasmtime_c_api_aarch64_musl = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-musl-c-api.tar.xz",
+            .hash = "N-V-__8AAI196wa9pwADoA2RbCDp5F7bKQg1iOPq6gIh8-FH",
+            .lazy = true,
+        },
+        .wasmtime_c_api_aarch64_windows = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-aarch64-windows-c-api.zip",
+            .hash = "N-V-__8AAC9u4wXfqd1Q6XyQaC8_DbQZClXux60Vu5743N05",
+            .lazy = true,
+        },
+        .wasmtime_c_api_armv7_linux = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-armv7-linux-c-api.tar.xz",
+            .hash = "N-V-__8AAHXe8gWs3s83Cc5G6SIq0_jWxj8fGTT5xG4vb6-x",
+            .lazy = true,
+        },
+        .wasmtime_c_api_i686_linux = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-i686-linux-c-api.tar.xz",
+            .hash = "N-V-__8AAN2pzgUUfulRCYnipSfis9IIYHoTHVlieLRmKuct",
+            .lazy = true,
+        },
+        .wasmtime_c_api_i686_windows = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-i686-windows-c-api.zip",
+            .hash = "N-V-__8AAJu0YAUUTFBLxFIOi-MSQVezA6MMkpoFtuaf2Quf",
+            .lazy = true,
+        },
+        .wasmtime_c_api_riscv64gc_linux = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-riscv64gc-linux-c-api.tar.xz",
+            .hash = "N-V-__8AAG8m-gc3E3AIImtTZ3l1c7HC6HUWazQ9OH5KACX4",
+            .lazy = true,
+        },
+        .wasmtime_c_api_s390x_linux = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-s390x-linux-c-api.tar.xz",
+            .hash = "N-V-__8AAH314gd-gE4IBp2uvAL3gHeuW1uUZjMiLLeUdXL_",
+            .lazy = true,
+        },
+        .wasmtime_c_api_x86_64_android = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-android-c-api.tar.xz",
+            .hash = "N-V-__8AAIPNRwfNkznebrcGb0IKUe7f35bkuZEYOjcx6q3f",
+            .lazy = true,
+        },
+        .wasmtime_c_api_x86_64_linux = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-linux-c-api.tar.xz",
+            .hash = "N-V-__8AAI8EDwcyTtk_Afhk47SEaqfpoRqGkJeZpGs69ChF",
+            .lazy = true,
+        },
+        .wasmtime_c_api_x86_64_macos = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-macos-c-api.tar.xz",
+            .hash = "N-V-__8AAGtGNgVaOpHSxC22IjrampbRIy6lLwscdcAE8nG1",
+            .lazy = true,
+        },
+        .wasmtime_c_api_x86_64_mingw = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-mingw-c-api.zip",
+            .hash = "N-V-__8AAPS2PAbVix50L6lnddlgazCPTz3whLUFk1qnRtnZ",
+            .lazy = true,
+        },
+        .wasmtime_c_api_x86_64_musl = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-musl-c-api.tar.xz",
+            .hash = "N-V-__8AAF-WEQe0nzvi09PgusM5i46FIuCKJmIDWUleWgQ3",
+            .lazy = true,
+        },
+        .wasmtime_c_api_x86_64_windows = .{
+            .url = "https://github.com/bytecodealliance/wasmtime/releases/download/v33.0.2/wasmtime-v33.0.2-x86_64-windows-c-api.zip",
+            .hash = "N-V-__8AAKGNXwbpJQsn0_6kwSIVDDWifSg8cBzf7T2RzsC9",
+            .lazy = true,
+        },
     },
 }

cli/build.rs

@@ -1,139 +0,0 @@
-use std::{
-    env,
-    ffi::OsStr,
-    fs,
-    path::{Path, PathBuf},
-    time::SystemTime,
-};
-
-fn main() {
-    if let Some(git_sha) = read_git_sha() {
-        println!("cargo:rustc-env=BUILD_SHA={git_sha}");
-    }
-
-    if web_playground_files_present() {
-        println!("cargo:rustc-cfg=TREE_SITTER_EMBED_WASM_BINDING");
-    }
-
-    let build_time = SystemTime::now()
-        .duration_since(SystemTime::UNIX_EPOCH)
-        .unwrap()
-        .as_secs_f64();
-    println!("cargo:rustc-env=BUILD_TIME={build_time}");
-
-    #[cfg(any(
-        target_os = "linux",
-        target_os = "android",
-        target_os = "freebsd",
-        target_os = "openbsd",
-        target_os = "netbsd",
-        target_os = "dragonfly",
-    ))]
-    {
-        let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()).join("dynamic-symbols.txt");
-        std::fs::write(
-            &out_dir,
-            "{
-                ts_current_malloc;
-                ts_current_calloc;
-                ts_current_realloc;
-                ts_current_free;
-            };",
-        )
-        .unwrap();
-        println!(
-            "cargo:rustc-link-arg=-Wl,--dynamic-list={}",
-            out_dir.display()
-        );
-    }
-}
-
-fn web_playground_files_present() -> bool {
-    let paths = [
-        "../docs/assets/js/playground.js",
-        "../lib/binding_web/tree-sitter.js",
-        "../lib/binding_web/tree-sitter.wasm",
-    ];
-
-    paths.iter().all(|p| Path::new(p).exists())
-}
-
-fn read_git_sha() -> Option<String> {
-    let mut repo_path = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
-
-    let mut git_path;
-    loop {
-        git_path = repo_path.join(".git");
-        if git_path.exists() {
-            break;
-        }
-        if !repo_path.pop() {
-            return None;
-        }
-    }
-
-    let git_dir_path;
-    if git_path.is_dir() {
-        git_dir_path = git_path;
-    } else if let Ok(git_path_content) = fs::read_to_string(&git_path) {
-        git_dir_path = repo_path.join(git_path_content.get("gitdir: ".len()..).unwrap().trim_end());
-    } else {
-        return None;
-    }
-    let git_head_path = git_dir_path.join("HEAD");
-    if let Some(path) = git_head_path.to_str() {
-        println!("cargo:rerun-if-changed={path}");
-    }
-    if let Ok(mut head_content) = fs::read_to_string(&git_head_path) {
-        if head_content.ends_with('\n') {
-            head_content.pop();
-        }
-
-        // If we're on a branch, read the SHA from the ref file.
-        if head_content.starts_with("ref: ") {
-            head_content.replace_range(0.."ref: ".len(), "");
-            let ref_filename = {
-                // Go to real non-worktree gitdir
-                let git_dir_path = git_dir_path
-                    .parent()
-                    .and_then(|p| {
-                        p.file_name()
-                            .map(|n| n == OsStr::new("worktrees"))
-                            .and_then(|x| x.then(|| p.parent()))
-                    })
-                    .flatten()
-                    .unwrap_or(&git_dir_path);
-
-                let file = git_dir_path.join(&head_content);
-                if file.is_file() {
-                    file
-                } else {
-                    let packed_refs = git_dir_path.join("packed-refs");
-                    if let Ok(packed_refs_content) = fs::read_to_string(&packed_refs) {
-                        for line in packed_refs_content.lines() {
-                            if let Some((hash, r#ref)) = line.split_once(' ') {
-                                if r#ref == head_content {
-                                    if let Some(path) = packed_refs.to_str() {
-                                        println!("cargo:rerun-if-changed={path}");
-                                    }
-                                    return Some(hash.to_string());
-                                }
-                            }
-                        }
-                    }
-                    return None;
-                }
-            };
-            if let Some(path) = ref_filename.to_str() {
-                println!("cargo:rerun-if-changed={path}");
-            }
-            return fs::read_to_string(&ref_filename).ok();
-        }
-        // If we're on a detached commit, then the `HEAD` file itself contains the sha.
-        if head_content.len() == 40 {
-            return Some(head_content);
-        }
-    }
-
-    None
-}

cli/loader/emscripten-version

@@ -1 +0,0 @@
-3.1.55

cli/src/generate/grammar_files.rs

@@ -1,657 +0,0 @@
-use std::{
-    fs,
-    fs::File,
-    io::BufReader,
-    path::{Path, PathBuf},
-    str,
-};
-
-use anyhow::{anyhow, Context, Result};
-use heck::{ToKebabCase, ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase};
-use indoc::indoc;
-use serde::Deserialize;
-use serde_json::{json, Map, Value};
-
-use super::write_file;
-
-const CLI_VERSION: &str = env!("CARGO_PKG_VERSION");
-const CLI_VERSION_PLACEHOLDER: &str = "CLI_VERSION";
-
-const PARSER_NAME_PLACEHOLDER: &str = "PARSER_NAME";
-const CAMEL_PARSER_NAME_PLACEHOLDER: &str = "CAMEL_PARSER_NAME";
-const UPPER_PARSER_NAME_PLACEHOLDER: &str = "UPPER_PARSER_NAME";
-const LOWER_PARSER_NAME_PLACEHOLDER: &str = "LOWER_PARSER_NAME";
-
-const GRAMMAR_JS_TEMPLATE: &str = include_str!("./templates/grammar.js");
-const PACKAGE_JSON_TEMPLATE: &str = include_str!("./templates/package.json");
-const GITIGNORE_TEMPLATE: &str = include_str!("./templates/gitignore");
-const GITATTRIBUTES_TEMPLATE: &str = include_str!("./templates/gitattributes");
-const EDITORCONFIG_TEMPLATE: &str = include_str!("./templates/.editorconfig");
-
-const RUST_BINDING_VERSION: &str = env!("CARGO_PKG_VERSION");
-const RUST_BINDING_VERSION_PLACEHOLDER: &str = "RUST_BINDING_VERSION";
-
-const LIB_RS_TEMPLATE: &str = include_str!("./templates/lib.rs");
-const BUILD_RS_TEMPLATE: &str = include_str!("./templates/build.rs");
-const CARGO_TOML_TEMPLATE: &str = include_str!("./templates/cargo.toml");
-
-const INDEX_JS_TEMPLATE: &str = include_str!("./templates/index.js");
-const INDEX_D_TS_TEMPLATE: &str = include_str!("./templates/index.d.ts");
-const JS_BINDING_CC_TEMPLATE: &str = include_str!("./templates/js-binding.cc");
-const BINDING_GYP_TEMPLATE: &str = include_str!("./templates/binding.gyp");
-const BINDING_TEST_JS_TEMPLATE: &str = include_str!("./templates/binding_test.js");
-
-const MAKEFILE_TEMPLATE: &str = include_str!("./templates/makefile");
-const PARSER_NAME_H_TEMPLATE: &str = include_str!("./templates/PARSER_NAME.h");
-const PARSER_NAME_PC_IN_TEMPLATE: &str = include_str!("./templates/PARSER_NAME.pc.in");
-
-const GO_MOD_TEMPLATE: &str = include_str!("./templates/go.mod");
-const BINDING_GO_TEMPLATE: &str = include_str!("./templates/binding.go");
-const BINDING_TEST_GO_TEMPLATE: &str = include_str!("./templates/binding_test.go");
-
-const SETUP_PY_TEMPLATE: &str = include_str!("./templates/setup.py");
-const INIT_PY_TEMPLATE: &str = include_str!("./templates/__init__.py");
-const INIT_PYI_TEMPLATE: &str = include_str!("./templates/__init__.pyi");
-const PYPROJECT_TOML_TEMPLATE: &str = include_str!("./templates/pyproject.toml");
-const PY_BINDING_C_TEMPLATE: &str = include_str!("./templates/py-binding.c");
-const TEST_BINDING_PY_TEMPLATE: &str = include_str!("./templates/test_binding.py");
-
-const PACKAGE_SWIFT_TEMPLATE: &str = include_str!("./templates/package.swift");
-const TESTS_SWIFT_TEMPLATE: &str = include_str!("./templates/tests.swift");
-
-#[derive(Deserialize, Debug)]
-struct LanguageConfiguration {}
-
-#[derive(Deserialize, Debug)]
-pub struct PackageJSON {
-    #[serde(rename = "tree-sitter")]
-    tree_sitter: Option<Vec<LanguageConfiguration>>,
-}
-
-pub fn path_in_ignore(repo_path: &Path) -> bool {
-    [
-        "bindings",
-        "build",
-        "examples",
-        "node_modules",
-        "queries",
-        "script",
-        "src",
-        "target",
-        "test",
-        "types",
-    ]
-    .iter()
-    .any(|dir| repo_path.ends_with(dir))
-}
-
-fn insert_after(
-    map: Map<String, Value>,
-    after: &str,
-    key: &str,
-    value: Value,
-) -> Map<String, Value> {
-    let mut entries = map.into_iter().collect::<Vec<_>>();
-    let after_index = entries
-        .iter()
-        .position(|(k, _)| k == after)
-        .unwrap_or(entries.len() - 1)
-        + 1;
-    entries.insert(after_index, (key.to_string(), value));
-    entries.into_iter().collect()
-}
-
-pub fn generate_grammar_files(
-    repo_path: &Path,
-    language_name: &str,
-    generate_bindings: bool,
-) -> Result<()> {
-    let dashed_language_name = language_name.to_kebab_case();
-
-    // TODO: remove legacy code updates in v0.24.0
-
-    // Create or update package.json
-    let package_json_path_state = missing_path_else(
-        repo_path.join("package.json"),
-        |path| generate_file(path, PACKAGE_JSON_TEMPLATE, dashed_language_name.as_str()),
-        |path| {
-            let package_json_str =
-                fs::read_to_string(path).with_context(|| "Failed to read package.json")?;
-            let mut package_json = serde_json::from_str::<Map<String, Value>>(&package_json_str)
-                .with_context(|| "Failed to parse package.json")?;
-            if generate_bindings {
-                let mut updated = false;
-
-                let dependencies = package_json
-                    .entry("dependencies".to_string())
-                    .or_insert_with(|| Value::Object(Map::new()))
-                    .as_object_mut()
-                    .unwrap();
-                if dependencies.remove("nan").is_some() {
-                    eprintln!("Replacing nan dependency with node-addon-api in package.json");
-                    dependencies.insert("node-addon-api".to_string(), "^8.0.0".into());
-                    updated = true;
-                }
-                if !dependencies.contains_key("node-gyp-build") {
-                    eprintln!("Adding node-gyp-build dependency to package.json");
-                    dependencies.insert("node-gyp-build".to_string(), "^4.8.1".into());
-                    updated = true;
-                }
-
-                let dev_dependencies = package_json
-                    .entry("devDependencies".to_string())
-                    .or_insert_with(|| Value::Object(Map::new()))
-                    .as_object_mut()
-                    .unwrap();
-                if !dev_dependencies.contains_key("prebuildify") {
-                    eprintln!("Adding prebuildify devDependency to package.json");
-                    dev_dependencies.insert("prebuildify".to_string(), "^6.0.1".into());
-                    updated = true;
-                }
-
-                let node_test = "node --test bindings/node/*_test.js";
-                let scripts = package_json
-                    .entry("scripts".to_string())
-                    .or_insert_with(|| Value::Object(Map::new()))
-                    .as_object_mut()
-                    .unwrap();
-                if !scripts.get("test").is_some_and(|v| v == node_test) {
-                    eprintln!("Updating package.json scripts");
-                    *scripts = Map::from_iter([
-                        ("install".to_string(), "node-gyp-build".into()),
-                        ("prestart".to_string(), "tree-sitter build --wasm".into()),
-                        ("start".to_string(), "tree-sitter playground".into()),
-                        ("test".to_string(), node_test.into()),
-                    ]);
-                    updated = true;
-                }
-
-                // insert `peerDependencies` after `dependencies`
-                if !package_json.contains_key("peerDependencies") {
-                    eprintln!("Adding peerDependencies to package.json");
-                    package_json = insert_after(
-                        package_json,
-                        "dependencies",
-                        "peerDependencies",
-                        json!({"tree-sitter": "^0.21.1"}),
-                    );
-
-                    package_json = insert_after(
-                        package_json,
-                        "peerDependencies",
-                        "peerDependenciesMeta",
-                        json!({"tree_sitter": {"optional": true}}),
-                    );
-                    updated = true;
-                }
-
-                // insert `types` right after `main`
-                if !package_json.contains_key("types") {
-                    eprintln!("Adding types to package.json");
-                    package_json =
-                        insert_after(package_json, "main", "types", "bindings/node".into());
-                    updated = true;
-                }
-
-                // insert `files` right after `keywords`
-                if !package_json.contains_key("files") {
-                    eprintln!("Adding files to package.json");
-                    package_json = insert_after(
-                        package_json,
-                        "keywords",
-                        "files",
-                        json!([
-                            "grammar.js",
-                            "binding.gyp",
-                            "prebuilds/**",
-                            "bindings/node/*",
-                            "queries/*",
-                            "src/**",
-                            "*.wasm"
-                        ]),
-                    );
-                    updated = true;
-                }
-
-                // insert `tree-sitter` at the end
-                if !package_json.contains_key("tree-sitter") {
-                    eprintln!("Adding a `tree-sitter` section to package.json");
-                    package_json.insert(
-                        "tree-sitter".to_string(),
-                        json!([{
-                            "scope": format!("source.{language_name}"),
-                            "injection-regex": format!("^{language_name}$"),
-                        }]),
-                    );
-                    updated = true;
-                }
-
-                if updated {
-                    let mut package_json_str = serde_json::to_string_pretty(&package_json)?;
-                    package_json_str.push('\n');
-                    write_file(path, package_json_str)?;
-                }
-            }
-
-            Ok(())
-        },
-    )?;
-
-    let package_json = match lookup_package_json_for_path(package_json_path_state.as_path()) {
-        Ok((_, p)) => p,
-        Err(e) if generate_bindings => return Err(e),
-        _ => return Ok(()),
-    };
-
-    // Do not create a grammar.js file in a repo with multiple language configs
-    if !package_json.has_multiple_language_configs() {
-        missing_path(repo_path.join("grammar.js"), |path| {
-            generate_file(path, GRAMMAR_JS_TEMPLATE, language_name)
-        })?;
-    }
-
-    if !generate_bindings {
-        // our job is done
-        return Ok(());
-    }
-
-    // Write .gitignore file
-    missing_path(repo_path.join(".gitignore"), |path| {
-        generate_file(path, GITIGNORE_TEMPLATE, language_name)
-    })?;
-
-    // Write .gitattributes file
-    missing_path(repo_path.join(".gitattributes"), |path| {
-        generate_file(path, GITATTRIBUTES_TEMPLATE, language_name)
-    })?;
-
-    // Write .editorconfig file
-    missing_path(repo_path.join(".editorconfig"), |path| {
-        generate_file(path, EDITORCONFIG_TEMPLATE, language_name)
-    })?;
-
-    let bindings_dir = repo_path.join("bindings");
-
-    // Generate Rust bindings
-    missing_path(bindings_dir.join("rust"), create_dir)?.apply(|path| {
-        missing_path_else(
-            path.join("lib.rs"),
-            |path| generate_file(path, LIB_RS_TEMPLATE, language_name),
-            |path| {
-                let lib_rs =
-                    fs::read_to_string(path).with_context(|| "Failed to read lib.rs")?;
-                if !lib_rs.contains("tree_sitter_language") {
-                    generate_file(path, LIB_RS_TEMPLATE, language_name)?;
-                    eprintln!("Updated lib.rs with `tree_sitter_language` dependency");
-                }
-                Ok(())
-            },
-        )?;
-
-        missing_path_else(
-            path.join("build.rs"),
-            |path| generate_file(path, BUILD_RS_TEMPLATE, language_name),
-            |path| {
-                let build_rs =
-                    fs::read_to_string(path).with_context(|| "Failed to read build.rs")?;
-                if !build_rs.contains("-utf-8") {
-                    let index = build_rs
-                        .find("    let parser_path = src_dir.join(\"parser.c\")")
-                        .ok_or_else(|| anyhow!(indoc!{
-                            "Failed to auto-update build.rs with the `/utf-8` flag for windows.
-                             To fix this, remove `bindings/rust/build.rs` and re-run `tree-sitter generate`"}))?;
-
-                    let build_rs = format!(
-                        "{}{}{}\n{}",
-                        &build_rs[..index],
-                        "    #[cfg(target_env = \"msvc\")]\n",
-                        "    c_config.flag(\"-utf-8\");\n",
-                        &build_rs[index..]
-                    );
-
-                    write_file(path, build_rs)?;
-                    eprintln!("Updated build.rs with the /utf-8 flag for Windows compilation");
-                }
-                Ok(())
-            },
-        )?;
-
-        missing_path_else(
-            repo_path.join("Cargo.toml"),
-            |path| generate_file(path, CARGO_TOML_TEMPLATE, dashed_language_name.as_str()),
-            |path| {
-                let cargo_toml =
-                    fs::read_to_string(path).with_context(|| "Failed to read Cargo.toml")?;
-                if !cargo_toml.contains("tree-sitter-language") {
-                    let start_index = cargo_toml
-                        .find("tree-sitter = \"")
-                        .ok_or_else(|| anyhow!("Failed to find the `tree-sitter` dependency in Cargo.toml"))?;
-
-                    let version_start_index = start_index + "tree-sitter = \"".len();
-                    let version_end_index = cargo_toml[version_start_index..]
-                        .find('\"')
-                        .map(|i| i + version_start_index)
-                        .ok_or_else(|| anyhow!("Failed to find the end of the `tree-sitter` version in Cargo.toml"))?;
-
-                    let cargo_toml = format!(
-                        "{}{}{}",
-                        &cargo_toml[..start_index],
-                        "tree-sitter-language = \"0.1.0\"",
-                        &cargo_toml[version_end_index + 1..],
-                    );
-
-                    write_file(path, cargo_toml)?;
-                    eprintln!("Updated Cargo.toml with the `tree-sitter-language` dependency");
-                }
-                Ok(())
-            },
-        )?;
-
-        Ok(())
-    })?;
-
-    // Generate Node bindings
-    missing_path(bindings_dir.join("node"), create_dir)?.apply(|path| {
-        missing_path_else(
-            path.join("index.js"),
-            |path| generate_file(path, INDEX_JS_TEMPLATE, language_name),
-            |path| {
-                let index_js =
-                    fs::read_to_string(path).with_context(|| "Failed to read index.js")?;
-                if index_js.contains("../../build/Release") {
-                    eprintln!("Replacing index.js with new binding API");
-                    generate_file(path, INDEX_JS_TEMPLATE, language_name)?;
-                }
-                Ok(())
-            },
-        )?;
-
-        missing_path(path.join("index.d.ts"), |path| {
-            generate_file(path, INDEX_D_TS_TEMPLATE, language_name)
-        })?;
-
-        missing_path(path.join("binding_test.js"), |path| {
-            generate_file(path, BINDING_TEST_JS_TEMPLATE, language_name)
-        })?;
-
-        missing_path_else(
-            path.join("binding.cc"),
-            |path| generate_file(path, JS_BINDING_CC_TEMPLATE, language_name),
-            |path| {
-                let binding_cc =
-                    fs::read_to_string(path).with_context(|| "Failed to read binding.cc")?;
-                if binding_cc.contains("NAN_METHOD(New) {}") {
-                    eprintln!("Replacing binding.cc with new binding API");
-                    generate_file(path, JS_BINDING_CC_TEMPLATE, language_name)?;
-                }
-                Ok(())
-            },
-        )?;
-
-        // Create binding.gyp, or update it with new binding API.
-        missing_path_else(
-            repo_path.join("binding.gyp"),
-            |path| generate_file(path, BINDING_GYP_TEMPLATE, language_name),
-            |path| {
-                let binding_gyp =
-                    fs::read_to_string(path).with_context(|| "Failed to read binding.gyp")?;
-                if binding_gyp.contains("require('nan')") {
-                    eprintln!("Replacing binding.gyp with new binding API");
-                    generate_file(path, BINDING_GYP_TEMPLATE, language_name)?;
-                }
-                Ok(())
-            },
-        )?;
-
-        Ok(())
-    })?;
-
-    // Generate C bindings
-    missing_path(bindings_dir.join("c"), create_dir)?.apply(|path| {
-        missing_path(
-            path.join(format!("tree-sitter-{language_name}.h")),
-            |path| generate_file(path, PARSER_NAME_H_TEMPLATE, language_name),
-        )?;
-
-        missing_path(
-            path.join(format!("tree-sitter-{language_name}.pc.in")),
-            |path| generate_file(path, PARSER_NAME_PC_IN_TEMPLATE, language_name),
-        )?;
-
-        missing_path(repo_path.join("Makefile"), |path| {
-            generate_file(path, MAKEFILE_TEMPLATE, language_name)
-        })?;
-
-        Ok(())
-    })?;
-
-    // Generate Go bindings
-    missing_path(bindings_dir.join("go"), create_dir)?.apply(|path| {
-        missing_path(path.join("binding.go"), |path| {
-            generate_file(path, BINDING_GO_TEMPLATE, language_name)
-        })?;
-
-        missing_path(path.join("binding_test.go"), |path| {
-            generate_file(path, BINDING_TEST_GO_TEMPLATE, language_name)
-        })?;
-
-        missing_path(path.join("go.mod"), |path| {
-            generate_file(path, GO_MOD_TEMPLATE, language_name)
-        })?;
-
-        Ok(())
-    })?;
-
-    // Generate Python bindings
-    missing_path(bindings_dir.join("python"), create_dir)?.apply(|path| {
-        let lang_path = path.join(format!("tree_sitter_{}", language_name.to_snake_case()));
-        missing_path(&lang_path, create_dir)?;
-
-        missing_path(lang_path.join("binding.c"), |path| {
-            generate_file(path, PY_BINDING_C_TEMPLATE, language_name)
-        })?;
-
-        missing_path(lang_path.join("__init__.py"), |path| {
-            generate_file(path, INIT_PY_TEMPLATE, language_name)
-        })?;
-
-        missing_path(lang_path.join("__init__.pyi"), |path| {
-            generate_file(path, INIT_PYI_TEMPLATE, language_name)
-        })?;
-
-        missing_path(lang_path.join("py.typed"), |path| {
-            generate_file(path, "", language_name) // py.typed is empty
-        })?;
-
-        missing_path(path.join("tests"), create_dir)?.apply(|path| {
-            missing_path(path.join("test_binding.py"), |path| {
-                generate_file(path, TEST_BINDING_PY_TEMPLATE, language_name)
-            })?;
-            Ok(())
-        })?;
-
-        missing_path(repo_path.join("setup.py"), |path| {
-            generate_file(path, SETUP_PY_TEMPLATE, language_name)
-        })?;
-
-        missing_path(repo_path.join("pyproject.toml"), |path| {
-            generate_file(path, PYPROJECT_TOML_TEMPLATE, dashed_language_name.as_str())
-        })?;
-
-        Ok(())
-    })?;
-
-    // Generate Swift bindings
-    missing_path(bindings_dir.join("swift"), create_dir)?.apply(|path| {
-        let lang_path = path.join(format!("TreeSitter{}", language_name.to_upper_camel_case()));
-        missing_path(&lang_path, create_dir)?;
-
-        missing_path(lang_path.join(format!("{language_name}.h")), |path| {
-            generate_file(path, PARSER_NAME_H_TEMPLATE, language_name)
-        })?;
-
-        missing_path(
-            path.join(format!(
-                "TreeSitter{}Tests",
-                language_name.to_upper_camel_case()
-            )),
-            create_dir,
-        )?
-        .apply(|path| {
-            missing_path(
-                path.join(format!(
-                    "TreeSitter{}Tests.swift",
-                    language_name.to_upper_camel_case()
-                )),
-                |path| generate_file(path, TESTS_SWIFT_TEMPLATE, language_name),
-            )?;
-
-            Ok(())
-        })?;
-
-        missing_path(repo_path.join("Package.swift"), |path| {
-            generate_file(path, PACKAGE_SWIFT_TEMPLATE, language_name)
-        })?;
-
-        Ok(())
-    })?;
-
-    Ok(())
-}
-
-pub fn lookup_package_json_for_path(path: &Path) -> Result<(PathBuf, PackageJSON)> {
-    let mut pathbuf = path.to_owned();
-    loop {
-        let package_json = pathbuf
-            .exists()
-            .then(|| -> Result<PackageJSON> {
-                let file =
-                    File::open(pathbuf.as_path()).with_context(|| "Failed to open package.json")?;
-                serde_json::from_reader(BufReader::new(file)).context(
-                    "Failed to parse package.json, is the `tree-sitter` section malformed?",
-                )
-            })
-            .transpose()?;
-        if let Some(package_json) = package_json {
-            if package_json.tree_sitter.is_some() {
-                return Ok((pathbuf, package_json));
-            }
-        }
-        pathbuf.pop(); // package.json
-        if !pathbuf.pop() {
-            return Err(anyhow!(concat!(
-                "Failed to locate a package.json file that has a \"tree-sitter\" section,",
-                " please ensure you have one, and if you don't then consult the docs",
-            )));
-        }
-        pathbuf.push("package.json");
-    }
-}
-
-fn generate_file(path: &Path, template: &str, language_name: &str) -> Result<()> {
-    write_file(
-        path,
-        template
-            .replace(
-                CAMEL_PARSER_NAME_PLACEHOLDER,
-                &language_name.to_upper_camel_case(),
-            )
-            .replace(
-                UPPER_PARSER_NAME_PLACEHOLDER,
-                &language_name.to_shouty_snake_case(),
-            )
-            .replace(
-                LOWER_PARSER_NAME_PLACEHOLDER,
-                &language_name.to_snake_case(),
-            )
-            .replace(PARSER_NAME_PLACEHOLDER, language_name)
-            .replace(CLI_VERSION_PLACEHOLDER, CLI_VERSION)
-            .replace(RUST_BINDING_VERSION_PLACEHOLDER, RUST_BINDING_VERSION),
-    )
-}
-
-fn create_dir(path: &Path) -> Result<()> {
-    fs::create_dir_all(path)
-        .with_context(|| format!("Failed to create {:?}", path.to_string_lossy()))
-}
-
-#[derive(PartialEq, Eq, Debug)]
-enum PathState<P>
-where
-    P: AsRef<Path>,
-{
-    Exists(P),
-    Missing(P),
-}
-
-#[allow(dead_code)]
-impl<P> PathState<P>
-where
-    P: AsRef<Path>,
-{
-    fn exists(&self, mut action: impl FnMut(&Path) -> Result<()>) -> Result<&Self> {
-        if let Self::Exists(path) = self {
-            action(path.as_ref())?;
-        }
-        Ok(self)
-    }
-
-    fn missing(&self, mut action: impl FnMut(&Path) -> Result<()>) -> Result<&Self> {
-        if let Self::Missing(path) = self {
-            action(path.as_ref())?;
-        }
-        Ok(self)
-    }
-
-    fn apply(&self, mut action: impl FnMut(&Path) -> Result<()>) -> Result<&Self> {
-        action(self.as_path())?;
-        Ok(self)
-    }
-
-    fn apply_state(&self, mut action: impl FnMut(&Self) -> Result<()>) -> Result<&Self> {
-        action(self)?;
-        Ok(self)
-    }
-
-    fn as_path(&self) -> &Path {
-        match self {
-            Self::Exists(path) | Self::Missing(path) => path.as_ref(),
-        }
-    }
-}
-
-fn missing_path<P, F>(path: P, mut action: F) -> Result<PathState<P>>
-where
-    P: AsRef<Path>,
-    F: FnMut(&Path) -> Result<()>,
-{
-    let path_ref = path.as_ref();
-    if !path_ref.exists() {
-        action(path_ref)?;
-        Ok(PathState::Missing(path))
-    } else {
-        Ok(PathState::Exists(path))
-    }
-}
-
-fn missing_path_else<P, T, F>(path: P, mut action: T, mut else_action: F) -> Result<PathState<P>>
-where
-    P: AsRef<Path>,
-    T: FnMut(&Path) -> Result<()>,
-    F: FnMut(&Path) -> Result<()>,
-{
-    let path_ref = path.as_ref();
-    if !path_ref.exists() {
-        action(path_ref)?;
-        Ok(PathState::Missing(path))
-    } else {
-        else_action(path_ref)?;
-        Ok(PathState::Exists(path))
-    }
-}
-
-impl PackageJSON {
-    fn has_multiple_language_configs(&self) -> bool {
-        self.tree_sitter.as_ref().is_some_and(|c| c.len() > 1)
-    }
-}

cli/src/generate/mod.rs

@@ -1,275 +0,0 @@
-use std::{
-    env, fs,
-    io::Write,
-    path::{Path, PathBuf},
-    process::{Command, Stdio},
-};
-
-use anyhow::{anyhow, Context, Result};
-use build_tables::build_tables;
-use grammar_files::path_in_ignore;
-use grammars::InputGrammar;
-use lazy_static::lazy_static;
-use parse_grammar::parse_grammar;
-use prepare_grammar::prepare_grammar;
-use regex::{Regex, RegexBuilder};
-use render::render_c_code;
-use semver::Version;
-
-mod build_tables;
-mod dedup;
-mod grammar_files;
-mod grammars;
-mod nfa;
-mod node_types;
-pub mod parse_grammar;
-mod prepare_grammar;
-mod render;
-mod rules;
-mod tables;
-
-pub use grammar_files::lookup_package_json_for_path;
-
-lazy_static! {
-    static ref JSON_COMMENT_REGEX: Regex = RegexBuilder::new("^\\s*//.*")
-        .multi_line(true)
-        .build()
-        .unwrap();
-}
-
-struct GeneratedParser {
-    c_code: String,
-    node_types_json: String,
-}
-
-pub const ALLOC_HEADER: &str = include_str!("./templates/alloc.h");
-
-pub fn generate_parser_in_directory(
-    repo_path: &Path,
-    grammar_path: Option<&str>,
-    abi_version: usize,
-    generate_bindings: bool,
-    report_symbol_name: Option<&str>,
-    js_runtime: Option<&str>,
-) -> Result<()> {
-    let mut repo_path = repo_path.to_owned();
-    let mut grammar_path = grammar_path;
-
-    // Populate a new empty grammar directory.
-    if let Some(path) = grammar_path {
-        let path = PathBuf::from(path);
-        if !path
-            .try_exists()
-            .with_context(|| "Some error with specified path")?
-        {
-            fs::create_dir_all(&path)?;
-            grammar_path = None;
-            repo_path = path;
-        }
-    }
-
-    if repo_path.is_dir() && !repo_path.join("grammar.js").exists() && !path_in_ignore(&repo_path) {
-        if let Some(dir_name) = repo_path
-            .file_name()
-            .map(|x| x.to_string_lossy().to_ascii_lowercase())
-        {
-            if let Some(language_name) = dir_name
-                .strip_prefix("tree-sitter-")
-                .or_else(|| Some(dir_name.as_ref()))
-            {
-                grammar_files::generate_grammar_files(&repo_path, language_name, false)?;
-            }
-        }
-    }
-
-    // Read the grammar.json.
-    let grammar_json = if let Some(path) = grammar_path {
-        load_grammar_file(path.as_ref(), js_runtime)?
-    } else {
-        let grammar_js_path =
-            grammar_path.map_or(repo_path.join("grammar.js"), std::convert::Into::into);
-        load_grammar_file(&grammar_js_path, js_runtime)?
-    };
-
-    let src_path = repo_path.join("src");
-    let header_path = src_path.join("tree_sitter");
-
-    // Ensure that the output directories exist.
-    fs::create_dir_all(&src_path)?;
-    fs::create_dir_all(&header_path)?;
-
-    if grammar_path.is_none() {
-        fs::write(src_path.join("grammar.json"), &grammar_json)
-            .with_context(|| format!("Failed to write grammar.json to {src_path:?}"))?;
-    }
-
-    // Parse and preprocess the grammar.
-    let input_grammar = parse_grammar(&grammar_json)?;
-
-    // Generate the parser and related files.
-    let GeneratedParser {
-        c_code,
-        node_types_json,
-    } = generate_parser_for_grammar_with_opts(&input_grammar, abi_version, report_symbol_name)?;
-
-    write_file(&src_path.join("parser.c"), c_code)?;
-    write_file(&src_path.join("node-types.json"), node_types_json)?;
-    write_file(&header_path.join("alloc.h"), ALLOC_HEADER)?;
-    write_file(&header_path.join("array.h"), tree_sitter::ARRAY_HEADER)?;
-    write_file(&header_path.join("parser.h"), tree_sitter::PARSER_HEADER)?;
-
-    if !path_in_ignore(&repo_path) {
-        grammar_files::generate_grammar_files(&repo_path, &input_grammar.name, generate_bindings)?;
-    }
-
-    Ok(())
-}
-
-pub fn generate_parser_for_grammar(grammar_json: &str) -> Result<(String, String)> {
-    let grammar_json = JSON_COMMENT_REGEX.replace_all(grammar_json, "\n");
-    let input_grammar = parse_grammar(&grammar_json)?;
-    let parser =
-        generate_parser_for_grammar_with_opts(&input_grammar, tree_sitter::LANGUAGE_VERSION, None)?;
-    Ok((input_grammar.name.clone(), parser.c_code))
-}
-
-fn generate_parser_for_grammar_with_opts(
-    input_grammar: &InputGrammar,
-    abi_version: usize,
-    report_symbol_name: Option<&str>,
-) -> Result<GeneratedParser> {
-    let (syntax_grammar, lexical_grammar, inlines, simple_aliases) =
-        prepare_grammar(input_grammar)?;
-    let variable_info =
-        node_types::get_variable_info(&syntax_grammar, &lexical_grammar, &simple_aliases)?;
-    let node_types_json = node_types::generate_node_types_json(
-        &syntax_grammar,
-        &lexical_grammar,
-        &simple_aliases,
-        &variable_info,
-    );
-    let tables = build_tables(
-        &syntax_grammar,
-        &lexical_grammar,
-        &simple_aliases,
-        &variable_info,
-        &inlines,
-        report_symbol_name,
-    )?;
-    let c_code = render_c_code(
-        &input_grammar.name,
-        tables,
-        syntax_grammar,
-        lexical_grammar,
-        simple_aliases,
-        abi_version,
-    );
-    Ok(GeneratedParser {
-        c_code,
-        node_types_json: serde_json::to_string_pretty(&node_types_json).unwrap(),
-    })
-}
-
-pub fn load_grammar_file(grammar_path: &Path, js_runtime: Option<&str>) -> Result<String> {
-    if grammar_path.is_dir() {
-        return Err(anyhow!(
-            "Path to a grammar file with `.js` or `.json` extension is required"
-        ));
-    }
-    match grammar_path.extension().and_then(|e| e.to_str()) {
-        Some("js") => Ok(load_js_grammar_file(grammar_path, js_runtime)
-            .with_context(|| "Failed to load grammar.js")?),
-        Some("json") => {
-            Ok(fs::read_to_string(grammar_path).with_context(|| "Failed to load grammar.json")?)
-        }
-        _ => Err(anyhow!("Unknown grammar file extension: {grammar_path:?}",)),
-    }
-}
-
-fn load_js_grammar_file(grammar_path: &Path, js_runtime: Option<&str>) -> Result<String> {
-    let grammar_path = fs::canonicalize(grammar_path)?;
-
-    #[cfg(windows)]
-    let grammar_path = url::Url::from_file_path(grammar_path)
-        .expect("Failed to convert path to URL")
-        .to_string();
-
-    let js_runtime = js_runtime.unwrap_or("node");
-
-    let mut js_command = Command::new(js_runtime);
-    match js_runtime {
-        "node" => {
-            js_command.args(["--input-type=module", "-"]);
-        }
-        "bun" => {
-            js_command.arg("-");
-        }
-        "deno" => {
-            js_command.args(["run", "--allow-all", "-"]);
-        }
-        _ => {}
-    }
-
-    let mut js_process = js_command
-        .env("TREE_SITTER_GRAMMAR_PATH", grammar_path)
-        .stdin(Stdio::piped())
-        .stdout(Stdio::piped())
-        .spawn()
-        .with_context(|| format!("Failed to run `{js_runtime}`"))?;
-
-    let mut js_stdin = js_process
-        .stdin
-        .take()
-        .with_context(|| format!("Failed to open stdin for {js_runtime}"))?;
-    let cli_version = Version::parse(env!("CARGO_PKG_VERSION"))
-        .with_context(|| "Could not parse this package's version as semver.")?;
-    write!(
-        js_stdin,
-        "globalThis.TREE_SITTER_CLI_VERSION_MAJOR = {};
-        globalThis.TREE_SITTER_CLI_VERSION_MINOR = {};
-        globalThis.TREE_SITTER_CLI_VERSION_PATCH = {};",
-        cli_version.major, cli_version.minor, cli_version.patch,
-    )
-    .with_context(|| format!("Failed to write tree-sitter version to {js_runtime}'s stdin"))?;
-    js_stdin
-        .write(include_bytes!("./dsl.js"))
-        .with_context(|| format!("Failed to write grammar dsl to {js_runtime}'s stdin"))?;
-    drop(js_stdin);
-
-    let output = js_process
-        .wait_with_output()
-        .with_context(|| format!("Failed to read output from {js_runtime}"))?;
-    match output.status.code() {
-        None => panic!("{js_runtime} process was killed"),
-        Some(0) => {
-            let stdout = String::from_utf8(output.stdout)
-                .with_context(|| format!("Got invalid UTF8 from {js_runtime}"))?;
-
-            let mut grammar_json = &stdout[..];
-
-            if let Some(pos) = stdout.rfind('\n') {
-                // If there's a newline, split the last line from the rest of the output
-                let node_output = &stdout[..pos];
-                grammar_json = &stdout[pos + 1..];
-
-                let mut stdout = std::io::stdout().lock();
-                stdout.write_all(node_output.as_bytes())?;
-                stdout.write_all(b"\n")?;
-                stdout.flush()?;
-            }
-
-            Ok(serde_json::to_string_pretty(
-                &serde_json::from_str::<serde_json::Value>(grammar_json)
-                    .with_context(|| "Failed to parse grammar JSON")?,
-            )
-            .with_context(|| "Failed to serialize grammar JSON")?
-                + "\n")
-        }
-        Some(code) => Err(anyhow!("{js_runtime} process exited with status {code}")),
-    }
-}
-
-fn write_file(path: &Path, body: impl AsRef<[u8]>) -> Result<()> {
-    fs::write(path, body)
-        .with_context(|| format!("Failed to write {:?}", path.file_name().unwrap()))
-}

cli/src/generate/parse_grammar.rs

@@ -1,258 +0,0 @@
-use anyhow::{anyhow, Result};
-use serde::Deserialize;
-use serde_json::{Map, Value};
-
-use super::{
-    grammars::{InputGrammar, PrecedenceEntry, Variable, VariableType},
-    rules::{Precedence, Rule},
-};
-
-#[derive(Deserialize)]
-#[serde(tag = "type")]
-#[allow(non_camel_case_types)]
-#[allow(clippy::upper_case_acronyms)]
-enum RuleJSON {
-    ALIAS {
-        content: Box<RuleJSON>,
-        named: bool,
-        value: String,
-    },
-    BLANK,
-    STRING {
-        value: String,
-    },
-    PATTERN {
-        value: String,
-        flags: Option<String>,
-    },
-    SYMBOL {
-        name: String,
-    },
-    CHOICE {
-        members: Vec<RuleJSON>,
-    },
-    FIELD {
-        name: String,
-        content: Box<RuleJSON>,
-    },
-    SEQ {
-        members: Vec<RuleJSON>,
-    },
-    REPEAT {
-        content: Box<RuleJSON>,
-    },
-    REPEAT1 {
-        content: Box<RuleJSON>,
-    },
-    PREC_DYNAMIC {
-        value: i32,
-        content: Box<RuleJSON>,
-    },
-    PREC_LEFT {
-        value: PrecedenceValueJSON,
-        content: Box<RuleJSON>,
-    },
-    PREC_RIGHT {
-        value: PrecedenceValueJSON,
-        content: Box<RuleJSON>,
-    },
-    PREC {
-        value: PrecedenceValueJSON,
-        content: Box<RuleJSON>,
-    },
-    TOKEN {
-        content: Box<RuleJSON>,
-    },
-    IMMEDIATE_TOKEN {
-        content: Box<RuleJSON>,
-    },
-}
-
-#[derive(Deserialize)]
-#[serde(untagged)]
-enum PrecedenceValueJSON {
-    Integer(i32),
-    Name(String),
-}
-
-#[derive(Deserialize)]
-pub(crate) struct GrammarJSON {
-    pub(crate) name: String,
-    rules: Map<String, Value>,
-    #[serde(default)]
-    precedences: Vec<Vec<RuleJSON>>,
-    #[serde(default)]
-    conflicts: Vec<Vec<String>>,
-    #[serde(default)]
-    externals: Vec<RuleJSON>,
-    #[serde(default)]
-    extras: Vec<RuleJSON>,
-    #[serde(default)]
-    inline: Vec<String>,
-    #[serde(default)]
-    supertypes: Vec<String>,
-    word: Option<String>,
-}
-
-pub(crate) fn parse_grammar(input: &str) -> Result<InputGrammar> {
-    let grammar_json = serde_json::from_str::<GrammarJSON>(input)?;
-
-    let mut variables = Vec::with_capacity(grammar_json.rules.len());
-    for (name, value) in grammar_json.rules {
-        variables.push(Variable {
-            name: name.clone(),
-            kind: VariableType::Named,
-            rule: parse_rule(serde_json::from_value(value)?),
-        });
-    }
-
-    let mut precedence_orderings = Vec::with_capacity(grammar_json.precedences.len());
-    for list in grammar_json.precedences {
-        let mut ordering = Vec::with_capacity(list.len());
-        for entry in list {
-            ordering.push(match entry {
-                RuleJSON::STRING { value } => PrecedenceEntry::Name(value),
-                RuleJSON::SYMBOL { name } => PrecedenceEntry::Symbol(name),
-                _ => {
-                    return Err(anyhow!(
-                        "Invalid rule in precedences array. Only strings and symbols are allowed"
-                    ))
-                }
-            });
-        }
-        precedence_orderings.push(ordering);
-    }
-
-    let extra_symbols = grammar_json
-        .extras
-        .into_iter()
-        .try_fold(Vec::new(), |mut acc, item| {
-            let rule = parse_rule(item);
-            if let Rule::String(ref value) = rule {
-                if value.is_empty() {
-                    return Err(anyhow!(
-                        "Rules in the `extras` array must not contain empty strings"
-                    ));
-                }
-            }
-            acc.push(rule);
-            Ok(acc)
-        })?;
-
-    let external_tokens = grammar_json.externals.into_iter().map(parse_rule).collect();
-
-    Ok(InputGrammar {
-        name: grammar_json.name,
-        word_token: grammar_json.word,
-        expected_conflicts: grammar_json.conflicts,
-        supertype_symbols: grammar_json.supertypes,
-        variables_to_inline: grammar_json.inline,
-        precedence_orderings,
-        variables,
-        extra_symbols,
-        external_tokens,
-    })
-}
-
-fn parse_rule(json: RuleJSON) -> Rule {
-    match json {
-        RuleJSON::ALIAS {
-            content,
-            value,
-            named,
-        } => Rule::alias(parse_rule(*content), value, named),
-        RuleJSON::BLANK => Rule::Blank,
-        RuleJSON::STRING { value } => Rule::String(value),
-        RuleJSON::PATTERN { value, flags } => Rule::Pattern(
-            value,
-            flags.map_or(String::new(), |f| {
-                f.matches(|c| {
-                    if c == 'i' {
-                        true
-                    } else {
-                        // silently ignore unicode flags
-                        if c != 'u' && c != 'v' {
-                            eprintln!("Warning: unsupported flag {c}");
-                        }
-                        false
-                    }
-                })
-                .collect()
-            }),
-        ),
-        RuleJSON::SYMBOL { name } => Rule::NamedSymbol(name),
-        RuleJSON::CHOICE { members } => Rule::choice(members.into_iter().map(parse_rule).collect()),
-        RuleJSON::FIELD { content, name } => Rule::field(name, parse_rule(*content)),
-        RuleJSON::SEQ { members } => Rule::seq(members.into_iter().map(parse_rule).collect()),
-        RuleJSON::REPEAT1 { content } => Rule::repeat(parse_rule(*content)),
-        RuleJSON::REPEAT { content } => {
-            Rule::choice(vec![Rule::repeat(parse_rule(*content)), Rule::Blank])
-        }
-        RuleJSON::PREC { value, content } => Rule::prec(value.into(), parse_rule(*content)),
-        RuleJSON::PREC_LEFT { value, content } => {
-            Rule::prec_left(value.into(), parse_rule(*content))
-        }
-        RuleJSON::PREC_RIGHT { value, content } => {
-            Rule::prec_right(value.into(), parse_rule(*content))
-        }
-        RuleJSON::PREC_DYNAMIC { value, content } => {
-            Rule::prec_dynamic(value, parse_rule(*content))
-        }
-        RuleJSON::TOKEN { content } => Rule::token(parse_rule(*content)),
-        RuleJSON::IMMEDIATE_TOKEN { content } => Rule::immediate_token(parse_rule(*content)),
-    }
-}
-
-impl From<PrecedenceValueJSON> for Precedence {
-    fn from(val: PrecedenceValueJSON) -> Self {
-        match val {
-            PrecedenceValueJSON::Integer(i) => Self::Integer(i),
-            PrecedenceValueJSON::Name(i) => Self::Name(i),
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_parse_grammar() {
-        let grammar = parse_grammar(
-            r#"{
-            "name": "my_lang",
-            "rules": {
-                "file": {
-                    "type": "REPEAT1",
-                    "content": {
-                        "type": "SYMBOL",
-                        "name": "statement"
-                    }
-                },
-                "statement": {
-                    "type": "STRING",
-                    "value": "foo"
-                }
-            }
-        }"#,
-        )
-        .unwrap();
-
-        assert_eq!(grammar.name, "my_lang");
-        assert_eq!(
-            grammar.variables,
-            vec![
-                Variable {
-                    name: "file".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::repeat(Rule::NamedSymbol("statement".to_string()))
-                },
-                Variable {
-                    name: "statement".to_string(),
-                    kind: VariableType::Named,
-                    rule: Rule::String("foo".to_string())
-                },
-            ]
-        );
-    }
-}

cli/src/generate/prepare_grammar/unicode-category-aliases.json

@@ -1 +0,0 @@
-{"Other":"C","Control":"Cc","cntrl":"Cc","Format":"Cf","Unassigned":"Cn","Private_Use":"Co","Surrogate":"Cs","Letter":"L","Cased_Letter":"LC","Lowercase_Letter":"Ll","Modifier_Letter":"Lm","Other_Letter":"Lo","Titlecase_Letter":"Lt","Uppercase_Letter":"Lu","Mark":"M","Combining_Mark":"M","Spacing_Mark":"Mc","Enclosing_Mark":"Me","Nonspacing_Mark":"Mn","Number":"N","Decimal_Number":"Nd","digit":"Nd","Letter_Number":"Nl","Other_Number":"No","Punctuation":"P","punct":"P","Connector_Punctuation":"Pc","Dash_Punctuation":"Pd","Close_Punctuation":"Pe","Final_Punctuation":"Pf","Initial_Punctuation":"Pi","Other_Punctuation":"Po","Open_Punctuation":"Ps","Symbol":"S","Currency_Symbol":"Sc","Modifier_Symbol":"Sk","Math_Symbol":"Sm","Other_Symbol":"So","Separator":"Z","Line_Separator":"Zl","Paragraph_Separator":"Zp","Space_Separator":"Zs"}

cli/src/generate/prepare_grammar/unicode-property-aliases.json

@@ -1 +0,0 @@
-{"cjkAccountingNumeric":"kAccountingNumeric","cjkOtherNumeric":"kOtherNumeric","cjkPrimaryNumeric":"kPrimaryNumeric","nv":"Numeric_Value","bmg":"Bidi_Mirroring_Glyph","bpb":"Bidi_Paired_Bracket","cf":"Case_Folding","cjkCompatibilityVariant":"kCompatibilityVariant","dm":"Decomposition_Mapping","EqUIdeo":"Equivalent_Unified_Ideograph","FC_NFKC":"FC_NFKC_Closure","lc":"Lowercase_Mapping","NFKC_CF":"NFKC_Casefold","NFKC_SCF":"NFKC_Simple_Casefold","scf":"Simple_Case_Folding","sfc":"Simple_Case_Folding","slc":"Simple_Lowercase_Mapping","stc":"Simple_Titlecase_Mapping","suc":"Simple_Uppercase_Mapping","tc":"Titlecase_Mapping","uc":"Uppercase_Mapping","cjkIICore":"kIICore","cjkIRG_GSource":"kIRG_GSource","cjkIRG_HSource":"kIRG_HSource","cjkIRG_JSource":"kIRG_JSource","cjkIRG_KPSource":"kIRG_KPSource","cjkIRG_KSource":"kIRG_KSource","cjkIRG_MSource":"kIRG_MSource","cjkIRG_SSource":"kIRG_SSource","cjkIRG_TSource":"kIRG_TSource","cjkIRG_UKSource":"kIRG_UKSource","cjkIRG_USource":"kIRG_USource","cjkIRG_VSource":"kIRG_VSource","cjkRSUnicode":"kRSUnicode","Unicode_Radical_Stroke":"kRSUnicode","URS":"kRSUnicode","isc":"ISO_Comment","JSN":"Jamo_Short_Name","na":"Name","na1":"Unicode_1_Name","Name_Alias":"Name_Alias","scx":"Script_Extensions","age":"Age","blk":"Block","sc":"Script","bc":"Bidi_Class","bpt":"Bidi_Paired_Bracket_Type","ccc":"Canonical_Combining_Class","dt":"Decomposition_Type","ea":"East_Asian_Width","gc":"General_Category","GCB":"Grapheme_Cluster_Break","hst":"Hangul_Syllable_Type","InCB":"Indic_Conjunct_Break","InPC":"Indic_Positional_Category","InSC":"Indic_Syllabic_Category","jg":"Joining_Group","jt":"Joining_Type","lb":"Line_Break","NFC_QC":"NFC_Quick_Check","NFD_QC":"NFD_Quick_Check","NFKC_QC":"NFKC_Quick_Check","NFKD_QC":"NFKD_Quick_Check","nt":"Numeric_Type","SB":"Sentence_Break","vo":"Vertical_Orientation","WB":"Word_Break","AHex":"ASCII_Hex_Digit","Alpha":"Alphabetic","Bidi_C":"Bidi_Control","Bidi_M":"Bidi_Mirrored","Cased":"Cased","CE":"Composition_Exclusion","CI":"Case_Ignorable","Comp_Ex":"Full_Composition_Exclusion","CWCF":"Changes_When_Casefolded","CWCM":"Changes_When_Casemapped","CWKCF":"Changes_When_NFKC_Casefolded","CWL":"Changes_When_Lowercased","CWT":"Changes_When_Titlecased","CWU":"Changes_When_Uppercased","Dash":"Dash","Dep":"Deprecated","DI":"Default_Ignorable_Code_Point","Dia":"Diacritic","EBase":"Emoji_Modifier_Base","EComp":"Emoji_Component","EMod":"Emoji_Modifier","Emoji":"Emoji","EPres":"Emoji_Presentation","Ext":"Extender","ExtPict":"Extended_Pictographic","Gr_Base":"Grapheme_Base","Gr_Ext":"Grapheme_Extend","Gr_Link":"Grapheme_Link","Hex":"Hex_Digit","Hyphen":"Hyphen","ID_Compat_Math_Continue":"ID_Compat_Math_Continue","ID_Compat_Math_Start":"ID_Compat_Math_Start","IDC":"ID_Continue","Ideo":"Ideographic","IDS":"ID_Start","IDSB":"IDS_Binary_Operator","IDST":"IDS_Trinary_Operator","IDSU":"IDS_Unary_Operator","Join_C":"Join_Control","LOE":"Logical_Order_Exception","Lower":"Lowercase","Math":"Math","NChar":"Noncharacter_Code_Point","OAlpha":"Other_Alphabetic","ODI":"Other_Default_Ignorable_Code_Point","OGr_Ext":"Other_Grapheme_Extend","OIDC":"Other_ID_Continue","OIDS":"Other_ID_Start","OLower":"Other_Lowercase","OMath":"Other_Math","OUpper":"Other_Uppercase","Pat_Syn":"Pattern_Syntax","Pat_WS":"Pattern_White_Space","PCM":"Prepended_Concatenation_Mark","QMark":"Quotation_Mark","Radical":"Radical","RI":"Regional_Indicator","SD":"Soft_Dotted","STerm":"Sentence_Terminal","Term":"Terminal_Punctuation","UIdeo":"Unified_Ideograph","Upper":"Uppercase","VS":"Variation_Selector","WSpace":"White_Space","space":"White_Space","XIDC":"XID_Continue","XIDS":"XID_Start","XO_NFC":"Expands_On_NFC","XO_NFD":"Expands_On_NFD","XO_NFKC":"Expands_On_NFKC","XO_NFKD":"Expands_On_NFKD"}

cli/src/generate/templates/PARSER_NAME.pc.in

@@ -1,11 +0,0 @@
-prefix=@PREFIX@
-libdir=@LIBDIR@
-includedir=@INCLUDEDIR@
-
-Name: tree-sitter-PARSER_NAME
-Description: CAMEL_PARSER_NAME grammar for tree-sitter
-URL: @URL@
-Version: @VERSION@
-Requires: @REQUIRES@
-Libs: -L${libdir} @ADDITIONAL_LIBS@ -ltree-sitter-PARSER_NAME
-Cflags: -I${includedir}

cli/src/generate/templates/__init__.py

@@ -1,42 +0,0 @@
-"""CAMEL_PARSER_NAME grammar for tree-sitter"""
-
-from importlib.resources import files as _files
-
-from ._binding import language
-
-
-def _get_query(name, file):
-    query = _files(f"{__package__}.queries") / file
-    globals()[name] = query.read_text()
-    return globals()[name]
-
-
-def __getattr__(name):
-    # NOTE: uncomment these to include any queries that this grammar contains:
-
-    # if name == "HIGHLIGHTS_QUERY":
-    #     return _get_query("HIGHLIGHTS_QUERY", "highlights.scm")
-    # if name == "INJECTIONS_QUERY":
-    #     return _get_query("INJECTIONS_QUERY", "injections.scm")
-    # if name == "LOCALS_QUERY":
-    #     return _get_query("LOCALS_QUERY", "locals.scm")
-    # if name == "TAGS_QUERY":
-    #     return _get_query("TAGS_QUERY", "tags.scm")
-
-    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
-
-
-__all__ = [
-    "language",
-    # "HIGHLIGHTS_QUERY",
-    # "INJECTIONS_QUERY",
-    # "LOCALS_QUERY",
-    # "TAGS_QUERY",
-]
-
-
-def __dir__():
-    return sorted(__all__ + [
-        "__all__", "__builtins__", "__cached__", "__doc__", "__file__",
-        "__loader__", "__name__", "__package__", "__path__", "__spec__",
-    ])

cli/src/generate/templates/__init__.pyi

@@ -1,10 +0,0 @@
-from typing import Final
-
-# NOTE: uncomment these to include any queries that this grammar contains:
-
-# HIGHLIGHTS_QUERY: Final[str]
-# INJECTIONS_QUERY: Final[str]
-# LOCALS_QUERY: Final[str]
-# TAGS_QUERY: Final[str]
-
-def language() -> int: ...

cli/src/generate/templates/binding_test.go

@@ -1,15 +0,0 @@
-package tree_sitter_PARSER_NAME_test
-
-import (
-	"testing"
-
-	tree_sitter "github.com/smacker/go-tree-sitter"
-	"github.com/tree-sitter/tree-sitter-PARSER_NAME"
-)
-
-func TestCanLoadGrammar(t *testing.T) {
-	language := tree_sitter.NewLanguage(tree_sitter_PARSER_NAME.Language())
-	if language == nil {
-		t.Errorf("Error loading CAMEL_PARSER_NAME grammar")
-	}
-}

cli/src/generate/templates/binding_test.js

@@ -1,9 +0,0 @@
-/// <reference types="node" />
-
-const assert = require("node:assert");
-const { test } = require("node:test");
-
-test("can load grammar", () => {
-  const parser = new (require("tree-sitter"))();
-  assert.doesNotThrow(() => parser.setLanguage(require(".")));
-});

cli/src/generate/templates/build.rs

@@ -1,22 +0,0 @@
-fn main() {
-    let src_dir = std::path::Path::new("src");
-
-    let mut c_config = cc::Build::new();
-    c_config.std("c11").include(src_dir);
-
-    #[cfg(target_env = "msvc")]
-    c_config.flag("-utf-8");
-
-    let parser_path = src_dir.join("parser.c");
-    c_config.file(&parser_path);
-    println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap());
-
-    // NOTE: if your language uses an external scanner, uncomment this block:
-    /*
-    let scanner_path = src_dir.join("scanner.c");
-    c_config.file(&scanner_path);
-    println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap());
-    */
-
-    c_config.compile("tree-sitter-PARSER_NAME");
-}

cli/src/generate/templates/cargo.toml

@@ -1,26 +0,0 @@
-[package]
-name = "tree-sitter-PARSER_NAME"
-description = "CAMEL_PARSER_NAME grammar for tree-sitter"
-version = "0.0.1"
-license = "MIT"
-readme = "README.md"
-keywords = ["incremental", "parsing", "tree-sitter", "PARSER_NAME"]
-categories = ["parsing", "text-editors"]
-repository = "https://github.com/tree-sitter/tree-sitter-PARSER_NAME"
-edition = "2021"
-autoexamples = false
-
-build = "bindings/rust/build.rs"
-include = ["bindings/rust/*", "grammar.js", "queries/*", "src/*"]
-
-[lib]
-path = "bindings/rust/lib.rs"
-
-[dependencies]
-tree-sitter-language = "0.1"
-
-[dev-dependencies]
-tree-sitter = { version = "0.22" }
-
-[build-dependencies]
-cc = "1.0.87"

cli/src/generate/templates/gitattributes

@@ -1,11 +0,0 @@
-* text=auto eol=lf
-
-src/*.json linguist-generated
-src/parser.c linguist-generated
-src/tree_sitter/* linguist-generated
-
-bindings/** linguist-generated
-binding.gyp linguist-generated
-setup.py linguist-generated
-Makefile linguist-generated
-Package.swift linguist-generated

cli/src/generate/templates/go.mod

@@ -1,5 +0,0 @@
-module github.com/tree-sitter/tree-sitter-PARSER_NAME
-
-go 1.22
-
-require github.com/smacker/go-tree-sitter v0.0.0-20230720070738-0d0a9f78d8f8

cli/src/generate/templates/index.d.ts

@@ -1,28 +0,0 @@
-type BaseNode = {
-  type: string;
-  named: boolean;
-};
-
-type ChildNode = {
-  multiple: boolean;
-  required: boolean;
-  types: BaseNode[];
-};
-
-type NodeInfo =
-  | (BaseNode & {
-      subtypes: BaseNode[];
-    })
-  | (BaseNode & {
-      fields: { [name: string]: ChildNode };
-      children: ChildNode[];
-    });
-
-type Language = {
-  name: string;
-  language: unknown;
-  nodeTypeInfo: NodeInfo[];
-};
-
-declare const language: Language;
-export = language;

cli/src/generate/templates/index.js

@@ -1,7 +0,0 @@
-const root = require("path").join(__dirname, "..", "..");
-
-module.exports = require("node-gyp-build")(root);
-
-try {
-  module.exports.nodeTypeInfo = require("../../src/node-types.json");
-} catch (_) {}

cli/src/generate/templates/lib.rs

@@ -1,53 +0,0 @@
-//! This crate provides CAMEL_PARSER_NAME language support for the [tree-sitter][] parsing library.
-//!
-//! Typically, you will use the [language][language func] function to add this language to a
-//! tree-sitter [Parser][], and then use the parser to parse some code:
-//!
-//! ```
-//! let code = r#"
-//! "#;
-//! let mut parser = tree_sitter::Parser::new();
-//! let language = tree_sitter_PARSER_NAME::LANGUAGE;
-//! parser
-//!     .set_language(&language.into())
-//!     .expect("Error loading CAMEL_PARSER_NAME parser");
-//! let tree = parser.parse(code, None).unwrap();
-//! assert!(!tree.root_node().has_error());
-//! ```
-//!
-//! [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
-//! [language func]: fn.language.html
-//! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
-//! [tree-sitter]: https://tree-sitter.github.io/
-
-use tree_sitter_language::LanguageFn;
-
-extern "C" {
-    fn tree_sitter_PARSER_NAME() -> *const ();
-}
-
-/// The tree-sitter [`LanguageFn`] for this grammar.
-pub const LANGUAGE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_PARSER_NAME) };
-
-/// The content of the [`node-types.json`][] file for this grammar.
-///
-/// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types
-pub const NODE_TYPES: &str = include_str!("../../src/node-types.json");
-
-// NOTE: uncomment these to include any queries that this grammar contains:
-
-// pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
-// pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm");
-// pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm");
-// pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm");
-
-#[cfg(test)]
-mod tests {
-    #[test]
-    fn test_can_load_grammar() {
-        let mut parser = tree_sitter::Parser::new();
-        parser
-            .set_language(&super::LANGUAGE.into())
-            .expect("Error loading CAMEL_PARSER_NAME parser");
-    }
-}

cli/src/generate/templates/makefile

@@ -1,112 +0,0 @@
-VERSION := 0.0.1
-
-LANGUAGE_NAME := tree-sitter-PARSER_NAME
-
-# repository
-SRC_DIR := src
-
-PARSER_REPO_URL := $(shell git -C $(SRC_DIR) remote get-url origin 2>/dev/null)
-
-ifeq ($(PARSER_URL),)
-	PARSER_URL := $(subst .git,,$(PARSER_REPO_URL))
-ifeq ($(shell echo $(PARSER_URL) | grep '^[a-z][-+.0-9a-z]*://'),)
-	PARSER_URL := $(subst :,/,$(PARSER_URL))
-	PARSER_URL := $(subst git@,https://,$(PARSER_URL))
-endif
-endif
-
-TS ?= tree-sitter
-
-# ABI versioning
-SONAME_MAJOR := $(word 1,$(subst ., ,$(VERSION)))
-SONAME_MINOR := $(word 2,$(subst ., ,$(VERSION)))
-
-# install directory layout
-PREFIX ?= /usr/local
-INCLUDEDIR ?= $(PREFIX)/include
-LIBDIR ?= $(PREFIX)/lib
-PCLIBDIR ?= $(LIBDIR)/pkgconfig
-
-# source/object files
-PARSER := $(SRC_DIR)/parser.c
-EXTRAS := $(filter-out $(PARSER),$(wildcard $(SRC_DIR)/*.c))
-OBJS := $(patsubst %.c,%.o,$(PARSER) $(EXTRAS))
-
-# flags
-ARFLAGS ?= rcs
-override CFLAGS += -I$(SRC_DIR) -std=c11 -fPIC
-
-# OS-specific bits
-ifeq ($(OS),Windows_NT)
-	$(error "Windows is not supported")
-else ifeq ($(shell uname),Darwin)
-	SOEXT = dylib
-	SOEXTVER_MAJOR = $(SONAME_MAJOR).dylib
-	SOEXTVER = $(SONAME_MAJOR).$(SONAME_MINOR).dylib
-	LINKSHARED := $(LINKSHARED)-dynamiclib -Wl,
-	ifneq ($(ADDITIONAL_LIBS),)
-	LINKSHARED := $(LINKSHARED)$(ADDITIONAL_LIBS),
-	endif
-	LINKSHARED := $(LINKSHARED)-install_name,$(LIBDIR)/lib$(LANGUAGE_NAME).$(SONAME_MAJOR).dylib,-rpath,@executable_path/../Frameworks
-else
-	SOEXT = so
-	SOEXTVER_MAJOR = so.$(SONAME_MAJOR)
-	SOEXTVER = so.$(SONAME_MAJOR).$(SONAME_MINOR)
-	LINKSHARED := $(LINKSHARED)-shared -Wl,
-	ifneq ($(ADDITIONAL_LIBS),)
-	LINKSHARED := $(LINKSHARED)$(ADDITIONAL_LIBS)
-	endif
-	LINKSHARED := $(LINKSHARED)-soname,lib$(LANGUAGE_NAME).so.$(SONAME_MAJOR)
-endif
-ifneq ($(filter $(shell uname),FreeBSD NetBSD DragonFly),)
-	PCLIBDIR := $(PREFIX)/libdata/pkgconfig
-endif
-
-all: lib$(LANGUAGE_NAME).a lib$(LANGUAGE_NAME).$(SOEXT) $(LANGUAGE_NAME).pc
-
-lib$(LANGUAGE_NAME).a: $(OBJS)
-	$(AR) $(ARFLAGS) $@ $^
-
-lib$(LANGUAGE_NAME).$(SOEXT): $(OBJS)
-	$(CC) $(LDFLAGS) $(LINKSHARED) $^ $(LDLIBS) -o $@
-ifneq ($(STRIP),)
-	$(STRIP) $@
-endif
-
-$(LANGUAGE_NAME).pc: bindings/c/$(LANGUAGE_NAME).pc.in
-	sed  -e 's|@URL@|$(PARSER_URL)|' \
-		-e 's|@VERSION@|$(VERSION)|' \
-		-e 's|@LIBDIR@|$(LIBDIR)|' \
-		-e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \
-		-e 's|@REQUIRES@|$(REQUIRES)|' \
-		-e 's|@ADDITIONAL_LIBS@|$(ADDITIONAL_LIBS)|' \
-		-e 's|=$(PREFIX)|=$${prefix}|' \
-		-e 's|@PREFIX@|$(PREFIX)|' $< > $@
-
-$(PARSER): $(SRC_DIR)/grammar.json
-	$(TS) generate --no-bindings $^
-
-install: all
-	install -d '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter '$(DESTDIR)$(PCLIBDIR)' '$(DESTDIR)$(LIBDIR)'
-	install -m644 bindings/c/$(LANGUAGE_NAME).h '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/$(LANGUAGE_NAME).h
-	install -m644 $(LANGUAGE_NAME).pc '$(DESTDIR)$(PCLIBDIR)'/$(LANGUAGE_NAME).pc
-	install -m644 lib$(LANGUAGE_NAME).a '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).a
-	install -m755 lib$(LANGUAGE_NAME).$(SOEXT) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER)
-	ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR)
-	ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXT)
-
-uninstall:
-	$(RM) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).a \
-		'$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER) \
-		'$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) \
-		'$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXT) \
-		'$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/$(LANGUAGE_NAME).h \
-		'$(DESTDIR)$(PCLIBDIR)'/$(LANGUAGE_NAME).pc
-
-clean:
-	$(RM) $(OBJS) $(LANGUAGE_NAME).pc lib$(LANGUAGE_NAME).a lib$(LANGUAGE_NAME).$(SOEXT)
-
-test:
-	$(TS) test
-
-.PHONY: all install uninstall clean test

cli/src/generate/templates/package.swift

@@ -1,60 +0,0 @@
-// swift-tools-version:5.3
-import PackageDescription
-
-let package = Package(
-    name: "TreeSitterCAMEL_PARSER_NAME",
-    products: [
-        .library(name: "TreeSitterCAMEL_PARSER_NAME", targets: ["TreeSitterCAMEL_PARSER_NAME"]),
-    ],
-    dependencies: [
-        .package(url: "https://github.com/ChimeHQ/SwiftTreeSitter", from: "0.8.0"),
-    ],
-    targets: [
-        .target(
-            name: "TreeSitterCAMEL_PARSER_NAME",
-            dependencies: [],
-            path: ".",
-            exclude: [
-                "Cargo.toml",
-                "Makefile",
-                "binding.gyp",
-                "bindings/c",
-                "bindings/go",
-                "bindings/node",
-                "bindings/python",
-                "bindings/rust",
-                "prebuilds",
-                "grammar.js",
-                "package.json",
-                "package-lock.json",
-                "pyproject.toml",
-                "setup.py",
-                "test",
-                "examples",
-                ".editorconfig",
-                ".github",
-                ".gitignore",
-                ".gitattributes",
-                ".gitmodules",
-            ],
-            sources: [
-                "src/parser.c",
-                // NOTE: if your language has an external scanner, add it here.
-            ],
-            resources: [
-                .copy("queries")
-            ],
-            publicHeadersPath: "bindings/swift",
-            cSettings: [.headerSearchPath("src")]
-        ),
-        .testTarget(
-            name: "TreeSitterCAMEL_PARSER_NAMETests",
-            dependencies: [
-                "SwiftTreeSitter",
-                "TreeSitterCAMEL_PARSER_NAME",
-            ],
-            path: "bindings/swift/TreeSitterCAMEL_PARSER_NAMETests"
-        )
-    ],
-    cLanguageStandard: .c11
-)

cli/src/generate/templates/setup.py

@@ -1,62 +0,0 @@
-from os.path import isdir, join
-from platform import system
-
-from setuptools import Extension, find_packages, setup
-from setuptools.command.build import build
-from wheel.bdist_wheel import bdist_wheel
-
-
-class Build(build):
-    def run(self):
-        if isdir("queries"):
-            dest = join(self.build_lib, "tree_sitter_PARSER_NAME", "queries")
-            self.copy_tree("queries", dest)
-        super().run()
-
-
-class BdistWheel(bdist_wheel):
-    def get_tag(self):
-        python, abi, platform = super().get_tag()
-        if python.startswith("cp"):
-            python, abi = "cp39", "abi3"
-        return python, abi, platform
-
-
-setup(
-    packages=find_packages("bindings/python"),
-    package_dir={"": "bindings/python"},
-    package_data={
-        "tree_sitter_LOWER_PARSER_NAME": ["*.pyi", "py.typed"],
-        "tree_sitter_LOWER_PARSER_NAME.queries": ["*.scm"],
-    },
-    ext_package="tree_sitter_LOWER_PARSER_NAME",
-    ext_modules=[
-        Extension(
-            name="_binding",
-            sources=[
-                "bindings/python/tree_sitter_LOWER_PARSER_NAME/binding.c",
-                "src/parser.c",
-                # NOTE: if your language uses an external scanner, add it here.
-            ],
-            extra_compile_args=[
-                "-std=c11",
-                "-fvisibility=hidden",
-            ] if system() != "Windows" else [
-                "/std:c11",
-                "/utf-8",
-            ],
-            define_macros=[
-                ("Py_LIMITED_API", "0x03090000"),
-                ("PY_SSIZE_T_CLEAN", None),
-                ("TREE_SITTER_HIDE_SYMBOLS", None),
-            ],
-            include_dirs=["src"],
-            py_limited_api=True,
-        )
-    ],
-    cmdclass={
-        "build": Build,
-        "bdist_wheel": BdistWheel
-    },
-    zip_safe=False
-)

cli/src/generate/templates/test_binding.py

@@ -1,11 +0,0 @@
-from unittest import TestCase
-
-import tree_sitter, tree_sitter_LOWER_PARSER_NAME
-
-
-class TestLanguage(TestCase):
-    def test_can_load_grammar(self):
-        try:
-            tree_sitter.Language(tree_sitter_LOWER_PARSER_NAME.language())
-        except Exception:
-            self.fail("Error loading CAMEL_PARSER_NAME grammar")

cli/src/logger.rs

@@ -1,30 +0,0 @@
-use log::{LevelFilter, Log, Metadata, Record};
-
-#[allow(dead_code)]
-struct Logger {
-    pub filter: Option<String>,
-}
-
-impl Log for Logger {
-    fn enabled(&self, _: &Metadata) -> bool {
-        true
-    }
-
-    fn log(&self, record: &Record) {
-        eprintln!(
-            "[{}] {}",
-            record
-                .module_path()
-                .unwrap_or_default()
-                .trim_start_matches("rust_tree_sitter_cli::"),
-            record.args()
-        );
-    }
-
-    fn flush(&self) {}
-}
-
-pub fn init() {
-    log::set_boxed_logger(Box::new(Logger { filter: None })).unwrap();
-    log::set_max_level(LevelFilter::Info);
-}

cli/src/parse.rs

@@ -1,488 +0,0 @@
-use std::{
-    fmt, fs,
-    io::{self, Write},
-    path::Path,
-    sync::atomic::AtomicUsize,
-    time::{Duration, Instant},
-};
-
-use anyhow::{anyhow, Context, Result};
-use tree_sitter::{ffi, InputEdit, Language, LogType, Parser, Point, Tree};
-
-use super::util;
-
-#[derive(Debug)]
-pub struct Edit {
-    pub position: usize,
-    pub deleted_length: usize,
-    pub inserted_text: Vec<u8>,
-}
-
-#[derive(Debug, Default)]
-pub struct Stats {
-    pub successful_parses: usize,
-    pub total_parses: usize,
-    pub total_bytes: usize,
-    pub total_duration: Duration,
-}
-
-impl fmt::Display for Stats {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let duration_us = self.total_duration.as_micros();
-        writeln!(
-            f,
-            "Total parses: {}; successful parses: {}; failed parses: {}; success percentage: {:.2}%; average speed: {} bytes/ms",
-            self.total_parses,
-            self.successful_parses,
-            self.total_parses - self.successful_parses,
-            ((self.successful_parses as f64) / (self.total_parses as f64)) * 100.0,
-            if duration_us != 0 {
-                ((self.total_bytes as u128) * 1_000) / duration_us
-            } else {
-                0
-            }
-        )
-    }
-}
-
-#[derive(Copy, Clone, PartialEq, Eq)]
-pub enum ParseOutput {
-    Normal,
-    Quiet,
-    Xml,
-    Dot,
-}
-
-pub struct ParseFileOptions<'a> {
-    pub language: Language,
-    pub path: &'a Path,
-    pub edits: &'a [&'a str],
-    pub max_path_length: usize,
-    pub output: ParseOutput,
-    pub print_time: bool,
-    pub timeout: u64,
-    pub debug: bool,
-    pub debug_graph: bool,
-    pub cancellation_flag: Option<&'a AtomicUsize>,
-    pub encoding: Option<u32>,
-    pub open_log: bool,
-}
-
-#[derive(Copy, Clone)]
-pub struct ParseResult {
-    pub successful: bool,
-    pub bytes: usize,
-    pub duration: Option<Duration>,
-}
-
-pub fn parse_file_at_path(parser: &mut Parser, opts: &ParseFileOptions) -> Result<ParseResult> {
-    let mut _log_session = None;
-    parser.set_language(&opts.language)?;
-    let mut source_code = fs::read(opts.path)
-        .with_context(|| format!("Error reading source file {:?}", opts.path))?;
-
-    // If the `--cancel` flag was passed, then cancel the parse
-    // when the user types a newline.
-    unsafe { parser.set_cancellation_flag(opts.cancellation_flag) };
-
-    // Set a timeout based on the `--time` flag.
-    parser.set_timeout_micros(opts.timeout);
-
-    // Render an HTML graph if `--debug-graph` was passed
-    if opts.debug_graph {
-        _log_session = Some(util::log_graphs(parser, "log.html", opts.open_log)?);
-    }
-    // Log to stderr if `--debug` was passed
-    else if opts.debug {
-        parser.set_logger(Some(Box::new(|log_type, message| {
-            if log_type == LogType::Lex {
-                io::stderr().write_all(b"  ").unwrap();
-            }
-            writeln!(&mut io::stderr(), "{message}").unwrap();
-        })));
-    }
-
-    let time = Instant::now();
-
-    #[inline(always)]
-    fn is_utf16_bom(bom_bytes: &[u8]) -> bool {
-        bom_bytes == [0xFF, 0xFE] || bom_bytes == [0xFE, 0xFF]
-    }
-
-    let tree = match opts.encoding {
-        Some(encoding) if encoding == ffi::TSInputEncodingUTF16 => {
-            let source_code_utf16 = source_code
-                .chunks_exact(2)
-                .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]]))
-                .collect::<Vec<_>>();
-            parser.parse_utf16(&source_code_utf16, None)
-        }
-        None if source_code.len() >= 2 && is_utf16_bom(&source_code[0..2]) => {
-            let source_code_utf16 = source_code
-                .chunks_exact(2)
-                .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]]))
-                .collect::<Vec<_>>();
-            parser.parse_utf16(&source_code_utf16, None)
-        }
-        _ => parser.parse(&source_code, None),
-    };
-
-    parser.stop_printing_dot_graphs();
-
-    let stdout = io::stdout();
-    let mut stdout = stdout.lock();
-
-    if let Some(mut tree) = tree {
-        if opts.debug_graph && !opts.edits.is_empty() {
-            println!("BEFORE:\n{}", String::from_utf8_lossy(&source_code));
-        }
-
-        for (i, edit) in opts.edits.iter().enumerate() {
-            let edit = parse_edit_flag(&source_code, edit)?;
-            perform_edit(&mut tree, &mut source_code, &edit)?;
-            tree = parser.parse(&source_code, Some(&tree)).unwrap();
-
-            if opts.debug_graph {
-                println!("AFTER {i}:\n{}", String::from_utf8_lossy(&source_code));
-            }
-        }
-
-        let duration = time.elapsed();
-        let duration_ms = duration.as_micros() as f64 / 1e3;
-        let mut cursor = tree.walk();
-
-        if opts.output == ParseOutput::Normal {
-            let mut needs_newline = false;
-            let mut indent_level = 0;
-            let mut did_visit_children = false;
-            loop {
-                let node = cursor.node();
-                let is_named = node.is_named();
-                if did_visit_children {
-                    if is_named {
-                        stdout.write_all(b")")?;
-                        needs_newline = true;
-                    }
-                    if cursor.goto_next_sibling() {
-                        did_visit_children = false;
-                    } else if cursor.goto_parent() {
-                        did_visit_children = true;
-                        indent_level -= 1;
-                    } else {
-                        break;
-                    }
-                } else {
-                    if is_named {
-                        if needs_newline {
-                            stdout.write_all(b"\n")?;
-                        }
-                        for _ in 0..indent_level {
-                            stdout.write_all(b"  ")?;
-                        }
-                        let start = node.start_position();
-                        let end = node.end_position();
-                        if let Some(field_name) = cursor.field_name() {
-                            write!(&mut stdout, "{field_name}: ")?;
-                        }
-                        write!(
-                            &mut stdout,
-                            "({} [{}, {}] - [{}, {}]",
-                            node.kind(),
-                            start.row,
-                            start.column,
-                            end.row,
-                            end.column
-                        )?;
-                        needs_newline = true;
-                    }
-                    if cursor.goto_first_child() {
-                        did_visit_children = false;
-                        indent_level += 1;
-                    } else {
-                        did_visit_children = true;
-                    }
-                }
-            }
-            cursor.reset(tree.root_node());
-            println!();
-        }
-
-        if opts.output == ParseOutput::Xml {
-            let mut needs_newline = false;
-            let mut indent_level = 0;
-            let mut did_visit_children = false;
-            let mut had_named_children = false;
-            let mut tags = Vec::<&str>::new();
-            writeln!(&mut stdout, "<?xml version=\"1.0\"?>")?;
-            loop {
-                let node = cursor.node();
-                let is_named = node.is_named();
-                if did_visit_children {
-                    if is_named {
-                        let tag = tags.pop();
-                        if had_named_children {
-                            for _ in 0..indent_level {
-                                stdout.write_all(b"  ")?;
-                            }
-                        }
-                        write!(&mut stdout, "</{}>", tag.expect("there is a tag"))?;
-                        // we only write a line in the case where it's the last sibling
-                        if let Some(parent) = node.parent() {
-                            if parent.child(parent.child_count() - 1).unwrap() == node {
-                                stdout.write_all(b"\n")?;
-                            }
-                        }
-                        needs_newline = true;
-                    }
-                    if cursor.goto_next_sibling() {
-                        did_visit_children = false;
-                        had_named_children = false;
-                    } else if cursor.goto_parent() {
-                        did_visit_children = true;
-                        had_named_children = is_named;
-                        indent_level -= 1;
-                        if !is_named && needs_newline {
-                            stdout.write_all(b"\n")?;
-                            for _ in 0..indent_level {
-                                stdout.write_all(b"  ")?;
-                            }
-                        }
-                    } else {
-                        break;
-                    }
-                } else {
-                    if is_named {
-                        if needs_newline {
-                            stdout.write_all(b"\n")?;
-                        }
-                        for _ in 0..indent_level {
-                            stdout.write_all(b"  ")?;
-                        }
-                        write!(&mut stdout, "<{}", node.kind())?;
-                        if let Some(field_name) = cursor.field_name() {
-                            write!(&mut stdout, " field=\"{field_name}\"")?;
-                        }
-                        let start = node.start_position();
-                        let end = node.end_position();
-                        write!(&mut stdout, " srow=\"{}\"", start.row)?;
-                        write!(&mut stdout, " scol=\"{}\"", start.column)?;
-                        write!(&mut stdout, " erow=\"{}\"", end.row)?;
-                        write!(&mut stdout, " ecol=\"{}\"", end.column)?;
-                        write!(&mut stdout, ">")?;
-                        tags.push(node.kind());
-                        needs_newline = true;
-                    }
-                    if cursor.goto_first_child() {
-                        did_visit_children = false;
-                        had_named_children = false;
-                        indent_level += 1;
-                    } else {
-                        did_visit_children = true;
-                        let start = node.start_byte();
-                        let end = node.end_byte();
-                        let value =
-                            std::str::from_utf8(&source_code[start..end]).expect("has a string");
-                        // if !is_named {
-                        //     for _ in 0..indent_level {
-                        //         stdout.write_all(b"  ")?;
-                        //     }
-                        // }
-                        if !is_named && needs_newline {
-                            stdout.write_all(b"\n")?;
-                            for _ in 0..indent_level {
-                                stdout.write_all(b"  ")?;
-                            }
-                        }
-                        write!(&mut stdout, "{}", html_escape::encode_text(value))?;
-                    }
-                }
-            }
-            cursor.reset(tree.root_node());
-            println!();
-        }
-
-        if opts.output == ParseOutput::Dot {
-            util::print_tree_graph(&tree, "log.html", opts.open_log).unwrap();
-        }
-
-        let mut first_error = None;
-        loop {
-            let node = cursor.node();
-            if node.has_error() {
-                if node.is_error() || node.is_missing() {
-                    first_error = Some(node);
-                    break;
-                }
-                if !cursor.goto_first_child() {
-                    break;
-                }
-            } else if !cursor.goto_next_sibling() {
-                break;
-            }
-        }
-
-        if first_error.is_some() || opts.print_time {
-            write!(
-                &mut stdout,
-                "{:width$}\t{duration_ms:>7.2} ms\t{:>6} bytes/ms",
-                opts.path.to_str().unwrap(),
-                (source_code.len() as u128 * 1_000_000) / duration.as_nanos(),
-                width = opts.max_path_length
-            )?;
-            if let Some(node) = first_error {
-                let start = node.start_position();
-                let end = node.end_position();
-                write!(&mut stdout, "\t(")?;
-                if node.is_missing() {
-                    if node.is_named() {
-                        write!(&mut stdout, "MISSING {}", node.kind())?;
-                    } else {
-                        write!(
-                            &mut stdout,
-                            "MISSING \"{}\"",
-                            node.kind().replace('\n', "\\n")
-                        )?;
-                    }
-                } else {
-                    write!(&mut stdout, "{}", node.kind())?;
-                }
-                write!(
-                    &mut stdout,
-                    " [{}, {}] - [{}, {}])",
-                    start.row, start.column, end.row, end.column
-                )?;
-            }
-            writeln!(&mut stdout)?;
-        }
-
-        return Ok(ParseResult {
-            successful: first_error.is_none(),
-            bytes: source_code.len(),
-            duration: Some(duration),
-        });
-    }
-
-    if opts.print_time {
-        let duration = time.elapsed();
-        let duration_ms = duration.as_micros() as f64 / 1e3;
-        writeln!(
-            &mut stdout,
-            "{:width$}\t{duration_ms:>7.2} ms\t(timed out)",
-            opts.path.to_str().unwrap(),
-            width = opts.max_path_length
-        )?;
-    }
-
-    Ok(ParseResult {
-        successful: false,
-        bytes: source_code.len(),
-        duration: None,
-    })
-}
-
-pub fn perform_edit(tree: &mut Tree, input: &mut Vec<u8>, edit: &Edit) -> Result<InputEdit> {
-    let start_byte = edit.position;
-    let old_end_byte = edit.position + edit.deleted_length;
-    let new_end_byte = edit.position + edit.inserted_text.len();
-    let start_position = position_for_offset(input, start_byte)?;
-    let old_end_position = position_for_offset(input, old_end_byte)?;
-    input.splice(start_byte..old_end_byte, edit.inserted_text.iter().copied());
-    let new_end_position = position_for_offset(input, new_end_byte)?;
-    let edit = InputEdit {
-        start_byte,
-        old_end_byte,
-        new_end_byte,
-        start_position,
-        old_end_position,
-        new_end_position,
-    };
-    tree.edit(&edit);
-    Ok(edit)
-}
-
-fn parse_edit_flag(source_code: &[u8], flag: &str) -> Result<Edit> {
-    let error = || {
-        anyhow!(concat!(
-            "Invalid edit string '{}'. ",
-            "Edit strings must match the pattern '<START_BYTE_OR_POSITION> <REMOVED_LENGTH> <NEW_TEXT>'"
-        ), flag)
-    };
-
-    // Three whitespace-separated parts:
-    // * edit position
-    // * deleted length
-    // * inserted text
-    let mut parts = flag.split(' ');
-    let position = parts.next().ok_or_else(error)?;
-    let deleted_length = parts.next().ok_or_else(error)?;
-    let inserted_text = parts.collect::<Vec<_>>().join(" ").into_bytes();
-
-    // Position can either be a byte_offset or row,column pair, separated by a comma
-    let position = if position == "$" {
-        source_code.len()
-    } else if position.contains(',') {
-        let mut parts = position.split(',');
-        let row = parts.next().ok_or_else(error)?;
-        let row = row.parse::<usize>().map_err(|_| error())?;
-        let column = parts.next().ok_or_else(error)?;
-        let column = column.parse::<usize>().map_err(|_| error())?;
-        offset_for_position(source_code, Point { row, column })?
-    } else {
-        position.parse::<usize>().map_err(|_| error())?
-    };
-
-    // Deleted length must be a byte count.
-    let deleted_length = deleted_length.parse::<usize>().map_err(|_| error())?;
-
-    Ok(Edit {
-        position,
-        deleted_length,
-        inserted_text,
-    })
-}
-
-pub fn offset_for_position(input: &[u8], position: Point) -> Result<usize> {
-    let mut row = 0;
-    let mut offset = 0;
-    let mut iter = memchr::memchr_iter(b'\n', input);
-    loop {
-        if let Some(pos) = iter.next() {
-            if row < position.row {
-                row += 1;
-                offset = pos;
-                continue;
-            }
-        }
-        offset += 1;
-        break;
-    }
-    if position.row - row > 0 {
-        return Err(anyhow!("Failed to address a row: {}", position.row));
-    }
-    if let Some(pos) = iter.next() {
-        if (pos - offset < position.column) || (input[offset] == b'\n' && position.column > 0) {
-            return Err(anyhow!("Failed to address a column: {}", position.column));
-        };
-    } else if input.len() - offset < position.column {
-        return Err(anyhow!("Failed to address a column over the end"));
-    }
-    Ok(offset + position.column)
-}
-
-pub fn position_for_offset(input: &[u8], offset: usize) -> Result<Point> {
-    if offset > input.len() {
-        return Err(anyhow!("Failed to address an offset: {offset}"));
-    }
-    let mut result = Point { row: 0, column: 0 };
-    let mut last = 0;
-    for pos in memchr::memchr_iter(b'\n', &input[..offset]) {
-        result.row += 1;
-        last = pos;
-    }
-    result.column = if result.row > 0 {
-        offset - last - 1
-    } else {
-        offset
-    };
-    Ok(result)
-}

cli/src/playground.html

@@ -1,176 +0,0 @@
-<head>
-  <meta charset="utf-8">
-  <title>tree-sitter THE_LANGUAGE_NAME</title>
-  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.45.0/codemirror.min.css">
-  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/clusterize.js/0.18.0/clusterize.min.css">
-  <link rel="icon" type="image/png" href="https://tree-sitter.github.io/tree-sitter/assets/images/favicon-32x32.png" sizes="32x32" />
-  <link rel="icon" type="image/png" href="https://tree-sitter.github.io/tree-sitter/assets/images/favicon-16x16.png" sizes="16x16" />
-</head>
-
-<body>
-  <div id="playground-container" style="visibility: hidden;">
-    <header>
-      <div class=header-item>
-        <bold>THE_LANGUAGE_NAME</bold>
-      </div>
-
-      <div class=header-item>
-        <label for="logging-checkbox">log</label>
-        <input id="logging-checkbox" type="checkbox"></input>
-      </div>
-
-      <div class=header-item>
-        <label for="query-checkbox">query</label>
-        <input id="query-checkbox" type="checkbox"></input>
-      </div>
-
-      <div class=header-item>
-        <label for="update-time">parse time: </label>
-        <span id="update-time"></span>
-      </div>
-
-      <div class=header-item>
-        <a href="https://tree-sitter.github.io/tree-sitter/playground#about">(?)</a>
-      </div>
-
-      <select id="language-select" style="display: none;">
-        <option value="parser">Parser</option>
-      </select>
-    </header>
-
-    <main>
-      <div id="input-pane">
-        <div id="code-container">
-          <textarea id="code-input"></textarea>
-        </div>
-
-        <div id="query-container" style="visibility: hidden; position: absolute;">
-          <textarea id="query-input"></textarea>
-        </div>
-      </div>
-
-      <div id="output-container-scroll">
-        <pre id="output-container" class="highlight"></pre>
-      </div>
-    </main>
-  </div>
-
-  <script
-    src="https://code.jquery.com/jquery-3.3.1.min.js"
-    crossorigin="anonymous">
-  </script>
-
-  <script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.45.0/codemirror.min.js"></script>
-  <script src="https://cdnjs.cloudflare.com/ajax/libs/clusterize.js/0.18.0/clusterize.min.js"></script>
-
-  <script>LANGUAGE_BASE_URL = "";</script>
-  <script src=tree-sitter.js></script>
-  <script src=playground.js></script>
-
-  <style>
-    body {
-      margin: 0;
-      padding: 0;
-    }
-
-    #playground-container {
-      width: 100%;
-      height: 100%;
-      display: flex;
-      flex-direction: column;
-    }
-
-    header {
-      box-sizing: border-box;
-      display: flex;
-      padding: 20px;
-      height: 60px;
-      border-bottom: 1px solid #aaa;
-    }
-
-    main {
-      flex: 1;
-      position: relative;
-    }
-
-    #input-pane {
-      position: absolute;
-      top: 0;
-      left: 0;
-      bottom: 0;
-      right: 50%;
-      display: flex;
-      flex-direction: column;
-    }
-
-    #code-container, #query-container {
-      flex: 1;
-      position: relative;
-      overflow: hidden;
-      border-right: 1px solid #aaa;
-      border-bottom: 1px solid #aaa;
-    }
-
-    #output-container-scroll {
-      position: absolute;
-      top: 0;
-      left: 50%;
-      bottom: 0;
-      right: 0;
-    }
-
-    .header-item {
-      margin-right: 30px;
-    }
-
-    #playground-container .CodeMirror {
-      position: absolute;
-      top: 0;
-      bottom: 0;
-      left: 0;
-      right: 0;
-      height: 100%;
-    }
-
-    #output-container-scroll {
-      flex: 1;
-      padding: 0;
-      overflow: auto;
-    }
-
-    #output-container {
-      padding: 0 10px;
-      margin: 0;
-    }
-
-    #logging-checkbox {
-      vertical-align: middle;
-    }
-
-    .CodeMirror div.CodeMirror-cursor {
-      border-left: 3px solid red;
-    }
-
-    a {
-      text-decoration: none;
-      color: #040404;
-      padding: 2px;
-    }
-
-    a:hover {
-      text-decoration: underline;
-    }
-
-    a.highlighted {
-      background-color: #d9d9d9;
-      color: red;
-      border-radius: 3px;
-      text-decoration: underline;
-    }
-
-    .query-error {
-      text-decoration: underline red dashed;
-      -webkit-text-decoration: underline red dashed;
-    }
-  </style>
-</body>

cli/src/query.rs

@@ -1,124 +0,0 @@
-use std::{
-    fs,
-    io::{self, Write},
-    ops::Range,
-    path::Path,
-    time::Instant,
-};
-
-use anyhow::{Context, Result};
-use tree_sitter::{Language, Parser, Point, Query, QueryCursor};
-
-use crate::query_testing;
-
-#[allow(clippy::too_many_arguments)]
-pub fn query_files_at_paths(
-    language: &Language,
-    paths: Vec<String>,
-    query_path: &Path,
-    ordered_captures: bool,
-    byte_range: Option<Range<usize>>,
-    point_range: Option<Range<Point>>,
-    should_test: bool,
-    quiet: bool,
-    print_time: bool,
-) -> Result<()> {
-    let stdout = io::stdout();
-    let mut stdout = stdout.lock();
-
-    let query_source = fs::read_to_string(query_path)
-        .with_context(|| format!("Error reading query file {query_path:?}"))?;
-    let query = Query::new(language, &query_source).with_context(|| "Query compilation failed")?;
-
-    let mut query_cursor = QueryCursor::new();
-    if let Some(range) = byte_range {
-        query_cursor.set_byte_range(range);
-    }
-    if let Some(range) = point_range {
-        query_cursor.set_point_range(range);
-    }
-
-    let mut parser = Parser::new();
-    parser.set_language(language)?;
-
-    for path in paths {
-        let mut results = Vec::new();
-
-        writeln!(&mut stdout, "{path}")?;
-
-        let source_code =
-            fs::read(&path).with_context(|| format!("Error reading source file {path:?}"))?;
-        let tree = parser.parse(&source_code, None).unwrap();
-
-        let start = Instant::now();
-        if ordered_captures {
-            for (mat, capture_index) in
-                query_cursor.captures(&query, tree.root_node(), source_code.as_slice())
-            {
-                let capture = mat.captures[capture_index];
-                let capture_name = &query.capture_names()[capture.index as usize];
-                if !quiet {
-                    writeln!(
-                        &mut stdout,
-                        "    pattern: {:>2}, capture: {} - {capture_name}, start: {}, end: {}, text: `{}`",
-                        mat.pattern_index,
-                        capture.index,
-                        capture.node.start_position(),
-                        capture.node.end_position(),
-                        capture.node.utf8_text(&source_code).unwrap_or("")
-                    )?;
-                }
-                results.push(query_testing::CaptureInfo {
-                    name: (*capture_name).to_string(),
-                    start: capture.node.start_position(),
-                    end: capture.node.end_position(),
-                });
-            }
-        } else {
-            for m in query_cursor.matches(&query, tree.root_node(), source_code.as_slice()) {
-                if !quiet {
-                    writeln!(&mut stdout, "  pattern: {}", m.pattern_index)?;
-                }
-                for capture in m.captures {
-                    let start = capture.node.start_position();
-                    let end = capture.node.end_position();
-                    let capture_name = &query.capture_names()[capture.index as usize];
-                    if !quiet {
-                        if end.row == start.row {
-                            writeln!(
-                                &mut stdout,
-                                "    capture: {} - {capture_name}, start: {start}, end: {end}, text: `{}`",
-                                capture.index,
-                                capture.node.utf8_text(&source_code).unwrap_or("")
-                            )?;
-                        } else {
-                            writeln!(
-                                &mut stdout,
-                                "    capture: {capture_name}, start: {start}, end: {end}",
-                            )?;
-                        }
-                    }
-                    results.push(query_testing::CaptureInfo {
-                        name: (*capture_name).to_string(),
-                        start: capture.node.start_position(),
-                        end: capture.node.end_position(),
-                    });
-                }
-            }
-        }
-        if query_cursor.did_exceed_match_limit() {
-            writeln!(
-                &mut stdout,
-                "  WARNING: Query exceeded maximum number of in-progress captures!"
-            )?;
-        }
-        if should_test {
-            query_testing::assert_expected_captures(&results, path, &mut parser, language)?;
-        }
-        if print_time {
-            writeln!(&mut stdout, "{:?}", start.elapsed())?;
-        }
-    }
-
-    Ok(())
-}

cli/src/tags.rs

@@ -1,98 +0,0 @@
-use std::{
-    fs,
-    io::{self, Write},
-    path::Path,
-    str,
-    time::Instant,
-};
-
-use anyhow::{anyhow, Result};
-use tree_sitter_loader::{Config, Loader};
-use tree_sitter_tags::TagsContext;
-
-use super::util;
-
-pub fn generate_tags(
-    loader: &Loader,
-    loader_config: &Config,
-    scope: Option<&str>,
-    paths: &[String],
-    quiet: bool,
-    time: bool,
-) -> Result<()> {
-    let mut lang = None;
-    if let Some(scope) = scope {
-        lang = loader.language_configuration_for_scope(scope)?;
-        if lang.is_none() {
-            return Err(anyhow!("Unknown scope '{scope}'"));
-        }
-    }
-
-    let mut context = TagsContext::new();
-    let cancellation_flag = util::cancel_on_signal();
-    let stdout = io::stdout();
-    let mut stdout = stdout.lock();
-
-    for path in paths {
-        let path = Path::new(&path);
-        let (language, language_config) = match lang.clone() {
-            Some(v) => v,
-            None => {
-                if let Some(v) = loader.language_configuration_for_file_name(path)? {
-                    v
-                } else {
-                    eprintln!("{}", util::lang_not_found_for_path(path, loader_config));
-                    continue;
-                }
-            }
-        };
-
-        if let Some(tags_config) = language_config.tags_config(language)? {
-            let indent = if paths.len() > 1 {
-                if !quiet {
-                    writeln!(&mut stdout, "{}", path.to_string_lossy())?;
-                }
-                "\t"
-            } else {
-                ""
-            };
-
-            let source = fs::read(path)?;
-            let t0 = Instant::now();
-            for tag in context
-                .generate_tags(tags_config, &source, Some(&cancellation_flag))?
-                .0
-            {
-                let tag = tag?;
-                if !quiet {
-                    write!(
-                        &mut stdout,
-                        "{indent}{:<10}\t | {:<8}\t{} {} - {} `{}`",
-                        str::from_utf8(&source[tag.name_range]).unwrap_or(""),
-                        &tags_config.syntax_type_name(tag.syntax_type_id),
-                        if tag.is_definition { "def" } else { "ref" },
-                        tag.span.start,
-                        tag.span.end,
-                        str::from_utf8(&source[tag.line_range]).unwrap_or(""),
-                    )?;
-                    if let Some(docs) = tag.docs {
-                        if docs.len() > 120 {
-                            write!(&mut stdout, "\t{:?}...", docs.get(0..120).unwrap_or(""))?;
-                        } else {
-                            write!(&mut stdout, "\t{:?}", &docs)?;
-                        }
-                    }
-                    writeln!(&mut stdout)?;
-                }
-            }
-
-            if time {
-                writeln!(&mut stdout, "{indent}time: {}ms", t0.elapsed().as_millis(),)?;
-            }
-        } else {
-            eprintln!("No tags config found for path {path:?}");
-        }
-    }
-
-    Ok(())
-}

cli/src/tests/async_context_test.rs

@@ -1,284 +0,0 @@
-use std::{
-    future::Future,
-    pin::{pin, Pin},
-    ptr,
-    task::{self, Context, Poll, RawWaker, RawWakerVTable, Waker},
-};
-
-use tree_sitter::Parser;
-
-use super::helpers::fixtures::get_language;
-
-#[test]
-fn test_node_in_fut() {
-    let (ret, pended) = tokio_like_spawn(async {
-        let mut parser = Parser::new();
-        let language = get_language("bash");
-        parser.set_language(&language).unwrap();
-
-        let tree = parser.parse("#", None).unwrap();
-
-        let root = tree.root_node();
-        let root_ref = &root;
-
-        let fut_val_fn = || async {
-            // eprintln!("fut_val_fn: {}", root.child(0).unwrap().kind());
-            yield_now().await;
-            root.child(0).unwrap().kind()
-        };
-
-        yield_now().await;
-
-        let fut_ref_fn = || async {
-            // eprintln!("fut_ref_fn: {}", root_ref.child(0).unwrap().kind());
-            yield_now().await;
-            root_ref.child(0).unwrap().kind()
-        };
-
-        let f1 = fut_val_fn().await;
-        let f2 = fut_ref_fn().await;
-        assert_eq!(f1, f2);
-
-        let fut_val = async {
-            // eprintln!("fut_val: {}", root.child(0).unwrap().kind());
-            yield_now().await;
-            root.child(0).unwrap().kind()
-        };
-
-        let fut_ref = async {
-            // eprintln!("fut_ref: {}", root_ref.child(0).unwrap().kind());
-            yield_now().await;
-            root_ref.child(0).unwrap().kind()
-        };
-
-        let f1 = fut_val.await;
-        let f2 = fut_ref.await;
-        assert_eq!(f1, f2);
-
-        f1
-    })
-    .join();
-    // eprintln!("pended: {pended:?}");
-    assert_eq!(ret, "comment");
-    assert_eq!(pended, 5);
-}
-
-#[test]
-fn test_node_and_cursor_ref_in_fut() {
-    let ((), pended) = tokio_like_spawn(async {
-        let mut parser = Parser::new();
-        let language = get_language("c");
-        parser.set_language(&language).unwrap();
-
-        let tree = parser.parse("#", None).unwrap();
-
-        let root = tree.root_node();
-        let root_ref = &root;
-
-        let mut cursor = tree.walk();
-        let cursor_ref = &mut cursor;
-
-        cursor_ref.goto_first_child();
-
-        let fut_val = async {
-            yield_now().await;
-            let _ = root.to_sexp();
-        };
-
-        yield_now().await;
-
-        let fut_ref = async {
-            yield_now().await;
-            let _ = root_ref.to_sexp();
-            cursor_ref.goto_first_child();
-        };
-
-        fut_val.await;
-        fut_ref.await;
-
-        cursor_ref.goto_first_child();
-    })
-    .join();
-    assert_eq!(pended, 3);
-}
-
-#[test]
-fn test_node_and_cursor_ref_in_fut_with_fut_fabrics() {
-    let ((), pended) = tokio_like_spawn(async {
-        let mut parser = Parser::new();
-        let language = get_language("javascript");
-        parser.set_language(&language).unwrap();
-
-        let tree = parser.parse("#", None).unwrap();
-
-        let root = tree.root_node();
-        let root_ref = &root;
-
-        let mut cursor = tree.walk();
-        let cursor_ref = &mut cursor;
-
-        cursor_ref.goto_first_child();
-
-        let fut_val = || async {
-            yield_now().await;
-            let _ = root.to_sexp();
-        };
-
-        yield_now().await;
-
-        let fut_ref = || async move {
-            yield_now().await;
-            let _ = root_ref.to_sexp();
-            cursor_ref.goto_first_child();
-        };
-
-        fut_val().await;
-        fut_val().await;
-        fut_ref().await;
-    })
-    .join();
-    assert_eq!(pended, 4);
-}
-
-#[test]
-fn test_node_and_cursor_ref_in_fut_with_inner_spawns() {
-    let (ret, pended) = tokio_like_spawn(async {
-        let mut parser = Parser::new();
-        let language = get_language("rust");
-        parser.set_language(&language).unwrap();
-
-        let tree = parser.parse("#", None).unwrap();
-
-        let mut cursor = tree.walk();
-        let cursor_ref = &mut cursor;
-
-        cursor_ref.goto_first_child();
-
-        let fut_val = || {
-            let tree = tree.clone();
-            async move {
-                let root = tree.root_node();
-                let mut cursor = tree.walk();
-                let cursor_ref = &mut cursor;
-                yield_now().await;
-                let _ = root.to_sexp();
-                cursor_ref.goto_first_child();
-            }
-        };
-
-        yield_now().await;
-
-        let fut_ref = || {
-            let tree = tree.clone();
-            async move {
-                let root = tree.root_node();
-                let root_ref = &root;
-                let mut cursor = tree.walk();
-                let cursor_ref = &mut cursor;
-                yield_now().await;
-                let _ = root_ref.to_sexp();
-                cursor_ref.goto_first_child();
-            }
-        };
-
-        let ((), p1) = tokio_like_spawn(fut_val()).await.unwrap();
-        let ((), p2) = tokio_like_spawn(fut_ref()).await.unwrap();
-
-        cursor_ref.goto_first_child();
-
-        fut_val().await;
-        fut_val().await;
-        fut_ref().await;
-
-        cursor_ref.goto_first_child();
-
-        p1 + p2
-    })
-    .join();
-    assert_eq!(pended, 4);
-    assert_eq!(ret, 2);
-}
-
-fn tokio_like_spawn<T>(future: T) -> JoinHandle<(T::Output, usize)>
-where
-    T: Future + Send + 'static,
-    T::Output: Send + 'static,
-{
-    // No runtime, just noop waker
-
-    let waker = noop_waker();
-    let mut cx = task::Context::from_waker(&waker);
-
-    let mut pending = 0;
-    let mut future = pin!(future);
-    let ret = loop {
-        match future.as_mut().poll(&mut cx) {
-            Poll::Pending => pending += 1,
-            Poll::Ready(r) => {
-                // eprintln!("ready, pended: {pending}");
-                break r;
-            }
-        }
-    };
-    JoinHandle::new((ret, pending))
-}
-
-async fn yield_now() {
-    struct SimpleYieldNow {
-        yielded: bool,
-    }
-
-    impl Future for SimpleYieldNow {
-        type Output = ();
-
-        fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> {
-            cx.waker().wake_by_ref();
-            if self.yielded {
-                return Poll::Ready(());
-            }
-            self.yielded = true;
-            Poll::Pending
-        }
-    }
-
-    SimpleYieldNow { yielded: false }.await;
-}
-
-pub fn noop_waker() -> Waker {
-    const VTABLE: RawWakerVTable = RawWakerVTable::new(
-        // Cloning just returns a new no-op raw waker
-        |_| RAW,
-        // `wake` does nothing
-        |_| {},
-        // `wake_by_ref` does nothing
-        |_| {},
-        // Dropping does nothing as we don't allocate anything
-        |_| {},
-    );
-    const RAW: RawWaker = RawWaker::new(ptr::null(), &VTABLE);
-    unsafe { Waker::from_raw(RAW) }
-}
-
-struct JoinHandle<T> {
-    data: Option<T>,
-}
-
-impl<T> JoinHandle<T> {
-    #[must_use]
-    const fn new(data: T) -> Self {
-        Self { data: Some(data) }
-    }
-
-    fn join(&mut self) -> T {
-        self.data.take().unwrap()
-    }
-}
-
-impl<T: Unpin> Future for JoinHandle<T> {
-    type Output = std::result::Result<T, ()>;
-
-    fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Self::Output> {
-        let data = self.get_mut().data.take().unwrap();
-        Poll::Ready(Ok(data))
-    }
-}

cli/src/tests/helpers/dirs.rs

@@ -1,47 +0,0 @@
-lazy_static! {
-    pub static ref ROOT_DIR: PathBuf = PathBuf::from(env!("CARGO_MANIFEST_DIR")).parent().unwrap().to_owned();
-    pub static ref FIXTURES_DIR: PathBuf = ROOT_DIR.join("test").join("fixtures");
-    pub static ref HEADER_DIR: PathBuf = ROOT_DIR.join("lib").join("include");
-    pub static ref GRAMMARS_DIR: PathBuf = ROOT_DIR.join("test").join("fixtures").join("grammars");
-    pub static ref SCRATCH_BASE_DIR: PathBuf = {
-        let result = ROOT_DIR.join("target").join("scratch");
-        fs::create_dir_all(&result).unwrap();
-        result
-    };
-    pub static ref WASM_DIR: PathBuf = ROOT_DIR.join("target").join("release");
-    pub static ref SCRATCH_DIR: PathBuf = {
-        // https://doc.rust-lang.org/reference/conditional-compilation.html
-        let vendor = if cfg!(target_vendor = "apple") {
-            "apple"
-        } else if cfg!(target_vendor = "fortanix") {
-            "fortanix"
-        } else if cfg!(target_vendor = "pc") {
-            "pc"
-        } else {
-            "unknown"
-        };
-        let env = if cfg!(target_env = "gnu") {
-            "gnu"
-        } else if cfg!(target_env = "msvc") {
-            "msvc"
-        } else if cfg!(target_env = "musl") {
-            "musl"
-        } else if cfg!(target_env = "sgx") {
-            "sgx"
-        } else {
-            "unknown"
-        };
-        let endian = if cfg!(target_endian = "little") {
-            "little"
-        } else if cfg!(target_endian = "big") {
-            "big"
-        } else {
-            "unknown"
-        };
-
-        let machine = format!("{}-{}-{vendor}-{env}-{endian}", std::env::consts::ARCH, std::env::consts::OS);
-        let result = SCRATCH_BASE_DIR.join(machine);
-        fs::create_dir_all(&result).unwrap();
-        result
-    };
-}

cli/src/tests/helpers/mod.rs

@@ -1,35 +0,0 @@
-pub(super) mod allocations;
-pub(super) mod edits;
-pub(super) mod fixtures;
-pub(super) mod query_helpers;
-pub(super) mod random;
-pub(super) mod scope_sequence;
-
-use std::env;
-
-use lazy_static::lazy_static;
-use rand::Rng;
-
-lazy_static! {
-    pub static ref LOG_ENABLED: bool = env::var("TREE_SITTER_LOG").is_ok();
-    pub static ref LOG_GRAPH_ENABLED: bool = env::var("TREE_SITTER_LOG_GRAPHS").is_ok();
-    pub static ref LANGUAGE_FILTER: Option<String> = env::var("TREE_SITTER_LANGUAGE").ok();
-    pub static ref EXAMPLE_FILTER: Option<String> = env::var("TREE_SITTER_EXAMPLE").ok();
-}
-
-lazy_static! {
-    pub static ref START_SEED: usize = new_seed();
-    pub static ref EDIT_COUNT: usize = int_env_var("TREE_SITTER_EDITS").unwrap_or(3);
-    pub static ref ITERATION_COUNT: usize = int_env_var("TREE_SITTER_ITERATIONS").unwrap_or(10);
-}
-
-fn int_env_var(name: &'static str) -> Option<usize> {
-    env::var(name).ok().and_then(|e| e.parse().ok())
-}
-
-pub fn new_seed() -> usize {
-    int_env_var("TREE_SITTER_SEED").unwrap_or_else(|| {
-        let mut rng = rand::thread_rng();
-        rng.gen::<usize>()
-    })
-}

cli/src/tests/language_test.rs

@@ -1,65 +0,0 @@
-use tree_sitter::Parser;
-
-use super::helpers::fixtures::get_language;
-
-#[test]
-fn test_lookahead_iterator() {
-    let mut parser = Parser::new();
-    let language = get_language("rust");
-    parser.set_language(&language).unwrap();
-
-    let tree = parser.parse("struct Stuff {}", None).unwrap();
-
-    let mut cursor = tree.walk();
-
-    assert!(cursor.goto_first_child()); // struct
-    assert!(cursor.goto_first_child()); // struct keyword
-
-    let next_state = cursor.node().next_parse_state();
-    assert_ne!(next_state, 0);
-    assert_eq!(
-        next_state,
-        language.next_state(cursor.node().parse_state(), cursor.node().grammar_id())
-    );
-    assert!((next_state as usize) < language.parse_state_count());
-    assert!(cursor.goto_next_sibling()); // type_identifier
-    assert_eq!(next_state, cursor.node().parse_state());
-    assert_eq!(cursor.node().grammar_name(), "identifier");
-    assert_ne!(cursor.node().grammar_id(), cursor.node().kind_id());
-
-    let expected_symbols = ["//", "/*", "identifier", "line_comment", "block_comment"];
-    let mut lookahead = language.lookahead_iterator(next_state).unwrap();
-    assert_eq!(*lookahead.language(), language);
-    assert!(lookahead.iter_names().eq(expected_symbols));
-
-    lookahead.reset_state(next_state);
-    assert!(lookahead.iter_names().eq(expected_symbols));
-
-    lookahead.reset(&language, next_state);
-    assert!(lookahead
-        .map(|s| language.node_kind_for_id(s).unwrap())
-        .eq(expected_symbols));
-}
-
-#[test]
-fn test_lookahead_iterator_modifiable_only_by_mut() {
-    let mut parser = Parser::new();
-    let language = get_language("rust");
-    parser.set_language(&language).unwrap();
-
-    let tree = parser.parse("struct Stuff {}", None).unwrap();
-
-    let mut cursor = tree.walk();
-
-    assert!(cursor.goto_first_child()); // struct
-    assert!(cursor.goto_first_child()); // struct keyword
-
-    let next_state = cursor.node().next_parse_state();
-    assert_ne!(next_state, 0);
-
-    let mut lookahead = language.lookahead_iterator(next_state).unwrap();
-    let _ = lookahead.next();
-
-    let mut names = lookahead.iter_names();
-    let _ = names.next();
-}

cli/src/tests/mod.rs

@@ -1,19 +0,0 @@
-mod async_context_test;
-mod corpus_test;
-mod detect_language;
-mod helpers;
-mod highlight_test;
-mod language_test;
-mod node_test;
-mod parser_hang_test;
-mod parser_test;
-mod pathological_test;
-mod query_test;
-mod tags_test;
-mod test_highlight_test;
-mod test_tags_test;
-mod text_provider_test;
-mod tree_test;
-
-#[cfg(feature = "wasm")]
-mod wasm_language_test;

cli/src/tests/parser_hang_test.rs

@@ -1,105 +0,0 @@
-// For some reasons `Command::spawn` doesn't work in CI env for many exotic arches.
-#![cfg(all(any(target_arch = "x86_64", target_arch = "x86"), not(sanitizing)))]
-
-use std::{
-    env::VarError,
-    process::{Command, Stdio},
-};
-
-use tree_sitter::Parser;
-
-use crate::{
-    generate::{generate_parser_for_grammar, load_grammar_file},
-    tests::helpers::fixtures::{fixtures_dir, get_test_language},
-};
-
-// The `sanitizing` cfg is required to don't run tests under specific sunitizer
-// because they don't work well with subprocesses _(it's an assumption)_.
-//
-// Below are two alternative examples of how to disable tests for some arches
-// if a way with excluding the whole mod from compilation wouldn't work well.
-//
-// XXX: Also may be it makes sense to keep such tests as ignored by default
-//      to omit surprises and enable them on CI by passing an extra option explicitly:
-//
-//        > cargo test -- --include-ignored
-//
-// #[cfg(all(any(target_arch = "x86_64", target_arch = "x86"), not(sanitizing)))]
-// #[cfg_attr(not(all(any(target_arch = "x86_64", target_arch = "x86"), not(sanitizing))), ignore)]
-//
-#[test]
-fn test_grammar_that_should_hang_and_not_segfault() {
-    let parent_sleep_millis = 1000;
-    let test_name = "test_grammar_that_should_hang_and_not_segfault";
-    let test_var = "CARGO_HANG_TEST";
-
-    eprintln!("  {test_name}");
-
-    let tests_exec_path = std::env::args()
-        .next()
-        .expect("Failed to get tests executable path");
-
-    match std::env::var(test_var) {
-        Ok(v) if v == test_name => {
-            eprintln!("    child process id {}", std::process::id());
-            hang_test();
-        }
-
-        Err(VarError::NotPresent) => {
-            eprintln!("    parent process id {}", std::process::id());
-            let mut command = Command::new(tests_exec_path);
-            command.arg(test_name).env(test_var, test_name);
-
-            if std::env::args().any(|x| x == "--nocapture") {
-                command.arg("--nocapture");
-            } else {
-                command.stdout(Stdio::null()).stderr(Stdio::null());
-            }
-
-            match command.spawn() {
-                Ok(mut child) => {
-                    std::thread::sleep(std::time::Duration::from_millis(parent_sleep_millis));
-                    match child.try_wait() {
-                        Ok(Some(status)) if status.success() => {
-                            panic!("Child didn't hang and exited successfully")
-                        }
-                        Ok(Some(status)) => panic!(
-                            "Child didn't hang and exited with status code: {:?}",
-                            status.code()
-                        ),
-                        _ => (),
-                    }
-                    if let Err(e) = child.kill() {
-                        eprintln!(
-                            "Failed to kill hang test's process id: {}, error: {e}",
-                            child.id()
-                        );
-                    }
-                }
-                Err(e) => panic!("{e}"),
-            }
-        }
-
-        Err(e) => panic!("Env var error: {e}"),
-
-        _ => unreachable!(),
-    }
-}
-
-fn hang_test() {
-    let test_grammar_dir = fixtures_dir()
-        .join("test_grammars")
-        .join("get_col_should_hang_not_crash");
-
-    let grammar_json = load_grammar_file(&test_grammar_dir.join("grammar.js"), None).unwrap();
-    let (parser_name, parser_code) = generate_parser_for_grammar(grammar_json.as_str()).unwrap();
-
-    let language = get_test_language(&parser_name, &parser_code, Some(test_grammar_dir.as_path()));
-
-    let mut parser = Parser::new();
-    parser.set_language(&language).unwrap();
-
-    let code_that_should_hang = "\nHello";
-
-    parser.parse(code_that_should_hang, None).unwrap();
-}

cli/src/tests/test_highlight_test.rs

@@ -1,62 +0,0 @@
-use tree_sitter::{Parser, Point};
-use tree_sitter_highlight::{Highlight, Highlighter};
-
-use super::helpers::fixtures::{get_highlight_config, get_language, test_loader};
-use crate::{
-    query_testing::{parse_position_comments, Assertion},
-    test_highlight::get_highlight_positions,
-};
-
-#[test]
-fn test_highlight_test_with_basic_test() {
-    let language = get_language("javascript");
-    let config = get_highlight_config(
-        "javascript",
-        Some("injections.scm"),
-        &[
-            "function".to_string(),
-            "variable".to_string(),
-            "keyword".to_string(),
-        ],
-    );
-    let source = [
-        "// hi",
-        "var abc = function(d) {",
-        "  // ^ function",
-        "  //       ^ keyword",
-        "  return d + e;",
-        "  //     ^ variable",
-        "  //       ^ !variable",
-        "};",
-    ]
-    .join("\n");
-
-    let assertions =
-        parse_position_comments(&mut Parser::new(), &language, source.as_bytes()).unwrap();
-    assert_eq!(
-        assertions,
-        &[
-            Assertion::new(1, 5, false, String::from("function")),
-            Assertion::new(1, 11, false, String::from("keyword")),
-            Assertion::new(4, 9, false, String::from("variable")),
-            Assertion::new(4, 11, true, String::from("variable")),
-        ]
-    );
-
-    let mut highlighter = Highlighter::new();
-    let highlight_positions =
-        get_highlight_positions(test_loader(), &mut highlighter, &config, source.as_bytes())
-            .unwrap();
-    assert_eq!(
-        highlight_positions,
-        &[
-            (Point::new(1, 0), Point::new(1, 3), Highlight(2)), // "var"
-            (Point::new(1, 4), Point::new(1, 7), Highlight(0)), // "abc"
-            (Point::new(1, 10), Point::new(1, 18), Highlight(2)), // "function"
-            (Point::new(1, 19), Point::new(1, 20), Highlight(1)), // "d"
-            (Point::new(4, 2), Point::new(4, 8), Highlight(2)), // "return"
-            (Point::new(4, 9), Point::new(4, 10), Highlight(1)), // "d"
-            (Point::new(4, 13), Point::new(4, 14), Highlight(1)), // "e"
-        ]
-    );
-}

cli/vendor/xterm-colors.json

@@ -1,258 +0,0 @@
-[
-  "#000000",
-  "#800000",
-  "#008000",
-  "#808000",
-  "#000080",
-  "#800080",
-  "#008080",
-  "#c0c0c0",
-  "#808080",
-  "#ff0000",
-  "#00ff00",
-  "#ffff00",
-  "#0000ff",
-  "#ff00ff",
-  "#00ffff",
-  "#ffffff",
-  "#000000",
-  "#00005f",
-  "#000087",
-  "#0000af",
-  "#0000d7",
-  "#0000ff",
-  "#005f00",
-  "#005f5f",
-  "#005f87",
-  "#005faf",
-  "#005fd7",
-  "#005fff",
-  "#008700",
-  "#00875f",
-  "#008787",
-  "#0087af",
-  "#0087d7",
-  "#0087ff",
-  "#00af00",
-  "#00af5f",
-  "#00af87",
-  "#00afaf",
-  "#00afd7",
-  "#00afff",
-  "#00d700",
-  "#00d75f",
-  "#00d787",
-  "#00d7af",
-  "#00d7d7",
-  "#00d7ff",
-  "#00ff00",
-  "#00ff5f",
-  "#00ff87",
-  "#00ffaf",
-  "#00ffd7",
-  "#00ffff",
-  "#5f0000",
-  "#5f005f",
-  "#5f0087",
-  "#5f00af",
-  "#5f00d7",
-  "#5f00ff",
-  "#5f5f00",
-  "#5f5f5f",
-  "#5f5f87",
-  "#5f5faf",
-  "#5f5fd7",
-  "#5f5fff",
-  "#5f8700",
-  "#5f875f",
-  "#5f8787",
-  "#5f87af",
-  "#5f87d7",
-  "#5f87ff",
-  "#5faf00",
-  "#5faf5f",
-  "#5faf87",
-  "#5fafaf",
-  "#5fafd7",
-  "#5fafff",
-  "#5fd700",
-  "#5fd75f",
-  "#5fd787",
-  "#5fd7af",
-  "#5fd7d7",
-  "#5fd7ff",
-  "#5fff00",
-  "#5fff5f",
-  "#5fff87",
-  "#5fffaf",
-  "#5fffd7",
-  "#5fffff",
-  "#870000",
-  "#87005f",
-  "#870087",
-  "#8700af",
-  "#8700d7",
-  "#8700ff",
-  "#875f00",
-  "#875f5f",
-  "#875f87",
-  "#875faf",
-  "#875fd7",
-  "#875fff",
-  "#878700",
-  "#87875f",
-  "#878787",
-  "#8787af",
-  "#8787d7",
-  "#8787ff",
-  "#87af00",
-  "#87af5f",
-  "#87af87",
-  "#87afaf",
-  "#87afd7",
-  "#87afff",
-  "#87d700",
-  "#87d75f",
-  "#87d787",
-  "#87d7af",
-  "#87d7d7",
-  "#87d7ff",
-  "#87ff00",
-  "#87ff5f",
-  "#87ff87",
-  "#87ffaf",
-  "#87ffd7",
-  "#87ffff",
-  "#af0000",
-  "#af005f",
-  "#af0087",
-  "#af00af",
-  "#af00d7",
-  "#af00ff",
-  "#af5f00",
-  "#af5f5f",
-  "#af5f87",
-  "#af5faf",
-  "#af5fd7",
-  "#af5fff",
-  "#af8700",
-  "#af875f",
-  "#af8787",
-  "#af87af",
-  "#af87d7",
-  "#af87ff",
-  "#afaf00",
-  "#afaf5f",
-  "#afaf87",
-  "#afafaf",
-  "#afafd7",
-  "#afafff",
-  "#afd700",
-  "#afd75f",
-  "#afd787",
-  "#afd7af",
-  "#afd7d7",
-  "#afd7ff",
-  "#afff00",
-  "#afff5f",
-  "#afff87",
-  "#afffaf",
-  "#afffd7",
-  "#afffff",
-  "#d70000",
-  "#d7005f",
-  "#d70087",
-  "#d700af",
-  "#d700d7",
-  "#d700ff",
-  "#d75f00",
-  "#d75f5f",
-  "#d75f87",
-  "#d75faf",
-  "#d75fd7",
-  "#d75fff",
-  "#d78700",
-  "#d7875f",
-  "#d78787",
-  "#d787af",
-  "#d787d7",
-  "#d787ff",
-  "#d7af00",
-  "#d7af5f",
-  "#d7af87",
-  "#d7afaf",
-  "#d7afd7",
-  "#d7afff",
-  "#d7d700",
-  "#d7d75f",
-  "#d7d787",
-  "#d7d7af",
-  "#d7d7d7",
-  "#d7d7ff",
-  "#d7ff00",
-  "#d7ff5f",
-  "#d7ff87",
-  "#d7ffaf",
-  "#d7ffd7",
-  "#d7ffff",
-  "#ff0000",
-  "#ff005f",
-  "#ff0087",
-  "#ff00af",
-  "#ff00d7",
-  "#ff00ff",
-  "#ff5f00",
-  "#ff5f5f",
-  "#ff5f87",
-  "#ff5faf",
-  "#ff5fd7",
-  "#ff5fff",
-  "#ff8700",
-  "#ff875f",
-  "#ff8787",
-  "#ff87af",
-  "#ff87d7",
-  "#ff87ff",
-  "#ffaf00",
-  "#ffaf5f",
-  "#ffaf87",
-  "#ffafaf",
-  "#ffafd7",
-  "#ffafff",
-  "#ffd700",
-  "#ffd75f",
-  "#ffd787",
-  "#ffd7af",
-  "#ffd7d7",
-  "#ffd7ff",
-  "#ffff00",
-  "#ffff5f",
-  "#ffff87",
-  "#ffffaf",
-  "#ffffd7",
-  "#ffffff",
-  "#080808",
-  "#121212",
-  "#1c1c1c",
-  "#262626",
-  "#303030",
-  "#3a3a3a",
-  "#444444",
-  "#4e4e4e",
-  "#585858",
-  "#626262",
-  "#6c6c6c",
-  "#767676",
-  "#808080",
-  "#8a8a8a",
-  "#949494",
-  "#9e9e9e",
-  "#a8a8a8",
-  "#b2b2b2",
-  "#bcbcbc",
-  "#c6c6c6",
-  "#d0d0d0",
-  "#dadada",
-  "#e4e4e4",
-  "#eeeeee"
-]

crates/cli/Cargo.toml

@@ -8,9 +8,17 @@ rust-version.workspace = true
 readme = "README.md"
 homepage.workspace = true
 repository.workspace = true
+documentation = "https://docs.rs/tree-sitter-cli"
 license.workspace = true
 keywords.workspace = true
 categories.workspace = true
+include = ["build.rs", "README.md", "LICENSE", "benches/*", "src/**"]
+
+[lints]
+workspace = true
+
+[lib]
+path = "src/tree_sitter_cli.rs"
 
 [[bin]]
 name = "tree-sitter"
@@ -22,51 +30,54 @@ name = "benchmark"
 harness = false
 
 [features]
+default = ["qjs-rt"]
 wasm = ["tree-sitter/wasm", "tree-sitter-loader/wasm"]
+qjs-rt = ["tree-sitter-generate/qjs-rt"]
 
 [dependencies]
+ansi_colours.workspace = true
 anstyle.workspace = true
 anyhow.workspace = true
+bstr.workspace = true
 clap.workspace = true
+clap_complete.workspace = true
+clap_complete_nushell.workspace = true
+crc32fast.workspace = true
+ctor.workspace = true
 ctrlc.workspace = true
-dirs.workspace = true
-filetime.workspace = true
+dialoguer.workspace = true
 glob.workspace = true
 heck.workspace = true
 html-escape.workspace = true
-indexmap.workspace = true
 indoc.workspace = true
-lazy_static.workspace = true
 log.workspace = true
 memchr.workspace = true
+rand.workspace = true
 regex.workspace = true
-regex-syntax.workspace = true
-rustc-hash.workspace = true
+schemars.workspace = true
 semver.workspace = true
 serde.workspace = true
-serde_derive.workspace = true
 serde_json.workspace = true
 similar.workspace = true
-smallbitvec.workspace = true
+streaming-iterator.workspace = true
+thiserror.workspace = true
 tiny_http.workspace = true
 walkdir.workspace = true
 wasmparser.workspace = true
 webbrowser.workspace = true
 
 tree-sitter.workspace = true
+tree-sitter-generate.workspace = true
 tree-sitter-config.workspace = true
 tree-sitter-highlight.workspace = true
 tree-sitter-loader.workspace = true
 tree-sitter-tags.workspace = true
 
-[target."cfg(windows)".dependencies]
-url = "2.5.0"
-
 [dev-dependencies]
+encoding_rs = "0.8.35"
+widestring = "1.2.1"
 tree_sitter_proc_macro = { path = "src/tests/proc_macro", package = "tree-sitter-tests-proc-macro" }
 
-rand.workspace = true
 tempfile.workspace = true
 pretty_assertions.workspace = true
-ctor.workspace = true
 unindent.workspace = true

crates/cli/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Max Brunsfeld
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

crates/cli/README.md

@@ -7,14 +7,15 @@
 [npmjs.com]: https://www.npmjs.org/package/tree-sitter-cli
 [npmjs.com badge]: https://img.shields.io/npm/v/tree-sitter-cli.svg?color=%23BF4A4A
 
-The Tree-sitter CLI allows you to develop, test, and use Tree-sitter grammars from the command line. It works on MacOS, Linux, and Windows.
+The Tree-sitter CLI allows you to develop, test, and use Tree-sitter grammars from the command line. It works on `MacOS`,
+`Linux`, and `Windows`.
 
 ### Installation
 
 You can install the `tree-sitter-cli` with `cargo`:
 
 ```sh
-cargo install tree-sitter-cli
+cargo install --locked tree-sitter-cli
 ```
 
 or with `npm`:
@@ -34,9 +35,11 @@ The `tree-sitter` binary itself has no dependencies, but specific commands have
 
 ### Commands
 
-* `generate` - The `tree-sitter generate` command will generate a Tree-sitter parser based on the grammar in the current working directory. See [the documentation] for more information.
+* `generate` - The `tree-sitter generate` command will generate a Tree-sitter parser based on the grammar in the current
+  working directory. See [the documentation] for more information.
 
-* `test` - The `tree-sitter test` command will run the unit tests for the Tree-sitter parser in the current working directory. See [the documentation] for more information.
+* `test` - The `tree-sitter test` command will run the unit tests for the Tree-sitter parser in the current working directory.
+  See [the documentation] for more information.
 
 * `parse` - The `tree-sitter parse` command will parse a file (or list of files) using Tree-sitter parsers.
 

crates/cli/benches/benchmark.rs

@@ -3,71 +3,77 @@ use std::{
     env, fs,
     path::{Path, PathBuf},
     str,
+    sync::LazyLock,
     time::Instant,
-    usize,
 };
 
 use anyhow::Context;
-use lazy_static::lazy_static;
+use log::info;
 use tree_sitter::{Language, Parser, Query};
 use tree_sitter_loader::{CompileConfig, Loader};
 
 include!("../src/tests/helpers/dirs.rs");
 
-lazy_static! {
-    static ref LANGUAGE_FILTER: Option<String> =
-        env::var("TREE_SITTER_BENCHMARK_LANGUAGE_FILTER").ok();
-    static ref EXAMPLE_FILTER: Option<String> =
-        env::var("TREE_SITTER_BENCHMARK_EXAMPLE_FILTER").ok();
-    static ref REPETITION_COUNT: usize = env::var("TREE_SITTER_BENCHMARK_REPETITION_COUNT")
+static LANGUAGE_FILTER: LazyLock<Option<String>> =
+    LazyLock::new(|| env::var("TREE_SITTER_BENCHMARK_LANGUAGE_FILTER").ok());
+static EXAMPLE_FILTER: LazyLock<Option<String>> =
+    LazyLock::new(|| env::var("TREE_SITTER_BENCHMARK_EXAMPLE_FILTER").ok());
+static REPETITION_COUNT: LazyLock<usize> = LazyLock::new(|| {
+    env::var("TREE_SITTER_BENCHMARK_REPETITION_COUNT")
         .map(|s| s.parse::<usize>().unwrap())
-        .unwrap_or(5);
-    static ref TEST_LOADER: Loader = Loader::with_parser_lib_path(SCRATCH_DIR.clone());
-    static ref EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR: BTreeMap<PathBuf, (Vec<PathBuf>, Vec<PathBuf>)> = {
-        fn process_dir(result: &mut BTreeMap<PathBuf, (Vec<PathBuf>, Vec<PathBuf>)>, dir: &Path) {
-            if dir.join("grammar.js").exists() {
-                let relative_path = dir.strip_prefix(GRAMMARS_DIR.as_path()).unwrap();
-                let (example_paths, query_paths) =
-                    result.entry(relative_path.to_owned()).or_default();
+        .unwrap_or(5)
+});
+static TEST_LOADER: LazyLock<Loader> =
+    LazyLock::new(|| Loader::with_parser_lib_path(SCRATCH_DIR.clone()));
 
-                if let Ok(example_files) = fs::read_dir(dir.join("examples")) {
-                    example_paths.extend(example_files.filter_map(|p| {
-                        let p = p.unwrap().path();
-                        if p.is_file() {
-                            Some(p)
-                        } else {
-                            None
-                        }
-                    }));
-                }
+#[allow(clippy::type_complexity)]
+static EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR: LazyLock<
+    BTreeMap<PathBuf, (Vec<PathBuf>, Vec<PathBuf>)>,
+> = LazyLock::new(|| {
+    fn process_dir(result: &mut BTreeMap<PathBuf, (Vec<PathBuf>, Vec<PathBuf>)>, dir: &Path) {
+        if dir.join("grammar.js").exists() {
+            let relative_path = dir.strip_prefix(GRAMMARS_DIR.as_path()).unwrap();
+            let (example_paths, query_paths) = result.entry(relative_path.to_owned()).or_default();
 
-                if let Ok(query_files) = fs::read_dir(dir.join("queries")) {
-                    query_paths.extend(query_files.filter_map(|p| {
-                        let p = p.unwrap().path();
-                        if p.is_file() {
-                            Some(p)
-                        } else {
-                            None
-                        }
-                    }));
-                }
-            } else {
-                for entry in fs::read_dir(dir).unwrap() {
-                    let entry = entry.unwrap().path();
-                    if entry.is_dir() {
-                        process_dir(result, &entry);
+            if let Ok(example_files) = fs::read_dir(dir.join("examples")) {
+                example_paths.extend(example_files.filter_map(|p| {
+                    let p = p.unwrap().path();
+                    if p.is_file() {
+                        Some(p)
+                    } else {
+                        None
                     }
+                }));
+            }
+
+            if let Ok(query_files) = fs::read_dir(dir.join("queries")) {
+                query_paths.extend(query_files.filter_map(|p| {
+                    let p = p.unwrap().path();
+                    if p.is_file() {
+                        Some(p)
+                    } else {
+                        None
+                    }
+                }));
+            }
+        } else {
+            for entry in fs::read_dir(dir).unwrap() {
+                let entry = entry.unwrap().path();
+                if entry.is_dir() {
+                    process_dir(result, &entry);
                 }
             }
         }
+    }
 
-        let mut result = BTreeMap::new();
-        process_dir(&mut result, &GRAMMARS_DIR);
-        result
-    };
-}
+    let mut result = BTreeMap::new();
+    process_dir(&mut result, &GRAMMARS_DIR);
+    result
+});
 
 fn main() {
+    tree_sitter_cli::logger::init();
+
     let max_path_length = EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR
         .values()
         .flat_map(|(e, q)| {
@@ -78,7 +84,7 @@ fn main() {
         .max()
         .unwrap_or(0);
 
-    eprintln!("Benchmarking with {} repetitions", *REPETITION_COUNT);
+    info!("Benchmarking with {} repetitions", *REPETITION_COUNT);
 
     let mut parser = Parser::new();
     let mut all_normal_speeds = Vec::new();
@@ -95,11 +101,11 @@ fn main() {
             }
         }
 
-        eprintln!("\nLanguage: {language_name}");
+        info!("\nLanguage: {language_name}");
         let language = get_language(language_path);
         parser.set_language(&language).unwrap();
 
-        eprintln!("  Constructing Queries");
+        info!("  Constructing Queries");
         for path in query_paths {
             if let Some(filter) = EXAMPLE_FILTER.as_ref() {
                 if !path.to_str().unwrap().contains(filter.as_str()) {
@@ -109,12 +115,12 @@ fn main() {
 
             parse(path, max_path_length, |source| {
                 Query::new(&language, str::from_utf8(source).unwrap())
-                    .with_context(|| format!("Query file path: {path:?}"))
+                    .with_context(|| format!("Query file path: {}", path.display()))
                     .expect("Failed to parse query");
             });
         }
 
-        eprintln!("  Parsing Valid Code:");
+        info!("  Parsing Valid Code:");
         let mut normal_speeds = Vec::new();
         for example_path in example_paths {
             if let Some(filter) = EXAMPLE_FILTER.as_ref() {
@@ -128,7 +134,7 @@ fn main() {
             }));
         }
 
-        eprintln!("  Parsing Invalid Code (mismatched languages):");
+        info!("  Parsing Invalid Code (mismatched languages):");
         let mut error_speeds = Vec::new();
         for (other_language_path, (example_paths, _)) in
             EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR.iter()
@@ -149,30 +155,30 @@ fn main() {
         }
 
         if let Some((average_normal, worst_normal)) = aggregate(&normal_speeds) {
-            eprintln!("  Average Speed (normal): {average_normal} bytes/ms");
-            eprintln!("  Worst Speed (normal):   {worst_normal} bytes/ms");
+            info!("  Average Speed (normal): {average_normal} bytes/ms");
+            info!("  Worst Speed (normal):   {worst_normal} bytes/ms");
         }
 
         if let Some((average_error, worst_error)) = aggregate(&error_speeds) {
-            eprintln!("  Average Speed (errors): {average_error} bytes/ms");
-            eprintln!("  Worst Speed (errors):   {worst_error} bytes/ms");
+            info!("  Average Speed (errors): {average_error} bytes/ms");
+            info!("  Worst Speed (errors):   {worst_error} bytes/ms");
         }
 
         all_normal_speeds.extend(normal_speeds);
         all_error_speeds.extend(error_speeds);
     }
 
-    eprintln!("\n  Overall");
+    info!("\n  Overall");
     if let Some((average_normal, worst_normal)) = aggregate(&all_normal_speeds) {
-        eprintln!("  Average Speed (normal): {average_normal} bytes/ms");
-        eprintln!("  Worst Speed (normal):   {worst_normal} bytes/ms");
+        info!("  Average Speed (normal): {average_normal} bytes/ms");
+        info!("  Worst Speed (normal):   {worst_normal} bytes/ms");
     }
 
     if let Some((average_error, worst_error)) = aggregate(&all_error_speeds) {
-        eprintln!("  Average Speed (errors): {average_error} bytes/ms");
-        eprintln!("  Worst Speed (errors):   {worst_error} bytes/ms");
+        info!("  Average Speed (errors): {average_error} bytes/ms");
+        info!("  Worst Speed (errors):   {worst_error} bytes/ms");
     }
-    eprintln!();
+    info!("");
 }
 
 fn aggregate(speeds: &[usize]) -> Option<(usize, usize)> {
@@ -191,14 +197,8 @@ fn aggregate(speeds: &[usize]) -> Option<(usize, usize)> {
 }
 
 fn parse(path: &Path, max_path_length: usize, mut action: impl FnMut(&[u8])) -> usize {
-    eprint!(
-        "    {:width$}\t",
-        path.file_name().unwrap().to_str().unwrap(),
-        width = max_path_length
-    );
-
     let source_code = fs::read(path)
-        .with_context(|| format!("Failed to read {path:?}"))
+        .with_context(|| format!("Failed to read {}", path.display()))
         .unwrap();
     let time = Instant::now();
     for _ in 0..*REPETITION_COUNT {
@@ -207,8 +207,9 @@ fn parse(path: &Path, max_path_length: usize, mut action: impl FnMut(&[u8])) ->
     let duration = time.elapsed() / (*REPETITION_COUNT as u32);
     let duration_ns = duration.as_nanos();
     let speed = ((source_code.len() as u128) * 1_000_000) / duration_ns;
-    eprintln!(
-        "time {:>7.2} ms\t\tspeed {speed:>6} bytes/ms",
+    info!(
+        "    {:max_path_length$}\ttime {:>7.2} ms\t\tspeed {speed:>6} bytes/ms",
+        path.file_name().unwrap().to_str().unwrap(),
         (duration_ns as f64) / 1e6,
     );
     speed as usize
@@ -218,6 +219,6 @@ fn get_language(path: &Path) -> Language {
     let src_path = GRAMMARS_DIR.join(path).join("src");
     TEST_LOADER
         .load_language_at_path(CompileConfig::new(&src_path, None, None))
-        .with_context(|| format!("Failed to load language at path {src_path:?}"))
+        .with_context(|| format!("Failed to load language at path {}", src_path.display()))
         .unwrap()
 }

crates/cli/build.rs

@@ -0,0 +1,80 @@
+use std::{
+    env,
+    path::{Path, PathBuf},
+    process::Command,
+    time::SystemTime,
+};
+
+fn main() {
+    if let Some(git_sha) = read_git_sha() {
+        println!("cargo:rustc-env=BUILD_SHA={git_sha}");
+    }
+
+    println!("cargo:rustc-check-cfg=cfg(sanitizing)");
+    println!("cargo:rustc-check-cfg=cfg(TREE_SITTER_EMBED_WASM_BINDING)");
+
+    if web_playground_files_present() {
+        println!("cargo:rustc-cfg=TREE_SITTER_EMBED_WASM_BINDING");
+    }
+
+    let build_time = SystemTime::now()
+        .duration_since(SystemTime::UNIX_EPOCH)
+        .unwrap()
+        .as_secs_f64();
+    println!("cargo:rustc-env=BUILD_TIME={build_time}");
+
+    #[cfg(any(
+        target_os = "linux",
+        target_os = "android",
+        target_os = "freebsd",
+        target_os = "openbsd",
+        target_os = "netbsd",
+        target_os = "dragonfly",
+    ))]
+    {
+        let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()).join("dynamic-symbols.txt");
+        std::fs::write(
+            &out_dir,
+            "{
+                ts_current_malloc;
+                ts_current_calloc;
+                ts_current_realloc;
+                ts_current_free;
+            };",
+        )
+        .unwrap();
+        println!(
+            "cargo:rustc-link-arg=-Wl,--dynamic-list={}",
+            out_dir.display()
+        );
+    }
+}
+
+fn web_playground_files_present() -> bool {
+    let paths = [
+        "../../docs/src/assets/js/playground.js",
+        "../../lib/binding_web/web-tree-sitter.js",
+        "../../lib/binding_web/web-tree-sitter.wasm",
+    ];
+
+    paths.iter().all(|p| Path::new(p).exists())
+}
+
+fn read_git_sha() -> Option<String> {
+    let crate_path = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
+
+    if !crate_path.parent().is_some_and(|p| p.join(".git").exists()) {
+        return None;
+    }
+
+    Command::new("git")
+        .args(["rev-parse", "HEAD"])
+        .current_dir(crate_path)
+        .output()
+        .map_or(None, |output| {
+            if !output.status.success() {
+                return None;
+            }
+            Some(String::from_utf8_lossy(&output.stdout).to_string())
+        })
+}

crates/cli/eslint/.gitignore

@@ -0,0 +1 @@
+LICENSE

crates/cli/eslint/index.js

@@ -0,0 +1,121 @@
+import globals from 'globals';
+import jsdoc from 'eslint-plugin-jsdoc';
+
+export default [
+  jsdoc.configs['flat/recommended'],
+  {
+    languageOptions: {
+      ecmaVersion: 'latest',
+      sourceType: 'module',
+      globals: {
+        ...globals.commonjs,
+        ...globals.es2021,
+      },
+    },
+    plugins: {
+      jsdoc,
+    },
+    rules: {
+      'no-cond-assign': 'off',
+      'no-irregular-whitespace': 'error',
+      'no-unexpected-multiline': 'error',
+      'curly': ['error', 'multi-line'],
+      'guard-for-in': 'error',
+      'no-caller': 'error',
+      'no-extend-native': 'error',
+      'no-extra-bind': 'error',
+      'no-invalid-this': 'error',
+      'no-multi-spaces': 'error',
+      'no-multi-str': 'error',
+      'no-new-wrappers': 'error',
+      'no-throw-literal': 'error',
+      'no-with': 'error',
+      'prefer-promise-reject-errors': 'error',
+      'no-unused-vars': ['error', { args: 'none' }],
+      'array-bracket-newline': 'off',
+      'array-bracket-spacing': ['error', 'never'],
+      'array-element-newline': 'off',
+      'block-spacing': ['error', 'never'],
+      'brace-style': 'error',
+      'camelcase': ['error', { properties: 'never' }],
+      'comma-dangle': ['error', 'always-multiline'],
+      'comma-spacing': 'error',
+      'comma-style': 'error',
+      'computed-property-spacing': 'error',
+      'eol-last': 'error',
+      'func-call-spacing': 'error',
+
+      'camelcase': 'off',
+      'indent': [
+        'error',
+        2,
+        {
+          'SwitchCase': 1,
+        },
+      ],
+      'key-spacing': 'error',
+      'keyword-spacing': 'error',
+      'linebreak-style': 'error',
+      'max-len': [
+        'error',
+        {
+          code: 160,
+          ignoreComments: true,
+          ignoreUrls: true,
+          ignoreStrings: true,
+        },
+      ],
+      'new-cap': 'error',
+      'no-array-constructor': 'error',
+      'no-mixed-spaces-and-tabs': 'error',
+      'no-multiple-empty-lines': ['error', { max: 2 }],
+      'no-new-object': 'error',
+      'no-tabs': 'error',
+      'no-trailing-spaces': 'error',
+      'object-curly-spacing': 'error',
+      'one-var': ['error', {
+        var: 'never',
+        let: 'never',
+        const: 'never',
+      }],
+      'operator-linebreak': ['error', 'after'],
+      'padded-blocks': ['error', 'never'],
+      'quote-props': ['error', 'consistent'],
+      'quotes': ['error', 'single', { allowTemplateLiterals: true }],
+      'semi': 'error',
+      'semi-spacing': 'error',
+      'space-before-blocks': 'error',
+      'space-before-function-paren': ['error', {
+        asyncArrow: 'always',
+        anonymous: 'never',
+        named: 'never',
+      }],
+      'spaced-comment': [
+        'error',
+        'always',
+        {
+          line: {
+            markers: ['/'],
+          },
+        },
+      ],
+      'switch-colon-spacing': 'error',
+      'arrow-parens': 'off',
+      'constructor-super': 'error',
+      'generator-star-spacing': ['error', 'after'],
+      'no-new-symbol': 'error',
+      'no-this-before-super': 'error',
+      'no-var': 'error',
+      'prefer-const': ['error', { destructuring: 'all' }],
+      'prefer-rest-params': 'error',
+      'prefer-spread': 'error',
+      'rest-spread-spacing': 'error',
+      'yield-star-spacing': ['error', 'after'],
+      'jsdoc/no-undefined-types': 'off',
+      'jsdoc/require-param-description': 'off',
+      'jsdoc/require-returns-description': 'off',
+      'jsdoc/require-returns': 'off',
+      'jsdoc/tag-lines': ['error', 'any', { startLines: 1 }],
+    },
+  },
+];

crates/cli/eslint/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "eslint-config-treesitter",
+  "version": "1.0.2",
+  "description": "Eslint configuration for Tree-sitter grammar files",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/tree-sitter/tree-sitter.git",
+    "directory": "crates/cli/eslint"
+  },
+  "license": "MIT",
+  "author": "Amaan Qureshi <amaanq12@gmail.com>",
+  "main": "index.js",
+  "type": "module",
+  "keywords": [
+    "eslint",
+    "eslintconfig",
+    "tree-sitter"
+  ],
+  "dependencies": {
+    "eslint-plugin-jsdoc": "^50.2.4"
+  },
+  "peerDependencies": {
+    "eslint": ">= 9"
+  },
+  "scripts": {
+    "prepack": "cp ../../../LICENSE .",
+    "postpack": "rm LICENSE"
+  }
+}

crates/cli/npm/dsl.d.ts

@@ -10,6 +10,7 @@ type PrecRightRule = { type: 'PREC_RIGHT'; content: Rule; value: number };
 type PrecRule = { type: 'PREC'; content: Rule; value: number };
 type Repeat1Rule = { type: 'REPEAT1'; content: Rule };
 type RepeatRule = { type: 'REPEAT'; content: Rule };
+type ReservedRule = { type: 'RESERVED'; content: Rule; context_name: string };
 type SeqRule = { type: 'SEQ'; members: Rule[] };
 type StringRule = { type: 'STRING'; value: string };
 type SymbolRule<Name extends string> = { type: 'SYMBOL'; name: Name };
@@ -28,12 +29,19 @@ type Rule =
   | PrecRule
   | Repeat1Rule
   | RepeatRule
+  | ReservedRule
   | SeqRule
   | StringRule
   | SymbolRule<string>
   | TokenRule;
 
-type RuleOrLiteral = Rule | RegExp | string;
+declare class RustRegex {
+  value: string;
+
+  constructor(pattern: string);
+}
+
+type RuleOrLiteral = Rule | RegExp | RustRegex | string;
 
 type GrammarSymbols<RuleName extends string> = {
   [name in RuleName]: SymbolRule<name>;
@@ -42,7 +50,7 @@ type GrammarSymbols<RuleName extends string> = {
 
 type RuleBuilder<RuleName extends string> = (
   $: GrammarSymbols<RuleName>,
-  previous: Rule,
+  previous?: Rule,
 ) => RuleOrLiteral;
 
 type RuleBuilders<
@@ -105,7 +113,7 @@ interface Grammar<
    * @param $ grammar rules
    * @param previous array of externals from the base schema, if any
    *
-   * @see https://tree-sitter.github.io/tree-sitter/creating-parsers#external-scanners
+   * @see https://tree-sitter.github.io/tree-sitter/creating-parsers/4-external-scanners
    */
   externals?: (
     $: Record<string, SymbolRule<string>>,
@@ -143,7 +151,7 @@ interface Grammar<
    *
    * @param $ grammar rules
    *
-   * @see https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types
+   * @see https://tree-sitter.github.io/tree-sitter/using-parsers/6-static-node-types
    */
   supertypes?: (
     $: GrammarSymbols<RuleName | BaseGrammarRuleName>,
@@ -156,9 +164,20 @@ interface Grammar<
    *
    * @param $ grammar rules
    *
-   * @see https://tree-sitter.github.io/tree-sitter/creating-parsers#keyword-extraction
+   * @see https://tree-sitter.github.io/tree-sitter/creating-parsers/3-writing-the-grammar#keyword-extraction
    */
   word?: ($: GrammarSymbols<RuleName | BaseGrammarRuleName>) => RuleOrLiteral;
+
+
+  /**
+   * Mapping of names to reserved word sets. The first reserved word set is the
+   * global word set, meaning it applies to every rule in every parse state.
+   * The other word sets can be used with the `reserved` function.
+   */
+  reserved?: Record<
+    string,
+    ($: GrammarSymbols<RuleName | BaseGrammarRuleName>) => RuleOrLiteral[]
+  >;
 }
 
 type GrammarSchema<RuleName extends string> = {
@@ -243,7 +262,7 @@ declare function optional(rule: RuleOrLiteral): ChoiceRule;
  * @see https://docs.oracle.com/cd/E19504-01/802-5880/6i9k05dh3/index.html
  */
 declare const prec: {
-  (value: String | number, rule: RuleOrLiteral): PrecRule;
+  (value: string | number, rule: RuleOrLiteral): PrecRule;
 
   /**
    * Marks the given rule as left-associative (and optionally applies a
@@ -259,7 +278,7 @@ declare const prec: {
    * @see https://docs.oracle.com/cd/E19504-01/802-5880/6i9k05dh3/index.html
    */
   left(rule: RuleOrLiteral): PrecLeftRule;
-  left(value: String | number, rule: RuleOrLiteral): PrecLeftRule;
+  left(value: string | number, rule: RuleOrLiteral): PrecLeftRule;
 
   /**
    * Marks the given rule as right-associative (and optionally applies a
@@ -275,7 +294,7 @@ declare const prec: {
    * @see https://docs.oracle.com/cd/E19504-01/802-5880/6i9k05dh3/index.html
    */
   right(rule: RuleOrLiteral): PrecRightRule;
-  right(value: String | number, rule: RuleOrLiteral): PrecRightRule;
+  right(value: string | number, rule: RuleOrLiteral): PrecRightRule;
 
   /**
    * Marks the given rule with a numerical precedence which will be used to
@@ -292,7 +311,7 @@ declare const prec: {
    *
    * @see https://www.gnu.org/software/bison/manual/html_node/Generalized-LR-Parsing.html
    */
-  dynamic(value: String | number, rule: RuleOrLiteral): PrecDynamicRule;
+  dynamic(value: string | number, rule: RuleOrLiteral): PrecDynamicRule;
 };
 
 /**
@@ -312,6 +331,15 @@ declare function repeat(rule: RuleOrLiteral): RepeatRule;
  */
 declare function repeat1(rule: RuleOrLiteral): Repeat1Rule;
 
+/**
+ * Overrides the global reserved word set for a given rule. The word set name
+ * should be defined in the `reserved` field in the grammar.
+ *
+ * @param wordset name of the reserved word set
+ * @param rule rule that will use the reserved word set
+ */
+declare function reserved(wordset: string, rule: RuleOrLiteral): ReservedRule;
+
 /**
  * Creates a rule that matches any number of other rules, one after another.
  * It is analogous to simply writing multiple symbols next to each other
@@ -330,7 +358,7 @@ declare function sym<Name extends string>(name: Name): SymbolRule<Name>;
 
 /**
  * Marks the given rule as producing only a single token. Tree-sitter's
- * default is to treat each String or RegExp literal in the grammar as a
+ * default is to treat each string or RegExp literal in the grammar as a
  * separate token. Each token is matched separately by the lexer and
  * returned as its own leaf node in the tree. The token function allows
  * you to express a complex rule using the DSL functions (rather

crates/cli/npm/install.js

@@ -6,7 +6,8 @@ const http = require('http');
 const https = require('https');
 const packageJSON = require('./package.json');
 
-// Look to a results table in https://github.com/tree-sitter/tree-sitter/issues/2196
+https.globalAgent.keepAlive = false;
+
 const matrix = {
   platform: {
     'darwin': {
@@ -101,12 +102,23 @@ function get(url, callback) {
   const requestPort = requestUrl.port || (requestUrl.protocol === 'https:' ? 443 : 80);
   const proxyUrl = new URL(proxyEnv);
   const request = proxyUrl.protocol === 'https:' ? https : http;
-  request.request({
+  const requestOption = {
     host: proxyUrl.hostname,
     port: proxyUrl.port || (proxyUrl.protocol === 'https:' ? 443 : 80),
     method: 'CONNECT',
     path: `${requestUrl.hostname}:${requestPort}`,
-  }).on('connect', (response, socket, _head) => {
+  };
+  if (proxyUrl.username || proxyUrl.password) {
+    const auth = `${decodeURIComponent(
+      proxyUrl.username
+    )}:${decodeURIComponent(proxyUrl.password)}`;
+    requestOption.headers = {
+      'Proxy-Authorization': `Basic ${Buffer.from(
+        auth
+      ).toString('base64')}`,
+    }
+  }
+  request.request(requestOption).on('connect', (response, socket, _head) => {
     if (response.statusCode !== 200) {
       // let caller handle error
       callback(response);

crates/cli/npm/package-lock.json

@@ -0,0 +1,20 @@
+{
+  "name": "tree-sitter-cli",
+  "version": "0.27.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "tree-sitter-cli",
+      "version": "0.27.0",
+      "hasInstallScript": true,
+      "license": "MIT",
+      "bin": {
+        "tree-sitter": "cli.js"
+      },
+      "engines": {
+        "node": ">=12.0.0"
+      }
+    }
+  }
+}

crates/cli/npm/package.json

@@ -1,24 +1,33 @@
 {
   "name": "tree-sitter-cli",
-  "version": "0.22.6",
-  "author": "Max Brunsfeld",
+  "version": "0.27.0",
+  "author": {
+    "name": "Max Brunsfeld",
+    "email": "maxbrunsfeld@gmail.com"
+  },
+  "maintainers": [
+    {
+      "name": "Amaan Qureshi",
+      "email": "amaanq12@gmail.com"
+    }
+  ],
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": "https://github.com/tree-sitter/tree-sitter.git"
+    "url": "git+https://github.com/tree-sitter/tree-sitter.git",
+    "directory": "crates/cli/npm"
   },
   "description": "CLI for generating fast incremental parsers",
   "keywords": [
     "parser",
     "lexer"
   ],
-  "main": "lib/api/index.js",
   "engines": {
     "node": ">=12.0.0"
   },
   "scripts": {
     "install": "node install.js",
-    "prepack": "cp ../../LICENSE ../README.md .",
+    "prepack": "cp ../../../LICENSE ../README.md .",
     "postpack": "rm LICENSE README.md"
   },
   "bin": {

crates/cli/package.nix

@@ -0,0 +1,69 @@
+{
+  lib,
+  src,
+  rustPlatform,
+  version,
+  clang,
+  libclang,
+  cmake,
+  pkg-config,
+  nodejs_22,
+  test-grammars,
+  stdenv,
+  installShellFiles,
+}:
+let
+  isCross = stdenv.targetPlatform == stdenv.buildPlatform;
+in
+rustPlatform.buildRustPackage {
+  pname = "tree-sitter-cli";
+
+  inherit src version;
+
+  cargoBuildFlags = [ "--all-features" ];
+
+  nativeBuildInputs = [
+    clang
+    cmake
+    pkg-config
+    nodejs_22
+  ]
+  ++ lib.optionals (!isCross) [ installShellFiles ];
+
+  cargoLock.lockFile = ../../Cargo.lock;
+
+  env.LIBCLANG_PATH = "${libclang.lib}/lib";
+
+  preBuild = ''
+    rm -rf test/fixtures
+    mkdir -p test/fixtures
+    cp -r ${test-grammars}/fixtures/* test/fixtures/
+    chmod -R u+w test/fixtures
+  '';
+
+  preCheck = "export HOME=$TMPDIR";
+  doCheck = !isCross;
+
+  postInstall = lib.optionalString (!isCross) ''
+    installShellCompletion --cmd tree-sitter \
+      --bash <($out/bin/tree-sitter complete --shell bash) \
+      --zsh  <($out/bin/tree-sitter complete --shell zsh) \
+      --fish <($out/bin/tree-sitter complete --shell fish)
+  '';
+
+  meta = {
+    description = "Tree-sitter CLI - A tool for developing, testing, and using Tree-sitter parsers";
+    longDescription = ''
+      Tree-sitter is a parser generator tool and an incremental parsing library.
+      It can build a concrete syntax tree for a source file and efficiently update
+      the syntax tree as the source file is edited. This package provides the CLI
+      tool for developing, testing, and using Tree-sitter parsers.
+    '';
+    homepage = "https://tree-sitter.github.io/tree-sitter";
+    changelog = "https://github.com/tree-sitter/tree-sitter/releases/tag/v${version}";
+    license = lib.licenses.mit;
+    maintainers = with lib.maintainers; [ amaanq ];
+    platforms = lib.platforms.all;
+    mainProgram = "tree-sitter";
+  };
+}

crates/cli/src/fuzz.rs

@@ -0,0 +1,396 @@
+use std::{
+    collections::HashMap,
+    env, fs,
+    path::{Path, PathBuf},
+    sync::LazyLock,
+};
+
+use log::{error, info};
+use rand::Rng;
+use regex::Regex;
+use tree_sitter::{Language, Parser};
+
+pub mod allocations;
+pub mod corpus_test;
+pub mod edits;
+pub mod random;
+pub mod scope_sequence;
+
+use crate::{
+    fuzz::{
+        corpus_test::{
+            check_changed_ranges, check_consistent_sizes, get_parser, set_included_ranges,
+        },
+        edits::{get_random_edit, invert_edit},
+        random::Rand,
+    },
+    parse::perform_edit,
+    test::{parse_tests, strip_sexp_fields, DiffKey, TestDiff, TestEntry},
+};
+
+pub static LOG_ENABLED: LazyLock<bool> = LazyLock::new(|| env::var("TREE_SITTER_LOG").is_ok());
+
+pub static LOG_GRAPH_ENABLED: LazyLock<bool> =
+    LazyLock::new(|| env::var("TREE_SITTER_LOG_GRAPHS").is_ok());
+
+pub static LANGUAGE_FILTER: LazyLock<Option<String>> =
+    LazyLock::new(|| env::var("TREE_SITTER_LANGUAGE").ok());
+
+pub static EXAMPLE_INCLUDE: LazyLock<Option<Regex>> =
+    LazyLock::new(|| regex_env_var("TREE_SITTER_EXAMPLE_INCLUDE"));
+
+pub static EXAMPLE_EXCLUDE: LazyLock<Option<Regex>> =
+    LazyLock::new(|| regex_env_var("TREE_SITTER_EXAMPLE_EXCLUDE"));
+
+pub static START_SEED: LazyLock<usize> = LazyLock::new(new_seed);
+
+pub static EDIT_COUNT: LazyLock<usize> =
+    LazyLock::new(|| int_env_var("TREE_SITTER_EDITS").unwrap_or(3));
+
+pub static ITERATION_COUNT: LazyLock<usize> =
+    LazyLock::new(|| int_env_var("TREE_SITTER_ITERATIONS").unwrap_or(10));
+
+fn int_env_var(name: &'static str) -> Option<usize> {
+    env::var(name).ok().and_then(|e| e.parse().ok())
+}
+
+fn regex_env_var(name: &'static str) -> Option<Regex> {
+    env::var(name).ok().and_then(|e| Regex::new(&e).ok())
+}
+
+#[must_use]
+pub fn new_seed() -> usize {
+    int_env_var("TREE_SITTER_SEED").unwrap_or_else(|| {
+        let mut rng = rand::thread_rng();
+        let seed = rng.gen::<usize>();
+        info!("Seed: {seed}");
+        seed
+    })
+}
+
+pub struct FuzzOptions {
+    pub skipped: Option<Vec<String>>,
+    pub subdir: Option<PathBuf>,
+    pub edits: usize,
+    pub iterations: usize,
+    pub include: Option<Regex>,
+    pub exclude: Option<Regex>,
+    pub log_graphs: bool,
+    pub log: bool,
+}
+
+pub fn fuzz_language_corpus(
+    language: &Language,
+    language_name: &str,
+    start_seed: usize,
+    grammar_dir: &Path,
+    options: &mut FuzzOptions,
+) {
+    fn retain(entry: &mut TestEntry, language_name: &str) -> bool {
+        match entry {
+            TestEntry::Example { attributes, .. } => {
+                attributes.languages[0].is_empty()
+                    || attributes
+                        .languages
+                        .iter()
+                        .any(|lang| lang.as_ref() == language_name)
+            }
+            TestEntry::Group {
+                ref mut children, ..
+            } => {
+                children.retain_mut(|child| retain(child, language_name));
+                !children.is_empty()
+            }
+        }
+    }
+
+    let subdir = options.subdir.take().unwrap_or_default();
+
+    let corpus_dir = grammar_dir.join(subdir).join("test").join("corpus");
+
+    if !corpus_dir.exists() || !corpus_dir.is_dir() {
+        error!("No corpus directory found, ensure that you have a `test/corpus` directory in your grammar directory with at least one test file.");
+        return;
+    }
+
+    if std::fs::read_dir(&corpus_dir).unwrap().count() == 0 {
+        error!("No corpus files found in `test/corpus`, ensure that you have at least one test file in your corpus directory.");
+        return;
+    }
+
+    let mut main_tests = parse_tests(&corpus_dir).unwrap();
+    match main_tests {
+        TestEntry::Group {
+            ref mut children, ..
+        } => {
+            children.retain_mut(|child| retain(child, language_name));
+        }
+        TestEntry::Example { .. } => unreachable!(),
+    }
+    let tests = flatten_tests(
+        main_tests,
+        options.include.as_ref(),
+        options.exclude.as_ref(),
+    );
+
+    let get_test_name = |test: &FlattenedTest| format!("{language_name} - {}", test.name);
+
+    let mut skipped = options
+        .skipped
+        .take()
+        .unwrap_or_default()
+        .into_iter()
+        .chain(tests.iter().filter(|x| x.skip).map(get_test_name))
+        .map(|x| (x, 0))
+        .collect::<HashMap<String, usize>>();
+
+    let mut failure_count = 0;
+
+    let log_seed = env::var("TREE_SITTER_LOG_SEED").is_ok();
+    let dump_edits = env::var("TREE_SITTER_DUMP_EDITS").is_ok();
+
+    if log_seed {
+        info!("  start seed: {start_seed}");
+    }
+
+    println!();
+    for (test_index, test) in tests.iter().enumerate() {
+        let test_name = get_test_name(test);
+        if let Some(counter) = skipped.get_mut(test_name.as_str()) {
+            println!("  {test_index}. {test_name} - SKIPPED");
+            *counter += 1;
+            continue;
+        }
+
+        println!("  {test_index}. {test_name}");
+
+        let passed = allocations::record_checked(|| {
+            let mut log_session = None;
+            let mut parser = get_parser(&mut log_session, "log.html");
+            parser.set_language(language).unwrap();
+            set_included_ranges(&mut parser, &test.input, test.template_delimiters);
+
+            let tree = parser.parse(&test.input, None).unwrap();
+
+            if test.error {
+                return true;
+            }
+
+            let mut actual_output = tree.root_node().to_sexp();
+            if !test.has_fields {
+                actual_output = strip_sexp_fields(&actual_output);
+            }
+
+            if actual_output != test.output {
+                println!("Incorrect initial parse for {test_name}");
+                DiffKey::print();
+                println!("{}", TestDiff::new(&actual_output, &test.output));
+                println!();
+                return false;
+            }
+
+            true
+        })
+        .unwrap_or_else(|e| {
+            error!("{e}");
+            false
+        });
+
+        if !passed {
+            failure_count += 1;
+            continue;
+        }
+
+        let mut parser = Parser::new();
+        parser.set_language(language).unwrap();
+        let tree = parser.parse(&test.input, None).unwrap();
+        drop(parser);
+
+        for trial in 0..options.iterations {
+            let seed = start_seed + trial;
+            let passed = allocations::record_checked(|| {
+                let mut rand = Rand::new(seed);
+                let mut log_session = None;
+                let mut parser = get_parser(&mut log_session, "log.html");
+                parser.set_language(language).unwrap();
+                let mut tree = tree.clone();
+                let mut input = test.input.clone();
+
+                if options.log_graphs {
+                    info!("{}\n", String::from_utf8_lossy(&input));
+                }
+
+                // Perform a random series of edits and reparse.
+                let edit_count = rand.unsigned(*EDIT_COUNT);
+                let mut undo_stack = Vec::with_capacity(edit_count);
+                for _ in 0..=edit_count {
+                    let edit = get_random_edit(&mut rand, &input);
+                    undo_stack.push(invert_edit(&input, &edit));
+                    perform_edit(&mut tree, &mut input, &edit).unwrap();
+                }
+
+                if log_seed {
+                    info!("   {test_index}.{trial:<2} seed: {seed}");
+                }
+
+                if dump_edits {
+                    fs::create_dir_all("fuzz").unwrap();
+                    fs::write(
+                        Path::new("fuzz")
+                            .join(format!("edit.{seed}.{test_index}.{trial} {test_name}")),
+                        &input,
+                    )
+                    .unwrap();
+                }
+
+                if options.log_graphs {
+                    info!("{}\n", String::from_utf8_lossy(&input));
+                }
+
+                set_included_ranges(&mut parser, &input, test.template_delimiters);
+                let mut tree2 = parser.parse(&input, Some(&tree)).unwrap();
+
+                // Check that the new tree is consistent.
+                check_consistent_sizes(&tree2, &input);
+                if let Err(message) = check_changed_ranges(&tree, &tree2, &input) {
+                    error!("\nUnexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n",);
+                    return false;
+                }
+
+                // Undo all of the edits and re-parse again.
+                while let Some(edit) = undo_stack.pop() {
+                    perform_edit(&mut tree2, &mut input, &edit).unwrap();
+                }
+                if options.log_graphs {
+                    info!("{}\n", String::from_utf8_lossy(&input));
+                }
+
+                set_included_ranges(&mut parser, &test.input, test.template_delimiters);
+                let tree3 = parser.parse(&input, Some(&tree2)).unwrap();
+
+                // Verify that the final tree matches the expectation from the corpus.
+                let mut actual_output = tree3.root_node().to_sexp();
+                if !test.has_fields {
+                    actual_output = strip_sexp_fields(&actual_output);
+                }
+
+                if actual_output != test.output && !test.error {
+                    println!("Incorrect parse for {test_name} - seed {seed}");
+                    DiffKey::print();
+                    println!("{}", TestDiff::new(&actual_output, &test.output));
+                    println!();
+                    return false;
+                }
+
+                // Check that the edited tree is consistent.
+                check_consistent_sizes(&tree3, &input);
+                if let Err(message) = check_changed_ranges(&tree2, &tree3, &input) {
+                    error!("Unexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n");
+                    return false;
+                }
+
+                true
+            }).unwrap_or_else(|e| {
+                error!("{e}");
+                false
+            });
+
+            if !passed {
+                failure_count += 1;
+                break;
+            }
+        }
+    }
+
+    if failure_count != 0 {
+        info!("{failure_count} {language_name} corpus tests failed fuzzing");
+    }
+
+    skipped.retain(|_, v| *v == 0);
+
+    if !skipped.is_empty() {
+        info!("Non matchable skip definitions:");
+        for k in skipped.keys() {
+            info!("  {k}");
+        }
+        panic!("Non matchable skip definitions need to be removed");
+    }
+}
+
+pub struct FlattenedTest {
+    pub name: String,
+    pub input: Vec<u8>,
+    pub output: String,
+    pub languages: Vec<Box<str>>,
+    pub error: bool,
+    pub skip: bool,
+    pub has_fields: bool,
+    pub template_delimiters: Option<(&'static str, &'static str)>,
+}
+
+#[must_use]
+pub fn flatten_tests(
+    test: TestEntry,
+    include: Option<&Regex>,
+    exclude: Option<&Regex>,
+) -> Vec<FlattenedTest> {
+    fn helper(
+        test: TestEntry,
+        include: Option<&Regex>,
+        exclude: Option<&Regex>,
+        is_root: bool,
+        prefix: &str,
+        result: &mut Vec<FlattenedTest>,
+    ) {
+        match test {
+            TestEntry::Example {
+                mut name,
+                input,
+                output,
+                has_fields,
+                attributes,
+                ..
+            } => {
+                if !prefix.is_empty() {
+                    name.insert_str(0, " - ");
+                    name.insert_str(0, prefix);
+                }
+
+                if let Some(include) = include {
+                    if !include.is_match(&name) {
+                        return;
+                    }
+                } else if let Some(exclude) = exclude {
+                    if exclude.is_match(&name) {
+                        return;
+                    }
+                }
+
+                result.push(FlattenedTest {
+                    name,
+                    input,
+                    output,
+                    has_fields,
+                    languages: attributes.languages,
+                    error: attributes.error,
+                    skip: attributes.skip,
+                    template_delimiters: None,
+                });
+            }
+            TestEntry::Group {
+                mut name, children, ..
+            } => {
+                if !is_root && !prefix.is_empty() {
+                    name.insert_str(0, " - ");
+                    name.insert_str(0, prefix);
+                }
+                for child in children {
+                    helper(child, include, exclude, false, &name, result);
+                }
+            }
+        }
+    }
+    let mut result = Vec::new();
+    helper(test, include, exclude, true, "", &mut result);
+    result
+}

crates/cli/src/fuzz/allocations.rs

@@ -41,6 +41,10 @@ extern "C" {
 }
 
 pub fn record<T>(f: impl FnOnce() -> T) -> T {
+    record_checked(f).unwrap()
+}
+
+pub fn record_checked<T>(f: impl FnOnce() -> T) -> Result<T, String> {
     RECORDER.with(|recorder| {
         recorder.enabled.store(true, SeqCst);
         recorder.allocation_count.store(0, SeqCst);
@@ -60,11 +64,12 @@ pub fn record<T>(f: impl FnOnce() -> T) -> T {
             .map(|e| e.1)
             .collect::<Vec<_>>()
     });
-    assert!(
-        outstanding_allocation_indices.is_empty(),
-        "Leaked allocation indices: {outstanding_allocation_indices:?}"
-    );
-    value
+    if !outstanding_allocation_indices.is_empty() {
+        return Err(format!(
+            "Leaked allocation indices: {outstanding_allocation_indices:?}",
+        ));
+    }
+    Ok(value)
 }
 
 fn record_alloc(ptr: *mut c_void) {
@@ -92,30 +97,49 @@ fn record_dealloc(ptr: *mut c_void) {
     });
 }
 
-unsafe extern "C" fn ts_record_malloc(size: usize) -> *mut c_void {
+/// # Safety
+///
+/// The caller must ensure that the returned pointer is eventually
+/// freed by calling `ts_record_free`.
+#[must_use]
+pub unsafe extern "C" fn ts_record_malloc(size: usize) -> *mut c_void {
     let result = malloc(size);
     record_alloc(result);
     result
 }
 
-unsafe extern "C" fn ts_record_calloc(count: usize, size: usize) -> *mut c_void {
+/// # Safety
+///
+/// The caller must ensure that the returned pointer is eventually
+/// freed by calling `ts_record_free`.
+#[must_use]
+pub unsafe extern "C" fn ts_record_calloc(count: usize, size: usize) -> *mut c_void {
     let result = calloc(count, size);
     record_alloc(result);
     result
 }
 
-unsafe extern "C" fn ts_record_realloc(ptr: *mut c_void, size: usize) -> *mut c_void {
+/// # Safety
+///
+/// The caller must ensure that the returned pointer is eventually
+/// freed by calling `ts_record_free`.
+#[must_use]
+pub unsafe extern "C" fn ts_record_realloc(ptr: *mut c_void, size: usize) -> *mut c_void {
     let result = realloc(ptr, size);
     if ptr.is_null() {
         record_alloc(result);
-    } else if ptr != result {
+    } else if !core::ptr::eq(ptr, result) {
         record_dealloc(ptr);
         record_alloc(result);
     }
     result
 }
 
-unsafe extern "C" fn ts_record_free(ptr: *mut c_void) {
+/// # Safety
+///
+/// The caller must ensure that `ptr` was allocated by a previous call
+/// to `ts_record_malloc`, `ts_record_calloc`, or `ts_record_realloc`.
+pub unsafe extern "C" fn ts_record_free(ptr: *mut c_void) {
     record_dealloc(ptr);
     free(ptr);
 }

crates/cli/src/fuzz/corpus_test.rs

@@ -0,0 +1,147 @@
+use tree_sitter::{LogType, Node, Parser, Point, Range, Tree};
+
+use super::{scope_sequence::ScopeSequence, LOG_ENABLED, LOG_GRAPH_ENABLED};
+use crate::util;
+
+pub fn check_consistent_sizes(tree: &Tree, input: &[u8]) {
+    fn check(node: Node, line_offsets: &[usize]) {
+        let start_byte = node.start_byte();
+        let end_byte = node.end_byte();
+        let start_point = node.start_position();
+        let end_point = node.end_position();
+
+        assert!(start_byte <= end_byte);
+        assert!(start_point <= end_point);
+        assert_eq!(
+            start_byte,
+            line_offsets[start_point.row] + start_point.column
+        );
+        assert_eq!(end_byte, line_offsets[end_point.row] + end_point.column);
+
+        let mut last_child_end_byte = start_byte;
+        let mut last_child_end_point = start_point;
+        let mut some_child_has_changes = false;
+        let mut actual_named_child_count = 0;
+        for i in 0..node.child_count() {
+            let child = node.child(i as u32).unwrap();
+            assert!(child.start_byte() >= last_child_end_byte);
+            assert!(child.start_position() >= last_child_end_point);
+            check(child, line_offsets);
+            if child.has_changes() {
+                some_child_has_changes = true;
+            }
+            if child.is_named() {
+                actual_named_child_count += 1;
+            }
+            last_child_end_byte = child.end_byte();
+            last_child_end_point = child.end_position();
+        }
+
+        assert_eq!(actual_named_child_count, node.named_child_count());
+
+        if node.child_count() > 0 {
+            assert!(end_byte >= last_child_end_byte);
+            assert!(end_point >= last_child_end_point);
+        }
+
+        if some_child_has_changes {
+            assert!(node.has_changes());
+        }
+    }
+
+    let mut line_offsets = vec![0];
+    for (i, c) in input.iter().enumerate() {
+        if *c == b'\n' {
+            line_offsets.push(i + 1);
+        }
+    }
+
+    check(tree.root_node(), &line_offsets);
+}
+
+pub fn check_changed_ranges(old_tree: &Tree, new_tree: &Tree, input: &[u8]) -> Result<(), String> {
+    let changed_ranges = old_tree.changed_ranges(new_tree).collect::<Vec<_>>();
+    let old_scope_sequence = ScopeSequence::new(old_tree);
+    let new_scope_sequence = ScopeSequence::new(new_tree);
+
+    let old_range = old_tree.root_node().range();
+    let new_range = new_tree.root_node().range();
+
+    let byte_range =
+        old_range.start_byte.min(new_range.start_byte)..old_range.end_byte.max(new_range.end_byte);
+    let point_range = old_range.start_point.min(new_range.start_point)
+        ..old_range.end_point.max(new_range.end_point);
+
+    for range in &changed_ranges {
+        if range.end_byte > byte_range.end || range.end_point > point_range.end {
+            return Err(format!(
+                "changed range extends outside of the old and new trees {range:?}",
+            ));
+        }
+    }
+
+    old_scope_sequence.check_changes(&new_scope_sequence, input, &changed_ranges)
+}
+
+pub fn set_included_ranges(parser: &mut Parser, input: &[u8], delimiters: Option<(&str, &str)>) {
+    if let Some((start, end)) = delimiters {
+        let mut ranges = Vec::new();
+        let mut ix = 0;
+        while ix < input.len() {
+            let Some(mut start_ix) = input[ix..]
+                .windows(2)
+                .position(|win| win == start.as_bytes())
+            else {
+                break;
+            };
+            start_ix += ix + start.len();
+            let end_ix = input[start_ix..]
+                .windows(2)
+                .position(|win| win == end.as_bytes())
+                .map_or(input.len(), |ix| start_ix + ix);
+            ix = end_ix;
+            ranges.push(Range {
+                start_byte: start_ix,
+                end_byte: end_ix,
+                start_point: point_for_offset(input, start_ix),
+                end_point: point_for_offset(input, end_ix),
+            });
+        }
+
+        parser.set_included_ranges(&ranges).unwrap();
+    } else {
+        parser.set_included_ranges(&[]).unwrap();
+    }
+}
+
+fn point_for_offset(text: &[u8], offset: usize) -> Point {
+    let mut point = Point::default();
+    for byte in &text[..offset] {
+        if *byte == b'\n' {
+            point.row += 1;
+            point.column = 0;
+        } else {
+            point.column += 1;
+        }
+    }
+    point
+}
+
+pub fn get_parser(session: &mut Option<util::LogSession>, log_filename: &str) -> Parser {
+    let mut parser = Parser::new();
+
+    if *LOG_ENABLED {
+        parser.set_logger(Some(Box::new(|log_type, msg| {
+            if log_type == LogType::Lex {
+                eprintln!("  {msg}");
+            } else {
+                eprintln!("{msg}");
+            }
+        })));
+    }
+    if *LOG_GRAPH_ENABLED {
+        *session = Some(util::log_graphs(&mut parser, log_filename, false).unwrap());
+    }
+
+    parser
+}