Compare commits

..

1 commit

Author SHA1 Message Date
Amaan Qureshi
e471646d2a
test windows 2025-09-29 03:24:44 -04:00
152 changed files with 3855 additions and 7335 deletions

View file

@ -14,10 +14,6 @@ updates:
groups: groups:
cargo: cargo:
patterns: ["*"] patterns: ["*"]
ignore:
- dependency-name: "*"
update-types: ["version-update:semver-major", "version-update:semver-minor"]
- package-ecosystem: "github-actions" - package-ecosystem: "github-actions"
directory: "/" directory: "/"
schedule: schedule:
@ -32,7 +28,6 @@ updates:
groups: groups:
actions: actions:
patterns: ["*"] patterns: ["*"]
- package-ecosystem: "npm" - package-ecosystem: "npm"
versioning-strategy: increase versioning-strategy: increase
directories: directories:

View file

@ -1,25 +0,0 @@
module.exports = async ({ github, context, core }) => {
if (context.eventName !== 'pull_request') return;
const prNumber = context.payload.pull_request.number;
const owner = context.repo.owner;
const repo = context.repo.repo;
const { data: files } = await github.rest.pulls.listFiles({
owner,
repo,
pull_number: prNumber
});
const changedFiles = files.map(file => file.filename);
const wasmStdLibSrc = 'crates/language/wasm/';
const dirChanged = changedFiles.some(file => file.startsWith(wasmStdLibSrc));
if (!dirChanged) return;
const wasmStdLibHeader = 'lib/src/wasm/wasm-stdlib.h';
const requiredChanged = changedFiles.includes(wasmStdLibHeader);
if (!requiredChanged) core.setFailed(`Changes detected in ${wasmStdLibSrc} but ${wasmStdLibHeader} was not modified.`);
};

View file

@ -14,7 +14,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Checkout repository - name: Checkout repository
uses: actions/checkout@v6 uses: actions/checkout@v5
- name: Create app token - name: Create app token
uses: actions/create-github-app-token@v2 uses: actions/create-github-app-token@v2
@ -24,7 +24,7 @@ jobs:
private-key: ${{ secrets.BACKPORT_KEY }} private-key: ${{ secrets.BACKPORT_KEY }}
- name: Create backport PR - name: Create backport PR
uses: korthout/backport-action@v4 uses: korthout/backport-action@v3
with: with:
pull_title: "${pull_title}" pull_title: "${pull_title}"
label_pattern: "^ci:backport ([^ ]+)$" label_pattern: "^ci:backport ([^ ]+)$"

View file

@ -16,7 +16,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Checkout repository - name: Checkout repository
uses: actions/checkout@v6 uses: actions/checkout@v5
- name: Set up stable Rust toolchain - name: Set up stable Rust toolchain
uses: actions-rust-lang/setup-rust-toolchain@v1 uses: actions-rust-lang/setup-rust-toolchain@v1

View file

@ -41,7 +41,7 @@ jobs:
- { platform: windows-x64 , target: x86_64-pc-windows-msvc , os: windows-2025 } - { platform: windows-x64 , target: x86_64-pc-windows-msvc , os: windows-2025 }
- { platform: windows-x86 , target: i686-pc-windows-msvc , os: windows-2025 } - { platform: windows-x86 , target: i686-pc-windows-msvc , os: windows-2025 }
- { platform: macos-arm64 , target: aarch64-apple-darwin , os: macos-15 } - { platform: macos-arm64 , target: aarch64-apple-darwin , os: macos-15 }
- { platform: macos-x64 , target: x86_64-apple-darwin , os: macos-15-intel } - { platform: macos-x64 , target: x86_64-apple-darwin , os: macos-13 }
- { platform: wasm32 , target: wasm32-unknown-unknown , os: ubuntu-24.04 } - { platform: wasm32 , target: wasm32-unknown-unknown , os: ubuntu-24.04 }
# Extra features # Extra features
@ -68,25 +68,24 @@ jobs:
steps: steps:
- name: Checkout repository - name: Checkout repository
uses: actions/checkout@v6 uses: actions/checkout@v5
- name: Set up cross-compilation - name: Set up environment
if: matrix.cross
run: | run: |
for target in armv7-unknown-linux-gnueabihf i686-unknown-linux-gnu powerpc64-unknown-linux-gnu; do printf 'EMSCRIPTEN_VERSION=%s\n' "$(<crates/loader/emscripten-version)" >> $GITHUB_ENV
camel_target=${target//-/_}; target_cc=${target/-unknown/}
printf 'CC_%s=%s\n' "$camel_target" "${target_cc/v7/}-gcc"
printf 'AR_%s=%s\n' "$camel_target" "${target_cc/v7/}-ar"
printf 'CARGO_TARGET_%s_LINKER=%s\n' "${camel_target^^}" "${target_cc/v7/}-gcc"
done >> $GITHUB_ENV
{
printf 'CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_RUNNER=qemu-arm -L /usr/arm-linux-gnueabihf\n'
printf 'CARGO_TARGET_POWERPC64_UNKNOWN_LINUX_GNU_RUNNER=qemu-ppc64 -L /usr/powerpc64-linux-gnu\n'
} >> $GITHUB_ENV
- name: Get emscripten version if [[ '${{ matrix.cross }}' == true ]]; then
if: contains(matrix.features, 'wasm') for target in armv7-unknown-linux-gnueabihf i686-unknown-linux-gnu powerpc64-unknown-linux-gnu; do
run: printf 'EMSCRIPTEN_VERSION=%s\n' "$(<crates/loader/emscripten-version)" >> $GITHUB_ENV camel_target=${target//-/_}; target_cc=${target/-unknown/}
printf 'CC_%s=%s\n' "$camel_target" "${target_cc/v7/}-gcc"
printf 'AR_%s=%s\n' "$camel_target" "${target_cc/v7/}-ar"
printf 'CARGO_TARGET_%s_LINKER=%s\n' "${camel_target^^}" "${target_cc/v7/}-gcc"
done >> $GITHUB_ENV
{
printf 'CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_RUNNER=qemu-arm -L /usr/arm-linux-gnueabihf\n'
printf 'CARGO_TARGET_POWERPC64_UNKNOWN_LINUX_GNU_RUNNER=qemu-ppc64 -L /usr/powerpc64-linux-gnu\n'
} >> $GITHUB_ENV
fi
- name: Install Emscripten - name: Install Emscripten
if: contains(matrix.features, 'wasm') if: contains(matrix.features, 'wasm')
@ -278,7 +277,7 @@ jobs:
- name: Upload CLI artifact - name: Upload CLI artifact
if: "!matrix.no-run" if: "!matrix.no-run"
uses: actions/upload-artifact@v6 uses: actions/upload-artifact@v4
with: with:
name: tree-sitter.${{ matrix.platform }} name: tree-sitter.${{ matrix.platform }}
path: target/${{ matrix.target }}/release/tree-sitter${{ contains(matrix.target, 'windows') && '.exe' || '' }} path: target/${{ matrix.target }}/release/tree-sitter${{ contains(matrix.target, 'windows') && '.exe' || '' }}
@ -287,7 +286,7 @@ jobs:
- name: Upload Wasm artifacts - name: Upload Wasm artifacts
if: matrix.platform == 'linux-x64' if: matrix.platform == 'linux-x64'
uses: actions/upload-artifact@v6 uses: actions/upload-artifact@v4
with: with:
name: tree-sitter.wasm name: tree-sitter.wasm
path: | path: |

View file

@ -9,7 +9,7 @@ on:
- LICENSE - LICENSE
- cli/src/templates - cli/src/templates
push: push:
branches: [master] # branches: [master]
paths-ignore: paths-ignore:
- docs/** - docs/**
- "**/README.md" - "**/README.md"
@ -26,7 +26,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Checkout repository - name: Checkout repository
uses: actions/checkout@v6 uses: actions/checkout@v5
- name: Set up stable Rust toolchain - name: Set up stable Rust toolchain
uses: actions-rust-lang/setup-rust-toolchain@v1 uses: actions-rust-lang/setup-rust-toolchain@v1
@ -44,6 +44,3 @@ jobs:
build: build:
uses: ./.github/workflows/build.yml uses: ./.github/workflows/build.yml
check-wasm-stdlib:
uses: ./.github/workflows/wasm_stdlib.yml

View file

@ -3,7 +3,6 @@ on:
push: push:
branches: [master] branches: [master]
paths: [docs/**] paths: [docs/**]
workflow_dispatch:
jobs: jobs:
deploy-docs: deploy-docs:
@ -16,7 +15,7 @@ jobs:
steps: steps:
- name: Checkout repository - name: Checkout repository
uses: actions/checkout@v6 uses: actions/checkout@v5
- name: Set up Rust - name: Set up Rust
uses: actions-rust-lang/setup-rust-toolchain@v1 uses: actions-rust-lang/setup-rust-toolchain@v1
@ -26,7 +25,7 @@ jobs:
GH_TOKEN: ${{ github.token }} GH_TOKEN: ${{ github.token }}
run: | run: |
jq_expr='.assets[] | select(.name | contains("x86_64-unknown-linux-gnu")) | .browser_download_url' jq_expr='.assets[] | select(.name | contains("x86_64-unknown-linux-gnu")) | .browser_download_url'
url=$(gh api repos/rust-lang/mdbook/releases/tags/v0.4.52 --jq "$jq_expr") url=$(gh api repos/rust-lang/mdbook/releases/latest --jq "$jq_expr")
mkdir mdbook mkdir mdbook
curl -sSL "$url" | tar -xz -C mdbook curl -sSL "$url" | tar -xz -C mdbook
printf '%s/mdbook\n' "$PWD" >> "$GITHUB_PATH" printf '%s/mdbook\n' "$PWD" >> "$GITHUB_PATH"

View file

@ -28,9 +28,9 @@ jobs:
NVIM: ${{ matrix.os == 'windows-latest' && 'nvim-win64\\bin\\nvim.exe' || 'nvim' }} NVIM: ${{ matrix.os == 'windows-latest' && 'nvim-win64\\bin\\nvim.exe' || 'nvim' }}
NVIM_TS_DIR: nvim-treesitter NVIM_TS_DIR: nvim-treesitter
steps: steps:
- uses: actions/checkout@v6 - uses: actions/checkout@v5
- uses: actions/checkout@v6 - uses: actions/checkout@v5
with: with:
repository: nvim-treesitter/nvim-treesitter repository: nvim-treesitter/nvim-treesitter
path: ${{ env.NVIM_TS_DIR }} path: ${{ env.NVIM_TS_DIR }}

View file

@ -17,15 +17,13 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: build needs: build
permissions: permissions:
id-token: write
attestations: write
contents: write contents: write
steps: steps:
- name: Checkout repository - name: Checkout repository
uses: actions/checkout@v6 uses: actions/checkout@v5
- name: Download build artifacts - name: Download build artifacts
uses: actions/download-artifact@v7 uses: actions/download-artifact@v5
with: with:
path: artifacts path: artifacts
@ -49,16 +47,9 @@ jobs:
rm -rf artifacts rm -rf artifacts
ls -l target/ ls -l target/
- name: Generate attestations
uses: actions/attest-build-provenance@v3
with:
subject-path: |
target/tree-sitter-*.gz
target/web-tree-sitter.tar.gz
- name: Create release - name: Create release
run: |- run: |-
gh release create $GITHUB_REF_NAME \ gh release create ${{ github.ref_name }} \
target/tree-sitter-*.gz \ target/tree-sitter-*.gz \
target/web-tree-sitter.tar.gz target/web-tree-sitter.tar.gz
env: env:
@ -67,34 +58,22 @@ jobs:
crates_io: crates_io:
name: Publish packages to Crates.io name: Publish packages to Crates.io
runs-on: ubuntu-latest runs-on: ubuntu-latest
environment: crates
permissions:
id-token: write
contents: read
needs: release needs: release
steps: steps:
- name: Checkout repository - name: Checkout repository
uses: actions/checkout@v6 uses: actions/checkout@v5
- name: Set up Rust - name: Set up Rust
uses: actions-rust-lang/setup-rust-toolchain@v1 uses: actions-rust-lang/setup-rust-toolchain@v1
- name: Set up registry token
id: auth
uses: rust-lang/crates-io-auth-action@v1
- name: Publish crates to Crates.io - name: Publish crates to Crates.io
uses: katyo/publish-crates@v2 uses: katyo/publish-crates@v2
with: with:
registry-token: ${{ steps.auth.outputs.token }} registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
npm: npm:
name: Publish packages to npmjs.com name: Publish packages to npmjs.com
runs-on: ubuntu-latest runs-on: ubuntu-latest
environment: npm
permissions:
id-token: write
contents: read
needs: release needs: release
strategy: strategy:
fail-fast: false fail-fast: false
@ -102,12 +81,12 @@ jobs:
directory: [crates/cli/npm, lib/binding_web] directory: [crates/cli/npm, lib/binding_web]
steps: steps:
- name: Checkout repository - name: Checkout repository
uses: actions/checkout@v6 uses: actions/checkout@v5
- name: Set up Node - name: Set up Node
uses: actions/setup-node@v6 uses: actions/setup-node@v5
with: with:
node-version: 24 node-version: 20
registry-url: https://registry.npmjs.org registry-url: https://registry.npmjs.org
- name: Set up Rust - name: Set up Rust
@ -127,3 +106,5 @@ jobs:
- name: Publish to npmjs.com - name: Publish to npmjs.com
working-directory: ${{ matrix.directory }} working-directory: ${{ matrix.directory }}
run: npm publish run: npm publish
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

View file

@ -17,7 +17,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Checkout script - name: Checkout script
uses: actions/checkout@v6 uses: actions/checkout@v5
with: with:
sparse-checkout: .github/scripts/close_unresponsive.js sparse-checkout: .github/scripts/close_unresponsive.js
sparse-checkout-cone-mode: false sparse-checkout-cone-mode: false
@ -35,7 +35,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Checkout script - name: Checkout script
uses: actions/checkout@v6 uses: actions/checkout@v5
with: with:
sparse-checkout: .github/scripts/remove_response_label.js sparse-checkout: .github/scripts/remove_response_label.js
sparse-checkout-cone-mode: false sparse-checkout-cone-mode: false

View file

@ -12,7 +12,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Checkout script - name: Checkout script
uses: actions/checkout@v6 uses: actions/checkout@v5
with: with:
sparse-checkout: .github/scripts/reviewers_remove.js sparse-checkout: .github/scripts/reviewers_remove.js
sparse-checkout-cone-mode: false sparse-checkout-cone-mode: false

View file

@ -15,7 +15,7 @@ jobs:
TREE_SITTER: ${{ github.workspace }}/target/release/tree-sitter TREE_SITTER: ${{ github.workspace }}/target/release/tree-sitter
steps: steps:
- name: Checkout repository - name: Checkout repository
uses: actions/checkout@v6 uses: actions/checkout@v5
- name: Install UBSAN library - name: Install UBSAN library
run: sudo apt-get update -y && sudo apt-get install -y libubsan1 run: sudo apt-get update -y && sudo apt-get install -y libubsan1

View file

@ -16,7 +16,7 @@ jobs:
if: github.event.label.name == 'spam' if: github.event.label.name == 'spam'
steps: steps:
- name: Checkout script - name: Checkout script
uses: actions/checkout@v6 uses: actions/checkout@v5
with: with:
sparse-checkout: .github/scripts/close_spam.js sparse-checkout: .github/scripts/close_spam.js
sparse-checkout-cone-mode: false sparse-checkout-cone-mode: false

View file

@ -18,7 +18,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Checkout repository - name: Checkout repository
uses: actions/checkout@v6 uses: actions/checkout@v5
- name: Set up stable Rust toolchain - name: Set up stable Rust toolchain
uses: actions-rust-lang/setup-rust-toolchain@v1 uses: actions-rust-lang/setup-rust-toolchain@v1

View file

@ -1,19 +0,0 @@
name: Check Wasm Stdlib build
on:
workflow_call:
jobs:
check:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v6
- name: Check directory changes
uses: actions/github-script@v8
with:
script: |
const scriptPath = `${process.env.GITHUB_WORKSPACE}/.github/scripts/wasm_stdlib.js`;
const script = require(scriptPath);
return script({ github, context, core });

View file

@ -1,7 +1,7 @@
cmake_minimum_required(VERSION 3.13) cmake_minimum_required(VERSION 3.13)
project(tree-sitter project(tree-sitter
VERSION "0.27.0" VERSION "0.26.0"
DESCRIPTION "An incremental parsing system for programming tools" DESCRIPTION "An incremental parsing system for programming tools"
HOMEPAGE_URL "https://tree-sitter.github.io/tree-sitter/" HOMEPAGE_URL "https://tree-sitter.github.io/tree-sitter/"
LANGUAGES C) LANGUAGES C)
@ -81,7 +81,7 @@ set_target_properties(tree-sitter
SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}" SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}"
DEFINE_SYMBOL "") DEFINE_SYMBOL "")
target_compile_definitions(tree-sitter PRIVATE _POSIX_C_SOURCE=200112L _DEFAULT_SOURCE _BSD_SOURCE _DARWIN_C_SOURCE) target_compile_definitions(tree-sitter PRIVATE _POSIX_C_SOURCE=200112L _DEFAULT_SOURCE _DARWIN_C_SOURCE)
include(GNUInstallDirs) include(GNUInstallDirs)

934
Cargo.lock generated

File diff suppressed because it is too large Load diff

View file

@ -14,13 +14,13 @@ members = [
resolver = "2" resolver = "2"
[workspace.package] [workspace.package]
version = "0.27.0" version = "0.26.0"
authors = [ authors = [
"Max Brunsfeld <maxbrunsfeld@gmail.com>", "Max Brunsfeld <maxbrunsfeld@gmail.com>",
"Amaan Qureshi <amaanq12@gmail.com>", "Amaan Qureshi <amaanq12@gmail.com>",
] ]
edition = "2021" edition = "2021"
rust-version = "1.85" rust-version = "1.84"
homepage = "https://tree-sitter.github.io/tree-sitter" homepage = "https://tree-sitter.github.io/tree-sitter"
repository = "https://github.com/tree-sitter/tree-sitter" repository = "https://github.com/tree-sitter/tree-sitter"
license = "MIT" license = "MIT"
@ -103,11 +103,11 @@ codegen-units = 256
[workspace.dependencies] [workspace.dependencies]
ansi_colours = "1.2.3" ansi_colours = "1.2.3"
anstyle = "1.0.13" anstyle = "1.0.11"
anyhow = "1.0.100" anyhow = "1.0.100"
bstr = "1.12.0" bstr = "1.12.0"
cc = "1.2.53" cc = "1.2.39"
clap = { version = "4.5.54", features = [ clap = { version = "4.5.48", features = [
"cargo", "cargo",
"derive", "derive",
"env", "env",
@ -115,49 +115,48 @@ clap = { version = "4.5.54", features = [
"string", "string",
"unstable-styles", "unstable-styles",
] } ] }
clap_complete = "4.5.65" clap_complete = "4.5.58"
clap_complete_nushell = "4.5.10" clap_complete_nushell = "4.5.8"
crc32fast = "1.5.0" crc32fast = "1.5.0"
ctor = "0.2.9" ctor = "0.2.9"
ctrlc = { version = "3.5.0", features = ["termination"] } ctrlc = { version = "3.5.0", features = ["termination"] }
dialoguer = { version = "0.11.0", features = ["fuzzy-select"] } dialoguer = { version = "0.11.0", features = ["fuzzy-select"] }
etcetera = "0.11.0" etcetera = "0.10.0"
fs4 = "0.12.0" fs4 = "0.12.0"
glob = "0.3.3" glob = "0.3.3"
heck = "0.5.0" heck = "0.5.0"
html-escape = "0.2.13" html-escape = "0.2.13"
indexmap = "2.12.1" indexmap = "2.11.4"
indoc = "2.0.6" indoc = "2.0.6"
libloading = "0.9.0" libloading = "0.8.9"
log = { version = "0.4.28", features = ["std"] } log = { version = "0.4.28", features = ["std"] }
memchr = "2.7.6" memchr = "2.7.5"
once_cell = "1.21.3" once_cell = "1.21.3"
pretty_assertions = "1.4.1" pretty_assertions = "1.4.1"
rand = "0.8.5" rand = "0.8.5"
regex = "1.11.3" regex = "1.11.2"
regex-syntax = "0.8.6" regex-syntax = "0.8.6"
rustc-hash = "2.1.1" rustc-hash = "2.1.1"
schemars = "1.0.5"
semver = { version = "1.0.27", features = ["serde"] } semver = { version = "1.0.27", features = ["serde"] }
serde = { version = "1.0.219", features = ["derive"] } serde = { version = "1.0.219", features = ["derive"] }
serde_json = { version = "1.0.149", features = ["preserve_order"] } serde_json = { version = "1.0.145", features = ["preserve_order"] }
similar = "2.7.0" similar = "2.7.0"
smallbitvec = "2.6.0" smallbitvec = "2.6.0"
streaming-iterator = "0.1.9" streaming-iterator = "0.1.9"
tempfile = "3.23.0" tempfile = "3.23.0"
thiserror = "2.0.17" thiserror = "2.0.16"
tiny_http = "0.12.0" tiny_http = "0.12.0"
topological-sort = "0.2.2" topological-sort = "0.2.2"
unindent = "0.2.4" unindent = "0.2.4"
walkdir = "2.5.0" walkdir = "2.5.0"
wasmparser = "0.243.0" wasmparser = "0.229.0"
webbrowser = "1.0.5" webbrowser = "1.0.5"
tree-sitter = { version = "0.27.0", path = "./lib" } tree-sitter = { version = "0.26.0", path = "./lib" }
tree-sitter-generate = { version = "0.27.0", path = "./crates/generate" } tree-sitter-generate = { version = "0.26.0", path = "./crates/generate" }
tree-sitter-loader = { version = "0.27.0", path = "./crates/loader" } tree-sitter-loader = { version = "0.26.0", path = "./crates/loader" }
tree-sitter-config = { version = "0.27.0", path = "./crates/config" } tree-sitter-config = { version = "0.26.0", path = "./crates/config" }
tree-sitter-highlight = { version = "0.27.0", path = "./crates/highlight" } tree-sitter-highlight = { version = "0.26.0", path = "./crates/highlight" }
tree-sitter-tags = { version = "0.27.0", path = "./crates/tags" } tree-sitter-tags = { version = "0.26.0", path = "./crates/tags" }
tree-sitter-language = { version = "0.1", path = "./crates/language" } tree-sitter-language = { version = "0.1.5", path = "./crates/language" }

View file

@ -1,4 +1,4 @@
VERSION := 0.27.0 VERSION := 0.26.0
DESCRIPTION := An incremental parsing system for programming tools DESCRIPTION := An incremental parsing system for programming tools
HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/ HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/
@ -24,7 +24,7 @@ OBJ := $(SRC:.c=.o)
ARFLAGS := rcs ARFLAGS := rcs
CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types
override CFLAGS += -std=c11 -fPIC -fvisibility=hidden override CFLAGS += -std=c11 -fPIC -fvisibility=hidden
override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE -D_BSD_SOURCE -D_DARWIN_C_SOURCE override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE -D_DARWIN_C_SOURCE
override CFLAGS += -Ilib/src -Ilib/src/wasm -Ilib/include override CFLAGS += -Ilib/src -Ilib/src/wasm -Ilib/include
# ABI versioning # ABI versioning

View file

@ -27,7 +27,6 @@ let package = Package(
.headerSearchPath("src"), .headerSearchPath("src"),
.define("_POSIX_C_SOURCE", to: "200112L"), .define("_POSIX_C_SOURCE", to: "200112L"),
.define("_DEFAULT_SOURCE"), .define("_DEFAULT_SOURCE"),
.define("_BSD_SOURCE"),
.define("_DARWIN_C_SOURCE"), .define("_DARWIN_C_SOURCE"),
]), ]),
], ],

View file

@ -40,7 +40,6 @@ pub fn build(b: *std.Build) !void {
lib.root_module.addCMacro("_POSIX_C_SOURCE", "200112L"); lib.root_module.addCMacro("_POSIX_C_SOURCE", "200112L");
lib.root_module.addCMacro("_DEFAULT_SOURCE", ""); lib.root_module.addCMacro("_DEFAULT_SOURCE", "");
lib.root_module.addCMacro("_BSD_SOURCE", "");
lib.root_module.addCMacro("_DARWIN_C_SOURCE", ""); lib.root_module.addCMacro("_DARWIN_C_SOURCE", "");
if (wasm) { if (wasm) {

View file

@ -1,7 +1,7 @@
.{ .{
.name = .tree_sitter, .name = .tree_sitter,
.fingerprint = 0x841224b447ac0d4f, .fingerprint = 0x841224b447ac0d4f,
.version = "0.27.0", .version = "0.26.0",
.minimum_zig_version = "0.14.1", .minimum_zig_version = "0.14.1",
.paths = .{ .paths = .{
"build.zig", "build.zig",

View file

@ -54,13 +54,11 @@ log.workspace = true
memchr.workspace = true memchr.workspace = true
rand.workspace = true rand.workspace = true
regex.workspace = true regex.workspace = true
schemars.workspace = true
semver.workspace = true semver.workspace = true
serde.workspace = true serde.workspace = true
serde_json.workspace = true serde_json.workspace = true
similar.workspace = true similar.workspace = true
streaming-iterator.workspace = true streaming-iterator.workspace = true
thiserror.workspace = true
tiny_http.workspace = true tiny_http.workspace = true
walkdir.workspace = true walkdir.workspace = true
wasmparser.workspace = true wasmparser.workspace = true
@ -75,7 +73,7 @@ tree-sitter-tags.workspace = true
[dev-dependencies] [dev-dependencies]
encoding_rs = "0.8.35" encoding_rs = "0.8.35"
widestring = "1.2.1" widestring = "1.2.0"
tree_sitter_proc_macro = { path = "src/tests/proc_macro", package = "tree-sitter-tests-proc-macro" } tree_sitter_proc_macro = { path = "src/tests/proc_macro", package = "tree-sitter-tests-proc-macro" }
tempfile.workspace = true tempfile.workspace = true

View file

@ -7,8 +7,7 @@
[npmjs.com]: https://www.npmjs.org/package/tree-sitter-cli [npmjs.com]: https://www.npmjs.org/package/tree-sitter-cli
[npmjs.com badge]: https://img.shields.io/npm/v/tree-sitter-cli.svg?color=%23BF4A4A [npmjs.com badge]: https://img.shields.io/npm/v/tree-sitter-cli.svg?color=%23BF4A4A
The Tree-sitter CLI allows you to develop, test, and use Tree-sitter grammars from the command line. It works on `MacOS`, The Tree-sitter CLI allows you to develop, test, and use Tree-sitter grammars from the command line. It works on `MacOS`, `Linux`, and `Windows`.
`Linux`, and `Windows`.
### Installation ### Installation
@ -35,11 +34,9 @@ The `tree-sitter` binary itself has no dependencies, but specific commands have
### Commands ### Commands
* `generate` - The `tree-sitter generate` command will generate a Tree-sitter parser based on the grammar in the current * `generate` - The `tree-sitter generate` command will generate a Tree-sitter parser based on the grammar in the current working directory. See [the documentation] for more information.
working directory. See [the documentation] for more information.
* `test` - The `tree-sitter test` command will run the unit tests for the Tree-sitter parser in the current working directory. * `test` - The `tree-sitter test` command will run the unit tests for the Tree-sitter parser in the current working directory. See [the documentation] for more information.
See [the documentation] for more information.
* `parse` - The `tree-sitter parse` command will parse a file (or list of files) using Tree-sitter parsers. * `parse` - The `tree-sitter parse` command will parse a file (or list of files) using Tree-sitter parsers.

View file

@ -805,9 +805,9 @@
"peer": true "peer": true
}, },
"node_modules/js-yaml": { "node_modules/js-yaml": {
"version": "4.1.1", "version": "4.1.0",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
"integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
"license": "MIT", "license": "MIT",
"peer": true, "peer": true,
"dependencies": { "dependencies": {

View file

@ -29,7 +29,6 @@ type Rule =
| PrecRule | PrecRule
| Repeat1Rule | Repeat1Rule
| RepeatRule | RepeatRule
| ReservedRule
| SeqRule | SeqRule
| StringRule | StringRule
| SymbolRule<string> | SymbolRule<string>

View file

@ -1,12 +1,12 @@
{ {
"name": "tree-sitter-cli", "name": "tree-sitter-cli",
"version": "0.27.0", "version": "0.26.0",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "tree-sitter-cli", "name": "tree-sitter-cli",
"version": "0.27.0", "version": "0.26.0",
"hasInstallScript": true, "hasInstallScript": true,
"license": "MIT", "license": "MIT",
"bin": { "bin": {

View file

@ -1,6 +1,6 @@
{ {
"name": "tree-sitter-cli", "name": "tree-sitter-cli",
"version": "0.27.0", "version": "0.26.0",
"author": { "author": {
"name": "Max Brunsfeld", "name": "Max Brunsfeld",
"email": "maxbrunsfeld@gmail.com" "email": "maxbrunsfeld@gmail.com"

View file

@ -25,7 +25,7 @@ use crate::{
random::Rand, random::Rand,
}, },
parse::perform_edit, parse::perform_edit,
test::{parse_tests, strip_sexp_fields, DiffKey, TestDiff, TestEntry}, test::{parse_tests, print_diff, print_diff_key, strip_sexp_fields, TestEntry},
}; };
pub static LOG_ENABLED: LazyLock<bool> = LazyLock::new(|| env::var("TREE_SITTER_LOG").is_ok()); pub static LOG_ENABLED: LazyLock<bool> = LazyLock::new(|| env::var("TREE_SITTER_LOG").is_ok());
@ -183,8 +183,8 @@ pub fn fuzz_language_corpus(
if actual_output != test.output { if actual_output != test.output {
println!("Incorrect initial parse for {test_name}"); println!("Incorrect initial parse for {test_name}");
DiffKey::print(); print_diff_key();
println!("{}", TestDiff::new(&actual_output, &test.output)); print_diff(&actual_output, &test.output, true);
println!(); println!();
return false; return false;
} }
@ -276,8 +276,8 @@ pub fn fuzz_language_corpus(
if actual_output != test.output && !test.error { if actual_output != test.output && !test.error {
println!("Incorrect parse for {test_name} - seed {seed}"); println!("Incorrect parse for {test_name} - seed {seed}");
DiffKey::print(); print_diff_key();
println!("{}", TestDiff::new(&actual_output, &test.output)); print_diff(&actual_output, &test.output, true);
println!(); println!();
return false; return false;
} }

View file

@ -8,17 +8,13 @@ use anyhow::{anyhow, Context, Result};
use crc32fast::hash as crc32; use crc32fast::hash as crc32;
use heck::{ToKebabCase, ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase}; use heck::{ToKebabCase, ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase};
use indoc::{formatdoc, indoc}; use indoc::{formatdoc, indoc};
use log::info; use log::warn;
use rand::{thread_rng, Rng}; use rand::{thread_rng, Rng};
use semver::Version; use semver::Version;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_json::{Map, Value}; use serde_json::{Map, Value};
use tree_sitter_generate::write_file; use tree_sitter_generate::write_file;
use tree_sitter_loader::{ use tree_sitter_loader::{Author, Bindings, Grammar, Links, Metadata, PathsJSON, TreeSitterJSON};
Author, Bindings, Grammar, Links, Metadata, PathsJSON, TreeSitterJSON,
DEFAULT_HIGHLIGHTS_QUERY_FILE_NAME, DEFAULT_INJECTIONS_QUERY_FILE_NAME,
DEFAULT_LOCALS_QUERY_FILE_NAME, DEFAULT_TAGS_QUERY_FILE_NAME,
};
const CLI_VERSION: &str = env!("CARGO_PKG_VERSION"); const CLI_VERSION: &str = env!("CARGO_PKG_VERSION");
const CLI_VERSION_PLACEHOLDER: &str = "CLI_VERSION"; const CLI_VERSION_PLACEHOLDER: &str = "CLI_VERSION";
@ -36,8 +32,6 @@ const PARSER_CLASS_NAME_PLACEHOLDER: &str = "PARSER_CLASS_NAME";
const PARSER_DESCRIPTION_PLACEHOLDER: &str = "PARSER_DESCRIPTION"; const PARSER_DESCRIPTION_PLACEHOLDER: &str = "PARSER_DESCRIPTION";
const PARSER_LICENSE_PLACEHOLDER: &str = "PARSER_LICENSE"; const PARSER_LICENSE_PLACEHOLDER: &str = "PARSER_LICENSE";
const PARSER_NS_PLACEHOLDER: &str = "PARSER_NS";
const PARSER_NS_CLEANED_PLACEHOLDER: &str = "PARSER_NS_CLEANED";
const PARSER_URL_PLACEHOLDER: &str = "PARSER_URL"; const PARSER_URL_PLACEHOLDER: &str = "PARSER_URL";
const PARSER_URL_STRIPPED_PLACEHOLDER: &str = "PARSER_URL_STRIPPED"; const PARSER_URL_STRIPPED_PLACEHOLDER: &str = "PARSER_URL_STRIPPED";
const PARSER_VERSION_PLACEHOLDER: &str = "PARSER_VERSION"; const PARSER_VERSION_PLACEHOLDER: &str = "PARSER_VERSION";
@ -60,22 +54,12 @@ const AUTHOR_BLOCK_RS: &str = "\nauthors = [";
const AUTHOR_NAME_PLACEHOLDER_RS: &str = "PARSER_AUTHOR_NAME"; const AUTHOR_NAME_PLACEHOLDER_RS: &str = "PARSER_AUTHOR_NAME";
const AUTHOR_EMAIL_PLACEHOLDER_RS: &str = " PARSER_AUTHOR_EMAIL"; const AUTHOR_EMAIL_PLACEHOLDER_RS: &str = " PARSER_AUTHOR_EMAIL";
const AUTHOR_BLOCK_JAVA: &str = "\n <developer>";
const AUTHOR_NAME_PLACEHOLDER_JAVA: &str = "\n <name>PARSER_AUTHOR_NAME</name>";
const AUTHOR_EMAIL_PLACEHOLDER_JAVA: &str = "\n <email>PARSER_AUTHOR_EMAIL</email>";
const AUTHOR_URL_PLACEHOLDER_JAVA: &str = "\n <url>PARSER_AUTHOR_URL</url>";
const AUTHOR_BLOCK_GRAMMAR: &str = "\n * @author "; const AUTHOR_BLOCK_GRAMMAR: &str = "\n * @author ";
const AUTHOR_NAME_PLACEHOLDER_GRAMMAR: &str = "PARSER_AUTHOR_NAME"; const AUTHOR_NAME_PLACEHOLDER_GRAMMAR: &str = "PARSER_AUTHOR_NAME";
const AUTHOR_EMAIL_PLACEHOLDER_GRAMMAR: &str = " PARSER_AUTHOR_EMAIL"; const AUTHOR_EMAIL_PLACEHOLDER_GRAMMAR: &str = " PARSER_AUTHOR_EMAIL";
const FUNDING_URL_PLACEHOLDER: &str = "FUNDING_URL"; const FUNDING_URL_PLACEHOLDER: &str = "FUNDING_URL";
const HIGHLIGHTS_QUERY_PATH_PLACEHOLDER: &str = "HIGHLIGHTS_QUERY_PATH";
const INJECTIONS_QUERY_PATH_PLACEHOLDER: &str = "INJECTIONS_QUERY_PATH";
const LOCALS_QUERY_PATH_PLACEHOLDER: &str = "LOCALS_QUERY_PATH";
const TAGS_QUERY_PATH_PLACEHOLDER: &str = "TAGS_QUERY_PATH";
const GRAMMAR_JS_TEMPLATE: &str = include_str!("./templates/grammar.js"); const GRAMMAR_JS_TEMPLATE: &str = include_str!("./templates/grammar.js");
const PACKAGE_JSON_TEMPLATE: &str = include_str!("./templates/package.json"); const PACKAGE_JSON_TEMPLATE: &str = include_str!("./templates/package.json");
const GITIGNORE_TEMPLATE: &str = include_str!("./templates/gitignore"); const GITIGNORE_TEMPLATE: &str = include_str!("./templates/gitignore");
@ -114,16 +98,12 @@ const TEST_BINDING_PY_TEMPLATE: &str = include_str!("./templates/test_binding.py
const PACKAGE_SWIFT_TEMPLATE: &str = include_str!("./templates/package.swift"); const PACKAGE_SWIFT_TEMPLATE: &str = include_str!("./templates/package.swift");
const TESTS_SWIFT_TEMPLATE: &str = include_str!("./templates/tests.swift"); const TESTS_SWIFT_TEMPLATE: &str = include_str!("./templates/tests.swift");
const POM_XML_TEMPLATE: &str = include_str!("./templates/pom.xml");
const BINDING_JAVA_TEMPLATE: &str = include_str!("./templates/binding.java");
const TEST_JAVA_TEMPLATE: &str = include_str!("./templates/test.java");
const BUILD_ZIG_TEMPLATE: &str = include_str!("./templates/build.zig"); const BUILD_ZIG_TEMPLATE: &str = include_str!("./templates/build.zig");
const BUILD_ZIG_ZON_TEMPLATE: &str = include_str!("./templates/build.zig.zon"); const BUILD_ZIG_ZON_TEMPLATE: &str = include_str!("./templates/build.zig.zon");
const ROOT_ZIG_TEMPLATE: &str = include_str!("./templates/root.zig"); const ROOT_ZIG_TEMPLATE: &str = include_str!("./templates/root.zig");
const TEST_ZIG_TEMPLATE: &str = include_str!("./templates/test.zig"); const TEST_ZIG_TEMPLATE: &str = include_str!("./templates/test.zig");
pub const TREE_SITTER_JSON_SCHEMA: &str = const TREE_SITTER_JSON_SCHEMA: &str =
"https://tree-sitter.github.io/tree-sitter/assets/schemas/config.schema.json"; "https://tree-sitter.github.io/tree-sitter/assets/schemas/config.schema.json";
#[derive(Serialize, Deserialize, Clone)] #[derive(Serialize, Deserialize, Clone)]
@ -145,7 +125,6 @@ pub struct JsonConfigOpts {
pub email: Option<String>, pub email: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
pub url: Option<String>, pub url: Option<String>,
pub namespace: Option<String>,
pub bindings: Bindings, pub bindings: Bindings,
} }
@ -186,7 +165,7 @@ impl JsonConfigOpts {
}), }),
funding: self.funding, funding: self.funding,
}), }),
namespace: self.namespace, namespace: None,
}, },
bindings: self.bindings, bindings: self.bindings,
} }
@ -209,7 +188,6 @@ impl Default for JsonConfigOpts {
author: String::new(), author: String::new(),
email: None, email: None,
url: None, url: None,
namespace: None,
bindings: Bindings::default(), bindings: Bindings::default(),
} }
} }
@ -227,11 +205,6 @@ struct GenerateOpts<'a> {
camel_parser_name: &'a str, camel_parser_name: &'a str,
title_parser_name: &'a str, title_parser_name: &'a str,
class_name: &'a str, class_name: &'a str,
highlights_query_path: &'a str,
injections_query_path: &'a str,
locals_query_path: &'a str,
tags_query_path: &'a str,
namespace: Option<&'a str>,
} }
pub fn generate_grammar_files( pub fn generate_grammar_files(
@ -282,11 +255,6 @@ pub fn generate_grammar_files(
.clone() .clone()
.unwrap_or_else(|| format!("TreeSitter{}", language_name.to_upper_camel_case())); .unwrap_or_else(|| format!("TreeSitter{}", language_name.to_upper_camel_case()));
let default_highlights_path = Path::new("queries").join(DEFAULT_HIGHLIGHTS_QUERY_FILE_NAME);
let default_injections_path = Path::new("queries").join(DEFAULT_INJECTIONS_QUERY_FILE_NAME);
let default_locals_path = Path::new("queries").join(DEFAULT_LOCALS_QUERY_FILE_NAME);
let default_tags_path = Path::new("queries").join(DEFAULT_TAGS_QUERY_FILE_NAME);
let generate_opts = GenerateOpts { let generate_opts = GenerateOpts {
author_name: authors author_name: authors
.map(|a| a.first().map(|a| a.name.as_str())) .map(|a| a.first().map(|a| a.name.as_str()))
@ -313,19 +281,6 @@ pub fn generate_grammar_files(
camel_parser_name: &camel_name, camel_parser_name: &camel_name,
title_parser_name: &title_name, title_parser_name: &title_name,
class_name: &class_name, class_name: &class_name,
highlights_query_path: tree_sitter_config.grammars[0]
.highlights
.to_variable_value(&default_highlights_path),
injections_query_path: tree_sitter_config.grammars[0]
.injections
.to_variable_value(&default_injections_path),
locals_query_path: tree_sitter_config.grammars[0]
.locals
.to_variable_value(&default_locals_path),
tags_query_path: tree_sitter_config.grammars[0]
.tags
.to_variable_value(&default_tags_path),
namespace: tree_sitter_config.metadata.namespace.as_deref(),
}; };
// Create package.json // Create package.json
@ -352,11 +307,11 @@ pub fn generate_grammar_files(
"tree-sitter-cli":"#}, "tree-sitter-cli":"#},
indoc! {r#" indoc! {r#"
"prebuildify": "^6.0.1", "prebuildify": "^6.0.1",
"tree-sitter": "^0.25.0", "tree-sitter": "^0.22.4",
"tree-sitter-cli":"#}, "tree-sitter-cli":"#},
); );
if !contents.contains("module") { if !contents.contains("module") {
info!("Migrating package.json to ESM"); warn!("Updating package.json");
contents = contents.replace( contents = contents.replace(
r#""repository":"#, r#""repository":"#,
indoc! {r#" indoc! {r#"
@ -378,7 +333,6 @@ pub fn generate_grammar_files(
|path| { |path| {
let mut contents = fs::read_to_string(path)?; let mut contents = fs::read_to_string(path)?;
if contents.contains("module.exports") { if contents.contains("module.exports") {
info!("Migrating grammars.js to ESM");
contents = contents.replace("module.exports =", "export default"); contents = contents.replace("module.exports =", "export default");
write_file(path, contents)?; write_file(path, contents)?;
} }
@ -394,16 +348,10 @@ pub fn generate_grammar_files(
allow_update, allow_update,
|path| generate_file(path, GITIGNORE_TEMPLATE, language_name, &generate_opts), |path| generate_file(path, GITIGNORE_TEMPLATE, language_name, &generate_opts),
|path| { |path| {
let mut contents = fs::read_to_string(path)?; let contents = fs::read_to_string(path)?;
if !contents.contains("Zig artifacts") { if !contents.contains("Zig artifacts") {
info!("Adding zig entries to .gitignore"); warn!("Replacing .gitignore");
contents.push('\n'); generate_file(path, GITIGNORE_TEMPLATE, language_name, &generate_opts)?;
contents.push_str(indoc! {"
# Zig artifacts
.zig-cache/
zig-cache/
zig-out/
"});
} }
Ok(()) Ok(())
}, },
@ -416,13 +364,8 @@ pub fn generate_grammar_files(
|path| generate_file(path, GITATTRIBUTES_TEMPLATE, language_name, &generate_opts), |path| generate_file(path, GITATTRIBUTES_TEMPLATE, language_name, &generate_opts),
|path| { |path| {
let mut contents = fs::read_to_string(path)?; let mut contents = fs::read_to_string(path)?;
let c_bindings_entry = "bindings/c/* "; contents = contents.replace("bindings/c/* ", "bindings/c/** ");
if contents.contains(c_bindings_entry) {
info!("Updating c bindings entry in .gitattributes");
contents = contents.replace(c_bindings_entry, "bindings/c/** ");
}
if !contents.contains("Zig bindings") { if !contents.contains("Zig bindings") {
info!("Adding zig entries to .gitattributes");
contents.push('\n'); contents.push('\n');
contents.push_str(indoc! {" contents.push_str(indoc! {"
# Zig bindings # Zig bindings
@ -445,48 +388,8 @@ pub fn generate_grammar_files(
// Generate Rust bindings // Generate Rust bindings
if tree_sitter_config.bindings.rust { if tree_sitter_config.bindings.rust {
missing_path(bindings_dir.join("rust"), create_dir)?.apply(|path| { missing_path(bindings_dir.join("rust"), create_dir)?.apply(|path| {
missing_path_else(path.join("lib.rs"), allow_update, |path| { missing_path(path.join("lib.rs"), |path| {
generate_file(path, LIB_RS_TEMPLATE, language_name, &generate_opts) generate_file(path, LIB_RS_TEMPLATE, language_name, &generate_opts)
}, |path| {
let mut contents = fs::read_to_string(path)?;
if !contents.contains("#[cfg(with_highlights_query)]") {
info!("Updating query constants in bindings/rust/lib.rs");
let replacement = indoc! {r#"
#[cfg(with_highlights_query)]
/// The syntax highlighting query for this grammar.
pub const HIGHLIGHTS_QUERY: &str = include_str!("../../HIGHLIGHTS_QUERY_PATH");
#[cfg(with_injections_query)]
/// The language injection query for this grammar.
pub const INJECTIONS_QUERY: &str = include_str!("../../INJECTIONS_QUERY_PATH");
#[cfg(with_locals_query)]
/// The local variable query for this grammar.
pub const LOCALS_QUERY: &str = include_str!("../../LOCALS_QUERY_PATH");
#[cfg(with_tags_query)]
/// The symbol tagging query for this grammar.
pub const TAGS_QUERY: &str = include_str!("../../TAGS_QUERY_PATH");
"#}
.replace("HIGHLIGHTS_QUERY_PATH", generate_opts.highlights_query_path)
.replace("INJECTIONS_QUERY_PATH", generate_opts.injections_query_path)
.replace("LOCALS_QUERY_PATH", generate_opts.locals_query_path)
.replace("TAGS_QUERY_PATH", generate_opts.tags_query_path);
contents = contents
.replace(
indoc! {r#"
// NOTE: uncomment these to include any queries that this grammar contains:
// pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
// pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm");
// pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm");
// pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm");
"#},
&replacement,
);
}
write_file(path, contents)?;
Ok(())
})?; })?;
missing_path_else( missing_path_else(
@ -494,76 +397,37 @@ pub fn generate_grammar_files(
allow_update, allow_update,
|path| generate_file(path, BUILD_RS_TEMPLATE, language_name, &generate_opts), |path| generate_file(path, BUILD_RS_TEMPLATE, language_name, &generate_opts),
|path| { |path| {
let replacement = indoc!{r#"
c_config.flag("-utf-8");
if std::env::var("TARGET").unwrap() == "wasm32-unknown-unknown" {
let Ok(wasm_headers) = std::env::var("DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS") else {
panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS must be set by the language crate");
};
let Ok(wasm_src) =
std::env::var("DEP_TREE_SITTER_LANGUAGE_WASM_SRC").map(std::path::PathBuf::from)
else {
panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_SRC must be set by the language crate");
};
c_config.include(&wasm_headers);
c_config.files([
wasm_src.join("stdio.c"),
wasm_src.join("stdlib.c"),
wasm_src.join("string.c"),
]);
}
"#};
let indented_replacement = replacement
.lines()
.map(|line| if line.is_empty() { line.to_string() } else { format!(" {line}") })
.collect::<Vec<_>>()
.join("\n");
let mut contents = fs::read_to_string(path)?; let mut contents = fs::read_to_string(path)?;
if !contents.contains("wasm32-unknown-unknown") { if !contents.contains("wasm32-unknown-unknown") {
info!("Adding wasm32-unknown-unknown target to bindings/rust/build.rs"); contents = contents.replace(r#" c_config.flag("-utf-8");"#, &indented_replacement);
let replacement = indoc!{r#"
c_config.flag("-utf-8");
if std::env::var("TARGET").unwrap() == "wasm32-unknown-unknown" {
let Ok(wasm_headers) = std::env::var("DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS") else {
panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS must be set by the language crate");
};
let Ok(wasm_src) =
std::env::var("DEP_TREE_SITTER_LANGUAGE_WASM_SRC").map(std::path::PathBuf::from)
else {
panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_SRC must be set by the language crate");
};
c_config.include(&wasm_headers);
c_config.files([
wasm_src.join("stdio.c"),
wasm_src.join("stdlib.c"),
wasm_src.join("string.c"),
]);
}
"#}
.lines()
.map(|line| if line.is_empty() { line.to_string() } else { format!(" {line}") })
.collect::<Vec<_>>()
.join("\n");
contents = contents.replace(r#" c_config.flag("-utf-8");"#, &replacement);
}
// Introduce configuration variables for dynamic query inclusion
if !contents.contains("with_highlights_query") {
info!("Adding support for dynamic query inclusion to bindings/rust/build.rs");
let replaced = indoc! {r#"
c_config.compile("tree-sitter-KEBAB_PARSER_NAME");
}"#}
.replace("KEBAB_PARSER_NAME", &language_name.to_kebab_case());
let replacement = indoc! {r#"
c_config.compile("tree-sitter-KEBAB_PARSER_NAME");
println!("cargo:rustc-check-cfg=cfg(with_highlights_query)");
if !"HIGHLIGHTS_QUERY_PATH".is_empty() && std::path::Path::new("HIGHLIGHTS_QUERY_PATH").exists() {
println!("cargo:rustc-cfg=with_highlights_query");
}
println!("cargo:rustc-check-cfg=cfg(with_injections_query)");
if !"INJECTIONS_QUERY_PATH".is_empty() && std::path::Path::new("INJECTIONS_QUERY_PATH").exists() {
println!("cargo:rustc-cfg=with_injections_query");
}
println!("cargo:rustc-check-cfg=cfg(with_locals_query)");
if !"LOCALS_QUERY_PATH".is_empty() && std::path::Path::new("LOCALS_QUERY_PATH").exists() {
println!("cargo:rustc-cfg=with_locals_query");
}
println!("cargo:rustc-check-cfg=cfg(with_tags_query)");
if !"TAGS_QUERY_PATH".is_empty() && std::path::Path::new("TAGS_QUERY_PATH").exists() {
println!("cargo:rustc-cfg=with_tags_query");
}
}"#}
.replace("KEBAB_PARSER_NAME", &language_name.to_kebab_case())
.replace("HIGHLIGHTS_QUERY_PATH", generate_opts.highlights_query_path)
.replace("INJECTIONS_QUERY_PATH", generate_opts.injections_query_path)
.replace("LOCALS_QUERY_PATH", generate_opts.locals_query_path)
.replace("TAGS_QUERY_PATH", generate_opts.tags_query_path);
contents = contents.replace(
&replaced,
&replacement,
);
} }
write_file(path, contents)?; write_file(path, contents)?;
@ -585,7 +449,6 @@ pub fn generate_grammar_files(
|path| { |path| {
let contents = fs::read_to_string(path)?; let contents = fs::read_to_string(path)?;
if contents.contains("\"LICENSE\"") { if contents.contains("\"LICENSE\"") {
info!("Adding LICENSE entry to bindings/rust/Cargo.toml");
write_file(path, contents.replace("\"LICENSE\"", "\"/LICENSE\""))?; write_file(path, contents.replace("\"LICENSE\"", "\"/LICENSE\""))?;
} }
Ok(()) Ok(())
@ -605,27 +468,17 @@ pub fn generate_grammar_files(
|path| generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts), |path| generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts),
|path| { |path| {
let contents = fs::read_to_string(path)?; let contents = fs::read_to_string(path)?;
if !contents.contains("Object.defineProperty") { if !contents.contains("new URL") {
info!("Replacing index.js"); warn!("Replacing index.js");
generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts)?; generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts)?;
} }
Ok(()) Ok(())
}, },
)?; )?;
missing_path_else( missing_path(path.join("index.d.ts"), |path| {
path.join("index.d.ts"), generate_file(path, INDEX_D_TS_TEMPLATE, language_name, &generate_opts)
allow_update, })?;
|path| generate_file(path, INDEX_D_TS_TEMPLATE, language_name, &generate_opts),
|path| {
let contents = fs::read_to_string(path)?;
if !contents.contains("export default binding") {
info!("Replacing index.d.ts");
generate_file(path, INDEX_D_TS_TEMPLATE, language_name, &generate_opts)?;
}
Ok(())
},
)?;
missing_path_else( missing_path_else(
path.join("binding_test.js"), path.join("binding_test.js"),
@ -641,7 +494,7 @@ pub fn generate_grammar_files(
|path| { |path| {
let contents = fs::read_to_string(path)?; let contents = fs::read_to_string(path)?;
if !contents.contains("import") { if !contents.contains("import") {
info!("Replacing binding_test.js"); warn!("Replacing binding_test.js");
generate_file( generate_file(
path, path,
BINDING_TEST_JS_TEMPLATE, BINDING_TEST_JS_TEMPLATE,
@ -664,7 +517,6 @@ pub fn generate_grammar_files(
|path| { |path| {
let contents = fs::read_to_string(path)?; let contents = fs::read_to_string(path)?;
if contents.contains("fs.exists(") { if contents.contains("fs.exists(") {
info!("Replacing `fs.exists` calls in binding.gyp");
write_file(path, contents.replace("fs.exists(", "fs.existsSync("))?; write_file(path, contents.replace("fs.exists(", "fs.existsSync("))?;
} }
Ok(()) Ok(())
@ -677,17 +529,14 @@ pub fn generate_grammar_files(
// Generate C bindings // Generate C bindings
if tree_sitter_config.bindings.c { if tree_sitter_config.bindings.c {
let kebab_case_name = language_name.to_kebab_case();
missing_path(bindings_dir.join("c"), create_dir)?.apply(|path| { missing_path(bindings_dir.join("c"), create_dir)?.apply(|path| {
let header_name = format!("tree-sitter-{kebab_case_name}.h"); let old_file = &path.join(format!("tree-sitter-{}.h", language_name.to_kebab_case()));
let old_file = &path.join(&header_name);
if allow_update && fs::exists(old_file).unwrap_or(false) { if allow_update && fs::exists(old_file).unwrap_or(false) {
info!("Removing bindings/c/{header_name}");
fs::remove_file(old_file)?; fs::remove_file(old_file)?;
} }
missing_path(path.join("tree_sitter"), create_dir)?.apply(|include_path| { missing_path(path.join("tree_sitter"), create_dir)?.apply(|include_path| {
missing_path( missing_path(
include_path.join(&header_name), include_path.join(format!("tree-sitter-{}.h", language_name.to_kebab_case())),
|path| { |path| {
generate_file(path, PARSER_NAME_H_TEMPLATE, language_name, &generate_opts) generate_file(path, PARSER_NAME_H_TEMPLATE, language_name, &generate_opts)
}, },
@ -696,7 +545,7 @@ pub fn generate_grammar_files(
})?; })?;
missing_path( missing_path(
path.join(format!("tree-sitter-{kebab_case_name}.pc.in")), path.join(format!("tree-sitter-{}.pc.in", language_name.to_kebab_case())),
|path| { |path| {
generate_file( generate_file(
path, path,
@ -716,27 +565,23 @@ pub fn generate_grammar_files(
|path| { |path| {
let mut contents = fs::read_to_string(path)?; let mut contents = fs::read_to_string(path)?;
if !contents.contains("cd '$(DESTDIR)$(LIBDIR)' && ln -sf") { if !contents.contains("cd '$(DESTDIR)$(LIBDIR)' && ln -sf") {
info!("Replacing Makefile"); warn!("Replacing Makefile");
generate_file(path, MAKEFILE_TEMPLATE, language_name, &generate_opts)?; generate_file(path, MAKEFILE_TEMPLATE, language_name, &generate_opts)?;
} else { } else {
let replaced = indoc! {r" contents = contents
$(PARSER): $(SRC_DIR)/grammar.json .replace(
$(TS) generate $^ indoc! {r"
"}; $(PARSER): $(SRC_DIR)/grammar.json
if contents.contains(replaced) { $(TS) generate $^
info!("Adding --no-parser target to Makefile"); "},
contents = contents indoc! {r"
.replace( $(SRC_DIR)/grammar.json: grammar.js
replaced, $(TS) generate --emit=json $^
indoc! {r"
$(SRC_DIR)/grammar.json: grammar.js
$(TS) generate --no-parser $^
$(PARSER): $(SRC_DIR)/grammar.json $(PARSER): $(SRC_DIR)/grammar.json
$(TS) generate $^ $(TS) generate --emit=parser $^
"} "}
); );
}
write_file(path, contents)?; write_file(path, contents)?;
} }
Ok(()) Ok(())
@ -748,8 +593,8 @@ pub fn generate_grammar_files(
allow_update, allow_update,
|path| generate_file(path, CMAKELISTS_TXT_TEMPLATE, language_name, &generate_opts), |path| generate_file(path, CMAKELISTS_TXT_TEMPLATE, language_name, &generate_opts),
|path| { |path| {
let contents = fs::read_to_string(path)?; let mut contents = fs::read_to_string(path)?;
let replaced_contents = contents contents = contents
.replace("add_custom_target(test", "add_custom_target(ts-test") .replace("add_custom_target(test", "add_custom_target(ts-test")
.replace( .replace(
&formatdoc! {r#" &formatdoc! {r#"
@ -780,27 +625,21 @@ pub fn generate_grammar_files(
"#}, "#},
indoc! {r#" indoc! {r#"
add_custom_command(OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/src/grammar.json" add_custom_command(OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/src/grammar.json"
"${CMAKE_CURRENT_SOURCE_DIR}/src/node-types.json"
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/grammar.js" DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/grammar.js"
COMMAND "${TREE_SITTER_CLI}" generate grammar.js --no-parser COMMAND "${TREE_SITTER_CLI}" generate grammar.js
--emit=json
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
COMMENT "Generating grammar.json") COMMENT "Generating grammar.json")
add_custom_command(OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/src/parser.c" add_custom_command(OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/src/parser.c"
BYPRODUCTS "${CMAKE_CURRENT_SOURCE_DIR}/src/tree_sitter/parser.h"
"${CMAKE_CURRENT_SOURCE_DIR}/src/tree_sitter/alloc.h"
"${CMAKE_CURRENT_SOURCE_DIR}/src/tree_sitter/array.h"
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/grammar.json" DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/grammar.json"
COMMAND "${TREE_SITTER_CLI}" generate src/grammar.json COMMAND "${TREE_SITTER_CLI}" generate src/grammar.json
--abi=${TREE_SITTER_ABI_VERSION} --emit=parser --abi=${TREE_SITTER_ABI_VERSION}
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
COMMENT "Generating parser.c") COMMENT "Generating parser.c")
"#} "#}
); );
if !replaced_contents.eq(&contents) { write_file(path, contents)?;
info!("Updating CMakeLists.txt");
write_file(path, replaced_contents)?;
}
Ok(()) Ok(())
}, },
)?; )?;
@ -836,8 +675,7 @@ pub fn generate_grammar_files(
// Generate Python bindings // Generate Python bindings
if tree_sitter_config.bindings.python { if tree_sitter_config.bindings.python {
missing_path(bindings_dir.join("python"), create_dir)?.apply(|path| { missing_path(bindings_dir.join("python"), create_dir)?.apply(|path| {
let snake_case_grammar_name = format!("tree_sitter_{}", language_name.to_snake_case()); let lang_path = path.join(format!("tree_sitter_{}", language_name.to_snake_case()));
let lang_path = path.join(&snake_case_grammar_name);
missing_path(&lang_path, create_dir)?; missing_path(&lang_path, create_dir)?;
missing_path_else( missing_path_else(
@ -847,7 +685,6 @@ pub fn generate_grammar_files(
|path| { |path| {
let mut contents = fs::read_to_string(path)?; let mut contents = fs::read_to_string(path)?;
if !contents.contains("PyModuleDef_Init") { if !contents.contains("PyModuleDef_Init") {
info!("Updating bindings/python/{snake_case_grammar_name}/binding.c");
contents = contents contents = contents
.replace("PyModule_Create", "PyModuleDef_Init") .replace("PyModule_Create", "PyModuleDef_Init")
.replace( .replace(
@ -880,21 +717,9 @@ pub fn generate_grammar_files(
}, },
)?; )?;
missing_path_else( missing_path(lang_path.join("__init__.py"), |path| {
lang_path.join("__init__.py"), generate_file(path, INIT_PY_TEMPLATE, language_name, &generate_opts)
allow_update, })?;
|path| {
generate_file(path, INIT_PY_TEMPLATE, language_name, &generate_opts)
},
|path| {
let contents = fs::read_to_string(path)?;
if !contents.contains("uncomment these to include any queries") {
info!("Replacing __init__.py");
generate_file(path, INIT_PY_TEMPLATE, language_name, &generate_opts)?;
}
Ok(())
},
)?;
missing_path_else( missing_path_else(
lang_path.join("__init__.pyi"), lang_path.join("__init__.pyi"),
@ -902,11 +727,7 @@ pub fn generate_grammar_files(
|path| generate_file(path, INIT_PYI_TEMPLATE, language_name, &generate_opts), |path| generate_file(path, INIT_PYI_TEMPLATE, language_name, &generate_opts),
|path| { |path| {
let mut contents = fs::read_to_string(path)?; let mut contents = fs::read_to_string(path)?;
if contents.contains("uncomment these to include any queries") { if !contents.contains("CapsuleType") {
info!("Replacing __init__.pyi");
generate_file(path, INIT_PYI_TEMPLATE, language_name, &generate_opts)?;
} else if !contents.contains("CapsuleType") {
info!("Updating __init__.pyi");
contents = contents contents = contents
.replace( .replace(
"from typing import Final", "from typing import Final",
@ -938,7 +759,6 @@ pub fn generate_grammar_files(
|path| { |path| {
let mut contents = fs::read_to_string(path)?; let mut contents = fs::read_to_string(path)?;
if !contents.contains("Parser(Language(") { if !contents.contains("Parser(Language(") {
info!("Updating Language function in bindings/python/tests/test_binding.py");
contents = contents contents = contents
.replace("tree_sitter.Language(", "Parser(Language(") .replace("tree_sitter.Language(", "Parser(Language(")
.replace(".language())\n", ".language()))\n") .replace(".language())\n", ".language()))\n")
@ -959,19 +779,11 @@ pub fn generate_grammar_files(
allow_update, allow_update,
|path| generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts), |path| generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts),
|path| { |path| {
let mut contents = fs::read_to_string(path)?; let contents = fs::read_to_string(path)?;
if !contents.contains("build_ext") { if !contents.contains("build_ext") {
info!("Replacing setup.py"); warn!("Replacing setup.py");
generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts)?; generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts)?;
} }
if !contents.contains(" and not get_config_var") {
info!("Updating Python free-threading support in setup.py");
contents = contents.replace(
r#"startswith("cp"):"#,
r#"startswith("cp") and not get_config_var("Py_GIL_DISABLED"):"#
);
write_file(path, contents)?;
}
Ok(()) Ok(())
}, },
)?; )?;
@ -990,7 +802,6 @@ pub fn generate_grammar_files(
|path| { |path| {
let mut contents = fs::read_to_string(path)?; let mut contents = fs::read_to_string(path)?;
if !contents.contains("cp310-*") { if !contents.contains("cp310-*") {
info!("Updating dependencies in pyproject.toml");
contents = contents contents = contents
.replace(r#"build = "cp39-*""#, r#"build = "cp310-*""#) .replace(r#"build = "cp39-*""#, r#"build = "cp310-*""#)
.replace(r#"python = ">=3.9""#, r#"python = ">=3.10""#) .replace(r#"python = ">=3.9""#, r#"python = ">=3.10""#)
@ -1028,18 +839,15 @@ pub fn generate_grammar_files(
allow_update, allow_update,
|path| generate_file(path, PACKAGE_SWIFT_TEMPLATE, language_name, &generate_opts), |path| generate_file(path, PACKAGE_SWIFT_TEMPLATE, language_name, &generate_opts),
|path| { |path| {
let contents = fs::read_to_string(path)?; let mut contents = fs::read_to_string(path)?;
let replaced_contents = contents contents = contents
.replace( .replace(
"https://github.com/ChimeHQ/SwiftTreeSitter", "https://github.com/ChimeHQ/SwiftTreeSitter",
"https://github.com/tree-sitter/swift-tree-sitter", "https://github.com/tree-sitter/swift-tree-sitter",
) )
.replace("version: \"0.8.0\")", "version: \"0.9.0\")") .replace("version: \"0.8.0\")", "version: \"0.9.0\")")
.replace("(url:", "(name: \"SwiftTreeSitter\", url:"); .replace("(url:", "(name: \"SwiftTreeSitter\", url:");
if !replaced_contents.eq(&contents) { write_file(path, contents)?;
info!("Updating tree-sitter dependency in Package.swift");
write_file(path, contents)?;
}
Ok(()) Ok(())
}, },
)?; )?;
@ -1057,7 +865,7 @@ pub fn generate_grammar_files(
|path| { |path| {
let contents = fs::read_to_string(path)?; let contents = fs::read_to_string(path)?;
if !contents.contains("b.pkg_hash.len") { if !contents.contains("b.pkg_hash.len") {
info!("Replacing build.zig"); warn!("Replacing build.zig");
generate_file(path, BUILD_ZIG_TEMPLATE, language_name, &generate_opts) generate_file(path, BUILD_ZIG_TEMPLATE, language_name, &generate_opts)
} else { } else {
Ok(()) Ok(())
@ -1072,7 +880,7 @@ pub fn generate_grammar_files(
|path| { |path| {
let contents = fs::read_to_string(path)?; let contents = fs::read_to_string(path)?;
if !contents.contains(".name = .tree_sitter_") { if !contents.contains(".name = .tree_sitter_") {
info!("Replacing build.zig.zon"); warn!("Replacing build.zig.zon");
generate_file(path, BUILD_ZIG_ZON_TEMPLATE, language_name, &generate_opts) generate_file(path, BUILD_ZIG_ZON_TEMPLATE, language_name, &generate_opts)
} else { } else {
Ok(()) Ok(())
@ -1088,7 +896,7 @@ pub fn generate_grammar_files(
|path| { |path| {
let contents = fs::read_to_string(path)?; let contents = fs::read_to_string(path)?;
if contents.contains("ts.Language") { if contents.contains("ts.Language") {
info!("Replacing root.zig"); warn!("Replacing root.zig");
generate_file(path, ROOT_ZIG_TEMPLATE, language_name, &generate_opts) generate_file(path, ROOT_ZIG_TEMPLATE, language_name, &generate_opts)
} else { } else {
Ok(()) Ok(())
@ -1104,45 +912,6 @@ pub fn generate_grammar_files(
})?; })?;
} }
// Generate Java bindings
if tree_sitter_config.bindings.java {
missing_path(repo_path.join("pom.xml"), |path| {
generate_file(path, POM_XML_TEMPLATE, language_name, &generate_opts)
})?;
missing_path(bindings_dir.join("java"), create_dir)?.apply(|path| {
missing_path(path.join("main"), create_dir)?.apply(|path| {
let package_path = generate_opts
.namespace
.unwrap_or("io.github.treesitter")
.replace(['-', '_'], "")
.split('.')
.fold(path.to_path_buf(), |path, dir| path.join(dir))
.join("jtreesitter")
.join(language_name.to_lowercase().replace('_', ""));
missing_path(package_path, create_dir)?.apply(|path| {
missing_path(path.join(format!("{class_name}.java")), |path| {
generate_file(path, BINDING_JAVA_TEMPLATE, language_name, &generate_opts)
})?;
Ok(())
})?;
Ok(())
})?;
missing_path(path.join("test"), create_dir)?.apply(|path| {
missing_path(path.join(format!("{class_name}Test.java")), |path| {
generate_file(path, TEST_JAVA_TEMPLATE, language_name, &generate_opts)
})?;
Ok(())
})?;
Ok(())
})?;
}
Ok(()) Ok(())
} }
@ -1192,15 +961,6 @@ fn generate_file(
) -> Result<()> { ) -> Result<()> {
let filename = path.file_name().unwrap().to_str().unwrap(); let filename = path.file_name().unwrap().to_str().unwrap();
let lower_parser_name = if path
.extension()
.is_some_and(|e| e.eq_ignore_ascii_case("java"))
{
language_name.to_snake_case().replace('_', "")
} else {
language_name.to_snake_case()
};
let mut replacement = template let mut replacement = template
.replace( .replace(
CAMEL_PARSER_NAME_PLACEHOLDER, CAMEL_PARSER_NAME_PLACEHOLDER,
@ -1214,11 +974,14 @@ fn generate_file(
UPPER_PARSER_NAME_PLACEHOLDER, UPPER_PARSER_NAME_PLACEHOLDER,
&language_name.to_shouty_snake_case(), &language_name.to_shouty_snake_case(),
) )
.replace(
LOWER_PARSER_NAME_PLACEHOLDER,
&language_name.to_snake_case(),
)
.replace( .replace(
KEBAB_PARSER_NAME_PLACEHOLDER, KEBAB_PARSER_NAME_PLACEHOLDER,
&language_name.to_kebab_case(), &language_name.to_kebab_case(),
) )
.replace(LOWER_PARSER_NAME_PLACEHOLDER, &lower_parser_name)
.replace(PARSER_NAME_PLACEHOLDER, language_name) .replace(PARSER_NAME_PLACEHOLDER, language_name)
.replace(CLI_VERSION_PLACEHOLDER, CLI_VERSION) .replace(CLI_VERSION_PLACEHOLDER, CLI_VERSION)
.replace(RUST_BINDING_VERSION_PLACEHOLDER, RUST_BINDING_VERSION) .replace(RUST_BINDING_VERSION_PLACEHOLDER, RUST_BINDING_VERSION)
@ -1227,20 +990,7 @@ fn generate_file(
PARSER_VERSION_PLACEHOLDER, PARSER_VERSION_PLACEHOLDER,
&generate_opts.version.to_string(), &generate_opts.version.to_string(),
) )
.replace(PARSER_CLASS_NAME_PLACEHOLDER, generate_opts.class_name) .replace(PARSER_CLASS_NAME_PLACEHOLDER, generate_opts.class_name);
.replace(
HIGHLIGHTS_QUERY_PATH_PLACEHOLDER,
generate_opts.highlights_query_path,
)
.replace(
INJECTIONS_QUERY_PATH_PLACEHOLDER,
generate_opts.injections_query_path,
)
.replace(
LOCALS_QUERY_PATH_PLACEHOLDER,
generate_opts.locals_query_path,
)
.replace(TAGS_QUERY_PATH_PLACEHOLDER, generate_opts.tags_query_path);
if let Some(name) = generate_opts.author_name { if let Some(name) = generate_opts.author_name {
replacement = replacement.replace(AUTHOR_NAME_PLACEHOLDER, name); replacement = replacement.replace(AUTHOR_NAME_PLACEHOLDER, name);
@ -1258,9 +1008,6 @@ fn generate_file(
"Cargo.toml" => { "Cargo.toml" => {
replacement = replacement.replace(AUTHOR_NAME_PLACEHOLDER_RS, ""); replacement = replacement.replace(AUTHOR_NAME_PLACEHOLDER_RS, "");
} }
"pom.xml" => {
replacement = replacement.replace(AUTHOR_NAME_PLACEHOLDER_JAVA, "");
}
_ => {} _ => {}
} }
} }
@ -1286,52 +1033,30 @@ fn generate_file(
"Cargo.toml" => { "Cargo.toml" => {
replacement = replacement.replace(AUTHOR_EMAIL_PLACEHOLDER_RS, ""); replacement = replacement.replace(AUTHOR_EMAIL_PLACEHOLDER_RS, "");
} }
"pom.xml" => {
replacement = replacement.replace(AUTHOR_EMAIL_PLACEHOLDER_JAVA, "");
}
_ => {} _ => {}
} }
} }
match (generate_opts.author_url, filename) { if filename == "package.json" {
(Some(url), "package.json" | "pom.xml") => { if let Some(url) = generate_opts.author_url {
replacement = replacement.replace(AUTHOR_URL_PLACEHOLDER, url); replacement = replacement.replace(AUTHOR_URL_PLACEHOLDER, url);
} } else {
(None, "package.json") => {
replacement = replacement.replace(AUTHOR_URL_PLACEHOLDER_JS, ""); replacement = replacement.replace(AUTHOR_URL_PLACEHOLDER_JS, "");
} }
(None, "pom.xml") => {
replacement = replacement.replace(AUTHOR_URL_PLACEHOLDER_JAVA, "");
}
_ => {}
} }
if generate_opts.author_name.is_none() if generate_opts.author_name.is_none()
&& generate_opts.author_email.is_none() && generate_opts.author_email.is_none()
&& generate_opts.author_url.is_none() && generate_opts.author_url.is_none()
&& filename == "package.json"
{ {
match filename { if let Some(start_idx) = replacement.find(AUTHOR_BLOCK_JS) {
"package.json" => { if let Some(end_idx) = replacement[start_idx..]
if let Some(start_idx) = replacement.find(AUTHOR_BLOCK_JS) { .find("},")
if let Some(end_idx) = replacement[start_idx..] .map(|i| i + start_idx + 2)
.find("},") {
.map(|i| i + start_idx + 2) replacement.replace_range(start_idx..end_idx, "");
{
replacement.replace_range(start_idx..end_idx, "");
}
}
} }
"pom.xml" => {
if let Some(start_idx) = replacement.find(AUTHOR_BLOCK_JAVA) {
if let Some(end_idx) = replacement[start_idx..]
.find("</developer>")
.map(|i| i + start_idx + 12)
{
replacement.replace_range(start_idx..end_idx, "");
}
}
}
_ => {}
} }
} else if generate_opts.author_name.is_none() && generate_opts.author_email.is_none() { } else if generate_opts.author_name.is_none() && generate_opts.author_email.is_none() {
match filename { match filename {
@ -1412,19 +1137,6 @@ fn generate_file(
); );
} }
if let Some(namespace) = generate_opts.namespace {
replacement = replacement
.replace(
PARSER_NS_CLEANED_PLACEHOLDER,
&namespace.replace(['-', '_'], ""),
)
.replace(PARSER_NS_PLACEHOLDER, namespace);
} else {
replacement = replacement
.replace(PARSER_NS_CLEANED_PLACEHOLDER, "io.github.treesitter")
.replace(PARSER_NS_PLACEHOLDER, "io.github.tree-sitter");
}
if let Some(funding_url) = generate_opts.funding { if let Some(funding_url) = generate_opts.funding {
match filename { match filename {
"pyproject.toml" | "package.json" => { "pyproject.toml" | "package.json" => {

View file

@ -20,16 +20,15 @@ use tree_sitter_cli::{
LOG_GRAPH_ENABLED, START_SEED, LOG_GRAPH_ENABLED, START_SEED,
}, },
highlight::{self, HighlightOptions}, highlight::{self, HighlightOptions},
init::{generate_grammar_files, JsonConfigOpts, TREE_SITTER_JSON_SCHEMA}, init::{generate_grammar_files, JsonConfigOpts},
input::{get_input, get_tmp_source_file, CliInput}, input::{get_input, get_tmp_source_file, CliInput},
logger, logger,
parse::{self, ParseDebugType, ParseFileOptions, ParseOutput, ParseTheme}, parse::{self, ParseDebugType, ParseFileOptions, ParseOutput, ParseTheme},
playground, playground, query,
query::{self, QueryFileOptions},
tags::{self, TagsOptions}, tags::{self, TagsOptions},
test::{self, TestOptions, TestStats, TestSummary}, test::{self, TestOptions, TestStats},
test_highlight, test_tags, util, test_highlight, test_tags, util, version,
version::{self, BumpLevel}, version::BumpLevel,
wasm, wasm,
}; };
use tree_sitter_config::Config; use tree_sitter_config::Config;
@ -89,6 +88,17 @@ struct Init {
pub grammar_path: Option<PathBuf>, pub grammar_path: Option<PathBuf>,
} }
#[derive(Clone, Debug, Default, ValueEnum, PartialEq, Eq)]
enum GenerationEmit {
/// Generate `grammar.json` and `node-types.json`
Json,
/// Generate `parser.c` and related files
#[default]
Parser,
/// Compile to a library
Lib,
}
#[derive(Args)] #[derive(Args)]
#[command(alias = "gen", alias = "g")] #[command(alias = "gen", alias = "g")]
struct Generate { struct Generate {
@ -111,38 +121,28 @@ struct Generate {
) )
)] )]
pub abi_version: Option<String>, pub abi_version: Option<String>,
/// Only generate `grammar.json` and `node-types.json` /// What generated files to emit
#[arg(long)] #[arg(long)]
pub no_parser: bool, #[clap(value_enum, default_value_t=GenerationEmit::Parser)]
/// Deprecated: use the `build` command pub emit: GenerationEmit,
#[arg(long, short = 'b')] /// Deprecated: use --emit=lib.
#[arg(long, short = 'b', conflicts_with = "emit")]
pub build: bool, pub build: bool,
/// Deprecated: use the `build` command /// Compile a parser in debug mode
#[arg(long, short = '0')] #[arg(long, short = '0')]
pub debug_build: bool, pub debug_build: bool,
/// Deprecated: use the `build` command /// The path to the directory containing the parser library
#[arg(long, value_name = "PATH")] #[arg(long, value_name = "PATH")]
pub libdir: Option<PathBuf>, pub libdir: Option<PathBuf>,
/// The path to output the generated source files /// The path to output the generated source files
#[arg(long, short, value_name = "DIRECTORY")] #[arg(long, short, value_name = "DIRECTORY")]
pub output: Option<PathBuf>, pub output: Option<PathBuf>,
/// Produce a report of the states for the given rule, use `-` to report every rule /// Produce a report of the states for the given rule, use `-` to report every rule
#[arg(long, conflicts_with = "json", conflicts_with = "json_summary")] #[arg(long)]
pub report_states_for_rule: Option<String>, pub report_states_for_rule: Option<String>,
/// Deprecated: use --json-summary
#[arg(
long,
conflicts_with = "json_summary",
conflicts_with = "report_states_for_rule"
)]
pub json: bool,
/// Report conflicts in a JSON format /// Report conflicts in a JSON format
#[arg( #[arg(long)]
long, pub json: bool,
conflicts_with = "json",
conflicts_with = "report_states_for_rule"
)]
pub json_summary: bool,
/// The name or path of the JavaScript runtime to use for generating parsers /// The name or path of the JavaScript runtime to use for generating parsers
#[cfg(not(feature = "qjs-rt"))] #[cfg(not(feature = "qjs-rt"))]
#[arg( #[arg(
@ -223,7 +223,7 @@ struct Parse {
#[arg(long, short = 'D')] #[arg(long, short = 'D')]
pub debug_graph: bool, pub debug_graph: bool,
/// Compile parsers to Wasm instead of native dynamic libraries /// Compile parsers to Wasm instead of native dynamic libraries
#[arg(long, hide = cfg!(not(feature = "wasm")))] #[arg(long)]
pub wasm: bool, pub wasm: bool,
/// Output the parse data with graphviz dot /// Output the parse data with graphviz dot
#[arg(long = "dot")] #[arg(long = "dot")]
@ -235,7 +235,7 @@ struct Parse {
#[arg(long = "cst", short = 'c')] #[arg(long = "cst", short = 'c')]
pub output_cst: bool, pub output_cst: bool,
/// Show parsing statistic /// Show parsing statistic
#[arg(long, short, conflicts_with = "json", conflicts_with = "json_summary")] #[arg(long, short)]
pub stat: bool, pub stat: bool,
/// Interrupt the parsing process by timeout (µs) /// Interrupt the parsing process by timeout (µs)
#[arg(long)] #[arg(long)]
@ -260,12 +260,9 @@ struct Parse {
/// Open `log.html` in the default browser, if `--debug-graph` is supplied /// Open `log.html` in the default browser, if `--debug-graph` is supplied
#[arg(long)] #[arg(long)]
pub open_log: bool, pub open_log: bool,
/// Deprecated: use --json-summary
#[arg(long, conflicts_with = "json_summary", conflicts_with = "stat")]
pub json: bool,
/// Output parsing results in a JSON format /// Output parsing results in a JSON format
#[arg(long, short = 'j', conflicts_with = "json", conflicts_with = "stat")] #[arg(long, short = 'j')]
pub json_summary: bool, pub json: bool,
/// The path to an alternative config.json file /// The path to an alternative config.json file
#[arg(long)] #[arg(long)]
pub config_path: Option<PathBuf>, pub config_path: Option<PathBuf>,
@ -323,7 +320,7 @@ struct Test {
#[arg(long, short = 'D')] #[arg(long, short = 'D')]
pub debug_graph: bool, pub debug_graph: bool,
/// Compile parsers to Wasm instead of native dynamic libraries /// Compile parsers to Wasm instead of native dynamic libraries
#[arg(long, hide = cfg!(not(feature = "wasm")))] #[arg(long)]
pub wasm: bool, pub wasm: bool,
/// Open `log.html` in the default browser, if `--debug-graph` is supplied /// Open `log.html` in the default browser, if `--debug-graph` is supplied
#[arg(long)] #[arg(long)]
@ -343,9 +340,6 @@ struct Test {
/// Show only the pass-fail overview tree /// Show only the pass-fail overview tree
#[arg(long)] #[arg(long)]
pub overview_only: bool, pub overview_only: bool,
/// Output the test summary in a JSON format
#[arg(long)]
pub json_summary: bool,
} }
#[derive(Args)] #[derive(Args)]
@ -448,14 +442,6 @@ struct Query {
/// The range of rows in which the query will be executed /// The range of rows in which the query will be executed
#[arg(long)] #[arg(long)]
pub row_range: Option<String>, pub row_range: Option<String>,
/// The range of byte offsets in which the query will be executed. Only the matches that are fully contained within the provided
/// byte range will be returned.
#[arg(long)]
pub containing_byte_range: Option<String>,
/// The range of rows in which the query will be executed. Only the matches that are fully contained within the provided row range
/// will be returned.
#[arg(long)]
pub containing_row_range: Option<String>,
/// Select a language by the scope instead of a file extension /// Select a language by the scope instead of a file extension
#[arg(long)] #[arg(long)]
pub scope: Option<String>, pub scope: Option<String>,
@ -597,20 +583,6 @@ pub enum Shell {
Nushell, Nushell,
} }
/// Complete `action` if the wasm feature is enabled, otherwise return an error
macro_rules! checked_wasm {
($action:block) => {
#[cfg(feature = "wasm")]
{
$action
}
#[cfg(not(feature = "wasm"))]
{
Err(anyhow!("--wasm flag specified, but this build of tree-sitter-cli does not include the wasm feature"))?;
}
};
}
impl InitConfig { impl InitConfig {
fn run() -> Result<()> { fn run() -> Result<()> {
if let Ok(Some(config_path)) = Config::find_config_file() { if let Ok(Some(config_path)) = Config::find_config_file() {
@ -772,14 +744,6 @@ impl Init {
.map(|e| Some(e.trim().to_string())) .map(|e| Some(e.trim().to_string()))
}; };
let namespace = || {
Input::<String>::with_theme(&ColorfulTheme::default())
.with_prompt("Package namespace")
.default("io.github.tree-sitter".to_string())
.allow_empty(true)
.interact()
};
let bindings = || { let bindings = || {
let languages = Bindings::default().languages(); let languages = Bindings::default().languages();
@ -809,7 +773,6 @@ impl Init {
"author", "author",
"email", "email",
"url", "url",
"namespace",
"bindings", "bindings",
"exit", "exit",
]; ];
@ -830,7 +793,6 @@ impl Init {
"author" => opts.author = author()?, "author" => opts.author = author()?,
"email" => opts.email = email()?, "email" => opts.email = email()?,
"url" => opts.url = url()?, "url" => opts.url = url()?,
"namespace" => opts.namespace = Some(namespace()?),
"bindings" => opts.bindings = bindings()?, "bindings" => opts.bindings = bindings()?,
"exit" => break, "exit" => break,
_ => unreachable!(), _ => unreachable!(),
@ -867,26 +829,10 @@ impl Init {
(opts.name.clone(), Some(opts)) (opts.name.clone(), Some(opts))
} else { } else {
let old_config = fs::read_to_string(current_dir.join("tree-sitter.json")) let mut json = serde_json::from_str::<TreeSitterJSON>(
.with_context(|| "Failed to read tree-sitter.json")?; &fs::read_to_string(current_dir.join("tree-sitter.json"))
.with_context(|| "Failed to read tree-sitter.json")?,
let mut json = serde_json::from_str::<TreeSitterJSON>(&old_config)?; )?;
if json.schema.is_none() {
json.schema = Some(TREE_SITTER_JSON_SCHEMA.to_string());
}
let new_config = format!("{}\n", serde_json::to_string_pretty(&json)?);
// Write the re-serialized config back, as newly added optional boolean fields
// will be included with explicit `false`s rather than implict `null`s
if self.update && !old_config.trim().eq(new_config.trim()) {
info!("Updating tree-sitter.json");
fs::write(
current_dir.join("tree-sitter.json"),
serde_json::to_string_pretty(&json)?,
)
.with_context(|| "Failed to write tree-sitter.json")?;
}
(json.grammars.swap_remove(0).name, None) (json.grammars.swap_remove(0).name, None)
}; };
@ -916,13 +862,9 @@ impl Generate {
version.parse().expect("invalid abi version flag") version.parse().expect("invalid abi version flag")
} }
}); });
if self.build {
let json_summary = if self.json { warn!("--build is deprecated, use --emit=lib instead");
warn!("--json is deprecated, use --json-summary instead"); }
true
} else {
self.json_summary
};
if let Err(err) = tree_sitter_generate::generate_parser_in_directory( if let Err(err) = tree_sitter_generate::generate_parser_in_directory(
current_dir, current_dir,
@ -931,14 +873,14 @@ impl Generate {
abi_version, abi_version,
self.report_states_for_rule.as_deref(), self.report_states_for_rule.as_deref(),
self.js_runtime.as_deref(), self.js_runtime.as_deref(),
!self.no_parser, self.emit != GenerationEmit::Json,
if self.disable_optimizations { if self.disable_optimizations {
OptLevel::empty() OptLevel::empty()
} else { } else {
OptLevel::default() OptLevel::default()
}, },
) { ) {
if json_summary { if self.json {
eprintln!("{}", serde_json::to_string_pretty(&err)?); eprintln!("{}", serde_json::to_string_pretty(&err)?);
// Exit early to prevent errors from being printed a second time in the caller // Exit early to prevent errors from being printed a second time in the caller
std::process::exit(1); std::process::exit(1);
@ -947,8 +889,7 @@ impl Generate {
Err(anyhow!(err.to_string())).with_context(|| "Error when generating parser")?; Err(anyhow!(err.to_string())).with_context(|| "Error when generating parser")?;
} }
} }
if self.build { if self.emit == GenerationEmit::Lib || self.build {
warn!("--build is deprecated, use the `build` command");
if let Some(path) = self.libdir { if let Some(path) = self.libdir {
loader = loader::Loader::with_parser_lib_path(path); loader = loader::Loader::with_parser_lib_path(path);
} }
@ -971,21 +912,11 @@ impl Build {
} else { } else {
let output_path = if let Some(ref path) = self.output { let output_path = if let Some(ref path) = self.output {
let path = Path::new(path); let path = Path::new(path);
let full_path = if path.is_absolute() { if path.is_absolute() {
path.to_path_buf() path.to_path_buf()
} else { } else {
current_dir.join(path) current_dir.join(path)
}; }
let parent_path = full_path
.parent()
.context("Output path must have a parent")?;
let name = full_path
.file_name()
.context("Ouput path must have a filename")?;
fs::create_dir_all(parent_path).context("Failed to create output path")?;
let mut canon_path = parent_path.canonicalize().context("Invalid output path")?;
canon_path.push(name);
canon_path
} else { } else {
let file_name = grammar_path let file_name = grammar_path
.file_stem() .file_stem()
@ -1008,9 +939,12 @@ impl Build {
loader.force_rebuild(true); loader.force_rebuild(true);
let config = Config::load(None)?;
let loader_config = config.get()?;
loader.find_all_languages(&loader_config).unwrap();
loader loader
.compile_parser_at_path(&grammar_path, output_path, flags) .compile_parser_at_path(&grammar_path, output_path, flags)
.context("Failed to compile parser")?; .unwrap();
} }
Ok(()) Ok(())
} }
@ -1020,19 +954,13 @@ impl Parse {
fn run(self, mut loader: loader::Loader, current_dir: &Path) -> Result<()> { fn run(self, mut loader: loader::Loader, current_dir: &Path) -> Result<()> {
let config = Config::load(self.config_path)?; let config = Config::load(self.config_path)?;
let color = env::var("NO_COLOR").map_or(true, |v| v != "1"); let color = env::var("NO_COLOR").map_or(true, |v| v != "1");
let json_summary = if self.json {
warn!("--json is deprecated, use --json-summary instead");
true
} else {
self.json_summary
};
let output = if self.output_dot { let output = if self.output_dot {
ParseOutput::Dot ParseOutput::Dot
} else if self.output_xml { } else if self.output_xml {
ParseOutput::Xml ParseOutput::Xml
} else if self.output_cst { } else if self.output_cst {
ParseOutput::Cst ParseOutput::Cst
} else if self.quiet || json_summary { } else if self.quiet || self.json {
ParseOutput::Quiet ParseOutput::Quiet
} else { } else {
ParseOutput::Normal ParseOutput::Normal
@ -1063,14 +991,13 @@ impl Parse {
loader.debug_build(self.debug_build); loader.debug_build(self.debug_build);
loader.force_rebuild(self.rebuild || self.grammar_path.is_some()); loader.force_rebuild(self.rebuild || self.grammar_path.is_some());
#[cfg(feature = "wasm")]
if self.wasm { if self.wasm {
checked_wasm!({ let engine = tree_sitter::wasmtime::Engine::default();
let engine = tree_sitter::wasmtime::Engine::default(); parser
parser .set_wasm_store(tree_sitter::WasmStore::new(&engine).unwrap())
.set_wasm_store(tree_sitter::WasmStore::new(&engine).unwrap()) .unwrap();
.unwrap(); loader.use_wasm(&engine);
loader.use_wasm(&engine);
});
} }
let timeout = self.timeout.unwrap_or_default(); let timeout = self.timeout.unwrap_or_default();
@ -1107,7 +1034,7 @@ impl Parse {
let mut update_stats = |stats: &mut parse::ParseStats| { let mut update_stats = |stats: &mut parse::ParseStats| {
let parse_result = stats.parse_summaries.last().unwrap(); let parse_result = stats.parse_summaries.last().unwrap();
if should_track_stats || json_summary { if should_track_stats {
stats.cumulative_stats.total_parses += 1; stats.cumulative_stats.total_parses += 1;
if parse_result.successful { if parse_result.successful {
stats.cumulative_stats.successful_parses += 1; stats.cumulative_stats.successful_parses += 1;
@ -1145,13 +1072,13 @@ impl Parse {
let path = Path::new(&path); let path = Path::new(&path);
let language = loader let language = loader
.select_language( .select_language(
Some(path), path,
current_dir, current_dir,
self.scope.as_deref(), self.scope.as_deref(),
lib_info.as_ref(), lib_info.as_ref(),
) )
.with_context(|| { .with_context(|| {
anyhow!("Failed to load language for path \"{}\"", path.display()) anyhow!("Failed to load langauge for path \"{}\"", path.display())
})?; })?;
parse::parse_file_at_path( parse::parse_file_at_path(
@ -1176,12 +1103,7 @@ impl Parse {
let language = if let Some(ref lib_path) = self.lib_path { let language = if let Some(ref lib_path) = self.lib_path {
&loader &loader
.select_language( .select_language(lib_path, current_dir, None, lib_info.as_ref())
None,
current_dir,
self.scope.as_deref(),
lib_info.as_ref(),
)
.with_context(|| { .with_context(|| {
anyhow!( anyhow!(
"Failed to load language for path \"{}\"", "Failed to load language for path \"{}\"",
@ -1215,12 +1137,8 @@ impl Parse {
let path = get_tmp_source_file(&contents)?; let path = get_tmp_source_file(&contents)?;
let name = "stdin"; let name = "stdin";
let language = loader.select_language( let language =
None, loader.select_language(&path, current_dir, None, lib_info.as_ref())?;
current_dir,
self.scope.as_deref(),
lib_info.as_ref(),
)?;
parse::parse_file_at_path( parse::parse_file_at_path(
&mut parser, &mut parser,
@ -1238,7 +1156,7 @@ impl Parse {
if should_track_stats { if should_track_stats {
println!("\n{}", stats.cumulative_stats); println!("\n{}", stats.cumulative_stats);
} }
if json_summary { if self.json {
println!("{}", serde_json::to_string_pretty(&stats)?); println!("{}", serde_json::to_string_pretty(&stats)?);
} }
@ -1250,28 +1168,6 @@ impl Parse {
} }
} }
/// In case an error is encountered, prints out the contents of `test_summary` and
/// propagates the error
fn check_test(
test_result: Result<()>,
test_summary: &TestSummary,
json_summary: bool,
) -> Result<()> {
if let Err(e) = test_result {
if json_summary {
let json_summary = serde_json::to_string_pretty(test_summary)
.expect("Failed to encode summary to JSON");
println!("{json_summary}");
} else {
println!("{test_summary}");
}
Err(e)?;
}
Ok(())
}
impl Test { impl Test {
fn run(self, mut loader: loader::Loader, current_dir: &Path) -> Result<()> { fn run(self, mut loader: loader::Loader, current_dir: &Path) -> Result<()> {
let config = Config::load(self.config_path)?; let config = Config::load(self.config_path)?;
@ -1283,14 +1179,13 @@ impl Test {
let mut parser = Parser::new(); let mut parser = Parser::new();
#[cfg(feature = "wasm")]
if self.wasm { if self.wasm {
checked_wasm!({ let engine = tree_sitter::wasmtime::Engine::default();
let engine = tree_sitter::wasmtime::Engine::default(); parser
parser .set_wasm_store(tree_sitter::WasmStore::new(&engine).unwrap())
.set_wasm_store(tree_sitter::WasmStore::new(&engine).unwrap()) .unwrap();
.unwrap(); loader.use_wasm(&engine);
loader.use_wasm(&engine);
});
} }
if self.lib_path.is_none() && self.lang_name.is_some() { if self.lib_path.is_none() && self.lang_name.is_some() {
@ -1301,7 +1196,7 @@ impl Test {
let lib_info = let lib_info =
get_lib_info(self.lib_path.as_ref(), self.lang_name.as_ref(), current_dir); get_lib_info(self.lib_path.as_ref(), self.lang_name.as_ref(), current_dir);
&loader &loader
.select_language(None, current_dir, None, lib_info.as_ref()) .select_language(lib_path, current_dir, None, lib_info.as_ref())
.with_context(|| { .with_context(|| {
anyhow!( anyhow!(
"Failed to load language for path \"{}\"", "Failed to load language for path \"{}\"",
@ -1317,18 +1212,15 @@ impl Test {
parser.set_language(language)?; parser.set_language(language)?;
let test_dir = current_dir.join("test"); let test_dir = current_dir.join("test");
let mut test_summary = TestSummary::new( let mut stats = parse::Stats::default();
color,
stat,
self.update,
self.overview_only,
self.json_summary,
);
// Run the corpus tests. Look for them in `test/corpus`. // Run the corpus tests. Look for them in `test/corpus`.
let test_corpus_dir = test_dir.join("corpus"); let test_corpus_dir = test_dir.join("corpus");
if test_corpus_dir.is_dir() { if test_corpus_dir.is_dir() {
let opts = TestOptions { let mut output = String::new();
let mut rates = Vec::new();
let mut opts = TestOptions {
output: &mut output,
path: test_corpus_dir, path: test_corpus_dir,
debug: self.debug, debug: self.debug,
debug_graph: self.debug_graph, debug_graph: self.debug_graph,
@ -1339,67 +1231,51 @@ impl Test {
open_log: self.open_log, open_log: self.open_log,
languages: languages.iter().map(|(l, n)| (n.as_str(), l)).collect(), languages: languages.iter().map(|(l, n)| (n.as_str(), l)).collect(),
color, color,
test_num: 1,
parse_rates: &mut rates,
stat_display: stat,
stats: &mut stats,
show_fields: self.show_fields, show_fields: self.show_fields,
overview_only: self.overview_only, overview_only: self.overview_only,
}; };
check_test( test::run_tests_at_path(&mut parser, &mut opts)?;
test::run_tests_at_path(&mut parser, &opts, &mut test_summary), println!("\n{stats}");
&test_summary,
self.json_summary,
)?;
test_summary.test_num = 1;
} }
// Check that all of the queries are valid. // Check that all of the queries are valid.
let query_dir = current_dir.join("queries"); test::check_queries_at_path(language, &current_dir.join("queries"))?;
check_test(
test::check_queries_at_path(language, &query_dir),
&test_summary,
self.json_summary,
)?;
test_summary.test_num = 1;
// Run the syntax highlighting tests. // Run the syntax highlighting tests.
let test_highlight_dir = test_dir.join("highlight"); let test_highlight_dir = test_dir.join("highlight");
if test_highlight_dir.is_dir() { if test_highlight_dir.is_dir() {
let mut highlighter = Highlighter::new(); let mut highlighter = Highlighter::new();
highlighter.parser = parser; highlighter.parser = parser;
check_test( test_highlight::test_highlights(
test_highlight::test_highlights( &loader,
&loader, &config.get()?,
&config.get()?, &mut highlighter,
&mut highlighter, &test_highlight_dir,
&test_highlight_dir, color,
&mut test_summary,
),
&test_summary,
self.json_summary,
)?; )?;
parser = highlighter.parser; parser = highlighter.parser;
test_summary.test_num = 1;
} }
let test_tag_dir = test_dir.join("tags"); let test_tag_dir = test_dir.join("tags");
if test_tag_dir.is_dir() { if test_tag_dir.is_dir() {
let mut tags_context = TagsContext::new(); let mut tags_context = TagsContext::new();
tags_context.parser = parser; tags_context.parser = parser;
check_test( test_tags::test_tags(
test_tags::test_tags( &loader,
&loader, &config.get()?,
&config.get()?, &mut tags_context,
&mut tags_context, &test_tag_dir,
&test_tag_dir, color,
&mut test_summary,
),
&test_summary,
self.json_summary,
)?; )?;
test_summary.test_num = 1;
} }
// For the rest of the queries, find their tests and run them // For the rest of the queries, find their tests and run them
for entry in walkdir::WalkDir::new(&query_dir) for entry in walkdir::WalkDir::new(current_dir.join("queries"))
.into_iter() .into_iter()
.filter_map(|e| e.ok()) .filter_map(|e| e.ok())
.filter(|e| e.file_type().is_file()) .filter(|e| e.file_type().is_file())
@ -1422,48 +1298,34 @@ impl Test {
}) })
.collect::<Vec<_>>(); .collect::<Vec<_>>();
if !entries.is_empty() { if !entries.is_empty() {
test_summary.query_results.add_group(stem); println!("{stem}:");
} }
test_summary.test_num = 1; for entry in entries {
let opts = QueryFileOptions::default();
for entry in &entries {
let path = entry.path(); let path = entry.path();
check_test( query::query_file_at_path(
query::query_file_at_path( language,
language, path,
path, &path.display().to_string(),
&path.display().to_string(), path,
path, false,
&opts, None,
Some(&mut test_summary), None,
), true,
&test_summary, false,
self.json_summary, false,
false,
)?; )?;
} }
if !entries.is_empty() {
test_summary.query_results.pop_traversal();
}
} }
} }
test_summary.test_num = 1;
if self.json_summary {
let json_summary = serde_json::to_string_pretty(&test_summary)
.expect("Failed to encode test summary to JSON");
println!("{json_summary}");
} else {
println!("{test_summary}");
}
Ok(()) Ok(())
} }
} }
impl Version { impl Version {
fn run(self, current_dir: PathBuf) -> Result<()> { fn run(self, current_dir: PathBuf) -> Result<()> {
Ok(version::Version::new(self.version, current_dir, self.bump).run()?) version::Version::new(self.version, current_dir, self.bump).run()
} }
} }
@ -1482,7 +1344,7 @@ impl Fuzz {
let lang_name = lib_info.1.to_string(); let lang_name = lib_info.1.to_string();
&( &(
loader loader
.select_language(None, current_dir, None, Some(&lib_info)) .select_language(lib_path, current_dir, None, Some(&lib_info))
.with_context(|| { .with_context(|| {
anyhow!( anyhow!(
"Failed to load language for path \"{}\"", "Failed to load language for path \"{}\"",
@ -1527,11 +1389,18 @@ impl Query {
loader.find_all_languages(&loader_config)?; loader.find_all_languages(&loader_config)?;
let query_path = Path::new(&self.query_path); let query_path = Path::new(&self.query_path);
let byte_range = parse_range(&self.byte_range, |x| x)?; let byte_range = self.byte_range.as_ref().and_then(|range| {
let point_range = parse_range(&self.row_range, |row| Point::new(row, 0))?; let mut parts = range.split(':');
let containing_byte_range = parse_range(&self.containing_byte_range, |x| x)?; let start = parts.next()?.parse().ok()?;
let containing_point_range = let end = parts.next().unwrap().parse().ok()?;
parse_range(&self.containing_row_range, |row| Point::new(row, 0))?; Some(start..end)
});
let point_range = self.row_range.as_ref().and_then(|range| {
let mut parts = range.split(':');
let start = parts.next()?.parse().ok()?;
let end = parts.next().unwrap().parse().ok()?;
Some(Point::new(start, 0)..Point::new(end, 0))
});
let cancellation_flag = util::cancel_on_signal(); let cancellation_flag = util::cancel_on_signal();
@ -1550,30 +1419,25 @@ impl Query {
match input { match input {
CliInput::Paths(paths) => { CliInput::Paths(paths) => {
let language = loader.select_language( let language = loader.select_language(
Some(Path::new(&paths[0])), Path::new(&paths[0]),
current_dir, current_dir,
self.scope.as_deref(), self.scope.as_deref(),
lib_info.as_ref(), lib_info.as_ref(),
)?; )?;
let opts = QueryFileOptions {
ordered_captures: self.captures,
byte_range,
point_range,
containing_byte_range,
containing_point_range,
quiet: self.quiet,
print_time: self.time,
stdin: false,
};
for path in paths { for path in paths {
query::query_file_at_path( query::query_file_at_path(
&language, &language,
&path, &path,
&path.display().to_string(), &path.display().to_string(),
query_path, query_path,
&opts, self.captures,
None, byte_range.clone(),
point_range.clone(),
self.test,
self.quiet,
self.time,
false,
)?; )?;
} }
} }
@ -1586,7 +1450,7 @@ impl Query {
let languages = loader.languages_at_path(current_dir)?; let languages = loader.languages_at_path(current_dir)?;
let language = if let Some(ref lib_path) = self.lib_path { let language = if let Some(ref lib_path) = self.lib_path {
&loader &loader
.select_language(None, current_dir, None, lib_info.as_ref()) .select_language(lib_path, current_dir, None, lib_info.as_ref())
.with_context(|| { .with_context(|| {
anyhow!( anyhow!(
"Failed to load language for path \"{}\"", "Failed to load language for path \"{}\"",
@ -1601,17 +1465,19 @@ impl Query {
.map(|(l, _)| l.clone()) .map(|(l, _)| l.clone())
.ok_or_else(|| anyhow!("No language found"))? .ok_or_else(|| anyhow!("No language found"))?
}; };
let opts = QueryFileOptions { query::query_file_at_path(
ordered_captures: self.captures, language,
&path,
&name,
query_path,
self.captures,
byte_range, byte_range,
point_range, point_range,
containing_byte_range, self.test,
containing_point_range, self.quiet,
quiet: self.quiet, self.time,
print_time: self.time, true,
stdin: true, )?;
};
query::query_file_at_path(language, &path, &name, query_path, &opts, None)?;
fs::remove_file(path)?; fs::remove_file(path)?;
} }
CliInput::Stdin(contents) => { CliInput::Stdin(contents) => {
@ -1620,18 +1486,20 @@ impl Query {
let path = get_tmp_source_file(&contents)?; let path = get_tmp_source_file(&contents)?;
let language = let language =
loader.select_language(None, current_dir, None, lib_info.as_ref())?; loader.select_language(&path, current_dir, None, lib_info.as_ref())?;
let opts = QueryFileOptions { query::query_file_at_path(
ordered_captures: self.captures, &language,
&path,
"stdin",
query_path,
self.captures,
byte_range, byte_range,
point_range, point_range,
containing_byte_range, self.test,
containing_point_range, self.quiet,
quiet: self.quiet, self.time,
print_time: self.time, true,
stdin: true, )?;
};
query::query_file_at_path(&language, &path, "stdin", query_path, &opts, None)?;
fs::remove_file(path)?; fs::remove_file(path)?;
} }
} }
@ -1648,7 +1516,6 @@ impl Highlight {
let loader_config = config.get()?; let loader_config = config.get()?;
loader.find_all_languages(&loader_config)?; loader.find_all_languages(&loader_config)?;
loader.force_rebuild(self.rebuild || self.grammar_path.is_some()); loader.force_rebuild(self.rebuild || self.grammar_path.is_some());
let languages = loader.languages_at_path(current_dir)?;
let cancellation_flag = util::cancel_on_signal(); let cancellation_flag = util::cancel_on_signal();
@ -1729,6 +1596,7 @@ impl Highlight {
} => { } => {
let path = get_tmp_source_file(&contents)?; let path = get_tmp_source_file(&contents)?;
let languages = loader.languages_at_path(current_dir)?;
let language = languages let language = languages
.iter() .iter()
.find(|(_, n)| language_names.contains(&Box::from(n.as_str()))) .find(|(_, n)| language_names.contains(&Box::from(n.as_str())))
@ -1759,6 +1627,7 @@ impl Highlight {
if let (Some(l), Some(lc)) = (language.clone(), language_configuration) { if let (Some(l), Some(lc)) = (language.clone(), language_configuration) {
(l, lc) (l, lc)
} else { } else {
let languages = loader.languages_at_path(current_dir)?;
let language = languages let language = languages
.first() .first()
.map(|(l, _)| l.clone()) .map(|(l, _)| l.clone())
@ -2131,32 +2000,3 @@ fn get_lib_info<'a>(
None None
} }
} }
/// Parse a range string of the form "start:end" into an optional Range<T>.
fn parse_range<T>(
range_str: &Option<String>,
make: impl Fn(usize) -> T,
) -> Result<Option<std::ops::Range<T>>> {
if let Some(range) = range_str.as_ref() {
let err_msg = format!("Invalid range '{range}', expected 'start:end'");
let mut parts = range.split(':');
let Some(part) = parts.next() else {
Err(anyhow!(err_msg))?
};
let Ok(start) = part.parse::<usize>() else {
Err(anyhow!(err_msg))?
};
let Some(part) = parts.next() else {
Err(anyhow!(err_msg))?
};
let Ok(end) = part.parse::<usize>() else {
Err(anyhow!(err_msg))?
};
Ok(Some(make(start)..make(end)))
} else {
Ok(None)
}
}

View file

@ -11,7 +11,6 @@ use anstyle::{AnsiColor, Color, RgbColor};
use anyhow::{anyhow, Context, Result}; use anyhow::{anyhow, Context, Result};
use clap::ValueEnum; use clap::ValueEnum;
use log::info; use log::info;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use tree_sitter::{ use tree_sitter::{
ffi, InputEdit, Language, LogType, ParseOptions, ParseState, Parser, Point, Range, Tree, ffi, InputEdit, Language, LogType, ParseOptions, ParseState, Parser, Point, Range, Tree,
@ -20,7 +19,7 @@ use tree_sitter::{
use crate::{fuzz::edits::Edit, logger::paint, util}; use crate::{fuzz::edits::Edit, logger::paint, util};
#[derive(Debug, Default, Serialize, JsonSchema)] #[derive(Debug, Default, Serialize)]
pub struct Stats { pub struct Stats {
pub successful_parses: usize, pub successful_parses: usize,
pub total_parses: usize, pub total_parses: usize,
@ -231,23 +230,13 @@ impl ParseSummary {
} }
} }
#[derive(Serialize, Debug)] #[derive(Serialize, Debug, Default)]
pub struct ParseStats { pub struct ParseStats {
pub parse_summaries: Vec<ParseSummary>, pub parse_summaries: Vec<ParseSummary>,
pub cumulative_stats: Stats, pub cumulative_stats: Stats,
pub source_count: usize, pub source_count: usize,
} }
impl Default for ParseStats {
fn default() -> Self {
Self {
parse_summaries: Vec::new(),
cumulative_stats: Stats::default(),
source_count: 1,
}
}
}
#[derive(Serialize, ValueEnum, Debug, Copy, Clone, Default, Eq, PartialEq)] #[derive(Serialize, ValueEnum, Debug, Copy, Clone, Default, Eq, PartialEq)]
pub enum ParseDebugType { pub enum ParseDebugType {
#[default] #[default]
@ -515,6 +504,7 @@ pub fn parse_file_at_path(
if opts.output == ParseOutput::Cst { if opts.output == ParseOutput::Cst {
render_cst(&source_code, &tree, &mut cursor, opts, &mut stdout)?; render_cst(&source_code, &tree, &mut cursor, opts, &mut stdout)?;
println!();
} }
if opts.output == ParseOutput::Xml { if opts.output == ParseOutput::Xml {
@ -674,9 +664,10 @@ pub fn parse_file_at_path(
width = max_path_length width = max_path_length
)?; )?;
if let Some(node) = first_error { if let Some(node) = first_error {
let node_kind = node.kind(); let start = node.start_position();
let mut node_text = String::with_capacity(node_kind.len()); let end = node.end_position();
for c in node_kind.chars() { let mut node_text = String::new();
for c in node.kind().chars() {
if let Some(escaped) = escape_invisible(c) { if let Some(escaped) = escape_invisible(c) {
node_text += escaped; node_text += escaped;
} else { } else {
@ -693,9 +684,6 @@ pub fn parse_file_at_path(
} else { } else {
write!(&mut stdout, "{node_text}")?; write!(&mut stdout, "{node_text}")?;
} }
let start = node.start_position();
let end = node.end_position();
write!( write!(
&mut stdout, &mut stdout,
" [{}, {}] - [{}, {}])", " [{}, {}] - [{}, {}])",
@ -784,7 +772,7 @@ pub fn render_cst<'a, 'b: 'a>(
.map(|(row, col)| (row as f64).log10() as usize + (col.len() as f64).log10() as usize + 1) .map(|(row, col)| (row as f64).log10() as usize + (col.len() as f64).log10() as usize + 1)
.max() .max()
.unwrap_or(1); .unwrap_or(1);
let mut indent_level = usize::from(!opts.no_ranges); let mut indent_level = 1;
let mut did_visit_children = false; let mut did_visit_children = false;
let mut in_error = false; let mut in_error = false;
loop { loop {
@ -882,24 +870,35 @@ fn write_node_text(
0 0
}; };
let formatted_line = render_line_feed(line, opts); let formatted_line = render_line_feed(line, opts);
write!( if !opts.no_ranges {
out, write!(
"{}{}{}{}{}{}", out,
if multiline { "\n" } else { " " }, "{}{}{}{}{}{}",
if multiline && !opts.no_ranges { if multiline { "\n" } else { "" },
render_node_range(opts, cursor, is_named, true, total_width, node_range) if multiline {
} else { render_node_range(opts, cursor, is_named, true, total_width, node_range)
String::new() } else {
}, String::new()
if multiline { },
" ".repeat(indent_level + 1) if multiline {
} else { " ".repeat(indent_level + 1)
String::new() } else {
}, String::new()
paint(quote_color, &String::from(quote)), },
paint(color, &render_node_text(&formatted_line)), paint(quote_color, &String::from(quote)),
paint(quote_color, &String::from(quote)), &paint(color, &render_node_text(&formatted_line)),
)?; paint(quote_color, &String::from(quote)),
)?;
} else {
write!(
out,
"\n{}{}{}{}",
" ".repeat(indent_level + 1),
paint(quote_color, &String::from(quote)),
&paint(color, &render_node_text(&formatted_line)),
paint(quote_color, &String::from(quote)),
)?;
}
} }
} }
@ -953,7 +952,7 @@ fn render_node_range(
fn cst_render_node( fn cst_render_node(
opts: &ParseFileOptions, opts: &ParseFileOptions,
cursor: &TreeCursor, cursor: &mut TreeCursor,
source_code: &[u8], source_code: &[u8],
out: &mut impl Write, out: &mut impl Write,
total_width: usize, total_width: usize,
@ -999,9 +998,10 @@ fn cst_render_node(
} else { } else {
opts.parse_theme.node_kind opts.parse_theme.node_kind
}; };
write!(out, "{}", paint(kind_color, node.kind()))?; write!(out, "{}", paint(kind_color, node.kind()),)?;
if node.child_count() == 0 { if node.child_count() == 0 {
write!(out, " ")?;
// Node text from a pattern or external scanner // Node text from a pattern or external scanner
write_node_text( write_node_text(
opts, opts,

View file

@ -19,7 +19,6 @@
--light-scrollbar-track: #f1f1f1; --light-scrollbar-track: #f1f1f1;
--light-scrollbar-thumb: #c1c1c1; --light-scrollbar-thumb: #c1c1c1;
--light-scrollbar-thumb-hover: #a8a8a8; --light-scrollbar-thumb-hover: #a8a8a8;
--light-tree-row-bg: #e3f2fd;
--dark-bg: #1d1f21; --dark-bg: #1d1f21;
--dark-border: #2d2d2d; --dark-border: #2d2d2d;
@ -29,7 +28,6 @@
--dark-scrollbar-track: #25282c; --dark-scrollbar-track: #25282c;
--dark-scrollbar-thumb: #4a4d51; --dark-scrollbar-thumb: #4a4d51;
--dark-scrollbar-thumb-hover: #5a5d61; --dark-scrollbar-thumb-hover: #5a5d61;
--dark-tree-row-bg: #373737;
--primary-color: #0550ae; --primary-color: #0550ae;
--primary-color-alpha: rgba(5, 80, 174, 0.1); --primary-color-alpha: rgba(5, 80, 174, 0.1);
@ -44,7 +42,6 @@
--text-color: var(--dark-text); --text-color: var(--dark-text);
--panel-bg: var(--dark-panel-bg); --panel-bg: var(--dark-panel-bg);
--code-bg: var(--dark-code-bg); --code-bg: var(--dark-code-bg);
--tree-row-bg: var(--dark-tree-row-bg);
} }
[data-theme="light"] { [data-theme="light"] {
@ -53,7 +50,6 @@
--text-color: var(--light-text); --text-color: var(--light-text);
--panel-bg: white; --panel-bg: white;
--code-bg: white; --code-bg: white;
--tree-row-bg: var(--light-tree-row-bg);
} }
/* Base Styles */ /* Base Styles */
@ -279,7 +275,7 @@
} }
#output-container a.highlighted { #output-container a.highlighted {
background-color: #cae2ff; background-color: #d9d9d9;
color: red; color: red;
border-radius: 3px; border-radius: 3px;
text-decoration: underline; text-decoration: underline;
@ -350,7 +346,7 @@
} }
& #output-container a.highlighted { & #output-container a.highlighted {
background-color: #656669; background-color: #373b41;
color: red; color: red;
} }
@ -377,9 +373,6 @@
color: var(--dark-text); color: var(--dark-text);
} }
} }
.tree-row:has(.highlighted) {
background-color: var(--tree-row-bg);
}
</style> </style>
</head> </head>

View file

@ -6,35 +6,30 @@ use std::{
time::Instant, time::Instant,
}; };
use anstyle::AnsiColor;
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use log::warn; use log::warn;
use streaming_iterator::StreamingIterator; use streaming_iterator::StreamingIterator;
use tree_sitter::{Language, Parser, Point, Query, QueryCursor}; use tree_sitter::{Language, Parser, Point, Query, QueryCursor};
use crate::{ use crate::{
logger::paint,
query_testing::{self, to_utf8_point}, query_testing::{self, to_utf8_point},
test::{TestInfo, TestOutcome, TestResult, TestSummary},
}; };
#[derive(Default)] #[allow(clippy::too_many_arguments)]
pub struct QueryFileOptions {
pub ordered_captures: bool,
pub byte_range: Option<Range<usize>>,
pub point_range: Option<Range<Point>>,
pub containing_byte_range: Option<Range<usize>>,
pub containing_point_range: Option<Range<Point>>,
pub quiet: bool,
pub print_time: bool,
pub stdin: bool,
}
pub fn query_file_at_path( pub fn query_file_at_path(
language: &Language, language: &Language,
path: &Path, path: &Path,
name: &str, name: &str,
query_path: &Path, query_path: &Path,
opts: &QueryFileOptions, ordered_captures: bool,
test_summary: Option<&mut TestSummary>, byte_range: Option<Range<usize>>,
point_range: Option<Range<Point>>,
should_test: bool,
quiet: bool,
print_time: bool,
stdin: bool,
) -> Result<()> { ) -> Result<()> {
let stdout = io::stdout(); let stdout = io::stdout();
let mut stdout = stdout.lock(); let mut stdout = stdout.lock();
@ -44,26 +39,19 @@ pub fn query_file_at_path(
let query = Query::new(language, &query_source).with_context(|| "Query compilation failed")?; let query = Query::new(language, &query_source).with_context(|| "Query compilation failed")?;
let mut query_cursor = QueryCursor::new(); let mut query_cursor = QueryCursor::new();
if let Some(ref range) = opts.byte_range { if let Some(range) = byte_range {
query_cursor.set_byte_range(range.clone()); query_cursor.set_byte_range(range);
} }
if let Some(ref range) = opts.point_range { if let Some(range) = point_range {
query_cursor.set_point_range(range.clone()); query_cursor.set_point_range(range);
}
if let Some(ref range) = opts.containing_byte_range {
query_cursor.set_containing_byte_range(range.clone());
}
if let Some(ref range) = opts.containing_point_range {
query_cursor.set_containing_point_range(range.clone());
} }
let mut parser = Parser::new(); let mut parser = Parser::new();
parser.set_language(language)?; parser.set_language(language)?;
let mut results = Vec::new(); let mut results = Vec::new();
let should_test = test_summary.is_some();
if !should_test && !opts.stdin { if !should_test && !stdin {
writeln!(&mut stdout, "{name}")?; writeln!(&mut stdout, "{name}")?;
} }
@ -72,12 +60,12 @@ pub fn query_file_at_path(
let tree = parser.parse(&source_code, None).unwrap(); let tree = parser.parse(&source_code, None).unwrap();
let start = Instant::now(); let start = Instant::now();
if opts.ordered_captures { if ordered_captures {
let mut captures = query_cursor.captures(&query, tree.root_node(), source_code.as_slice()); let mut captures = query_cursor.captures(&query, tree.root_node(), source_code.as_slice());
while let Some((mat, capture_index)) = captures.next() { while let Some((mat, capture_index)) = captures.next() {
let capture = mat.captures[*capture_index]; let capture = mat.captures[*capture_index];
let capture_name = &query.capture_names()[capture.index as usize]; let capture_name = &query.capture_names()[capture.index as usize];
if !opts.quiet && !should_test { if !quiet && !should_test {
writeln!( writeln!(
&mut stdout, &mut stdout,
" pattern: {:>2}, capture: {} - {capture_name}, start: {}, end: {}, text: `{}`", " pattern: {:>2}, capture: {} - {capture_name}, start: {}, end: {}, text: `{}`",
@ -88,25 +76,23 @@ pub fn query_file_at_path(
capture.node.utf8_text(&source_code).unwrap_or("") capture.node.utf8_text(&source_code).unwrap_or("")
)?; )?;
} }
if should_test { results.push(query_testing::CaptureInfo {
results.push(query_testing::CaptureInfo { name: (*capture_name).to_string(),
name: (*capture_name).to_string(), start: to_utf8_point(capture.node.start_position(), source_code.as_slice()),
start: to_utf8_point(capture.node.start_position(), source_code.as_slice()), end: to_utf8_point(capture.node.end_position(), source_code.as_slice()),
end: to_utf8_point(capture.node.end_position(), source_code.as_slice()), });
});
}
} }
} else { } else {
let mut matches = query_cursor.matches(&query, tree.root_node(), source_code.as_slice()); let mut matches = query_cursor.matches(&query, tree.root_node(), source_code.as_slice());
while let Some(m) = matches.next() { while let Some(m) = matches.next() {
if !opts.quiet && !should_test { if !quiet && !should_test {
writeln!(&mut stdout, " pattern: {}", m.pattern_index)?; writeln!(&mut stdout, " pattern: {}", m.pattern_index)?;
} }
for capture in m.captures { for capture in m.captures {
let start = capture.node.start_position(); let start = capture.node.start_position();
let end = capture.node.end_position(); let end = capture.node.end_position();
let capture_name = &query.capture_names()[capture.index as usize]; let capture_name = &query.capture_names()[capture.index as usize];
if !opts.quiet && !should_test { if !quiet && !should_test {
if end.row == start.row { if end.row == start.row {
writeln!( writeln!(
&mut stdout, &mut stdout,
@ -121,52 +107,38 @@ pub fn query_file_at_path(
)?; )?;
} }
} }
if should_test { results.push(query_testing::CaptureInfo {
results.push(query_testing::CaptureInfo { name: (*capture_name).to_string(),
name: (*capture_name).to_string(), start: to_utf8_point(capture.node.start_position(), source_code.as_slice()),
start: to_utf8_point(capture.node.start_position(), source_code.as_slice()), end: to_utf8_point(capture.node.end_position(), source_code.as_slice()),
end: to_utf8_point(capture.node.end_position(), source_code.as_slice()), });
});
}
} }
} }
} }
if query_cursor.did_exceed_match_limit() { if !query_cursor.did_exceed_match_limit() {
warn!("Query exceeded maximum number of in-progress captures!"); warn!("Query exceeded maximum number of in-progress captures!");
} }
if should_test { if should_test {
let path_name = if opts.stdin { let path_name = if stdin {
"stdin" "stdin"
} else { } else {
Path::new(&path).file_name().unwrap().to_str().unwrap() Path::new(&path).file_name().unwrap().to_str().unwrap()
}; };
// Invariant: `test_summary` will always be `Some` when `should_test` is true
let test_summary = test_summary.unwrap();
match query_testing::assert_expected_captures(&results, path, &mut parser, language) { match query_testing::assert_expected_captures(&results, path, &mut parser, language) {
Ok(assertion_count) => { Ok(assertion_count) => {
test_summary.query_results.add_case(TestResult { println!(
name: path_name.to_string(), " ✓ {} ({} assertions)",
info: TestInfo::AssertionTest { paint(Some(AnsiColor::Green), path_name),
outcome: TestOutcome::AssertionPassed { assertion_count }, assertion_count
test_num: test_summary.test_num, );
},
});
} }
Err(e) => { Err(e) => {
test_summary.query_results.add_case(TestResult { println!("{}", paint(Some(AnsiColor::Red), path_name));
name: path_name.to_string(),
info: TestInfo::AssertionTest {
outcome: TestOutcome::AssertionFailed {
error: e.to_string(),
},
test_num: test_summary.test_num,
},
});
return Err(e); return Err(e);
} }
} }
} }
if opts.print_time { if print_time {
writeln!(&mut stdout, "{:?}", start.elapsed())?; writeln!(&mut stdout, "{:?}", start.elapsed())?;
} }

View file

@ -3,11 +3,11 @@ root = true
[*] [*]
charset = utf-8 charset = utf-8
[*.{json,toml,yml,gyp,xml}] [*.{json,toml,yml,gyp}]
indent_style = space indent_style = space
indent_size = 2 indent_size = 2
[*.{js,ts}] [*.js]
indent_style = space indent_style = space
indent_size = 2 indent_size = 2
@ -31,10 +31,6 @@ indent_size = 4
indent_style = space indent_style = space
indent_size = 4 indent_size = 4
[*.java]
indent_style = space
indent_size = 4
[*.go] [*.go]
indent_style = tab indent_style = tab
indent_size = 8 indent_size = 8

View file

@ -6,33 +6,32 @@ from ._binding import language
def _get_query(name, file): def _get_query(name, file):
try: query = _files(f"{__package__}.queries") / file
query = _files(f"{__package__}") / file globals()[name] = query.read_text()
globals()[name] = query.read_text()
except FileNotFoundError:
globals()[name] = None
return globals()[name] return globals()[name]
def __getattr__(name): def __getattr__(name):
if name == "HIGHLIGHTS_QUERY": # NOTE: uncomment these to include any queries that this grammar contains:
return _get_query("HIGHLIGHTS_QUERY", "HIGHLIGHTS_QUERY_PATH")
if name == "INJECTIONS_QUERY": # if name == "HIGHLIGHTS_QUERY":
return _get_query("INJECTIONS_QUERY", "INJECTIONS_QUERY_PATH") # return _get_query("HIGHLIGHTS_QUERY", "highlights.scm")
if name == "LOCALS_QUERY": # if name == "INJECTIONS_QUERY":
return _get_query("LOCALS_QUERY", "LOCALS_QUERY_PATH") # return _get_query("INJECTIONS_QUERY", "injections.scm")
if name == "TAGS_QUERY": # if name == "LOCALS_QUERY":
return _get_query("TAGS_QUERY", "TAGS_QUERY_PATH") # return _get_query("LOCALS_QUERY", "locals.scm")
# if name == "TAGS_QUERY":
# return _get_query("TAGS_QUERY", "tags.scm")
raise AttributeError(f"module {__name__!r} has no attribute {name!r}") raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
__all__ = [ __all__ = [
"language", "language",
"HIGHLIGHTS_QUERY", # "HIGHLIGHTS_QUERY",
"INJECTIONS_QUERY", # "INJECTIONS_QUERY",
"LOCALS_QUERY", # "LOCALS_QUERY",
"TAGS_QUERY", # "TAGS_QUERY",
] ]

View file

@ -1,17 +1,11 @@
from typing import Final from typing import Final
from typing_extensions import CapsuleType from typing_extensions import CapsuleType
HIGHLIGHTS_QUERY: Final[str] | None # NOTE: uncomment these to include any queries that this grammar contains:
"""The syntax highlighting query for this grammar."""
INJECTIONS_QUERY: Final[str] | None # HIGHLIGHTS_QUERY: Final[str]
"""The language injection query for this grammar.""" # INJECTIONS_QUERY: Final[str]
# LOCALS_QUERY: Final[str]
# TAGS_QUERY: Final[str]
LOCALS_QUERY: Final[str] | None def language() -> CapsuleType: ...
"""The local variable query for this grammar."""
TAGS_QUERY: Final[str] | None
"""The symbol tagging query for this grammar."""
def language() -> CapsuleType:
"""The tree-sitter language function for this grammar."""

View file

@ -1,65 +0,0 @@
package PARSER_NS_CLEANED.jtreesitter.LOWER_PARSER_NAME;
import java.lang.foreign.*;
public final class PARSER_CLASS_NAME {
private static final ValueLayout VOID_PTR =
ValueLayout.ADDRESS.withTargetLayout(MemoryLayout.sequenceLayout(Long.MAX_VALUE, ValueLayout.JAVA_BYTE));
private static final FunctionDescriptor FUNC_DESC = FunctionDescriptor.of(VOID_PTR);
private static final Linker LINKER = Linker.nativeLinker();
private static final PARSER_CLASS_NAME INSTANCE = new PARSER_CLASS_NAME();
private final Arena arena = Arena.ofAuto();
private volatile SymbolLookup lookup = null;
private PARSER_CLASS_NAME() {}
/**
* Get the tree-sitter language for this grammar.
*/
public static MemorySegment language() {
if (INSTANCE.lookup == null)
INSTANCE.lookup = INSTANCE.findLibrary();
return language(INSTANCE.lookup);
}
/**
* Get the tree-sitter language for this grammar.
*
* <strong>The {@linkplain Arena} used in the {@code lookup}
* must not be closed while the language is being used.</strong>
*/
public static MemorySegment language(SymbolLookup lookup) {
return call(lookup, "tree_sitter_PARSER_NAME");
}
private SymbolLookup findLibrary() {
try {
var library = System.mapLibraryName("tree-sitter-KEBAB_PARSER_NAME");
return SymbolLookup.libraryLookup(library, arena);
} catch (IllegalArgumentException ex1) {
try {
System.loadLibrary("tree-sitter-KEBAB_PARSER_NAME");
return SymbolLookup.loaderLookup();
} catch (UnsatisfiedLinkError ex2) {
ex1.addSuppressed(ex2);
throw ex1;
}
}
}
private static UnsatisfiedLinkError unresolved(String name) {
return new UnsatisfiedLinkError("Unresolved symbol: %s".formatted(name));
}
@SuppressWarnings("SameParameterValue")
private static MemorySegment call(SymbolLookup lookup, String name) throws UnsatisfiedLinkError {
var address = lookup.find(name).orElseThrow(() -> unresolved(name));
try {
var function = LINKER.downcallHandle(address, FUNC_DESC);
return (MemorySegment) function.invokeExact();
} catch (Throwable e) {
throw new RuntimeException("Call to %s failed".formatted(name), e);
}
}
}

View file

@ -36,21 +36,4 @@ fn main() {
} }
c_config.compile("tree-sitter-KEBAB_PARSER_NAME"); c_config.compile("tree-sitter-KEBAB_PARSER_NAME");
println!("cargo:rustc-check-cfg=cfg(with_highlights_query)");
if !"HIGHLIGHTS_QUERY_PATH".is_empty() && std::path::Path::new("HIGHLIGHTS_QUERY_PATH").exists() {
println!("cargo:rustc-cfg=with_highlights_query");
}
println!("cargo:rustc-check-cfg=cfg(with_injections_query)");
if !"INJECTIONS_QUERY_PATH".is_empty() && std::path::Path::new("INJECTIONS_QUERY_PATH").exists() {
println!("cargo:rustc-cfg=with_injections_query");
}
println!("cargo:rustc-check-cfg=cfg(with_locals_query)");
if !"LOCALS_QUERY_PATH".is_empty() && std::path::Path::new("LOCALS_QUERY_PATH").exists() {
println!("cargo:rustc-cfg=with_locals_query");
}
println!("cargo:rustc-check-cfg=cfg(with_tags_query)");
if !"TAGS_QUERY_PATH".is_empty() && std::path::Path::new("TAGS_QUERY_PATH").exists() {
println!("cargo:rustc-cfg=with_tags_query");
}
} }

View file

@ -20,19 +20,16 @@ include(GNUInstallDirs)
find_program(TREE_SITTER_CLI tree-sitter DOC "Tree-sitter CLI") find_program(TREE_SITTER_CLI tree-sitter DOC "Tree-sitter CLI")
add_custom_command(OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/src/grammar.json" add_custom_command(OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/src/grammar.json"
"${CMAKE_CURRENT_SOURCE_DIR}/src/node-types.json"
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/grammar.js" DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/grammar.js"
COMMAND "${TREE_SITTER_CLI}" generate grammar.js --no-parser COMMAND "${TREE_SITTER_CLI}" generate grammar.js
--emit=json
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
COMMENT "Generating grammar.json") COMMENT "Generating grammar.json")
add_custom_command(OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/src/parser.c" add_custom_command(OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/src/parser.c"
BYPRODUCTS "${CMAKE_CURRENT_SOURCE_DIR}/src/tree_sitter/parser.h"
"${CMAKE_CURRENT_SOURCE_DIR}/src/tree_sitter/alloc.h"
"${CMAKE_CURRENT_SOURCE_DIR}/src/tree_sitter/array.h"
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/grammar.json" DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/grammar.json"
COMMAND "${TREE_SITTER_CLI}" generate src/grammar.json COMMAND "${TREE_SITTER_CLI}" generate src/grammar.json
--abi=${TREE_SITTER_ABI_VERSION} --emit=parser --abi=${TREE_SITTER_ABI_VERSION}
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
COMMENT "Generating parser.c") COMMENT "Generating parser.c")

View file

@ -40,7 +40,3 @@ Package.resolved linguist-generated
bindings/zig/* linguist-generated bindings/zig/* linguist-generated
build.zig linguist-generated build.zig linguist-generated
build.zig.zon linguist-generated build.zig.zon linguist-generated
# Java bindings
pom.xml linguist-generated
bindings/java/** linguist-generated

View file

@ -45,4 +45,3 @@ zig-out/
*.tar.gz *.tar.gz
*.tgz *.tgz
*.zip *.zip
*.jar

View file

@ -18,43 +18,10 @@ type NodeInfo =
children: ChildNode[]; children: ChildNode[];
}); });
/** type Language = {
* The tree-sitter language object for this grammar.
*
* @see {@linkcode https://tree-sitter.github.io/node-tree-sitter/interfaces/Parser.Language.html Parser.Language}
*
* @example
* import Parser from "tree-sitter";
* import CAMEL_PARSER_NAME from "tree-sitter-KEBAB_PARSER_NAME";
*
* const parser = new Parser();
* parser.setLanguage(CAMEL_PARSER_NAME);
*/
declare const binding: {
/**
* The inner language object.
* @private
*/
language: unknown; language: unknown;
/**
* The content of the `node-types.json` file for this grammar.
*
* @see {@linkplain https://tree-sitter.github.io/tree-sitter/using-parsers/6-static-node-types Static Node Types}
*/
nodeTypeInfo: NodeInfo[]; nodeTypeInfo: NodeInfo[];
/** The syntax highlighting query for this grammar. */
HIGHLIGHTS_QUERY?: string;
/** The language injection query for this grammar. */
INJECTIONS_QUERY?: string;
/** The local variable query for this grammar. */
LOCALS_QUERY?: string;
/** The symbol tagging query for this grammar. */
TAGS_QUERY?: string;
}; };
export default binding; declare const language: Language;
export = language;

View file

@ -1,4 +1,3 @@
import { readFileSync } from "node:fs";
import { fileURLToPath } from "node:url"; import { fileURLToPath } from "node:url";
const root = fileURLToPath(new URL("../..", import.meta.url)); const root = fileURLToPath(new URL("../..", import.meta.url));
@ -9,29 +8,8 @@ const binding = typeof process.versions.bun === "string"
: (await import("node-gyp-build")).default(root); : (await import("node-gyp-build")).default(root);
try { try {
const nodeTypes = await import(`${root}/src/node-types.json`, { with: { type: "json" } }); const nodeTypes = await import(`${root}/src/node-types.json`, {with: {type: "json"}});
binding.nodeTypeInfo = nodeTypes.default; binding.nodeTypeInfo = nodeTypes.default;
} catch { } } catch (_) {}
const queries = [
["HIGHLIGHTS_QUERY", `${root}/HIGHLIGHTS_QUERY_PATH`],
["INJECTIONS_QUERY", `${root}/INJECTIONS_QUERY_PATH`],
["LOCALS_QUERY", `${root}/LOCALS_QUERY_PATH`],
["TAGS_QUERY", `${root}/TAGS_QUERY_PATH`],
];
for (const [prop, path] of queries) {
Object.defineProperty(binding, prop, {
configurable: true,
enumerable: true,
get() {
delete binding[prop];
try {
binding[prop] = readFileSync(path, "utf8");
} catch { }
return binding[prop];
}
});
}
export default binding; export default binding;

View file

@ -32,21 +32,12 @@ pub const LANGUAGE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_PARSE
/// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers/6-static-node-types /// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers/6-static-node-types
pub const NODE_TYPES: &str = include_str!("../../src/node-types.json"); pub const NODE_TYPES: &str = include_str!("../../src/node-types.json");
#[cfg(with_highlights_query)] // NOTE: uncomment these to include any queries that this grammar contains:
/// The syntax highlighting query for this grammar.
pub const HIGHLIGHTS_QUERY: &str = include_str!("../../HIGHLIGHTS_QUERY_PATH");
#[cfg(with_injections_query)] // pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
/// The language injection query for this grammar. // pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm");
pub const INJECTIONS_QUERY: &str = include_str!("../../INJECTIONS_QUERY_PATH"); // pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm");
// pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm");
#[cfg(with_locals_query)]
/// The local variable query for this grammar.
pub const LOCALS_QUERY: &str = include_str!("../../LOCALS_QUERY_PATH");
#[cfg(with_tags_query)]
/// The symbol tagging query for this grammar.
pub const TAGS_QUERY: &str = include_str!("../../TAGS_QUERY_PATH");
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {

View file

@ -73,10 +73,10 @@ $(LANGUAGE_NAME).pc: bindings/c/$(LANGUAGE_NAME).pc.in
-e 's|@CMAKE_INSTALL_PREFIX@|$(PREFIX)|' $< > $@ -e 's|@CMAKE_INSTALL_PREFIX@|$(PREFIX)|' $< > $@
$(SRC_DIR)/grammar.json: grammar.js $(SRC_DIR)/grammar.json: grammar.js
$(TS) generate --no-parser $^ $(TS) generate --emit=json $^
$(PARSER): $(SRC_DIR)/grammar.json $(PARSER): $(SRC_DIR)/grammar.json
$(TS) generate $^ $(TS) generate --emit=parser $^
install: all install: all
install -d '$(DESTDIR)$(DATADIR)'/tree-sitter/queries/KEBAB_PARSER_NAME '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter '$(DESTDIR)$(PCLIBDIR)' '$(DESTDIR)$(LIBDIR)' install -d '$(DESTDIR)$(DATADIR)'/tree-sitter/queries/KEBAB_PARSER_NAME '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter '$(DESTDIR)$(PCLIBDIR)' '$(DESTDIR)$(LIBDIR)'

View file

@ -38,11 +38,11 @@
}, },
"devDependencies": { "devDependencies": {
"prebuildify": "^6.0.1", "prebuildify": "^6.0.1",
"tree-sitter": "^0.25.0", "tree-sitter": "^0.22.4",
"tree-sitter-cli": "^CLI_VERSION" "tree-sitter-cli": "^CLI_VERSION"
}, },
"peerDependencies": { "peerDependencies": {
"tree-sitter": "^0.25.0" "tree-sitter": "^0.22.4"
}, },
"peerDependenciesMeta": { "peerDependenciesMeta": {
"tree-sitter": { "tree-sitter": {

View file

@ -1,154 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>PARSER_NS</groupId>
<artifactId>jtreesitter-KEBAB_PARSER_NAME</artifactId>
<name>JTreeSitter CAMEL_PARSER_NAME</name>
<version>PARSER_VERSION</version>
<description>PARSER_DESCRIPTION</description>
<url>PARSER_URL</url>
<licenses>
<license>
<name>PARSER_LICENSE</name>
<url>https://spdx.org/licenses/PARSER_LICENSE.html</url>
</license>
</licenses>
<developers>
<developer>
<name>PARSER_AUTHOR_NAME</name>
<email>PARSER_AUTHOR_EMAIL</email>
<url>PARSER_AUTHOR_URL</url>
</developer>
</developers>
<scm>
<url>PARSER_URL</url>
<connection>scm:git:git://PARSER_URL_STRIPPED.git</connection>
<developerConnection>scm:git:ssh://PARSER_URL_STRIPPED.git</developerConnection>
</scm>
<properties>
<maven.compiler.release>23</maven.compiler.release>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.deploy.skip>true</maven.deploy.skip>
<gpg.skip>true</gpg.skip>
<publish.auto>false</publish.auto>
<publish.skip>true</publish.skip>
</properties>
<dependencies>
<dependency>
<groupId>io.github.tree-sitter</groupId>
<artifactId>jtreesitter</artifactId>
<version>0.26.0</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<version>6.0.1</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<sourceDirectory>bindings/java/main</sourceDirectory>
<testSourceDirectory>bindings/java/test</testSourceDirectory>
<plugins>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>3.5.4</version>
<configuration>
<reportsDirectory>
${project.build.directory}/reports/surefire
</reportsDirectory>
<argLine>--enable-native-access=ALL-UNNAMED</argLine>
</configuration>
</plugin>
<plugin>
<artifactId>maven-javadoc-plugin</artifactId>
<version>3.12.0</version>
<executions>
<execution>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
<configuration>
<show>public</show>
<nohelp>true</nohelp>
<noqualifier>true</noqualifier>
<doclint>all,-missing</doclint>
</configuration>
</plugin>
<plugin>
<artifactId>maven-source-plugin</artifactId>
<version>3.3.1</version>
<executions>
<execution>
<goals>
<goal>jar-no-fork</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-gpg-plugin</artifactId>
<version>3.2.8</version>
<executions>
<execution>
<phase>verify</phase>
<goals>
<goal>sign</goal>
</goals>
<configuration>
<bestPractices>true</bestPractices>
<gpgArguments>
<arg>--no-tty</arg>
<arg>--pinentry-mode</arg>
<arg>loopback</arg>
</gpgArguments>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>io.github.mavenplugins</groupId>
<artifactId>central-publishing-maven-plugin</artifactId>
<version>1.1.1</version>
<executions>
<execution>
<phase>deploy</phase>
<goals>
<goal>publish</goal>
</goals>
<configuration>
<waitUntil>validated</waitUntil>
<autoPublish>${publish.auto}</autoPublish>
<skipPublishing>${publish.skip}</skipPublishing>
<outputFilename>${project.artifactId}-${project.version}.zip</outputFilename>
<deploymentName>${project.artifactId}-${project.version}.zip</deploymentName>
</configuration>
</execution>
</executions>
<extensions>true</extensions>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>ci</id>
<activation>
<property>
<name>env.CI</name>
<value>true</value>
</property>
</activation>
<properties>
<gpg.skip>false</gpg.skip>
<publish.auto>true</publish.auto>
<publish.skip>false</publish.skip>
</properties>
</profile>
</profiles>
</project>

View file

@ -32,7 +32,7 @@ class BuildExt(build_ext):
class BdistWheel(bdist_wheel): class BdistWheel(bdist_wheel):
def get_tag(self): def get_tag(self):
python, abi, platform = super().get_tag() python, abi, platform = super().get_tag()
if python.startswith("cp") and not get_config_var("Py_GIL_DISABLED"): if python.startswith("cp"):
python, abi = "cp310", "abi3" python, abi = "cp310", "abi3"
return python, abi, platform return python, abi, platform

View file

@ -1,12 +0,0 @@
import io.github.treesitter.jtreesitter.Language;
import PARSER_NS_CLEANED.jtreesitter.LOWER_PARSER_NAME.PARSER_CLASS_NAME;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
public class PARSER_CLASS_NAMETest {
@Test
public void testCanLoadLanguage() {
assertDoesNotThrow(() -> new Language(PARSER_CLASS_NAME.language()));
}
}

File diff suppressed because it is too large Load diff

View file

@ -1,13 +1,14 @@
use std::{fs, path::Path}; use std::{fs, path::Path};
use anstyle::AnsiColor;
use anyhow::{anyhow, Result}; use anyhow::{anyhow, Result};
use tree_sitter::Point; use tree_sitter::Point;
use tree_sitter_highlight::{Highlight, HighlightConfiguration, HighlightEvent, Highlighter}; use tree_sitter_highlight::{Highlight, HighlightConfiguration, HighlightEvent, Highlighter};
use tree_sitter_loader::{Config, Loader}; use tree_sitter_loader::{Config, Loader};
use crate::{ use crate::{
logger::paint,
query_testing::{parse_position_comments, to_utf8_point, Assertion, Utf8Point}, query_testing::{parse_position_comments, to_utf8_point, Assertion, Utf8Point},
test::{TestInfo, TestOutcome, TestResult, TestSummary},
util, util,
}; };
@ -47,7 +48,19 @@ pub fn test_highlights(
loader_config: &Config, loader_config: &Config,
highlighter: &mut Highlighter, highlighter: &mut Highlighter,
directory: &Path, directory: &Path,
test_summary: &mut TestSummary, use_color: bool,
) -> Result<()> {
println!("syntax highlighting:");
test_highlights_indented(loader, loader_config, highlighter, directory, use_color, 2)
}
fn test_highlights_indented(
loader: &Loader,
loader_config: &Config,
highlighter: &mut Highlighter,
directory: &Path,
use_color: bool,
indent_level: usize,
) -> Result<()> { ) -> Result<()> {
let mut failed = false; let mut failed = false;
@ -55,22 +68,25 @@ pub fn test_highlights(
let highlight_test_file = highlight_test_file?; let highlight_test_file = highlight_test_file?;
let test_file_path = highlight_test_file.path(); let test_file_path = highlight_test_file.path();
let test_file_name = highlight_test_file.file_name(); let test_file_name = highlight_test_file.file_name();
print!(
"{indent:indent_level$}",
indent = "",
indent_level = indent_level * 2
);
if test_file_path.is_dir() && test_file_path.read_dir()?.next().is_some() { if test_file_path.is_dir() && test_file_path.read_dir()?.next().is_some() {
test_summary println!("{}:", test_file_name.to_string_lossy());
.highlight_results if test_highlights_indented(
.add_group(test_file_name.to_string_lossy().as_ref());
if test_highlights(
loader, loader,
loader_config, loader_config,
highlighter, highlighter,
&test_file_path, &test_file_path,
test_summary, use_color,
indent_level + 1,
) )
.is_err() .is_err()
{ {
failed = true; failed = true;
} }
test_summary.highlight_results.pop_traversal();
} else { } else {
let (language, language_config) = loader let (language, language_config) = loader
.language_configuration_for_file_name(&test_file_path)? .language_configuration_for_file_name(&test_file_path)?
@ -95,28 +111,30 @@ pub fn test_highlights(
fs::read(&test_file_path)?.as_slice(), fs::read(&test_file_path)?.as_slice(),
) { ) {
Ok(assertion_count) => { Ok(assertion_count) => {
test_summary.highlight_results.add_case(TestResult { println!(
name: test_file_name.to_string_lossy().to_string(), "✓ {} ({assertion_count} assertions)",
info: TestInfo::AssertionTest { paint(
outcome: TestOutcome::AssertionPassed { assertion_count }, use_color.then_some(AnsiColor::Green),
test_num: test_summary.test_num, test_file_name.to_string_lossy().as_ref()
}, ),
}); );
} }
Err(e) => { Err(e) => {
test_summary.highlight_results.add_case(TestResult { println!(
name: test_file_name.to_string_lossy().to_string(), "✗ {}",
info: TestInfo::AssertionTest { paint(
outcome: TestOutcome::AssertionFailed { use_color.then_some(AnsiColor::Red),
error: e.to_string(), test_file_name.to_string_lossy().as_ref()
}, )
test_num: test_summary.test_num, );
}, println!(
}); "{indent:indent_level$} {e}",
indent = "",
indent_level = indent_level * 2
);
failed = true; failed = true;
} }
} }
test_summary.test_num += 1;
} }
} }

View file

@ -1,12 +1,13 @@
use std::{fs, path::Path}; use std::{fs, path::Path};
use anstyle::AnsiColor;
use anyhow::{anyhow, Result}; use anyhow::{anyhow, Result};
use tree_sitter_loader::{Config, Loader}; use tree_sitter_loader::{Config, Loader};
use tree_sitter_tags::{TagsConfiguration, TagsContext}; use tree_sitter_tags::{TagsConfiguration, TagsContext};
use crate::{ use crate::{
logger::paint,
query_testing::{parse_position_comments, to_utf8_point, Assertion, Utf8Point}, query_testing::{parse_position_comments, to_utf8_point, Assertion, Utf8Point},
test::{TestInfo, TestOutcome, TestResult, TestSummary},
util, util,
}; };
@ -46,7 +47,19 @@ pub fn test_tags(
loader_config: &Config, loader_config: &Config,
tags_context: &mut TagsContext, tags_context: &mut TagsContext,
directory: &Path, directory: &Path,
test_summary: &mut TestSummary, use_color: bool,
) -> Result<()> {
println!("tags:");
test_tags_indented(loader, loader_config, tags_context, directory, use_color, 2)
}
pub fn test_tags_indented(
loader: &Loader,
loader_config: &Config,
tags_context: &mut TagsContext,
directory: &Path,
use_color: bool,
indent_level: usize,
) -> Result<()> { ) -> Result<()> {
let mut failed = false; let mut failed = false;
@ -54,22 +67,25 @@ pub fn test_tags(
let tag_test_file = tag_test_file?; let tag_test_file = tag_test_file?;
let test_file_path = tag_test_file.path(); let test_file_path = tag_test_file.path();
let test_file_name = tag_test_file.file_name(); let test_file_name = tag_test_file.file_name();
print!(
"{indent:indent_level$}",
indent = "",
indent_level = indent_level * 2
);
if test_file_path.is_dir() && test_file_path.read_dir()?.next().is_some() { if test_file_path.is_dir() && test_file_path.read_dir()?.next().is_some() {
test_summary println!("{}:", test_file_name.to_string_lossy());
.tag_results if test_tags_indented(
.add_group(test_file_name.to_string_lossy().as_ref());
if test_tags(
loader, loader,
loader_config, loader_config,
tags_context, tags_context,
&test_file_path, &test_file_path,
test_summary, use_color,
indent_level + 1,
) )
.is_err() .is_err()
{ {
failed = true; failed = true;
} }
test_summary.tag_results.pop_traversal();
} else { } else {
let (language, language_config) = loader let (language, language_config) = loader
.language_configuration_for_file_name(&test_file_path)? .language_configuration_for_file_name(&test_file_path)?
@ -88,28 +104,30 @@ pub fn test_tags(
fs::read(&test_file_path)?.as_slice(), fs::read(&test_file_path)?.as_slice(),
) { ) {
Ok(assertion_count) => { Ok(assertion_count) => {
test_summary.tag_results.add_case(TestResult { println!(
name: test_file_name.to_string_lossy().to_string(), "✓ {} ({assertion_count} assertions)",
info: TestInfo::AssertionTest { paint(
outcome: TestOutcome::AssertionPassed { assertion_count }, use_color.then_some(AnsiColor::Green),
test_num: test_summary.test_num, test_file_name.to_string_lossy().as_ref()
}, ),
}); );
} }
Err(e) => { Err(e) => {
test_summary.tag_results.add_case(TestResult { println!(
name: test_file_name.to_string_lossy().to_string(), "✗ {}",
info: TestInfo::AssertionTest { paint(
outcome: TestOutcome::AssertionFailed { use_color.then_some(AnsiColor::Red),
error: e.to_string(), test_file_name.to_string_lossy().as_ref()
}, )
test_num: test_summary.test_num, );
}, println!(
}); "{indent:indent_level$} {e}",
indent = "",
indent_level = indent_level * 2
);
failed = true; failed = true;
} }
} }
test_summary.test_num += 1;
} }
} }

View file

@ -26,8 +26,6 @@ pub use crate::fuzz::{
ITERATION_COUNT, ITERATION_COUNT,
}; };
pub use helpers::fixtures::get_language;
/// This is a simple wrapper around [`tree_sitter_generate::generate_parser_for_grammar`], because /// This is a simple wrapper around [`tree_sitter_generate::generate_parser_for_grammar`], because
/// our tests do not need to pass in a version number, only the grammar JSON. /// our tests do not need to pass in a version number, only the grammar JSON.
fn generate_parser(grammar_json: &str) -> GenerateResult<(String, String)> { fn generate_parser(grammar_json: &str) -> GenerateResult<(String, String)> {

View file

@ -16,7 +16,7 @@ use crate::{
LOG_GRAPH_ENABLED, START_SEED, LOG_GRAPH_ENABLED, START_SEED,
}, },
parse::perform_edit, parse::perform_edit,
test::{parse_tests, strip_sexp_fields, DiffKey, TestDiff}, test::{parse_tests, print_diff, print_diff_key, strip_sexp_fields},
tests::{ tests::{
allocations, allocations,
helpers::fixtures::{fixtures_dir, get_language, get_test_language, SCRATCH_BASE_DIR}, helpers::fixtures::{fixtures_dir, get_language, get_test_language, SCRATCH_BASE_DIR},
@ -209,8 +209,8 @@ pub fn test_language_corpus(
if actual_output != test.output { if actual_output != test.output {
println!("Incorrect initial parse for {test_name}"); println!("Incorrect initial parse for {test_name}");
DiffKey::print(); print_diff_key();
println!("{}", TestDiff::new(&actual_output, &test.output)); print_diff(&actual_output, &test.output, true);
println!(); println!();
return false; return false;
} }
@ -297,8 +297,8 @@ pub fn test_language_corpus(
if actual_output != test.output { if actual_output != test.output {
println!("Incorrect parse for {test_name} - seed {seed}"); println!("Incorrect parse for {test_name} - seed {seed}");
DiffKey::print(); print_diff_key();
println!("{}", TestDiff::new(&actual_output, &test.output)); print_diff(&actual_output, &test.output, true);
println!(); println!();
return false; return false;
} }
@ -428,8 +428,8 @@ fn test_feature_corpus_files() {
if actual_output == test.output { if actual_output == test.output {
true true
} else { } else {
DiffKey::print(); print_diff_key();
print!("{}", TestDiff::new(&actual_output, &test.output)); print_diff(&actual_output, &test.output, true);
println!(); println!();
false false
} }

View file

@ -90,7 +90,7 @@ fn detect_language_by_first_line_regex() {
} }
#[test] #[test]
fn detect_language_by_double_barrel_file_extension() { fn detect_langauge_by_double_barrel_file_extension() {
let blade_dir = tree_sitter_dir( let blade_dir = tree_sitter_dir(
r#"{ r#"{
"grammars": [ "grammars": [

View file

@ -12,7 +12,7 @@ pub struct Pattern {
named: bool, named: bool,
field: Option<&'static str>, field: Option<&'static str>,
capture: Option<String>, capture: Option<String>,
children: Vec<Self>, children: Vec<Pattern>,
} }
#[derive(Clone, Debug, PartialEq, Eq)] #[derive(Clone, Debug, PartialEq, Eq)]
@ -225,7 +225,7 @@ impl Pattern {
} }
// Find every matching combination of child patterns and child nodes. // Find every matching combination of child patterns and child nodes.
let mut finished_matches = Vec::<Match<'_, 'tree>>::new(); let mut finished_matches = Vec::<Match>::new();
if cursor.goto_first_child() { if cursor.goto_first_child() {
let mut match_states = vec![(0, mat)]; let mut match_states = vec![(0, mat)];
loop { loop {

View file

@ -2669,64 +2669,6 @@ fn test_query_matches_within_range_of_long_repetition() {
}); });
} }
#[test]
fn test_query_matches_contained_within_range() {
allocations::record(|| {
let language = get_language("json");
let query = Query::new(
&language,
r#"
("[" @l_bracket "]" @r_bracket)
("{" @l_brace "}" @r_brace)
"#,
)
.unwrap();
let source = r#"
[
{"key1": "value1"},
{"key2": "value2"},
{"key3": "value3"},
{"key4": "value4"},
{"key5": "value5"},
{"key6": "value6"},
{"key7": "value7"},
{"key8": "value8"},
{"key9": "value9"},
{"key10": "value10"},
{"key11": "value11"},
{"key12": "value12"},
]
"#
.unindent();
let mut parser = Parser::new();
parser.set_language(&language).unwrap();
let tree = parser.parse(&source, None).unwrap();
let expected_matches = [
(1, vec![("l_brace", "{"), ("r_brace", "}")]),
(1, vec![("l_brace", "{"), ("r_brace", "}")]),
];
{
let mut cursor = QueryCursor::new();
let matches = cursor
.set_containing_point_range(Point::new(5, 0)..Point::new(7, 0))
.matches(&query, tree.root_node(), source.as_bytes());
assert_eq!(collect_matches(matches, &query, &source), &expected_matches);
}
{
let mut cursor = QueryCursor::new();
let matches = cursor.set_containing_byte_range(78..120).matches(
&query,
tree.root_node(),
source.as_bytes(),
);
assert_eq!(collect_matches(matches, &query, &source), &expected_matches);
}
});
}
#[test] #[test]
fn test_query_matches_different_queries_same_cursor() { fn test_query_matches_different_queries_same_cursor() {
allocations::record(|| { allocations::record(|| {
@ -5090,26 +5032,6 @@ fn test_query_quantified_captures() {
("comment.documentation", "// quuz"), ("comment.documentation", "// quuz"),
], ],
}, },
Row {
description: "multiple quantifiers should not hang query parsing",
language: get_language("c"),
code: indoc! {"
// foo
// bar
// baz
"},
pattern: r"
((comment) ?+ @comment)
",
// This should be identical to the `*` quantifier.
captures: &[
("comment", "// foo"),
("comment", "// foo"),
("comment", "// foo"),
("comment", "// bar"),
("comment", "// baz"),
],
},
]; ];
allocations::record(|| { allocations::record(|| {

View file

@ -1,5 +1,6 @@
use std::{fs, path::PathBuf, process::Command}; use std::{fs, path::PathBuf, process::Command};
use anyhow::{anyhow, Context, Result};
use clap::ValueEnum; use clap::ValueEnum;
use log::{info, warn}; use log::{info, warn};
use regex::Regex; use regex::Regex;
@ -21,36 +22,6 @@ pub struct Version {
pub bump: Option<BumpLevel>, pub bump: Option<BumpLevel>,
} }
#[derive(thiserror::Error, Debug)]
pub enum VersionError {
#[error(transparent)]
Json(#[from] serde_json::Error),
#[error(transparent)]
Io(#[from] std::io::Error),
#[error("Failed to update one or more files:\n\n{0}")]
Update(UpdateErrors),
}
#[derive(thiserror::Error, Debug)]
pub struct UpdateErrors(Vec<UpdateError>);
impl std::fmt::Display for UpdateErrors {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
for error in &self.0 {
writeln!(f, "{error}\n")?;
}
Ok(())
}
}
#[derive(thiserror::Error, Debug)]
pub enum UpdateError {
#[error("Failed to update {1}:\n{0}")]
Io(std::io::Error, PathBuf),
#[error("Failed to run `{0}`:\n{1}")]
Command(&'static str, String),
}
impl Version { impl Version {
#[must_use] #[must_use]
pub const fn new( pub const fn new(
@ -65,7 +36,7 @@ impl Version {
} }
} }
pub fn run(mut self) -> Result<(), VersionError> { pub fn run(mut self) -> Result<()> {
let tree_sitter_json = self.current_dir.join("tree-sitter.json"); let tree_sitter_json = self.current_dir.join("tree-sitter.json");
let tree_sitter_json = let tree_sitter_json =
@ -113,101 +84,98 @@ impl Version {
let is_multigrammar = tree_sitter_json.grammars.len() > 1; let is_multigrammar = tree_sitter_json.grammars.len() > 1;
let mut errors = Vec::new(); self.update_treesitter_json().with_context(|| {
format!(
// Helper to push errors into the errors vector, returns true if an error was pushed "Failed to update tree-sitter.json at {}",
let mut push_err = |result: Result<(), UpdateError>| -> bool { self.current_dir.display()
if let Err(e) = result { )
errors.push(e); })?;
return true; self.update_cargo_toml().with_context(|| {
} format!(
false "Failed to update Cargo.toml at {}",
}; self.current_dir.display()
)
push_err(self.update_treesitter_json()); })?;
self.update_package_json().with_context(|| {
// Only update Cargo.lock if Cargo.toml was updated format!(
push_err(self.update_cargo_toml()).then(|| push_err(self.update_cargo_lock())); "Failed to update package.json at {}",
self.current_dir.display()
// Only update package-lock.json if package.json was updated )
push_err(self.update_package_json()).then(|| push_err(self.update_package_lock_json())); })?;
self.update_makefile(is_multigrammar).with_context(|| {
push_err(self.update_makefile(is_multigrammar)); format!(
push_err(self.update_cmakelists_txt()); "Failed to update Makefile at {}",
push_err(self.update_pyproject_toml()); self.current_dir.display()
push_err(self.update_zig_zon()); )
})?;
if errors.is_empty() { self.update_cmakelists_txt().with_context(|| {
Ok(()) format!(
} else { "Failed to update CMakeLists.txt at {}",
Err(VersionError::Update(UpdateErrors(errors))) self.current_dir.display()
} )
} })?;
self.update_pyproject_toml().with_context(|| {
fn update_file_with<F>(&self, path: &PathBuf, update_fn: F) -> Result<(), UpdateError> format!(
where "Failed to update pyproject.toml at {}",
F: Fn(&str) -> String, self.current_dir.display()
{ )
let content = fs::read_to_string(path).map_err(|e| UpdateError::Io(e, path.clone()))?;
let updated_content = update_fn(&content);
fs::write(path, updated_content).map_err(|e| UpdateError::Io(e, path.clone()))
}
fn update_treesitter_json(&self) -> Result<(), UpdateError> {
let json_path = self.current_dir.join("tree-sitter.json");
self.update_file_with(&json_path, |content| {
content
.lines()
.map(|line| {
if line.contains("\"version\":") {
let prefix_index =
line.find("\"version\":").unwrap() + "\"version\":".len();
let start_quote =
line[prefix_index..].find('"').unwrap() + prefix_index + 1;
let end_quote =
line[start_quote + 1..].find('"').unwrap() + start_quote + 1;
format!(
"{}{}{}",
&line[..start_quote],
self.version.as_ref().unwrap(),
&line[end_quote..]
)
} else {
line.to_string()
}
})
.collect::<Vec<_>>()
.join("\n")
+ "\n"
})
}
fn update_cargo_toml(&self) -> Result<(), UpdateError> {
let cargo_toml_path = self.current_dir.join("Cargo.toml");
if !cargo_toml_path.exists() {
return Ok(());
}
self.update_file_with(&cargo_toml_path, |content| {
content
.lines()
.map(|line| {
if line.starts_with("version =") {
format!("version = \"{}\"", self.version.as_ref().unwrap())
} else {
line.to_string()
}
})
.collect::<Vec<_>>()
.join("\n")
+ "\n"
})?; })?;
Ok(()) Ok(())
} }
fn update_cargo_lock(&self) -> Result<(), UpdateError> { fn update_treesitter_json(&self) -> Result<()> {
let tree_sitter_json = &fs::read_to_string(self.current_dir.join("tree-sitter.json"))?;
let tree_sitter_json = tree_sitter_json
.lines()
.map(|line| {
if line.contains("\"version\":") {
let prefix_index = line.find("\"version\":").unwrap() + "\"version\":".len();
let start_quote = line[prefix_index..].find('"').unwrap() + prefix_index + 1;
let end_quote = line[start_quote + 1..].find('"').unwrap() + start_quote + 1;
format!(
"{}{}{}",
&line[..start_quote],
self.version.as_ref().unwrap(),
&line[end_quote..]
)
} else {
line.to_string()
}
})
.collect::<Vec<_>>()
.join("\n")
+ "\n";
fs::write(self.current_dir.join("tree-sitter.json"), tree_sitter_json)?;
Ok(())
}
fn update_cargo_toml(&self) -> Result<()> {
if !self.current_dir.join("Cargo.toml").exists() {
return Ok(());
}
let cargo_toml = fs::read_to_string(self.current_dir.join("Cargo.toml"))?;
let cargo_toml = cargo_toml
.lines()
.map(|line| {
if line.starts_with("version =") {
format!("version = \"{}\"", self.version.as_ref().unwrap())
} else {
line.to_string()
}
})
.collect::<Vec<_>>()
.join("\n")
+ "\n";
fs::write(self.current_dir.join("Cargo.toml"), cargo_toml)?;
if self.current_dir.join("Cargo.lock").exists() { if self.current_dir.join("Cargo.lock").exists() {
let Ok(cmd) = Command::new("cargo") let Ok(cmd) = Command::new("cargo")
.arg("generate-lockfile") .arg("generate-lockfile")
@ -220,9 +188,8 @@ impl Version {
if !cmd.status.success() { if !cmd.status.success() {
let stderr = String::from_utf8_lossy(&cmd.stderr); let stderr = String::from_utf8_lossy(&cmd.stderr);
return Err(UpdateError::Command( return Err(anyhow!(
"cargo generate-lockfile", "Failed to run `cargo generate-lockfile`:\n{stderr}"
stderr.to_string(),
)); ));
} }
} }
@ -230,43 +197,37 @@ impl Version {
Ok(()) Ok(())
} }
fn update_package_json(&self) -> Result<(), UpdateError> { fn update_package_json(&self) -> Result<()> {
let package_json_path = self.current_dir.join("package.json"); if !self.current_dir.join("package.json").exists() {
if !package_json_path.exists() {
return Ok(()); return Ok(());
} }
self.update_file_with(&package_json_path, |content| { let package_json = &fs::read_to_string(self.current_dir.join("package.json"))?;
content
.lines()
.map(|line| {
if line.contains("\"version\":") {
let prefix_index =
line.find("\"version\":").unwrap() + "\"version\":".len();
let start_quote =
line[prefix_index..].find('"').unwrap() + prefix_index + 1;
let end_quote =
line[start_quote + 1..].find('"').unwrap() + start_quote + 1;
format!( let package_json = package_json
"{}{}{}", .lines()
&line[..start_quote], .map(|line| {
self.version.as_ref().unwrap(), if line.contains("\"version\":") {
&line[end_quote..] let prefix_index = line.find("\"version\":").unwrap() + "\"version\":".len();
) let start_quote = line[prefix_index..].find('"').unwrap() + prefix_index + 1;
} else { let end_quote = line[start_quote + 1..].find('"').unwrap() + start_quote + 1;
line.to_string()
}
})
.collect::<Vec<_>>()
.join("\n")
+ "\n"
})?;
Ok(()) format!(
} "{}{}{}",
&line[..start_quote],
self.version.as_ref().unwrap(),
&line[end_quote..]
)
} else {
line.to_string()
}
})
.collect::<Vec<_>>()
.join("\n")
+ "\n";
fs::write(self.current_dir.join("package.json"), package_json)?;
fn update_package_lock_json(&self) -> Result<(), UpdateError> {
if self.current_dir.join("package-lock.json").exists() { if self.current_dir.join("package-lock.json").exists() {
let Ok(cmd) = Command::new("npm") let Ok(cmd) = Command::new("npm")
.arg("install") .arg("install")
@ -279,117 +240,82 @@ impl Version {
if !cmd.status.success() { if !cmd.status.success() {
let stderr = String::from_utf8_lossy(&cmd.stderr); let stderr = String::from_utf8_lossy(&cmd.stderr);
return Err(UpdateError::Command("npm install", stderr.to_string())); return Err(anyhow!("Failed to run `npm install`:\n{stderr}"));
} }
} }
Ok(()) Ok(())
} }
fn update_makefile(&self, is_multigrammar: bool) -> Result<(), UpdateError> { fn update_makefile(&self, is_multigrammar: bool) -> Result<()> {
let makefile_path = if is_multigrammar { let makefile = if is_multigrammar {
self.current_dir.join("common").join("common.mak") if !self.current_dir.join("common").join("common.mak").exists() {
return Ok(());
}
fs::read_to_string(self.current_dir.join("Makefile"))?
} else { } else {
self.current_dir.join("Makefile") if !self.current_dir.join("Makefile").exists() {
return Ok(());
}
fs::read_to_string(self.current_dir.join("Makefile"))?
}; };
self.update_file_with(&makefile_path, |content| { let makefile = makefile
content .lines()
.lines() .map(|line| {
.map(|line| { if line.starts_with("VERSION") {
if line.starts_with("VERSION") { format!("VERSION := {}", self.version.as_ref().unwrap())
format!("VERSION := {}", self.version.as_ref().unwrap()) } else {
} else { line.to_string()
line.to_string() }
} })
}) .collect::<Vec<_>>()
.collect::<Vec<_>>() .join("\n")
.join("\n") + "\n";
+ "\n"
})?; fs::write(self.current_dir.join("Makefile"), makefile)?;
Ok(()) Ok(())
} }
fn update_cmakelists_txt(&self) -> Result<(), UpdateError> { fn update_cmakelists_txt(&self) -> Result<()> {
let cmake_lists_path = self.current_dir.join("CMakeLists.txt"); if !self.current_dir.join("CMakeLists.txt").exists() {
if !cmake_lists_path.exists() {
return Ok(()); return Ok(());
} }
self.update_file_with(&cmake_lists_path, |content| { let cmake = fs::read_to_string(self.current_dir.join("CMakeLists.txt"))?;
let re = Regex::new(r#"(\s*VERSION\s+)"[0-9]+\.[0-9]+\.[0-9]+""#)
.expect("Failed to compile regex"); let re = Regex::new(r#"(\s*VERSION\s+)"[0-9]+\.[0-9]+\.[0-9]+""#)?;
re.replace( let cmake = re.replace(&cmake, format!(r#"$1"{}""#, self.version.as_ref().unwrap()));
content,
format!(r#"$1"{}""#, self.version.as_ref().unwrap()), fs::write(self.current_dir.join("CMakeLists.txt"), cmake.as_bytes())?;
)
.to_string()
})?;
Ok(()) Ok(())
} }
fn update_pyproject_toml(&self) -> Result<(), UpdateError> { fn update_pyproject_toml(&self) -> Result<()> {
let pyproject_toml_path = self.current_dir.join("pyproject.toml"); if !self.current_dir.join("pyproject.toml").exists() {
if !pyproject_toml_path.exists() {
return Ok(()); return Ok(());
} }
self.update_file_with(&pyproject_toml_path, |content| { let pyproject_toml = fs::read_to_string(self.current_dir.join("pyproject.toml"))?;
content
.lines()
.map(|line| {
if line.starts_with("version =") {
format!("version = \"{}\"", self.version.as_ref().unwrap())
} else {
line.to_string()
}
})
.collect::<Vec<_>>()
.join("\n")
+ "\n"
})?;
Ok(()) let pyproject_toml = pyproject_toml
} .lines()
.map(|line| {
if line.starts_with("version =") {
format!("version = \"{}\"", self.version.as_ref().unwrap())
} else {
line.to_string()
}
})
.collect::<Vec<_>>()
.join("\n")
+ "\n";
fn update_zig_zon(&self) -> Result<(), UpdateError> { fs::write(self.current_dir.join("pyproject.toml"), pyproject_toml)?;
let zig_zon_path = self.current_dir.join("build.zig.zon");
if !zig_zon_path.exists() {
return Ok(());
}
self.update_file_with(&zig_zon_path, |content| {
let zig_version_prefix = ".version =";
content
.lines()
.map(|line| {
if line
.trim_start_matches(|c: char| c.is_ascii_whitespace())
.starts_with(zig_version_prefix)
{
let prefix_index =
line.find(zig_version_prefix).unwrap() + zig_version_prefix.len();
let start_quote =
line[prefix_index..].find('"').unwrap() + prefix_index + 1;
let end_quote =
line[start_quote + 1..].find('"').unwrap() + start_quote + 1;
format!(
"{}{}{}",
&line[..start_quote],
self.version.as_ref().unwrap(),
&line[end_quote..]
)
} else {
line.to_string()
}
})
.collect::<Vec<_>>()
.join("\n")
+ "\n"
})?;
Ok(()) Ok(())
} }

View file

@ -20,8 +20,8 @@ path = "src/tree_sitter_config.rs"
workspace = true workspace = true
[dependencies] [dependencies]
anyhow.workspace = true
etcetera.workspace = true etcetera.workspace = true
log.workspace = true log.workspace = true
serde.workspace = true serde.workspace = true
serde_json.workspace = true serde_json.workspace = true
thiserror.workspace = true

View file

@ -1,54 +1,12 @@
#![cfg_attr(not(any(test, doctest)), doc = include_str!("../README.md"))] #![cfg_attr(not(any(test, doctest)), doc = include_str!("../README.md"))]
use std::{ use std::{env, fs, path::PathBuf};
env, fs,
path::{Path, PathBuf},
};
use anyhow::{Context, Result};
use etcetera::BaseStrategy as _; use etcetera::BaseStrategy as _;
use log::warn; use log::warn;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_json::Value; use serde_json::Value;
use thiserror::Error;
pub type ConfigResult<T> = Result<T, ConfigError>;
#[derive(Debug, Error)]
pub enum ConfigError {
#[error("Bad JSON config {0} -- {1}")]
ConfigRead(String, serde_json::Error),
#[error(transparent)]
HomeDir(#[from] etcetera::HomeDirError),
#[error(transparent)]
IO(IoError),
#[error(transparent)]
Serialization(#[from] serde_json::Error),
}
#[derive(Debug, Error)]
pub struct IoError {
pub error: std::io::Error,
pub path: Option<String>,
}
impl IoError {
fn new(error: std::io::Error, path: Option<&Path>) -> Self {
Self {
error,
path: path.map(|p| p.to_string_lossy().to_string()),
}
}
}
impl std::fmt::Display for IoError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.error)?;
if let Some(ref path) = self.path {
write!(f, " ({path})")?;
}
Ok(())
}
}
/// Holds the contents of tree-sitter's configuration file. /// Holds the contents of tree-sitter's configuration file.
/// ///
@ -65,7 +23,7 @@ pub struct Config {
} }
impl Config { impl Config {
pub fn find_config_file() -> ConfigResult<Option<PathBuf>> { pub fn find_config_file() -> Result<Option<PathBuf>> {
if let Ok(path) = env::var("TREE_SITTER_DIR") { if let Ok(path) = env::var("TREE_SITTER_DIR") {
let mut path = PathBuf::from(path); let mut path = PathBuf::from(path);
path.push("config.json"); path.push("config.json");
@ -88,12 +46,8 @@ impl Config {
.join("tree-sitter") .join("tree-sitter")
.join("config.json"); .join("config.json");
if legacy_apple_path.is_file() { if legacy_apple_path.is_file() {
let xdg_dir = xdg_path.parent().unwrap(); fs::create_dir_all(xdg_path.parent().unwrap())?;
fs::create_dir_all(xdg_dir) fs::rename(&legacy_apple_path, &xdg_path)?;
.map_err(|e| ConfigError::IO(IoError::new(e, Some(xdg_dir))))?;
fs::rename(&legacy_apple_path, &xdg_path).map_err(|e| {
ConfigError::IO(IoError::new(e, Some(legacy_apple_path.as_path())))
})?;
warn!( warn!(
"Your config.json file has been automatically migrated from \"{}\" to \"{}\"", "Your config.json file has been automatically migrated from \"{}\" to \"{}\"",
legacy_apple_path.display(), legacy_apple_path.display(),
@ -113,7 +67,7 @@ impl Config {
Ok(None) Ok(None)
} }
fn xdg_config_file() -> ConfigResult<PathBuf> { fn xdg_config_file() -> Result<PathBuf> {
let xdg_path = etcetera::choose_base_strategy()? let xdg_path = etcetera::choose_base_strategy()?
.config_dir() .config_dir()
.join("tree-sitter") .join("tree-sitter")
@ -130,7 +84,7 @@ impl Config {
/// [`etcetera::choose_base_strategy`](https://docs.rs/etcetera/*/etcetera/#basestrategy) /// [`etcetera::choose_base_strategy`](https://docs.rs/etcetera/*/etcetera/#basestrategy)
/// - `$HOME/.tree-sitter/config.json` as a fallback from where tree-sitter _used_ to store /// - `$HOME/.tree-sitter/config.json` as a fallback from where tree-sitter _used_ to store
/// its configuration /// its configuration
pub fn load(path: Option<PathBuf>) -> ConfigResult<Self> { pub fn load(path: Option<PathBuf>) -> Result<Self> {
let location = if let Some(path) = path { let location = if let Some(path) = path {
path path
} else if let Some(path) = Self::find_config_file()? { } else if let Some(path) = Self::find_config_file()? {
@ -140,9 +94,9 @@ impl Config {
}; };
let content = fs::read_to_string(&location) let content = fs::read_to_string(&location)
.map_err(|e| ConfigError::IO(IoError::new(e, Some(location.as_path()))))?; .with_context(|| format!("Failed to read {}", location.to_string_lossy()))?;
let config = serde_json::from_str(&content) let config = serde_json::from_str(&content)
.map_err(|e| ConfigError::ConfigRead(location.to_string_lossy().to_string(), e))?; .with_context(|| format!("Bad JSON config {}", location.to_string_lossy()))?;
Ok(Self { location, config }) Ok(Self { location, config })
} }
@ -152,7 +106,7 @@ impl Config {
/// disk. /// disk.
/// ///
/// (Note that this is typically only done by the `tree-sitter init-config` command.) /// (Note that this is typically only done by the `tree-sitter init-config` command.)
pub fn initial() -> ConfigResult<Self> { pub fn initial() -> Result<Self> {
let location = if let Ok(path) = env::var("TREE_SITTER_DIR") { let location = if let Ok(path) = env::var("TREE_SITTER_DIR") {
let mut path = PathBuf::from(path); let mut path = PathBuf::from(path);
path.push("config.json"); path.push("config.json");
@ -165,20 +119,17 @@ impl Config {
} }
/// Saves this configuration to the file that it was originally loaded from. /// Saves this configuration to the file that it was originally loaded from.
pub fn save(&self) -> ConfigResult<()> { pub fn save(&self) -> Result<()> {
let json = serde_json::to_string_pretty(&self.config)?; let json = serde_json::to_string_pretty(&self.config)?;
let config_dir = self.location.parent().unwrap(); fs::create_dir_all(self.location.parent().unwrap())?;
fs::create_dir_all(config_dir) fs::write(&self.location, json)?;
.map_err(|e| ConfigError::IO(IoError::new(e, Some(config_dir))))?;
fs::write(&self.location, json)
.map_err(|e| ConfigError::IO(IoError::new(e, Some(self.location.as_path()))))?;
Ok(()) Ok(())
} }
/// Parses a component-specific configuration from the configuration file. The type `C` must /// Parses a component-specific configuration from the configuration file. The type `C` must
/// be [deserializable](https://docs.rs/serde/*/serde/trait.Deserialize.html) from a JSON /// be [deserializable](https://docs.rs/serde/*/serde/trait.Deserialize.html) from a JSON
/// object, and must only include the fields relevant to that component. /// object, and must only include the fields relevant to that component.
pub fn get<C>(&self) -> ConfigResult<C> pub fn get<C>(&self) -> Result<C>
where where
C: for<'de> Deserialize<'de>, C: for<'de> Deserialize<'de>,
{ {
@ -189,7 +140,7 @@ impl Config {
/// Adds a component-specific configuration to the configuration file. The type `C` must be /// Adds a component-specific configuration to the configuration file. The type `C` must be
/// [serializable](https://docs.rs/serde/*/serde/trait.Serialize.html) into a JSON object, and /// [serializable](https://docs.rs/serde/*/serde/trait.Serialize.html) into a JSON object, and
/// must only include the fields relevant to that component. /// must only include the fields relevant to that component.
pub fn add<C>(&mut self, config: C) -> ConfigResult<()> pub fn add<C>(&mut self, config: C) -> Result<()>
where where
C: Serialize, C: Serialize,
{ {

View file

@ -25,6 +25,7 @@ load = ["dep:semver"]
qjs-rt = ["load", "rquickjs", "pathdiff"] qjs-rt = ["load", "rquickjs", "pathdiff"]
[dependencies] [dependencies]
anyhow.workspace = true
bitflags = "2.9.4" bitflags = "2.9.4"
dunce = "1.0.5" dunce = "1.0.5"
indexmap.workspace = true indexmap.workspace = true
@ -33,7 +34,7 @@ log.workspace = true
pathdiff = { version = "0.2.3", optional = true } pathdiff = { version = "0.2.3", optional = true }
regex.workspace = true regex.workspace = true
regex-syntax.workspace = true regex-syntax.workspace = true
rquickjs = { version = "0.11.0", optional = true, features = [ rquickjs = { version = "0.9.0", optional = true, features = [
"bindgen", "bindgen",
"loader", "loader",
"macro", "macro",

View file

@ -81,7 +81,7 @@ pub enum ParseTableBuilderError {
StateCount(usize), StateCount(usize),
} }
#[derive(Default, Debug, Serialize, Error)] #[derive(Default, Debug, Serialize)]
pub struct ConflictError { pub struct ConflictError {
pub symbol_sequence: Vec<String>, pub symbol_sequence: Vec<String>,
pub conflicting_lookahead: String, pub conflicting_lookahead: String,
@ -89,7 +89,7 @@ pub struct ConflictError {
pub possible_resolutions: Vec<Resolution>, pub possible_resolutions: Vec<Resolution>,
} }
#[derive(Default, Debug, Serialize, Error)] #[derive(Default, Debug, Serialize)]
pub struct Interpretation { pub struct Interpretation {
pub preceding_symbols: Vec<String>, pub preceding_symbols: Vec<String>,
pub variable_name: String, pub variable_name: String,
@ -108,7 +108,7 @@ pub enum Resolution {
AddConflict { symbols: Vec<String> }, AddConflict { symbols: Vec<String> },
} }
#[derive(Debug, Serialize, Error)] #[derive(Debug, Serialize)]
pub struct AmbiguousExtraError { pub struct AmbiguousExtraError {
pub parent_symbols: Vec<String>, pub parent_symbols: Vec<String>,
} }
@ -238,6 +238,9 @@ impl std::fmt::Display for AmbiguousExtraError {
} }
} }
impl std::error::Error for ConflictError {}
impl std::error::Error for AmbiguousExtraError {}
impl<'a> ParseTableBuilder<'a> { impl<'a> ParseTableBuilder<'a> {
fn new( fn new(
syntax_grammar: &'a SyntaxGrammar, syntax_grammar: &'a SyntaxGrammar,

View file

@ -204,7 +204,7 @@ impl fmt::Display for ParseItemDisplay<'_> {
|| step.reserved_word_set_id != ReservedWordSetId::default() || step.reserved_word_set_id != ReservedWordSetId::default()
{ {
write!(f, " (")?; write!(f, " (")?;
if !step.precedence.is_none() { if step.precedence.is_none() {
write!(f, " {}", step.precedence)?; write!(f, " {}", step.precedence)?;
} }
if let Some(associativity) = step.associativity { if let Some(associativity) = step.associativity {

View file

@ -306,7 +306,9 @@ impl Minimizer<'_> {
return true; return true;
} }
for (action1, action2) in actions1.iter().zip(actions2.iter()) { for (i, action1) in actions1.iter().enumerate() {
let action2 = &actions2[i];
// Two shift actions are equivalent if their destinations are in the same group. // Two shift actions are equivalent if their destinations are in the same group.
if let ( if let (
ParseAction::Shift { ParseAction::Shift {

View file

@ -28,7 +28,7 @@ pub struct TokenConflictMap<'a> {
impl<'a> TokenConflictMap<'a> { impl<'a> TokenConflictMap<'a> {
/// Create a token conflict map based on a lexical grammar, which describes the structure /// Create a token conflict map based on a lexical grammar, which describes the structure
/// of each token, and a `following_token` map, which indicates which tokens may be appear /// each token, and a `following_token` map, which indicates which tokens may be appear
/// immediately after each other token. /// immediately after each other token.
/// ///
/// This analyzes the possible kinds of overlap between each pair of tokens and stores /// This analyzes the possible kinds of overlap between each pair of tokens and stores

View file

@ -1,4 +1,4 @@
use std::{collections::BTreeMap, sync::LazyLock}; use std::{collections::HashMap, sync::LazyLock};
#[cfg(feature = "load")] #[cfg(feature = "load")]
use std::{ use std::{
env, fs, env, fs,
@ -7,6 +7,7 @@ use std::{
process::{Command, Stdio}, process::{Command, Stdio},
}; };
use anyhow::Result;
use bitflags::bitflags; use bitflags::bitflags;
use log::warn; use log::warn;
use node_types::VariableInfo; use node_types::VariableInfo;
@ -56,7 +57,7 @@ struct JSONOutput {
syntax_grammar: SyntaxGrammar, syntax_grammar: SyntaxGrammar,
lexical_grammar: LexicalGrammar, lexical_grammar: LexicalGrammar,
inlines: InlinedProductionMap, inlines: InlinedProductionMap,
simple_aliases: BTreeMap<Symbol, Alias>, simple_aliases: HashMap<Symbol, Alias>,
variable_info: Vec<VariableInfo>, variable_info: Vec<VariableInfo>,
} }
@ -80,8 +81,8 @@ pub type GenerateResult<T> = Result<T, GenerateError>;
pub enum GenerateError { pub enum GenerateError {
#[error("Error with specified path -- {0}")] #[error("Error with specified path -- {0}")]
GrammarPath(String), GrammarPath(String),
#[error(transparent)] #[error("{0}")]
IO(IoError), IO(String),
#[cfg(feature = "load")] #[cfg(feature = "load")]
#[error(transparent)] #[error(transparent)]
LoadGrammarFile(#[from] LoadGrammarError), LoadGrammarFile(#[from] LoadGrammarError),
@ -100,28 +101,9 @@ pub enum GenerateError {
SuperTypeCycle(#[from] SuperTypeCycleError), SuperTypeCycle(#[from] SuperTypeCycleError),
} }
#[derive(Debug, Error, Serialize)] impl From<std::io::Error> for GenerateError {
pub struct IoError { fn from(value: std::io::Error) -> Self {
pub error: String, Self::IO(value.to_string())
pub path: Option<String>,
}
impl IoError {
fn new(error: &std::io::Error, path: Option<&Path>) -> Self {
Self {
error: error.to_string(),
path: path.map(|p| p.to_string_lossy().to_string()),
}
}
}
impl std::fmt::Display for IoError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.error)?;
if let Some(ref path) = self.path {
write!(f, " ({path})")?;
}
Ok(())
} }
} }
@ -136,11 +118,18 @@ pub enum LoadGrammarError {
#[error("Failed to load grammar.js -- {0}")] #[error("Failed to load grammar.js -- {0}")]
LoadJSGrammarFile(#[from] JSError), LoadJSGrammarFile(#[from] JSError),
#[error("Failed to load grammar.json -- {0}")] #[error("Failed to load grammar.json -- {0}")]
IO(IoError), IO(String),
#[error("Unknown grammar file extension: {0:?}")] #[error("Unknown grammar file extension: {0:?}")]
FileExtension(PathBuf), FileExtension(PathBuf),
} }
#[cfg(feature = "load")]
impl From<std::io::Error> for LoadGrammarError {
fn from(value: std::io::Error) -> Self {
Self::IO(value.to_string())
}
}
#[cfg(feature = "load")] #[cfg(feature = "load")]
#[derive(Debug, Error, Serialize)] #[derive(Debug, Error, Serialize)]
pub enum ParseVersionError { pub enum ParseVersionError {
@ -148,8 +137,8 @@ pub enum ParseVersionError {
Version(String), Version(String),
#[error("{0}")] #[error("{0}")]
JSON(String), JSON(String),
#[error(transparent)] #[error("{0}")]
IO(IoError), IO(String),
} }
#[cfg(feature = "load")] #[cfg(feature = "load")]
@ -164,21 +153,8 @@ pub enum JSError {
JSRuntimeUtf8 { runtime: String, error: String }, JSRuntimeUtf8 { runtime: String, error: String },
#[error("`{runtime}` process exited with status {code}")] #[error("`{runtime}` process exited with status {code}")]
JSRuntimeExit { runtime: String, code: i32 }, JSRuntimeExit { runtime: String, code: i32 },
#[error("Failed to open stdin for `{runtime}`")] #[error("{0}")]
JSRuntimeStdin { runtime: String }, IO(String),
#[error("Failed to write {item} to `{runtime}`'s stdin -- {error}")]
JSRuntimeWrite {
runtime: String,
item: String,
error: String,
},
#[error("Failed to read output from `{runtime}` -- {error}")]
JSRuntimeRead { runtime: String, error: String },
#[error(transparent)]
IO(IoError),
#[cfg(feature = "qjs-rt")]
#[error("Failed to get relative path")]
RelativePath,
#[error("Could not parse this package's version as semver -- {0}")] #[error("Could not parse this package's version as semver -- {0}")]
Semver(String), Semver(String),
#[error("Failed to serialze grammar JSON -- {0}")] #[error("Failed to serialze grammar JSON -- {0}")]
@ -188,6 +164,13 @@ pub enum JSError {
QuickJS(String), QuickJS(String),
} }
#[cfg(feature = "load")]
impl From<std::io::Error> for JSError {
fn from(value: std::io::Error) -> Self {
Self::IO(value.to_string())
}
}
#[cfg(feature = "load")] #[cfg(feature = "load")]
impl From<serde_json::Error> for JSError { impl From<serde_json::Error> for JSError {
fn from(value: serde_json::Error) -> Self { fn from(value: serde_json::Error) -> Self {
@ -248,8 +231,7 @@ where
.try_exists() .try_exists()
.map_err(|e| GenerateError::GrammarPath(e.to_string()))? .map_err(|e| GenerateError::GrammarPath(e.to_string()))?
{ {
fs::create_dir_all(&path_buf) fs::create_dir_all(&path_buf)?;
.map_err(|e| GenerateError::IO(IoError::new(&e, Some(path_buf.as_path()))))?;
repo_path = path_buf; repo_path = path_buf;
repo_path.join("grammar.js") repo_path.join("grammar.js")
} else { } else {
@ -266,12 +248,15 @@ where
let header_path = src_path.join("tree_sitter"); let header_path = src_path.join("tree_sitter");
// Ensure that the output directory exists // Ensure that the output directory exists
fs::create_dir_all(&src_path) fs::create_dir_all(&src_path)?;
.map_err(|e| GenerateError::IO(IoError::new(&e, Some(src_path.as_path()))))?;
if grammar_path.file_name().unwrap() != "grammar.json" { if grammar_path.file_name().unwrap() != "grammar.json" {
fs::write(src_path.join("grammar.json"), &grammar_json) fs::write(src_path.join("grammar.json"), &grammar_json).map_err(|e| {
.map_err(|e| GenerateError::IO(IoError::new(&e, Some(src_path.as_path()))))?; GenerateError::IO(format!(
"Failed to write grammar.json to {} -- {e}",
src_path.display()
))
})?;
} }
// If our job is only to generate `grammar.json` and not `parser.c`, stop here. // If our job is only to generate `grammar.json` and not `parser.c`, stop here.
@ -313,8 +298,7 @@ where
write_file(&src_path.join("parser.c"), c_code)?; write_file(&src_path.join("parser.c"), c_code)?;
write_file(&src_path.join("node-types.json"), node_types_json)?; write_file(&src_path.join("node-types.json"), node_types_json)?;
fs::create_dir_all(&header_path) fs::create_dir_all(&header_path)?;
.map_err(|e| GenerateError::IO(IoError::new(&e, Some(header_path.as_path()))))?;
write_file(&header_path.join("alloc.h"), ALLOC_HEADER)?; write_file(&header_path.join("alloc.h"), ALLOC_HEADER)?;
write_file(&header_path.join("array.h"), ARRAY_HEADER)?; write_file(&header_path.join("array.h"), ARRAY_HEADER)?;
write_file(&header_path.join("parser.h"), PARSER_HEADER)?; write_file(&header_path.join("parser.h"), PARSER_HEADER)?;
@ -430,8 +414,9 @@ fn read_grammar_version(repo_path: &Path) -> Result<Option<Version>, ParseVersio
let json = path let json = path
.exists() .exists()
.then(|| { .then(|| {
let contents = fs::read_to_string(path.as_path()) let contents = fs::read_to_string(path.as_path()).map_err(|e| {
.map_err(|e| ParseVersionError::IO(IoError::new(&e, Some(path.as_path()))))?; ParseVersionError::IO(format!("Failed to read `{}` -- {e}", path.display()))
})?;
serde_json::from_str::<TreeSitterJson>(&contents).map_err(|e| { serde_json::from_str::<TreeSitterJson>(&contents).map_err(|e| {
ParseVersionError::JSON(format!("Failed to parse `{}` -- {e}", path.display())) ParseVersionError::JSON(format!("Failed to parse `{}` -- {e}", path.display()))
}) })
@ -465,16 +450,14 @@ pub fn load_grammar_file(
} }
match grammar_path.extension().and_then(|e| e.to_str()) { match grammar_path.extension().and_then(|e| e.to_str()) {
Some("js") => Ok(load_js_grammar_file(grammar_path, js_runtime)?), Some("js") => Ok(load_js_grammar_file(grammar_path, js_runtime)?),
Some("json") => Ok(fs::read_to_string(grammar_path) Some("json") => Ok(fs::read_to_string(grammar_path)?),
.map_err(|e| LoadGrammarError::IO(IoError::new(&e, Some(grammar_path))))?),
_ => Err(LoadGrammarError::FileExtension(grammar_path.to_owned()))?, _ => Err(LoadGrammarError::FileExtension(grammar_path.to_owned()))?,
} }
} }
#[cfg(feature = "load")] #[cfg(feature = "load")]
fn load_js_grammar_file(grammar_path: &Path, js_runtime: Option<&str>) -> JSResult<String> { fn load_js_grammar_file(grammar_path: &Path, js_runtime: Option<&str>) -> JSResult<String> {
let grammar_path = dunce::canonicalize(grammar_path) let grammar_path = dunce::canonicalize(grammar_path)?;
.map_err(|e| JSError::IO(IoError::new(&e, Some(grammar_path))))?;
#[cfg(feature = "qjs-rt")] #[cfg(feature = "qjs-rt")]
if js_runtime == Some("native") { if js_runtime == Some("native") {
@ -515,9 +498,7 @@ fn load_js_grammar_file(grammar_path: &Path, js_runtime: Option<&str>) -> JSResu
let mut js_stdin = js_process let mut js_stdin = js_process
.stdin .stdin
.take() .take()
.ok_or_else(|| JSError::JSRuntimeStdin { .ok_or_else(|| JSError::IO(format!("Failed to open stdin for `{js_runtime}`")))?;
runtime: js_runtime.to_string(),
})?;
let cli_version = Version::parse(env!("CARGO_PKG_VERSION"))?; let cli_version = Version::parse(env!("CARGO_PKG_VERSION"))?;
write!( write!(
@ -527,26 +508,21 @@ fn load_js_grammar_file(grammar_path: &Path, js_runtime: Option<&str>) -> JSResu
globalThis.TREE_SITTER_CLI_VERSION_PATCH = {};", globalThis.TREE_SITTER_CLI_VERSION_PATCH = {};",
cli_version.major, cli_version.minor, cli_version.patch, cli_version.major, cli_version.minor, cli_version.patch,
) )
.map_err(|e| JSError::JSRuntimeWrite { .map_err(|e| {
runtime: js_runtime.to_string(), JSError::IO(format!(
item: "tree-sitter version".to_string(), "Failed to write tree-sitter version to `{js_runtime}`'s stdin -- {e}"
error: e.to_string(), ))
})?;
js_stdin.write(include_bytes!("./dsl.js")).map_err(|e| {
JSError::IO(format!(
"Failed to write grammar dsl to `{js_runtime}`'s stdin -- {e}"
))
})?; })?;
js_stdin
.write(include_bytes!("./dsl.js"))
.map_err(|e| JSError::JSRuntimeWrite {
runtime: js_runtime.to_string(),
item: "grammar dsl".to_string(),
error: e.to_string(),
})?;
drop(js_stdin); drop(js_stdin);
let output = js_process let output = js_process
.wait_with_output() .wait_with_output()
.map_err(|e| JSError::JSRuntimeRead { .map_err(|e| JSError::IO(format!("Failed to read output from `{js_runtime}` -- {e}")))?;
runtime: js_runtime.to_string(),
error: e.to_string(),
})?;
match output.status.code() { match output.status.code() {
Some(0) => { Some(0) => {
let stdout = String::from_utf8(output.stdout).map_err(|e| JSError::JSRuntimeUtf8 { let stdout = String::from_utf8(output.stdout).map_err(|e| JSError::JSRuntimeUtf8 {
@ -562,15 +538,9 @@ fn load_js_grammar_file(grammar_path: &Path, js_runtime: Option<&str>) -> JSResu
grammar_json = &stdout[pos + 1..]; grammar_json = &stdout[pos + 1..];
let mut stdout = std::io::stdout().lock(); let mut stdout = std::io::stdout().lock();
stdout stdout.write_all(node_output.as_bytes())?;
.write_all(node_output.as_bytes()) stdout.write_all(b"\n")?;
.map_err(|e| JSError::IO(IoError::new(&e, None)))?; stdout.flush()?;
stdout
.write_all(b"\n")
.map_err(|e| JSError::IO(IoError::new(&e, None)))?;
stdout
.flush()
.map_err(|e| JSError::IO(IoError::new(&e, None)))?;
} }
Ok(serde_json::to_string_pretty(&serde_json::from_str::< Ok(serde_json::to_string_pretty(&serde_json::from_str::<
@ -590,7 +560,8 @@ fn load_js_grammar_file(grammar_path: &Path, js_runtime: Option<&str>) -> JSResu
#[cfg(feature = "load")] #[cfg(feature = "load")]
pub fn write_file(path: &Path, body: impl AsRef<[u8]>) -> GenerateResult<()> { pub fn write_file(path: &Path, body: impl AsRef<[u8]>) -> GenerateResult<()> {
fs::write(path, body).map_err(|e| GenerateError::IO(IoError::new(&e, Some(path)))) fs::write(path, body)
.map_err(|e| GenerateError::IO(format!("Failed to write {:?} -- {e}", path.file_name())))
} }
#[cfg(test)] #[cfg(test)]

View file

@ -1,5 +1,6 @@
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::collections::{BTreeMap, HashMap, HashSet};
use anyhow::Result;
use serde::Serialize; use serde::Serialize;
use thiserror::Error; use thiserror::Error;
@ -377,11 +378,11 @@ pub fn get_variable_info(
fn get_aliases_by_symbol( fn get_aliases_by_symbol(
syntax_grammar: &SyntaxGrammar, syntax_grammar: &SyntaxGrammar,
default_aliases: &AliasMap, default_aliases: &AliasMap,
) -> HashMap<Symbol, BTreeSet<Option<Alias>>> { ) -> HashMap<Symbol, HashSet<Option<Alias>>> {
let mut aliases_by_symbol = HashMap::new(); let mut aliases_by_symbol = HashMap::new();
for (symbol, alias) in default_aliases { for (symbol, alias) in default_aliases {
aliases_by_symbol.insert(*symbol, { aliases_by_symbol.insert(*symbol, {
let mut aliases = BTreeSet::new(); let mut aliases = HashSet::new();
aliases.insert(Some(alias.clone())); aliases.insert(Some(alias.clone()));
aliases aliases
}); });
@ -390,7 +391,7 @@ fn get_aliases_by_symbol(
if !default_aliases.contains_key(extra_symbol) { if !default_aliases.contains_key(extra_symbol) {
aliases_by_symbol aliases_by_symbol
.entry(*extra_symbol) .entry(*extra_symbol)
.or_insert_with(BTreeSet::new) .or_insert_with(HashSet::new)
.insert(None); .insert(None);
} }
} }
@ -399,7 +400,7 @@ fn get_aliases_by_symbol(
for step in &production.steps { for step in &production.steps {
aliases_by_symbol aliases_by_symbol
.entry(step.symbol) .entry(step.symbol)
.or_insert_with(BTreeSet::new) .or_insert_with(HashSet::new)
.insert( .insert(
step.alias step.alias
.as_ref() .as_ref()
@ -530,7 +531,7 @@ pub fn generate_node_types_json(
let aliases_by_symbol = get_aliases_by_symbol(syntax_grammar, default_aliases); let aliases_by_symbol = get_aliases_by_symbol(syntax_grammar, default_aliases);
let empty = BTreeSet::new(); let empty = HashSet::new();
let extra_names = syntax_grammar let extra_names = syntax_grammar
.extra_symbols .extra_symbols
.iter() .iter()
@ -589,7 +590,7 @@ pub fn generate_node_types_json(
} else if !syntax_grammar.variables_to_inline.contains(&symbol) { } else if !syntax_grammar.variables_to_inline.contains(&symbol) {
// If a rule is aliased under multiple names, then its information // If a rule is aliased under multiple names, then its information
// contributes to multiple entries in the final JSON. // contributes to multiple entries in the final JSON.
for alias in aliases_by_symbol.get(&symbol).unwrap_or(&BTreeSet::new()) { for alias in aliases_by_symbol.get(&symbol).unwrap_or(&HashSet::new()) {
let kind; let kind;
let is_named; let is_named;
if let Some(alias) = alias { if let Some(alias) = alias {
@ -783,9 +784,6 @@ pub fn generate_node_types_json(
a_is_leaf.cmp(&b_is_leaf) a_is_leaf.cmp(&b_is_leaf)
}) })
.then_with(|| a.kind.cmp(&b.kind)) .then_with(|| a.kind.cmp(&b.kind))
.then_with(|| a.named.cmp(&b.named))
.then_with(|| a.root.cmp(&b.root))
.then_with(|| a.extra.cmp(&b.extra))
}); });
result.dedup(); result.dedup();
Ok(result) Ok(result)
@ -828,12 +826,12 @@ fn extend_sorted<'a, T>(vec: &mut Vec<T>, values: impl IntoIterator<Item = &'a T
where where
T: 'a + Clone + Eq + Ord, T: 'a + Clone + Eq + Ord,
{ {
values.into_iter().fold(false, |acc, value| { values.into_iter().any(|value| {
if let Err(i) = vec.binary_search(value) { if let Err(i) = vec.binary_search(value) {
vec.insert(i, value.clone()); vec.insert(i, value.clone());
true true
} else { } else {
acc false
} }
}) })
} }

View file

@ -1,5 +1,6 @@
use std::collections::HashSet; use std::collections::HashSet;
use anyhow::Result;
use log::warn; use log::warn;
use regex::Regex; use regex::Regex;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@ -17,7 +18,7 @@ use crate::{
#[allow(clippy::upper_case_acronyms)] #[allow(clippy::upper_case_acronyms)]
enum RuleJSON { enum RuleJSON {
ALIAS { ALIAS {
content: Box<Self>, content: Box<RuleJSON>,
named: bool, named: bool,
value: String, value: String,
}, },
@ -33,46 +34,46 @@ enum RuleJSON {
name: String, name: String,
}, },
CHOICE { CHOICE {
members: Vec<Self>, members: Vec<RuleJSON>,
}, },
FIELD { FIELD {
name: String, name: String,
content: Box<Self>, content: Box<RuleJSON>,
}, },
SEQ { SEQ {
members: Vec<Self>, members: Vec<RuleJSON>,
}, },
REPEAT { REPEAT {
content: Box<Self>, content: Box<RuleJSON>,
}, },
REPEAT1 { REPEAT1 {
content: Box<Self>, content: Box<RuleJSON>,
}, },
PREC_DYNAMIC { PREC_DYNAMIC {
value: i32, value: i32,
content: Box<Self>, content: Box<RuleJSON>,
}, },
PREC_LEFT { PREC_LEFT {
value: PrecedenceValueJSON, value: PrecedenceValueJSON,
content: Box<Self>, content: Box<RuleJSON>,
}, },
PREC_RIGHT { PREC_RIGHT {
value: PrecedenceValueJSON, value: PrecedenceValueJSON,
content: Box<Self>, content: Box<RuleJSON>,
}, },
PREC { PREC {
value: PrecedenceValueJSON, value: PrecedenceValueJSON,
content: Box<Self>, content: Box<RuleJSON>,
}, },
TOKEN { TOKEN {
content: Box<Self>, content: Box<RuleJSON>,
}, },
IMMEDIATE_TOKEN { IMMEDIATE_TOKEN {
content: Box<Self>, content: Box<RuleJSON>,
}, },
RESERVED { RESERVED {
context_name: String, context_name: String,
content: Box<Self>, content: Box<RuleJSON>,
}, },
} }

View file

@ -12,6 +12,7 @@ use std::{
mem, mem,
}; };
use anyhow::Result;
pub use expand_tokens::ExpandTokensError; pub use expand_tokens::ExpandTokensError;
pub use extract_tokens::ExtractTokensError; pub use extract_tokens::ExtractTokensError;
pub use flatten_grammar::FlattenGrammarError; pub use flatten_grammar::FlattenGrammarError;

View file

@ -1,3 +1,4 @@
use anyhow::Result;
use regex_syntax::{ use regex_syntax::{
hir::{Class, Hir, HirKind}, hir::{Class, Hir, HirKind},
ParserBuilder, ParserBuilder,
@ -26,7 +27,7 @@ pub enum ExpandTokensError {
"The rule `{0}` matches the empty string. "The rule `{0}` matches the empty string.
Tree-sitter does not support syntactic rules that match the empty string Tree-sitter does not support syntactic rules that match the empty string
unless they are used only as the grammar's start rule. unless they are used only as the grammar's start rule.
" "
)] )]
EmptyString(String), EmptyString(String),
#[error(transparent)] #[error(transparent)]
@ -188,7 +189,7 @@ impl NfaBuilder {
} }
Rule::String(s) => { Rule::String(s) => {
for c in s.chars().rev() { for c in s.chars().rev() {
self.push_advance(CharacterSet::from_char(c), next_state_id); self.push_advance(CharacterSet::empty().add_char(c), next_state_id);
next_state_id = self.nfa.last_state_id(); next_state_id = self.nfa.last_state_id();
} }
Ok(!s.is_empty()) Ok(!s.is_empty())

View file

@ -69,7 +69,9 @@ pub(super) fn extract_default_aliases(
SymbolType::External => &mut external_status_list[symbol.index], SymbolType::External => &mut external_status_list[symbol.index],
SymbolType::NonTerminal => &mut non_terminal_status_list[symbol.index], SymbolType::NonTerminal => &mut non_terminal_status_list[symbol.index],
SymbolType::Terminal => &mut terminal_status_list[symbol.index], SymbolType::Terminal => &mut terminal_status_list[symbol.index],
SymbolType::End | SymbolType::EndOfNonTerminalExtra => panic!("Unexpected end token"), SymbolType::End | SymbolType::EndOfNonTerminalExtra => {
panic!("Unexpected end token")
}
}; };
status.appears_unaliased = true; status.appears_unaliased = true;
} }

View file

@ -1,5 +1,6 @@
use std::collections::HashMap; use std::collections::HashMap;
use anyhow::Result;
use serde::Serialize; use serde::Serialize;
use thiserror::Error; use thiserror::Error;
@ -152,7 +153,7 @@ pub(super) fn extract_tokens(
} }
} }
let mut external_tokens = Vec::with_capacity(grammar.external_tokens.len()); let mut external_tokens = Vec::new();
for external_token in grammar.external_tokens { for external_token in grammar.external_tokens {
let rule = symbol_replacer.replace_symbols_in_rule(&external_token.rule); let rule = symbol_replacer.replace_symbols_in_rule(&external_token.rule);
if let Rule::Symbol(symbol) = rule { if let Rule::Symbol(symbol) = rule {

View file

@ -1,5 +1,6 @@
use std::collections::HashMap; use std::collections::HashMap;
use anyhow::Result;
use serde::Serialize; use serde::Serialize;
use thiserror::Error; use thiserror::Error;

View file

@ -1,3 +1,4 @@
use anyhow::Result;
use log::warn; use log::warn;
use serde::Serialize; use serde::Serialize;
use thiserror::Error; use thiserror::Error;

View file

@ -1,5 +1,6 @@
use std::collections::HashMap; use std::collections::HashMap;
use anyhow::Result;
use serde::Serialize; use serde::Serialize;
use thiserror::Error; use thiserror::Error;
@ -70,13 +71,12 @@ impl InlinedProductionMapBuilder {
let production_map = production_indices_by_step_id let production_map = production_indices_by_step_id
.into_iter() .into_iter()
.map(|(step_id, production_indices)| { .map(|(step_id, production_indices)| {
let production = let production = step_id.variable_index.map_or_else(
core::ptr::from_ref::<Production>(step_id.variable_index.map_or_else( || &productions[step_id.production_index],
|| &productions[step_id.production_index], |variable_index| {
|variable_index| { &grammar.variables[variable_index].productions[step_id.production_index]
&grammar.variables[variable_index].productions[step_id.production_index] },
}, ) as *const Production;
));
((production, step_id.step_index as u32), production_indices) ((production, step_id.step_index as u32), production_indices)
}) })
.collect(); .collect();

View file

@ -10,7 +10,7 @@ use rquickjs::{
Context, Ctx, Function, Module, Object, Runtime, Type, Value, Context, Ctx, Function, Module, Object, Runtime, Type, Value,
}; };
use super::{IoError, JSError, JSResult}; use super::{JSError, JSResult};
const DSL: &[u8] = include_bytes!("dsl.js"); const DSL: &[u8] = include_bytes!("dsl.js");
@ -95,27 +95,9 @@ impl Console {
Type::Module => "module".to_string(), Type::Module => "module".to_string(),
Type::BigInt => v.get::<String>().unwrap_or_else(|_| "BigInt".to_string()), Type::BigInt => v.get::<String>().unwrap_or_else(|_| "BigInt".to_string()),
Type::Unknown => "unknown".to_string(), Type::Unknown => "unknown".to_string(),
Type::Array => {
let js_vals = v
.as_array()
.unwrap()
.iter::<Value<'_>>()
.filter_map(|x| x.ok())
.map(|x| {
if x.is_string() {
format!("'{}'", Self::format_args(&[x]))
} else {
Self::format_args(&[x])
}
})
.collect::<Vec<_>>()
.join(", ");
format!("[ {js_vals} ]")
}
Type::Symbol Type::Symbol
| Type::Object | Type::Object
| Type::Proxy | Type::Array
| Type::Function | Type::Function
| Type::Constructor | Type::Constructor
| Type::Promise | Type::Promise
@ -215,11 +197,11 @@ fn try_resolve_path(path: &Path) -> rquickjs::Result<PathBuf> {
} }
#[allow(clippy::needless_pass_by_value)] #[allow(clippy::needless_pass_by_value)]
fn require_from_module<'js>( fn require_from_module<'a>(
ctx: Ctx<'js>, ctx: Ctx<'a>,
module_path: String, module_path: String,
from_module: &str, from_module: &str,
) -> rquickjs::Result<Value<'js>> { ) -> rquickjs::Result<Value<'a>> {
let current_module = PathBuf::from(from_module); let current_module = PathBuf::from(from_module);
let current_dir = if current_module.is_file() { let current_dir = if current_module.is_file() {
current_module.parent().unwrap_or(Path::new(".")) current_module.parent().unwrap_or(Path::new("."))
@ -234,13 +216,13 @@ fn require_from_module<'js>(
load_module_from_content(&ctx, &resolved_path, &contents) load_module_from_content(&ctx, &resolved_path, &contents)
} }
fn load_module_from_content<'js>( fn load_module_from_content<'a>(
ctx: &Ctx<'js>, ctx: &Ctx<'a>,
path: &Path, path: &Path,
contents: &str, contents: &str,
) -> rquickjs::Result<Value<'js>> { ) -> rquickjs::Result<Value<'a>> {
if path.extension().is_some_and(|ext| ext == "json") { if path.extension().is_some_and(|ext| ext == "json") {
return ctx.eval::<Value<'js>, _>(format!("JSON.parse({contents:?})")); return ctx.eval::<Value, _>(format!("JSON.parse({contents:?})"));
} }
let exports = Object::new(ctx.clone())?; let exports = Object::new(ctx.clone())?;
@ -256,7 +238,7 @@ fn load_module_from_content<'js>(
let module_path = filename.clone(); let module_path = filename.clone();
let require = Function::new( let require = Function::new(
ctx.clone(), ctx.clone(),
move |ctx_inner: Ctx<'js>, target_path: String| -> rquickjs::Result<Value<'js>> { move |ctx_inner: Ctx<'a>, target_path: String| -> rquickjs::Result<Value<'a>> {
require_from_module(ctx_inner, target_path, &module_path) require_from_module(ctx_inner, target_path, &module_path)
}, },
)?; )?;
@ -264,8 +246,8 @@ fn load_module_from_content<'js>(
let wrapper = let wrapper =
format!("(function(exports, require, module, __filename, __dirname) {{ {contents} }})"); format!("(function(exports, require, module, __filename, __dirname) {{ {contents} }})");
let module_func = ctx.eval::<Function<'js>, _>(wrapper)?; let module_func = ctx.eval::<Function, _>(wrapper)?;
module_func.call::<_, Value<'js>>((exports, require, module_obj.clone(), filename, dirname))?; module_func.call::<_, Value>((exports, require, module_obj.clone(), filename, dirname))?;
module_obj.get("exports") module_obj.get("exports")
} }
@ -279,16 +261,15 @@ pub fn execute_native_runtime(grammar_path: &Path) -> JSResult<String> {
let context = Context::full(&runtime)?; let context = Context::full(&runtime)?;
let resolver = FileResolver::default() let resolver = FileResolver::default()
.with_path("./node_modules")
.with_path("./") .with_path("./")
.with_pattern("{}.mjs"); .with_pattern("{}.mjs");
let loader = ScriptLoader::default().with_extension("mjs"); let loader = ScriptLoader::default().with_extension("mjs");
runtime.set_loader(resolver, loader); runtime.set_loader(resolver, loader);
let cwd = std::env::current_dir().map_err(|e| JSError::IO(IoError::new(&e, None)))?; let cwd = std::env::current_dir()?;
let relative_path = pathdiff::diff_paths(grammar_path, &cwd) let relative_path = pathdiff::diff_paths(grammar_path, &cwd)
.map(|p| p.to_string_lossy().to_string()) .map(|p| p.to_string_lossy().to_string())
.ok_or(JSError::RelativePath)?; .ok_or_else(|| JSError::IO("Failed to get relative path".to_string()))?;
context.with(|ctx| -> JSResult<String> { context.with(|ctx| -> JSResult<String> {
let globals = ctx.globals(); let globals = ctx.globals();

View file

@ -34,8 +34,6 @@ macro_rules! add {
macro_rules! add_whitespace { macro_rules! add_whitespace {
($this:tt) => {{ ($this:tt) => {{
// 4 bytes per char, 2 spaces per indent level
$this.buffer.reserve(4 * 2 * $this.indent_level);
for _ in 0..$this.indent_level { for _ in 0..$this.indent_level {
write!(&mut $this.buffer, " ").unwrap(); write!(&mut $this.buffer, " ").unwrap();
} }
@ -690,14 +688,13 @@ impl Generator {
flat_field_map.push((field_name.clone(), *location)); flat_field_map.push((field_name.clone(), *location));
} }
} }
let field_map_len = flat_field_map.len();
field_map_ids.push(( field_map_ids.push((
self.get_field_map_id( self.get_field_map_id(
flat_field_map, flat_field_map.clone(),
&mut flat_field_maps, &mut flat_field_maps,
&mut next_flat_field_map_index, &mut next_flat_field_map_index,
), ),
field_map_len, flat_field_map.len(),
)); ));
} }
} }
@ -965,7 +962,10 @@ impl Generator {
large_char_set_ix = Some(char_set_ix); large_char_set_ix = Some(char_set_ix);
} }
let line_break = format!("\n{}", " ".repeat(self.indent_level + 2)); let mut line_break = "\n".to_string();
for _ in 0..self.indent_level + 2 {
line_break.push_str(" ");
}
let has_positive_condition = large_char_set_ix.is_some() || !asserted_chars.is_empty(); let has_positive_condition = large_char_set_ix.is_some() || !asserted_chars.is_empty();
let has_negative_condition = !negated_chars.is_empty(); let has_negative_condition = !negated_chars.is_empty();

View file

@ -1,4 +1,4 @@
use std::{collections::BTreeMap, fmt}; use std::{collections::HashMap, fmt};
use serde::Serialize; use serde::Serialize;
use smallbitvec::SmallBitVec; use smallbitvec::SmallBitVec;
@ -34,7 +34,7 @@ pub enum Precedence {
Name(String), Name(String),
} }
pub type AliasMap = BTreeMap<Symbol, Alias>; pub type AliasMap = HashMap<Symbol, Alias>;
#[derive(Clone, Debug, Default, PartialEq, Eq, Hash, Serialize)] #[derive(Clone, Debug, Default, PartialEq, Eq, Hash, Serialize)]
pub struct MetadataParams { pub struct MetadataParams {
@ -60,15 +60,15 @@ pub enum Rule {
Pattern(String, String), Pattern(String, String),
NamedSymbol(String), NamedSymbol(String),
Symbol(Symbol), Symbol(Symbol),
Choice(Vec<Self>), Choice(Vec<Rule>),
Metadata { Metadata {
params: MetadataParams, params: MetadataParams,
rule: Box<Self>, rule: Box<Rule>,
}, },
Repeat(Box<Self>), Repeat(Box<Rule>),
Seq(Vec<Self>), Seq(Vec<Rule>),
Reserved { Reserved {
rule: Box<Self>, rule: Box<Rule>,
context_name: String, context_name: String,
}, },
} }

View file

@ -189,7 +189,7 @@ struct HighlightIterLayer<'a> {
depth: usize, depth: usize,
} }
pub struct _QueryCaptures<'query, 'tree, T: TextProvider<I>, I: AsRef<[u8]>> { pub struct _QueryCaptures<'query, 'tree: 'query, T: TextProvider<I>, I: AsRef<[u8]>> {
ptr: *mut ffi::TSQueryCursor, ptr: *mut ffi::TSQueryCursor,
query: &'query Query, query: &'query Query,
text_provider: T, text_provider: T,
@ -225,7 +225,7 @@ impl<'tree> _QueryMatch<'_, 'tree> {
} }
} }
impl<'query, 'tree, T: TextProvider<I>, I: AsRef<[u8]>> Iterator impl<'query, 'tree: 'query, T: TextProvider<I>, I: AsRef<[u8]>> Iterator
for _QueryCaptures<'query, 'tree, T, I> for _QueryCaptures<'query, 'tree, T, I>
{ {
type Item = (QueryMatch<'query, 'tree>, usize); type Item = (QueryMatch<'query, 'tree>, usize);
@ -344,13 +344,11 @@ impl HighlightConfiguration {
locals_query: &str, locals_query: &str,
) -> Result<Self, QueryError> { ) -> Result<Self, QueryError> {
// Concatenate the query strings, keeping track of the start offset of each section. // Concatenate the query strings, keeping track of the start offset of each section.
let mut query_source = String::with_capacity( let mut query_source = String::new();
injection_query.len() + locals_query.len() + highlights_query.len(),
);
query_source.push_str(injection_query); query_source.push_str(injection_query);
let locals_query_offset = injection_query.len(); let locals_query_offset = query_source.len();
query_source.push_str(locals_query); query_source.push_str(locals_query);
let highlights_query_offset = injection_query.len() + locals_query.len(); let highlights_query_offset = query_source.len();
query_source.push_str(highlights_query); query_source.push_str(highlights_query);
// Construct a single query by concatenating the three query strings, but record the // Construct a single query by concatenating the three query strings, but record the
@ -594,7 +592,6 @@ impl<'a> HighlightIterLayer<'a> {
} }
} }
// SAFETY:
// The `captures` iterator borrows the `Tree` and the `QueryCursor`, which // The `captures` iterator borrows the `Tree` and the `QueryCursor`, which
// prevents them from being moved. But both of these values are really just // prevents them from being moved. But both of these values are really just
// pointers, so it's actually ok to move them. // pointers, so it's actually ok to move them.

View file

@ -1,7 +1,7 @@
[package] [package]
name = "tree-sitter-language" name = "tree-sitter-language"
description = "The tree-sitter Language type, used by the library and by language implementations" description = "The tree-sitter Language type, used by the library and by language implementations"
version = "0.1.7" version = "0.1.5"
authors.workspace = true authors.workspace = true
edition.workspace = true edition.workspace = true
rust-version = "1.77" rust-version = "1.77"

View file

@ -23,15 +23,9 @@ typedef long unsigned int size_t;
typedef long unsigned int uintptr_t; typedef long unsigned int uintptr_t;
#define INT8_MAX 127
#define INT16_MAX 32767
#define INT32_MAX 2147483647L
#define INT64_MAX 9223372036854775807LL
#define UINT8_MAX 255
#define UINT16_MAX 65535 #define UINT16_MAX 65535
#define UINT32_MAX 4294967295U #define UINT32_MAX 4294967295U
#define UINT64_MAX 18446744073709551615ULL
#if defined(__wasm32__) #if defined(__wasm32__)

View file

@ -13,6 +13,4 @@ void *memset(void *dst, int value, size_t count);
int strncmp(const char *left, const char *right, size_t n); int strncmp(const char *left, const char *right, size_t n);
size_t strlen(const char *str);
#endif // TREE_SITTER_WASM_STRING_H_ #endif // TREE_SITTER_WASM_STRING_H_

View file

@ -1,5 +1,4 @@
#include <stdio.h> #include <stdio.h>
#include <string.h>
typedef struct { typedef struct {
bool left_justify; // - bool left_justify; // -
@ -106,6 +105,12 @@ static int ptr_to_str(void *ptr, char *buffer) {
return 2 + len; return 2 + len;
} }
size_t strlen(const char *str) {
const char *s = str;
while (*s) s++;
return s - str;
}
char *strncpy(char *dest, const char *src, size_t n) { char *strncpy(char *dest, const char *src, size_t n) {
char *d = dest; char *d = dest;
const char *s = src; const char *s = src;

View file

@ -58,9 +58,3 @@ int strncmp(const char *left, const char *right, size_t n) {
} }
return 0; return 0;
} }
size_t strlen(const char *str) {
const char *s = str;
while (*s) s++;
return s - str;
}

View file

@ -28,6 +28,7 @@ wasm = ["tree-sitter/wasm"]
default = ["tree-sitter-highlight", "tree-sitter-tags"] default = ["tree-sitter-highlight", "tree-sitter-tags"]
[dependencies] [dependencies]
anyhow.workspace = true
cc.workspace = true cc.workspace = true
etcetera.workspace = true etcetera.workspace = true
fs4.workspace = true fs4.workspace = true
@ -40,7 +41,6 @@ semver.workspace = true
serde.workspace = true serde.workspace = true
serde_json.workspace = true serde_json.workspace = true
tempfile.workspace = true tempfile.workspace = true
thiserror.workspace = true
tree-sitter = { workspace = true } tree-sitter = { workspace = true }
tree-sitter-highlight = { workspace = true, optional = true } tree-sitter-highlight = { workspace = true, optional = true }

File diff suppressed because it is too large Load diff

View file

@ -1 +0,0 @@
29.0

View file

@ -313,7 +313,6 @@ impl TagsContext {
) )
.ok_or(Error::Cancelled)?; .ok_or(Error::Cancelled)?;
// SAFETY:
// The `matches` iterator borrows the `Tree`, which prevents it from being // The `matches` iterator borrows the `Tree`, which prevents it from being
// moved. But the tree is really just a pointer, so it's actually ok to // moved. But the tree is really just a pointer, so it's actually ok to
// move it. // move it.

View file

@ -19,13 +19,9 @@ anstyle.workspace = true
anyhow.workspace = true anyhow.workspace = true
bindgen = { version = "0.72.0" } bindgen = { version = "0.72.0" }
clap.workspace = true clap.workspace = true
etcetera.workspace = true
indoc.workspace = true indoc.workspace = true
regex.workspace = true regex.workspace = true
schemars.workspace = true
semver.workspace = true semver.workspace = true
serde_json.workspace = true serde_json.workspace = true
tree-sitter-cli = { path = "../cli/" }
tree-sitter-loader = { path = "../loader/" }
notify = "8.2.0" notify = "8.2.0"
notify-debouncer-full = "0.6.0" notify-debouncer-full = "0.6.0"

View file

@ -8,15 +8,13 @@ use std::{
time::Duration, time::Duration,
}; };
use anyhow::{anyhow, Result}; use anyhow::{anyhow, Error, Result};
use etcetera::BaseStrategy as _;
use indoc::indoc; use indoc::indoc;
use notify::{ use notify::{
event::{AccessKind, AccessMode}, event::{AccessKind, AccessMode},
EventKind, RecursiveMode, EventKind, RecursiveMode,
}; };
use notify_debouncer_full::new_debouncer; use notify_debouncer_full::new_debouncer;
use tree_sitter_loader::{IoError, LoaderError, WasiSDKClangError};
use crate::{ use crate::{
bail_on_err, embed_sources::embed_sources_in_map, watch_wasm, BuildWasm, EMSCRIPTEN_TAG, bail_on_err, embed_sources::embed_sources_in_map, watch_wasm, BuildWasm, EMSCRIPTEN_TAG,
@ -52,8 +50,6 @@ const EXPORTED_RUNTIME_METHODS: [&str; 20] = [
"LE_HEAP_STORE_I64", "LE_HEAP_STORE_I64",
]; ];
const WASI_SDK_VERSION: &str = include_str!("../../loader/wasi-sdk-version").trim_ascii();
pub fn run_wasm(args: &BuildWasm) -> Result<()> { pub fn run_wasm(args: &BuildWasm) -> Result<()> {
let mut emscripten_flags = if args.debug { let mut emscripten_flags = if args.debug {
vec!["-O0", "--minify", "0"] vec!["-O0", "--minify", "0"]
@ -199,7 +195,6 @@ pub fn run_wasm(args: &BuildWasm) -> Result<()> {
"-D", "NDEBUG=", "-D", "NDEBUG=",
"-D", "_POSIX_C_SOURCE=200112L", "-D", "_POSIX_C_SOURCE=200112L",
"-D", "_DEFAULT_SOURCE=", "-D", "_DEFAULT_SOURCE=",
"-D", "_BSD_SOURCE=",
"-D", "_DARWIN_C_SOURCE=", "-D", "_DARWIN_C_SOURCE=",
"-I", "lib/src", "-I", "lib/src",
"-I", "lib/include", "-I", "lib/include",
@ -314,17 +309,9 @@ fn build_wasm(cmd: &mut Command, edit_tsd: bool) -> Result<()> {
Ok(()) Ok(())
} }
/// This ensures that the wasi-sdk is available, downloading and extracting it if necessary, /// This gets the path to the `clang` binary in the WASI SDK specified by the
/// and returns the path to the `clang` executable. /// `TREE_SITTER_WASI_SDK_PATH` environment variable.
/// fn get_wasi_binary() -> Result<PathBuf, Error> {
/// If `TREE_SITTER_WASI_SDK_PATH` is set, it will use that path to look for the clang executable.
///
/// Note that this is just a minimially modified version of
/// `tree_sitter_loader::ensure_wasi_sdk_exists`. In the loader, this functionality is implemented
/// as a private method of `Loader`. Rather than add this to the public API, we just
/// re-implement it. Any fixes and/or modifications made to the loader's copy should be reflected
/// here.
pub fn ensure_wasi_sdk_exists() -> Result<PathBuf> {
let possible_executables = if cfg!(windows) { let possible_executables = if cfg!(windows) {
vec![ vec![
"clang.exe", "clang.exe",
@ -345,122 +332,19 @@ pub fn ensure_wasi_sdk_exists() -> Result<PathBuf> {
} }
} }
Err(LoaderError::WasiSDKClang(WasiSDKClangError { return Err(anyhow!(
wasi_sdk_dir: wasi_sdk_dir.to_string_lossy().to_string(), "TREE_SITTER_WASI_SDK_PATH is set to '{}', but no clang executable found in 'bin/' directory. \
possible_executables: possible_executables.clone(), Looked for: {}",
download: false, wasi_sdk_dir.display(),
}))?; possible_executables.join(", ")
));
} }
let cache_dir = etcetera::choose_base_strategy()? Err(anyhow!(
.cache_dir() "TREE_SITTER_WASI_SDK_PATH environment variable is not set. \
.join("tree-sitter"); Please install the WASI SDK from https://github.com/WebAssembly/wasi-sdk/releases \
fs::create_dir_all(&cache_dir).map_err(|error| { and set TREE_SITTER_WASI_SDK_PATH to the installation directory."
LoaderError::IO(IoError { ))
error,
path: Some(cache_dir.to_string_lossy().to_string()),
})
})?;
let wasi_sdk_dir = cache_dir.join("wasi-sdk");
for exe in &possible_executables {
let clang_exe = wasi_sdk_dir.join("bin").join(exe);
if clang_exe.exists() {
return Ok(clang_exe);
}
}
fs::create_dir_all(&wasi_sdk_dir).map_err(|error| {
LoaderError::IO(IoError {
error,
path: Some(wasi_sdk_dir.to_string_lossy().to_string()),
})
})?;
let arch_os = if cfg!(target_os = "macos") {
if cfg!(target_arch = "aarch64") {
"arm64-macos"
} else {
"x86_64-macos"
}
} else if cfg!(target_os = "windows") {
if cfg!(target_arch = "aarch64") {
"arm64-windows"
} else {
"x86_64-windows"
}
} else if cfg!(target_os = "linux") {
if cfg!(target_arch = "aarch64") {
"arm64-linux"
} else {
"x86_64-linux"
}
} else {
Err(LoaderError::WasiSDKPlatform)?
};
let sdk_filename = format!("wasi-sdk-{WASI_SDK_VERSION}-{arch_os}.tar.gz");
let wasi_sdk_major_version = WASI_SDK_VERSION
.trim_end_matches(char::is_numeric) // trim minor version...
.trim_end_matches('.'); // ...and '.' separator
let sdk_url = format!(
"https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-{wasi_sdk_major_version}/{sdk_filename}",
);
eprintln!("Downloading wasi-sdk from {sdk_url}...");
let temp_tar_path = cache_dir.join(sdk_filename);
let status = Command::new("curl")
.arg("-f")
.arg("-L")
.arg("-o")
.arg(&temp_tar_path)
.arg(&sdk_url)
.status()
.map_err(|e| LoaderError::Curl(sdk_url.clone(), e))?;
if !status.success() {
Err(LoaderError::WasiSDKDownload(sdk_url))?;
}
eprintln!("Extracting wasi-sdk to {}...", wasi_sdk_dir.display());
extract_tar_gz_with_strip(&temp_tar_path, &wasi_sdk_dir)?;
fs::remove_file(temp_tar_path).ok();
for exe in &possible_executables {
let clang_exe = wasi_sdk_dir.join("bin").join(exe);
if clang_exe.exists() {
return Ok(clang_exe);
}
}
Err(LoaderError::WasiSDKClang(WasiSDKClangError {
wasi_sdk_dir: wasi_sdk_dir.to_string_lossy().to_string(),
possible_executables,
download: true,
}))?
}
/// Extracts a tar.gz archive with `tar`, stripping the first path component.
fn extract_tar_gz_with_strip(archive_path: &Path, destination: &Path) -> Result<()> {
let status = Command::new("tar")
.arg("-xzf")
.arg(archive_path)
.arg("--strip-components=1")
.arg("-C")
.arg(destination)
.status()
.map_err(|e| LoaderError::Tar(archive_path.to_string_lossy().to_string(), e))?;
if !status.success() {
Err(LoaderError::Extraction(
archive_path.to_string_lossy().to_string(),
destination.to_string_lossy().to_string(),
))?;
}
Ok(())
} }
pub fn run_wasm_stdlib() -> Result<()> { pub fn run_wasm_stdlib() -> Result<()> {
@ -469,7 +353,7 @@ pub fn run_wasm_stdlib() -> Result<()> {
.map(|line| format!("-Wl,--export={}", &line[1..line.len() - 2])) .map(|line| format!("-Wl,--export={}", &line[1..line.len() - 2]))
.collect::<Vec<String>>(); .collect::<Vec<String>>();
let clang_exe = ensure_wasi_sdk_exists()?; let clang_exe = get_wasi_binary()?;
let output = Command::new(&clang_exe) let output = Command::new(&clang_exe)
.args([ .args([

View file

@ -2,7 +2,7 @@ use std::{cmp::Ordering, path::Path};
use anyhow::{anyhow, Context, Result}; use anyhow::{anyhow, Context, Result};
use indoc::indoc; use indoc::indoc;
use semver::{Prerelease, Version}; use semver::{BuildMetadata, Prerelease, Version};
use crate::{create_commit, BumpVersion}; use crate::{create_commit, BumpVersion};
@ -48,6 +48,7 @@ pub fn run(args: BumpVersion) -> Result<()> {
String::from_utf8_lossy(&output.stderr) String::from_utf8_lossy(&output.stderr)
); );
} }
let latest_tag_sha = String::from_utf8(output.stdout)?.trim().to_string();
let workspace_toml_version = Version::parse(&fetch_workspace_version()?)?; let workspace_toml_version = Version::parse(&fetch_workspace_version()?)?;
@ -64,7 +65,102 @@ pub fn run(args: BumpVersion) -> Result<()> {
return Ok(()); return Ok(());
} }
let next_version = args.version; let output = std::process::Command::new("git")
.args(["rev-list", &format!("{latest_tag_sha}..HEAD")])
.output()?;
if !output.status.success() {
anyhow::bail!(
"Failed to get commits: {}",
String::from_utf8_lossy(&output.stderr)
);
}
let commits = String::from_utf8(output.stdout)?
.lines()
.map(|s| s.to_string())
.collect::<Vec<_>>();
let mut should_increment_patch = false;
let mut should_increment_minor = false;
for commit_sha in commits {
let output = std::process::Command::new("git")
.args(["log", "-1", "--format=%s", &commit_sha])
.output()?;
if !output.status.success() {
continue;
}
let message = String::from_utf8(output.stdout)?.trim().to_string();
let output = std::process::Command::new("git")
.args([
"diff-tree",
"--no-commit-id",
"--name-only",
"-r",
&commit_sha,
])
.output()?;
if !output.status.success() {
continue;
}
let mut source_code_changed = false;
for path in String::from_utf8(output.stdout)?.lines() {
let path = Path::new(path);
if path.extension().is_some_and(|ext| {
ext.eq_ignore_ascii_case("rs")
|| ext.eq_ignore_ascii_case("js")
|| ext.eq_ignore_ascii_case("c")
}) {
source_code_changed = true;
break;
}
}
if source_code_changed {
should_increment_patch = true;
let Some((prefix, _)) = message.split_once(':') else {
continue;
};
let convention = if prefix.contains('(') {
prefix.split_once('(').unwrap().0
} else {
prefix
};
if ["feat", "feat!"].contains(&convention) || prefix.ends_with('!') {
should_increment_minor = true;
}
}
}
let next_version = if let Some(version) = args.version {
version
} else {
let mut next_version = current_version.clone();
if should_increment_minor {
next_version.minor += 1;
next_version.patch = 0;
next_version.pre = Prerelease::EMPTY;
next_version.build = BuildMetadata::EMPTY;
} else if should_increment_patch {
next_version.patch += 1;
next_version.pre = Prerelease::EMPTY;
next_version.build = BuildMetadata::EMPTY;
} else {
return Err(anyhow!(format!(
"No source code changed since {current_version}"
)));
}
next_version
};
if next_version <= current_version {
return Err(anyhow!(format!(
"Next version {next_version} must be greater than current version {current_version}"
)));
}
println!("Bumping from {current_version} to {next_version}"); println!("Bumping from {current_version} to {next_version}");
update_crates(&current_version, &next_version)?; update_crates(&current_version, &next_version)?;

View file

@ -16,7 +16,7 @@ use notify_debouncer_full::new_debouncer;
use crate::{bail_on_err, watch_wasm, CheckWasmExports}; use crate::{bail_on_err, watch_wasm, CheckWasmExports};
const EXCLUDES: [&str; 25] = [ const EXCLUDES: [&str; 23] = [
// Unneeded because the JS side has its own way of implementing it // Unneeded because the JS side has its own way of implementing it
"ts_node_child_by_field_name", "ts_node_child_by_field_name",
"ts_node_edit", "ts_node_edit",
@ -44,8 +44,6 @@ const EXCLUDES: [&str; 25] = [
"ts_query_cursor_delete", "ts_query_cursor_delete",
"ts_query_cursor_match_limit", "ts_query_cursor_match_limit",
"ts_query_cursor_remove_match", "ts_query_cursor_remove_match",
"ts_query_cursor_set_point_range",
"ts_query_cursor_set_containing_byte_range",
]; ];
pub fn run(args: &CheckWasmExports) -> Result<()> { pub fn run(args: &CheckWasmExports) -> Result<()> {

View file

@ -7,7 +7,6 @@ mod embed_sources;
mod fetch; mod fetch;
mod generate; mod generate;
mod test; mod test;
mod test_schema;
mod upgrade_wasmtime; mod upgrade_wasmtime;
use std::{path::Path, process::Command}; use std::{path::Path, process::Command};
@ -41,8 +40,6 @@ enum Commands {
GenerateBindings, GenerateBindings,
/// Generates the fixtures for testing tree-sitter. /// Generates the fixtures for testing tree-sitter.
GenerateFixtures(GenerateFixtures), GenerateFixtures(GenerateFixtures),
/// Generates the JSON schema for the test runner summary.
GenerateTestSchema,
/// Generate the list of exports from Tree-sitter Wasm files. /// Generate the list of exports from Tree-sitter Wasm files.
GenerateWasmExports, GenerateWasmExports,
/// Run the test suite /// Run the test suite
@ -97,8 +94,8 @@ struct BuildWasm {
#[derive(Args)] #[derive(Args)]
struct BumpVersion { struct BumpVersion {
/// The version to bump to. /// The version to bump to.
#[arg(index = 1, required = true)] #[arg(long, short)]
version: Version, version: Option<Version>,
} }
#[derive(Args)] #[derive(Args)]
@ -239,7 +236,6 @@ fn run() -> Result<()> {
Commands::GenerateFixtures(generate_fixtures_options) => { Commands::GenerateFixtures(generate_fixtures_options) => {
generate::run_fixtures(&generate_fixtures_options)?; generate::run_fixtures(&generate_fixtures_options)?;
} }
Commands::GenerateTestSchema => test_schema::run_test_schema()?,
Commands::GenerateWasmExports => generate::run_wasm_exports()?, Commands::GenerateWasmExports => generate::run_wasm_exports()?,
Commands::Test(test_options) => test::run(&test_options)?, Commands::Test(test_options) => test::run(&test_options)?,
Commands::TestWasm => test::run_wasm()?, Commands::TestWasm => test::run_wasm()?,

View file

@ -73,6 +73,9 @@ pub fn run(args: &Test) -> Result<()> {
.arg("--no-run") .arg("--no-run")
.arg("--message-format=json"); .arg("--message-format=json");
#[cfg(target_os = "windows")]
cargo_cmd.arg("--").arg("--test-threads=1");
let cargo_cmd = cargo_cmd.stdout(Stdio::piped()).spawn()?; let cargo_cmd = cargo_cmd.stdout(Stdio::piped()).spawn()?;
let jq_cmd = Command::new("jq") let jq_cmd = Command::new("jq")
@ -100,6 +103,9 @@ pub fn run(args: &Test) -> Result<()> {
} }
cargo_cmd.args(&args.args); cargo_cmd.args(&args.args);
#[cfg(target_os = "windows")]
cargo_cmd.arg("--").arg("--test-threads=1");
if args.nocapture { if args.nocapture {
#[cfg(not(target_os = "windows"))] #[cfg(not(target_os = "windows"))]
cargo_cmd.arg("--"); cargo_cmd.arg("--");

View file

@ -1,25 +0,0 @@
use std::path::PathBuf;
use anyhow::Result;
use serde_json::to_writer_pretty;
use tree_sitter_cli::test::TestSummary;
pub fn run_test_schema() -> Result<()> {
let schema = schemars::schema_for!(TestSummary);
let xtask_path: PathBuf = env!("CARGO_MANIFEST_DIR").into();
let schema_path = xtask_path
.parent()
.unwrap()
.parent()
.unwrap()
.join("docs")
.join("src")
.join("assets")
.join("schemas")
.join("test-summary.schema.json");
let mut file = std::fs::File::create(schema_path)?;
Ok(to_writer_pretty(&mut file, &schema)?)
}

View file

@ -4,6 +4,7 @@ authors = [
"Amaan Qureshi <amaanq12@gmail.com>", "Amaan Qureshi <amaanq12@gmail.com>",
] ]
language = "en" language = "en"
multilingual = false
src = "src" src = "src"
title = "Tree-sitter" title = "Tree-sitter"

View file

@ -73,8 +73,9 @@ The behaviors of these three files are described in the next section.
## Queries ## Queries
Tree-sitter's syntax highlighting system is based on *tree queries*, which are a general system for pattern-matching on Tree-sitter's syntax highlighting system is based on *tree queries*, which are a general system for pattern-matching on Tree-sitter's
Tree-sitter's syntax trees. See [this section][pattern matching] of the documentation for more information about tree queries. syntax trees. See [this section][pattern matching] of the documentation for more information
about tree queries.
Syntax highlighting is controlled by *three* different types of query files that are usually included in the `queries` folder. Syntax highlighting is controlled by *three* different types of query files that are usually included in the `queries` folder.
The default names for the query files use the `.scm` file. We chose this extension because it commonly used for files written The default names for the query files use the `.scm` file. We chose this extension because it commonly used for files written

View file

@ -3,8 +3,7 @@
Tree-sitter can be used in conjunction with its [query language][query language] as a part of code navigation systems. Tree-sitter can be used in conjunction with its [query language][query language] as a part of code navigation systems.
An example of such a system can be seen in the `tree-sitter tags` command, which emits a textual dump of the interesting An example of such a system can be seen in the `tree-sitter tags` command, which emits a textual dump of the interesting
syntactic nodes in its file argument. A notable application of this is GitHub's support for [search-based code navigation][gh search]. syntactic nodes in its file argument. A notable application of this is GitHub's support for [search-based code navigation][gh search].
This document exists to describe how to integrate with such systems, and how to extend this functionality to any language This document exists to describe how to integrate with such systems, and how to extend this functionality to any language with a Tree-sitter grammar.
with a Tree-sitter grammar.
## Tagging and captures ## Tagging and captures
@ -13,9 +12,9 @@ entities. Having found them, you use a syntax capture to label the entity and it
The essence of a given tag lies in two pieces of data: the _role_ of the entity that is matched The essence of a given tag lies in two pieces of data: the _role_ of the entity that is matched
(i.e. whether it is a definition or a reference) and the _kind_ of that entity, which describes how the entity is used (i.e. whether it is a definition or a reference) and the _kind_ of that entity, which describes how the entity is used
(i.e. whether it's a class definition, function call, variable reference, and so on). Our convention is to use a syntax (i.e. whether it's a class definition, function call, variable reference, and so on). Our convention is to use a syntax capture
capture following the `@role.kind` capture name format, and another inner capture, always called `@name`, that pulls out following the `@role.kind` capture name format, and another inner capture, always called `@name`, that pulls out the name
the name of a given identifier. of a given identifier.
You may optionally include a capture named `@doc` to bind a docstring. For convenience purposes, the tagging system provides You may optionally include a capture named `@doc` to bind a docstring. For convenience purposes, the tagging system provides
two built-in functions, `#select-adjacent!` and `#strip!` that are convenient for removing comment syntax from a docstring. two built-in functions, `#select-adjacent!` and `#strip!` that are convenient for removing comment syntax from a docstring.

View file

@ -51,7 +51,7 @@ cargo install --path crates/cli
If you're going to be in a fast iteration cycle and would like the CLI to build faster, you can use the `release-dev` profile: If you're going to be in a fast iteration cycle and would like the CLI to build faster, you can use the `release-dev` profile:
```sh ```sh
cargo build --profile release-dev cargo build --release --profile release-dev
# or # or
cargo install --path crates/cli --profile release-dev cargo install --path crates/cli --profile release-dev
``` ```
@ -83,19 +83,6 @@ cargo xtask generate-fixtures --wasm
cargo xtask test-wasm cargo xtask test-wasm
``` ```
#### Wasm Stdlib
The tree-sitter Wasm stdlib can be built via xtask:
```sh
cargo xtask build-wasm-stdlib
```
This command looks for the [Wasi SDK][wasi_sdk] indicated by the `TREE_SITTER_WASI_SDK_PATH`
environment variable. If you don't have the binary, it can be downloaded from wasi-sdk's [releases][wasi-sdk-releases]
page. Note that any changes to `crates/language/wasm/**` requires rebuilding the tree-sitter Wasm stdlib via
`cargo xtask build-wasm-stdlib`.
### Debugging ### Debugging
The test script has a number of useful flags. You can list them all by running `cargo xtask test -h`. The test script has a number of useful flags. You can list them all by running `cargo xtask test -h`.
@ -233,6 +220,4 @@ and the tree-sitter module is fetched from [here][js url]. This, along with the
[pypi]: https://pypi.org [pypi]: https://pypi.org
[rust]: https://rustup.rs [rust]: https://rustup.rs
[ts repo]: https://github.com/tree-sitter/tree-sitter [ts repo]: https://github.com/tree-sitter/tree-sitter
[wasi_sdk]: https://github.com/WebAssembly/wasi-sdk
[wasi-sdk-releases]: https://github.com/WebAssembly/wasi-sdk/releases
[web-ts]: https://www.npmjs.com/package/web-tree-sitter [web-ts]: https://www.npmjs.com/package/web-tree-sitter

Some files were not shown because too many files have changed in this diff Show more