Compare commits

...
Sign in to create a new pull request.

44 commits

Author SHA1 Message Date
dependabot[bot]
6739742fb6 build(deps): bump cc from 1.2.52 to 1.2.53 in the cargo group
Bumps the cargo group with 1 update: [cc](https://github.com/rust-lang/cc-rs).


Updates `cc` from 1.2.52 to 1.2.53
- [Release notes](https://github.com/rust-lang/cc-rs/releases)
- [Changelog](https://github.com/rust-lang/cc-rs/blob/main/CHANGELOG.md)
- [Commits](https://github.com/rust-lang/cc-rs/compare/cc-v1.2.52...cc-v1.2.53)

---
updated-dependencies:
- dependency-name: cc
  dependency-version: 1.2.53
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cargo
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-01-20 10:34:48 +01:00
dependabot[bot]
d251226a3c ci: bump actions/github-script from 7 to 8 in the actions group
Bumps the actions group with 1 update: [actions/github-script](https://github.com/actions/github-script).


Updates `actions/github-script` from 7 to 8
- [Release notes](https://github.com/actions/github-script/releases)
- [Commits](https://github.com/actions/github-script/compare/v7...v8)

---
updated-dependencies:
- dependency-name: actions/github-script
  dependency-version: '8'
  dependency-type: direct:production
  update-type: version-update:semver-major
  dependency-group: actions
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-01-20 10:34:40 +01:00
Tam1SH
ae8184b8b9 docs(playground): highlight full row for highlighted nodes 2026-01-18 23:48:58 -05:00
Will Lillis
470ecf8996 feat(ci): ensure wasm-stdlib.h is regenerated when wasm stdlib source
files are modified.
2026-01-18 22:19:52 -05:00
Will Lillis
0cdb6bef7b fix(cli): warn user when nm can't be run to verify the symbols inside
the parser being built
2026-01-18 22:19:19 -05:00
theanarkh
cd603fa981
feat: free memory automatically (#5225) 2026-01-18 14:39:52 -08:00
DanikVitek
b12009a746 fix: Clarify/fix lifetimes
- One has to think about lifetimes if a type has one:
  - `<&'a Node<'tree>>::language` now returns `LanguageRef<'tree>` instead of
    `LanguageRef<'a>`, as it should;
- Remove explicit "outlives" requirements from `QueryMatches`, `QueryCaptures`,
  and their impl blocks, because they're inferred
- Removed unnecessary `&mut` from `cst_render_node`'s `cursor` parameter
2026-01-17 00:14:21 -05:00
DanikVitek
9f9a0bc410 fix: Renamed TreeCursor<'cursor> into TreeCursor<'tree>,
to be consistant with the usages and reduse confusion
2026-01-17 00:14:21 -05:00
Christian Clason
5d290a2a75 fix(wasm): regenerate stdlib with wasm-opt
Problem: Output of `cargo xtask build-wasm-stdlib` depends on whether
`wasm-opt` is installed (since `clang` will use it by default if it
finds it).

Solution: Install it and rerun the xtask.
2026-01-15 15:33:37 +01:00
Will Lillis
5808350bfe fix(docs): appease clippy regarding spacing in README 2026-01-15 10:38:57 +01:00
Will Lillis
e64e74d5ed docs: adhere to 120 new word column limit for docs 2026-01-14 18:11:42 -05:00
Will Lillis
1a88b26a10 docs: note requirement to rebuild wasm stdlib 2026-01-14 18:11:42 -05:00
dependabot[bot]
6c05cdfb0c build(deps): bump the cargo group with 3 updates
Bumps the cargo group with 3 updates: [cc](https://github.com/rust-lang/cc-rs), [clap_complete](https://github.com/clap-rs/clap) and [serde_json](https://github.com/serde-rs/json).


Updates `cc` from 1.2.51 to 1.2.52
- [Release notes](https://github.com/rust-lang/cc-rs/releases)
- [Changelog](https://github.com/rust-lang/cc-rs/blob/main/CHANGELOG.md)
- [Commits](https://github.com/rust-lang/cc-rs/compare/cc-v1.2.51...cc-v1.2.52)

Updates `clap_complete` from 4.5.64 to 4.5.65
- [Release notes](https://github.com/clap-rs/clap/releases)
- [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md)
- [Commits](https://github.com/clap-rs/clap/compare/clap_complete-v4.5.64...clap_complete-v4.5.65)

Updates `serde_json` from 1.0.148 to 1.0.149
- [Release notes](https://github.com/serde-rs/json/releases)
- [Commits](https://github.com/serde-rs/json/compare/v1.0.148...v1.0.149)

---
updated-dependencies:
- dependency-name: cc
  dependency-version: 1.2.52
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cargo
- dependency-name: clap_complete
  dependency-version: 4.5.65
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cargo
- dependency-name: serde_json
  dependency-version: 1.0.149
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cargo
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-01-13 10:13:03 +01:00
Will Lillis
aefae11c0d fix(build): define _BSD_SOURCE
System endian conversion macros are gated behind this feature flag for
older versions of GLIBC. `_BSD_SOURCE` and `_SVID_SOURCE` were
deprecated and replaced with `_DEFAULT_SOURCE` starting with GLIBC 2.19.
2026-01-12 19:41:58 -05:00
Kevin Wang
630fa52717 fix(templates): fix python free-threading compatibility 2026-01-09 11:44:41 +02:00
dependabot[bot]
eea85f4eff build(deps): bump clap from 4.5.53 to 4.5.54 in the cargo group
Bumps the cargo group with 1 update: [clap](https://github.com/clap-rs/clap).


Updates `clap` from 4.5.53 to 4.5.54
- [Release notes](https://github.com/clap-rs/clap/releases)
- [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md)
- [Commits](https://github.com/clap-rs/clap/compare/clap_complete-v4.5.53...clap_complete-v4.5.54)

---
updated-dependencies:
- dependency-name: clap
  dependency-version: 4.5.54
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cargo
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-01-07 16:42:00 +00:00
Christian Clason
cd6672701b fix(wasm): update wasm-stdlib.h 2026-01-06 17:28:39 +01:00
Trim21
f4ca3d95ca
fix(wasm) add common definitions to stdlib (#5199)
Also expose `strlen` through `string.h` instead of `stdio.h`.
2026-01-06 12:01:37 +01:00
skewb1k
17e3c7a5c5 fix(cli): restore test summary output for tree-sitter test
Problem:
After commit f02d7e7e33
the `tree-sitter test` command no longer printed the final test summary,
leaving empty line. The `Stats` struct was embedded into `TestSummary`,
and the explicit call to print it was removed.

Solution:
Print `parse_stats` from `TestSummary.fmt()` implementation.
2026-01-04 21:31:11 -08:00
WillLillis
dd60d5cff0 feat(cli): fill in missing fields to tree-sitter.json when running
`tree-sitter init -u`
2025-12-31 14:08:09 -05:00
WillLillis
f1288ea5c9 fix(cli): increase verbosity of tree-sitter init -u updates
Also, use `info` logs rather than `warn`
2025-12-31 14:08:09 -05:00
Christian Clason
47ae060966 feat(quickjs): add console support for Array 2025-12-31 13:32:09 +01:00
Christian Clason
a1893b4420 build(deps): update rquickjs to 0.11.0 2025-12-31 13:32:09 +01:00
skewb1k
999e041d49 docs: add tip about using test --update flag 2025-12-31 01:43:48 -05:00
skewb1k
0d4d854809 feat(cli): make test --update rewrite all corpus files 2025-12-31 01:43:48 -05:00
WillLillis
93d793d249 fix(cli): canonicalize build --output path
This fixes a potential issue with the new lock file hashing mechanism,
in which two different path literals pointing to the same location would
hash to separate lock files, allowing a race condition.
2025-12-30 17:07:04 +01:00
dependabot[bot]
82486d4b0a build(deps): bump the cargo group with 2 updates
Bumps the cargo group with 2 updates: [cc](https://github.com/rust-lang/cc-rs) and [serde_json](https://github.com/serde-rs/json).


Updates `cc` from 1.2.50 to 1.2.51
- [Release notes](https://github.com/rust-lang/cc-rs/releases)
- [Changelog](https://github.com/rust-lang/cc-rs/blob/main/CHANGELOG.md)
- [Commits](https://github.com/rust-lang/cc-rs/compare/cc-v1.2.50...cc-v1.2.51)

Updates `serde_json` from 1.0.145 to 1.0.147
- [Release notes](https://github.com/serde-rs/json/releases)
- [Commits](https://github.com/serde-rs/json/compare/v1.0.145...v1.0.147)

---
updated-dependencies:
- dependency-name: cc
  dependency-version: 1.2.51
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cargo
- dependency-name: serde_json
  dependency-version: 1.0.147
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cargo
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-12-30 17:06:34 +01:00
Firas al-Khalil
5d9605a91e feat(cli): concurrent build of same grammar on different paths 2025-12-29 00:20:05 -05:00
Firas al-Khalil
5293dd683e fix(cli): report library load failure
Instead of panicking somehere else.

This happens on concurrent builds of the the same grammar.
2025-12-29 00:20:05 -05:00
Firas al-Khalil
62effdf128 fix(cli): report context on compile fail 2025-12-29 00:20:05 -05:00
WillLillis
8e4f21aba0 fix(rust): address nightly clippy lint 2025-12-27 17:05:53 -05:00
WillLillis
5208299bbb fix(cli): set language in cwd for all usages of highlight command 2025-12-27 17:05:53 -05:00
Christian Clason
ba7350c7ee docs(cli): better description of files generated by init 2025-12-25 13:16:57 +01:00
skewb1k
f96d518ebf fix(cli): remove extra newline with --cst
Makes CST output consistent with other formats.
2025-12-24 15:06:48 +01:00
skewb1k
d5b82fbbab fix(cli): remove extra indentation with --cst --no-ranges 2025-12-24 15:06:48 +01:00
kevin-hua-kraken
a7d8c0cbb2
fix(playground): update query API 2025-12-23 10:36:40 +02:00
dependabot[bot]
24007727d4 build(deps): bump the cargo group with 2 updates
Bumps the cargo group with 2 updates: [cc](https://github.com/rust-lang/cc-rs) and [clap_complete](https://github.com/clap-rs/clap).


Updates `cc` from 1.2.49 to 1.2.50
- [Release notes](https://github.com/rust-lang/cc-rs/releases)
- [Changelog](https://github.com/rust-lang/cc-rs/blob/main/CHANGELOG.md)
- [Commits](https://github.com/rust-lang/cc-rs/compare/cc-v1.2.49...cc-v1.2.50)

Updates `clap_complete` from 4.5.61 to 4.5.62
- [Release notes](https://github.com/clap-rs/clap/releases)
- [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md)
- [Commits](https://github.com/clap-rs/clap/compare/clap_complete-v4.5.61...clap_complete-v4.5.62)

---
updated-dependencies:
- dependency-name: cc
  dependency-version: 1.2.50
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cargo
- dependency-name: clap_complete
  dependency-version: 4.5.62
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: cargo
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-12-22 22:18:42 +01:00
dependabot[bot]
6aa63a7213 ci: bump korthout/backport-action from 3 to 4 in the actions group
Bumps the actions group with 1 update: [korthout/backport-action](https://github.com/korthout/backport-action).


Updates `korthout/backport-action` from 3 to 4
- [Release notes](https://github.com/korthout/backport-action/releases)
- [Commits](https://github.com/korthout/backport-action/compare/v3...v4)

---
updated-dependencies:
- dependency-name: korthout/backport-action
  dependency-version: '4'
  dependency-type: direct:production
  update-type: version-update:semver-major
  dependency-group: actions
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-12-22 22:18:09 +01:00
Will Lillis
eacb95c85d fix(cli): correct discrepancy with cst for --no-ranges 2025-12-16 21:34:03 -05:00
dependabot[bot]
6967640571 ci: bump the actions group with 2 updates
Bumps the actions group with 2 updates: [actions/upload-artifact](https://github.com/actions/upload-artifact) and [actions/download-artifact](https://github.com/actions/download-artifact).


Updates `actions/upload-artifact` from 5 to 6
- [Release notes](https://github.com/actions/upload-artifact/releases)
- [Commits](https://github.com/actions/upload-artifact/compare/v5...v6)

Updates `actions/download-artifact` from 6 to 7
- [Release notes](https://github.com/actions/download-artifact/releases)
- [Commits](https://github.com/actions/download-artifact/compare/v6...v7)

---
updated-dependencies:
- dependency-name: actions/upload-artifact
  dependency-version: '6'
  dependency-type: direct:production
  update-type: version-update:semver-major
  dependency-group: actions
- dependency-name: actions/download-artifact
  dependency-version: '7'
  dependency-type: direct:production
  update-type: version-update:semver-major
  dependency-group: actions
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-12-15 22:26:50 +01:00
skewb1k
4ac2d5d276 fix(cli): trailing whitespace after multiline text nodes in CST
Problem:
The CST printer emits trailing whitespace after multiline text nodes.
With 1704c604bf and `:cst` corpus tests
this causes trailing spaces to appear on `test --update`.
These spaces cannot be removed afterward, as the test runner
expects an exact character-for-character match for CST tests.

Solution:
Print whitespace only if node is not multiline.
2025-12-14 21:34:50 -05:00
skewb1k
642b56d9af fix(docs): remove conflicting --release cargo flag in contributing.md
The argument '--release' cannot be used with '--profile <PROFILE-NAME>'
2025-12-14 20:35:13 +01:00
Christian Clason
0574fcf256 docs(cli): include information on generated files 2025-12-14 14:26:49 +01:00
Christian Clason
98de2bc1a8 feat: start working on v0.27
* bump tree-sitter crates to 0.27.0
* bump tree-sitter-language to 0.1.7
2025-12-13 14:14:33 +01:00
79 changed files with 1907 additions and 1505 deletions

25
.github/scripts/wasm_stdlib.js vendored Normal file
View file

@ -0,0 +1,25 @@
module.exports = async ({ github, context, core }) => {
if (context.eventName !== 'pull_request') return;
const prNumber = context.payload.pull_request.number;
const owner = context.repo.owner;
const repo = context.repo.repo;
const { data: files } = await github.rest.pulls.listFiles({
owner,
repo,
pull_number: prNumber
});
const changedFiles = files.map(file => file.filename);
const wasmStdLibSrc = 'crates/language/wasm/';
const dirChanged = changedFiles.some(file => file.startsWith(wasmStdLibSrc));
if (!dirChanged) return;
const wasmStdLibHeader = 'lib/src/wasm/wasm-stdlib.h';
const requiredChanged = changedFiles.includes(wasmStdLibHeader);
if (!requiredChanged) core.setFailed(`Changes detected in ${wasmStdLibSrc} but ${wasmStdLibHeader} was not modified.`);
};

View file

@ -24,7 +24,7 @@ jobs:
private-key: ${{ secrets.BACKPORT_KEY }} private-key: ${{ secrets.BACKPORT_KEY }}
- name: Create backport PR - name: Create backport PR
uses: korthout/backport-action@v3 uses: korthout/backport-action@v4
with: with:
pull_title: "${pull_title}" pull_title: "${pull_title}"
label_pattern: "^ci:backport ([^ ]+)$" label_pattern: "^ci:backport ([^ ]+)$"

View file

@ -278,7 +278,7 @@ jobs:
- name: Upload CLI artifact - name: Upload CLI artifact
if: "!matrix.no-run" if: "!matrix.no-run"
uses: actions/upload-artifact@v5 uses: actions/upload-artifact@v6
with: with:
name: tree-sitter.${{ matrix.platform }} name: tree-sitter.${{ matrix.platform }}
path: target/${{ matrix.target }}/release/tree-sitter${{ contains(matrix.target, 'windows') && '.exe' || '' }} path: target/${{ matrix.target }}/release/tree-sitter${{ contains(matrix.target, 'windows') && '.exe' || '' }}
@ -287,7 +287,7 @@ jobs:
- name: Upload Wasm artifacts - name: Upload Wasm artifacts
if: matrix.platform == 'linux-x64' if: matrix.platform == 'linux-x64'
uses: actions/upload-artifact@v5 uses: actions/upload-artifact@v6
with: with:
name: tree-sitter.wasm name: tree-sitter.wasm
path: | path: |

View file

@ -44,3 +44,6 @@ jobs:
build: build:
uses: ./.github/workflows/build.yml uses: ./.github/workflows/build.yml
check-wasm-stdlib:
uses: ./.github/workflows/wasm_stdlib.yml

View file

@ -25,7 +25,7 @@ jobs:
uses: actions/checkout@v6 uses: actions/checkout@v6
- name: Download build artifacts - name: Download build artifacts
uses: actions/download-artifact@v6 uses: actions/download-artifact@v7
with: with:
path: artifacts path: artifacts

19
.github/workflows/wasm_stdlib.yml vendored Normal file
View file

@ -0,0 +1,19 @@
name: Check Wasm Stdlib build
on:
workflow_call:
jobs:
check:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v6
- name: Check directory changes
uses: actions/github-script@v8
with:
script: |
const scriptPath = `${process.env.GITHUB_WORKSPACE}/.github/scripts/wasm_stdlib.js`;
const script = require(scriptPath);
return script({ github, context, core });

View file

@ -1,7 +1,7 @@
cmake_minimum_required(VERSION 3.13) cmake_minimum_required(VERSION 3.13)
project(tree-sitter project(tree-sitter
VERSION "0.26.3" VERSION "0.27.0"
DESCRIPTION "An incremental parsing system for programming tools" DESCRIPTION "An incremental parsing system for programming tools"
HOMEPAGE_URL "https://tree-sitter.github.io/tree-sitter/" HOMEPAGE_URL "https://tree-sitter.github.io/tree-sitter/"
LANGUAGES C) LANGUAGES C)
@ -81,7 +81,7 @@ set_target_properties(tree-sitter
SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}" SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}"
DEFINE_SYMBOL "") DEFINE_SYMBOL "")
target_compile_definitions(tree-sitter PRIVATE _POSIX_C_SOURCE=200112L _DEFAULT_SOURCE _DARWIN_C_SOURCE) target_compile_definitions(tree-sitter PRIVATE _POSIX_C_SOURCE=200112L _DEFAULT_SOURCE _BSD_SOURCE _DARWIN_C_SOURCE)
include(GNUInstallDirs) include(GNUInstallDirs)

138
Cargo.lock generated
View file

@ -166,9 +166,9 @@ dependencies = [
[[package]] [[package]]
name = "bumpalo" name = "bumpalo"
version = "3.19.0" version = "3.19.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510"
dependencies = [ dependencies = [
"allocator-api2", "allocator-api2",
] ]
@ -187,9 +187,9 @@ checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3"
[[package]] [[package]]
name = "cc" name = "cc"
version = "1.2.49" version = "1.2.53"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90583009037521a116abf44494efecd645ba48b6622457080f080b85544e2215" checksum = "755d2fce177175ffca841e9a06afdb2c4ab0f593d53b4dee48147dfaade85932"
dependencies = [ dependencies = [
"find-msvc-tools", "find-msvc-tools",
"shlex", "shlex",
@ -241,9 +241,9 @@ dependencies = [
[[package]] [[package]]
name = "clap" name = "clap"
version = "4.5.53" version = "4.5.54"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8" checksum = "c6e6ff9dcd79cff5cd969a17a545d79e84ab086e444102a591e288a8aa3ce394"
dependencies = [ dependencies = [
"clap_builder", "clap_builder",
"clap_derive", "clap_derive",
@ -251,9 +251,9 @@ dependencies = [
[[package]] [[package]]
name = "clap_builder" name = "clap_builder"
version = "4.5.53" version = "4.5.54"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00" checksum = "fa42cf4d2b7a41bc8f663a7cab4031ebafa1bf3875705bfaf8466dc60ab52c00"
dependencies = [ dependencies = [
"anstream", "anstream",
"anstyle", "anstyle",
@ -263,9 +263,9 @@ dependencies = [
[[package]] [[package]]
name = "clap_complete" name = "clap_complete"
version = "4.5.61" version = "4.5.65"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39615915e2ece2550c0149addac32fb5bd312c657f43845bb9088cb9c8a7c992" checksum = "430b4dc2b5e3861848de79627b2bedc9f3342c7da5173a14eaa5d0f8dc18ae5d"
dependencies = [ dependencies = [
"clap", "clap",
] ]
@ -338,9 +338,9 @@ dependencies = [
[[package]] [[package]]
name = "convert_case" name = "convert_case"
version = "0.8.0" version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baaaa0ecca5b51987b9423ccdc971514dd8b0bb7b4060b983d3664dad3f1f89f" checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9"
dependencies = [ dependencies = [
"unicode-segmentation", "unicode-segmentation",
] ]
@ -664,9 +664,9 @@ dependencies = [
[[package]] [[package]]
name = "find-msvc-tools" name = "find-msvc-tools"
version = "0.1.5" version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db"
[[package]] [[package]]
name = "fnv" name = "fnv"
@ -980,9 +980,9 @@ dependencies = [
[[package]] [[package]]
name = "itoa" name = "itoa"
version = "1.0.15" version = "1.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
[[package]] [[package]]
name = "jni" name = "jni"
@ -1119,7 +1119,7 @@ version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad38eb12aea514a0466ea40a80fd8cc83637065948eb4a426e4aa46261175227" checksum = "ad38eb12aea514a0466ea40a80fd8cc83637065948eb4a426e4aa46261175227"
dependencies = [ dependencies = [
"rustix 1.1.2", "rustix 1.1.3",
] ]
[[package]] [[package]]
@ -1371,9 +1371,9 @@ dependencies = [
[[package]] [[package]]
name = "proc-macro2" name = "proc-macro2"
version = "1.0.103" version = "1.0.104"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" checksum = "9695f8df41bb4f3d222c95a67532365f569318332d03d5f3f67f37b20e6ebdf0"
dependencies = [ dependencies = [
"unicode-ident", "unicode-ident",
] ]
@ -1527,9 +1527,9 @@ dependencies = [
[[package]] [[package]]
name = "rquickjs" name = "rquickjs"
version = "0.10.0" version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a135375fbac5ba723bb6a48f432a72f81539cedde422f0121a86c7c4e96d8e0d" checksum = "c50dc6d6c587c339edb4769cf705867497a2baf0eca8b4645fa6ecd22f02c77a"
dependencies = [ dependencies = [
"rquickjs-core", "rquickjs-core",
"rquickjs-macro", "rquickjs-macro",
@ -1537,9 +1537,9 @@ dependencies = [
[[package]] [[package]]
name = "rquickjs-core" name = "rquickjs-core"
version = "0.10.0" version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bccb7121a123865c8ace4dea42e7ed84d78b90cbaf4ca32c59849d8d210c9672" checksum = "b8bf7840285c321c3ab20e752a9afb95548c75cd7f4632a0627cea3507e310c1"
dependencies = [ dependencies = [
"hashbrown 0.16.1", "hashbrown 0.16.1",
"phf", "phf",
@ -1549,9 +1549,9 @@ dependencies = [
[[package]] [[package]]
name = "rquickjs-macro" name = "rquickjs-macro"
version = "0.10.0" version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89f93602cc3112c7f30bf5f29e722784232138692c7df4c52ebbac7e035d900d" checksum = "7106215ff41a5677b104906a13e1a440b880f4b6362b5dc4f3978c267fad2b80"
dependencies = [ dependencies = [
"convert_case", "convert_case",
"fnv", "fnv",
@ -1568,9 +1568,9 @@ dependencies = [
[[package]] [[package]]
name = "rquickjs-sys" name = "rquickjs-sys"
version = "0.10.0" version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57b1b6528590d4d65dc86b5159eae2d0219709546644c66408b2441696d1d725" checksum = "27344601ef27460e82d6a4e1ecb9e7e99f518122095f3c51296da8e9be2b9d83"
dependencies = [ dependencies = [
"bindgen", "bindgen",
"cc", "cc",
@ -1597,9 +1597,9 @@ dependencies = [
[[package]] [[package]]
name = "rustix" name = "rustix"
version = "1.1.2" version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34"
dependencies = [ dependencies = [
"bitflags 2.10.0", "bitflags 2.10.0",
"errno", "errno",
@ -1614,12 +1614,6 @@ version = "1.0.22"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
[[package]]
name = "ryu"
version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
[[package]] [[package]]
name = "same-file" name = "same-file"
version = "1.0.6" version = "1.0.6"
@ -1631,9 +1625,9 @@ dependencies = [
[[package]] [[package]]
name = "schemars" name = "schemars"
version = "1.1.0" version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9558e172d4e8533736ba97870c4b2cd63f84b382a3d6eb063da41b91cce17289" checksum = "54e910108742c57a770f492731f99be216a52fadd361b06c8fb59d74ccc267d2"
dependencies = [ dependencies = [
"dyn-clone", "dyn-clone",
"ref-cast", "ref-cast",
@ -1644,9 +1638,9 @@ dependencies = [
[[package]] [[package]]
name = "schemars_derive" name = "schemars_derive"
version = "1.1.0" version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "301858a4023d78debd2353c7426dc486001bddc91ae31a76fb1f55132f7e2633" checksum = "4908ad288c5035a8eb12cfdf0d49270def0a268ee162b75eeee0f85d155a7c45"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@ -1707,16 +1701,16 @@ dependencies = [
[[package]] [[package]]
name = "serde_json" name = "serde_json"
version = "1.0.145" version = "1.0.149"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
dependencies = [ dependencies = [
"indexmap", "indexmap",
"itoa", "itoa",
"memchr", "memchr",
"ryu",
"serde", "serde",
"serde_core", "serde_core",
"zmij",
] ]
[[package]] [[package]]
@ -1784,9 +1778,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]] [[package]]
name = "syn" name = "syn"
version = "2.0.111" version = "2.0.112"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" checksum = "21f182278bf2d2bcb3c88b1b08a37df029d71ce3d3ae26168e3c653b213b99d4"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@ -1806,20 +1800,20 @@ dependencies = [
[[package]] [[package]]
name = "target-lexicon" name = "target-lexicon"
version = "0.13.3" version = "0.13.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df7f62577c25e07834649fc3b39fafdc597c0a3527dc1c60129201ccfcbaa50c" checksum = "b1dd07eb858a2067e2f3c7155d54e929265c264e6f37efe3ee7a8d1b5a1dd0ba"
[[package]] [[package]]
name = "tempfile" name = "tempfile"
version = "3.23.0" version = "3.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c"
dependencies = [ dependencies = [
"fastrand", "fastrand",
"getrandom 0.3.4", "getrandom 0.3.4",
"once_cell", "once_cell",
"rustix 1.1.2", "rustix 1.1.3",
"windows-sys 0.61.2", "windows-sys 0.61.2",
] ]
@ -1905,18 +1899,18 @@ dependencies = [
[[package]] [[package]]
name = "toml_datetime" name = "toml_datetime"
version = "0.7.3" version = "0.7.5+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2cdb639ebbc97961c51720f858597f7f24c4fc295327923af55b74c3c724533" checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347"
dependencies = [ dependencies = [
"serde_core", "serde_core",
] ]
[[package]] [[package]]
name = "toml_edit" name = "toml_edit"
version = "0.23.9" version = "0.23.10+spec-1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d7cbc3b4b49633d57a0509303158ca50de80ae32c265093b24c414705807832" checksum = "84c8b9f757e028cee9fa244aea147aab2a9ec09d5325a9b01e0a49730c2b5269"
dependencies = [ dependencies = [
"indexmap", "indexmap",
"toml_datetime", "toml_datetime",
@ -1926,9 +1920,9 @@ dependencies = [
[[package]] [[package]]
name = "toml_parser" name = "toml_parser"
version = "1.0.4" version = "1.0.6+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0cbe268d35bdb4bb5a56a2de88d0ad0eb70af5384a99d648cd4b3d04039800e" checksum = "a3198b4b0a8e11f09dd03e133c0280504d0801269e9afa46362ffde1cbeebf44"
dependencies = [ dependencies = [
"winnow", "winnow",
] ]
@ -1941,9 +1935,9 @@ checksum = "ea68304e134ecd095ac6c3574494fc62b909f416c4fca77e440530221e549d3d"
[[package]] [[package]]
name = "tracing" name = "tracing"
version = "0.1.43" version = "0.1.44"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"
dependencies = [ dependencies = [
"pin-project-lite", "pin-project-lite",
"tracing-attributes", "tracing-attributes",
@ -1963,16 +1957,16 @@ dependencies = [
[[package]] [[package]]
name = "tracing-core" name = "tracing-core"
version = "0.1.35" version = "0.1.36"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a04e24fab5c89c6a36eb8558c9656f30d81de51dfa4d3b45f26b21d61fa0a6c" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a"
dependencies = [ dependencies = [
"once_cell", "once_cell",
] ]
[[package]] [[package]]
name = "tree-sitter" name = "tree-sitter"
version = "0.26.3" version = "0.27.0"
dependencies = [ dependencies = [
"bindgen", "bindgen",
"cc", "cc",
@ -1986,7 +1980,7 @@ dependencies = [
[[package]] [[package]]
name = "tree-sitter-cli" name = "tree-sitter-cli"
version = "0.26.3" version = "0.27.0"
dependencies = [ dependencies = [
"ansi_colours", "ansi_colours",
"anstyle", "anstyle",
@ -2034,7 +2028,7 @@ dependencies = [
[[package]] [[package]]
name = "tree-sitter-config" name = "tree-sitter-config"
version = "0.26.3" version = "0.27.0"
dependencies = [ dependencies = [
"etcetera", "etcetera",
"log", "log",
@ -2045,7 +2039,7 @@ dependencies = [
[[package]] [[package]]
name = "tree-sitter-generate" name = "tree-sitter-generate"
version = "0.26.3" version = "0.27.0"
dependencies = [ dependencies = [
"bitflags 2.10.0", "bitflags 2.10.0",
"dunce", "dunce",
@ -2068,7 +2062,7 @@ dependencies = [
[[package]] [[package]]
name = "tree-sitter-highlight" name = "tree-sitter-highlight"
version = "0.26.3" version = "0.27.0"
dependencies = [ dependencies = [
"regex", "regex",
"streaming-iterator", "streaming-iterator",
@ -2078,11 +2072,11 @@ dependencies = [
[[package]] [[package]]
name = "tree-sitter-language" name = "tree-sitter-language"
version = "0.1.6" version = "0.1.7"
[[package]] [[package]]
name = "tree-sitter-loader" name = "tree-sitter-loader"
version = "0.26.3" version = "0.27.0"
dependencies = [ dependencies = [
"cc", "cc",
"etcetera", "etcetera",
@ -2104,7 +2098,7 @@ dependencies = [
[[package]] [[package]]
name = "tree-sitter-tags" name = "tree-sitter-tags"
version = "0.26.3" version = "0.27.0"
dependencies = [ dependencies = [
"memchr", "memchr",
"regex", "regex",
@ -2316,7 +2310,7 @@ dependencies = [
"postcard", "postcard",
"psm", "psm",
"pulley-interpreter", "pulley-interpreter",
"rustix 1.1.2", "rustix 1.1.3",
"serde", "serde",
"serde_derive", "serde_derive",
"smallvec", "smallvec",
@ -2425,7 +2419,7 @@ dependencies = [
"anyhow", "anyhow",
"cc", "cc",
"cfg-if", "cfg-if",
"rustix 1.1.2", "rustix 1.1.3",
"wasmtime-asm-macros", "wasmtime-asm-macros",
"wasmtime-versioned-export-macros", "wasmtime-versioned-export-macros",
"windows-sys 0.59.0", "windows-sys 0.59.0",
@ -2932,3 +2926,9 @@ dependencies = [
"quote", "quote",
"syn", "syn",
] ]
[[package]]
name = "zmij"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9747e91771f56fd7893e1164abd78febd14a670ceec257caad15e051de35f06"

View file

@ -14,13 +14,13 @@ members = [
resolver = "2" resolver = "2"
[workspace.package] [workspace.package]
version = "0.26.3" version = "0.27.0"
authors = [ authors = [
"Max Brunsfeld <maxbrunsfeld@gmail.com>", "Max Brunsfeld <maxbrunsfeld@gmail.com>",
"Amaan Qureshi <amaanq12@gmail.com>", "Amaan Qureshi <amaanq12@gmail.com>",
] ]
edition = "2021" edition = "2021"
rust-version = "1.84" rust-version = "1.85"
homepage = "https://tree-sitter.github.io/tree-sitter" homepage = "https://tree-sitter.github.io/tree-sitter"
repository = "https://github.com/tree-sitter/tree-sitter" repository = "https://github.com/tree-sitter/tree-sitter"
license = "MIT" license = "MIT"
@ -106,8 +106,8 @@ ansi_colours = "1.2.3"
anstyle = "1.0.13" anstyle = "1.0.13"
anyhow = "1.0.100" anyhow = "1.0.100"
bstr = "1.12.0" bstr = "1.12.0"
cc = "1.2.48" cc = "1.2.53"
clap = { version = "4.5.53", features = [ clap = { version = "4.5.54", features = [
"cargo", "cargo",
"derive", "derive",
"env", "env",
@ -115,7 +115,7 @@ clap = { version = "4.5.53", features = [
"string", "string",
"unstable-styles", "unstable-styles",
] } ] }
clap_complete = "4.5.61" clap_complete = "4.5.65"
clap_complete_nushell = "4.5.10" clap_complete_nushell = "4.5.10"
crc32fast = "1.5.0" crc32fast = "1.5.0"
ctor = "0.2.9" ctor = "0.2.9"
@ -140,7 +140,7 @@ rustc-hash = "2.1.1"
schemars = "1.0.5" schemars = "1.0.5"
semver = { version = "1.0.27", features = ["serde"] } semver = { version = "1.0.27", features = ["serde"] }
serde = { version = "1.0.219", features = ["derive"] } serde = { version = "1.0.219", features = ["derive"] }
serde_json = { version = "1.0.145", features = ["preserve_order"] } serde_json = { version = "1.0.149", features = ["preserve_order"] }
similar = "2.7.0" similar = "2.7.0"
smallbitvec = "2.6.0" smallbitvec = "2.6.0"
streaming-iterator = "0.1.9" streaming-iterator = "0.1.9"
@ -153,11 +153,11 @@ walkdir = "2.5.0"
wasmparser = "0.243.0" wasmparser = "0.243.0"
webbrowser = "1.0.5" webbrowser = "1.0.5"
tree-sitter = { version = "0.26.3", path = "./lib" } tree-sitter = { version = "0.27.0", path = "./lib" }
tree-sitter-generate = { version = "0.26.3", path = "./crates/generate" } tree-sitter-generate = { version = "0.27.0", path = "./crates/generate" }
tree-sitter-loader = { version = "0.26.3", path = "./crates/loader" } tree-sitter-loader = { version = "0.27.0", path = "./crates/loader" }
tree-sitter-config = { version = "0.26.3", path = "./crates/config" } tree-sitter-config = { version = "0.27.0", path = "./crates/config" }
tree-sitter-highlight = { version = "0.26.3", path = "./crates/highlight" } tree-sitter-highlight = { version = "0.27.0", path = "./crates/highlight" }
tree-sitter-tags = { version = "0.26.3", path = "./crates/tags" } tree-sitter-tags = { version = "0.27.0", path = "./crates/tags" }
tree-sitter-language = { version = "0.1", path = "./crates/language" } tree-sitter-language = { version = "0.1", path = "./crates/language" }

View file

@ -1,4 +1,4 @@
VERSION := 0.26.3 VERSION := 0.27.0
DESCRIPTION := An incremental parsing system for programming tools DESCRIPTION := An incremental parsing system for programming tools
HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/ HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/
@ -24,7 +24,7 @@ OBJ := $(SRC:.c=.o)
ARFLAGS := rcs ARFLAGS := rcs
CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types
override CFLAGS += -std=c11 -fPIC -fvisibility=hidden override CFLAGS += -std=c11 -fPIC -fvisibility=hidden
override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE -D_DARWIN_C_SOURCE override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE -D_BSD_SOURCE -D_DARWIN_C_SOURCE
override CFLAGS += -Ilib/src -Ilib/src/wasm -Ilib/include override CFLAGS += -Ilib/src -Ilib/src/wasm -Ilib/include
# ABI versioning # ABI versioning

View file

@ -27,6 +27,7 @@ let package = Package(
.headerSearchPath("src"), .headerSearchPath("src"),
.define("_POSIX_C_SOURCE", to: "200112L"), .define("_POSIX_C_SOURCE", to: "200112L"),
.define("_DEFAULT_SOURCE"), .define("_DEFAULT_SOURCE"),
.define("_BSD_SOURCE"),
.define("_DARWIN_C_SOURCE"), .define("_DARWIN_C_SOURCE"),
]), ]),
], ],

View file

@ -40,6 +40,7 @@ pub fn build(b: *std.Build) !void {
lib.root_module.addCMacro("_POSIX_C_SOURCE", "200112L"); lib.root_module.addCMacro("_POSIX_C_SOURCE", "200112L");
lib.root_module.addCMacro("_DEFAULT_SOURCE", ""); lib.root_module.addCMacro("_DEFAULT_SOURCE", "");
lib.root_module.addCMacro("_BSD_SOURCE", "");
lib.root_module.addCMacro("_DARWIN_C_SOURCE", ""); lib.root_module.addCMacro("_DARWIN_C_SOURCE", "");
if (wasm) { if (wasm) {

View file

@ -1,7 +1,7 @@
.{ .{
.name = .tree_sitter, .name = .tree_sitter,
.fingerprint = 0x841224b447ac0d4f, .fingerprint = 0x841224b447ac0d4f,
.version = "0.26.3", .version = "0.27.0",
.minimum_zig_version = "0.14.1", .minimum_zig_version = "0.14.1",
.paths = .{ .paths = .{
"build.zig", "build.zig",

View file

@ -7,7 +7,8 @@
[npmjs.com]: https://www.npmjs.org/package/tree-sitter-cli [npmjs.com]: https://www.npmjs.org/package/tree-sitter-cli
[npmjs.com badge]: https://img.shields.io/npm/v/tree-sitter-cli.svg?color=%23BF4A4A [npmjs.com badge]: https://img.shields.io/npm/v/tree-sitter-cli.svg?color=%23BF4A4A
The Tree-sitter CLI allows you to develop, test, and use Tree-sitter grammars from the command line. It works on `MacOS`, `Linux`, and `Windows`. The Tree-sitter CLI allows you to develop, test, and use Tree-sitter grammars from the command line. It works on `MacOS`,
`Linux`, and `Windows`.
### Installation ### Installation
@ -34,9 +35,11 @@ The `tree-sitter` binary itself has no dependencies, but specific commands have
### Commands ### Commands
* `generate` - The `tree-sitter generate` command will generate a Tree-sitter parser based on the grammar in the current working directory. See [the documentation] for more information. * `generate` - The `tree-sitter generate` command will generate a Tree-sitter parser based on the grammar in the current
working directory. See [the documentation] for more information.
* `test` - The `tree-sitter test` command will run the unit tests for the Tree-sitter parser in the current working directory. See [the documentation] for more information. * `test` - The `tree-sitter test` command will run the unit tests for the Tree-sitter parser in the current working directory.
See [the documentation] for more information.
* `parse` - The `tree-sitter parse` command will parse a file (or list of files) using Tree-sitter parsers. * `parse` - The `tree-sitter parse` command will parse a file (or list of files) using Tree-sitter parsers.

View file

@ -1,12 +1,12 @@
{ {
"name": "tree-sitter-cli", "name": "tree-sitter-cli",
"version": "0.26.3", "version": "0.27.0",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "tree-sitter-cli", "name": "tree-sitter-cli",
"version": "0.26.3", "version": "0.27.0",
"hasInstallScript": true, "hasInstallScript": true,
"license": "MIT", "license": "MIT",
"bin": { "bin": {

View file

@ -1,6 +1,6 @@
{ {
"name": "tree-sitter-cli", "name": "tree-sitter-cli",
"version": "0.26.3", "version": "0.27.0",
"author": { "author": {
"name": "Max Brunsfeld", "name": "Max Brunsfeld",
"email": "maxbrunsfeld@gmail.com" "email": "maxbrunsfeld@gmail.com"

View file

@ -8,7 +8,7 @@ use anyhow::{anyhow, Context, Result};
use crc32fast::hash as crc32; use crc32fast::hash as crc32;
use heck::{ToKebabCase, ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase}; use heck::{ToKebabCase, ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase};
use indoc::{formatdoc, indoc}; use indoc::{formatdoc, indoc};
use log::warn; use log::info;
use rand::{thread_rng, Rng}; use rand::{thread_rng, Rng};
use semver::Version; use semver::Version;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@ -123,7 +123,7 @@ const BUILD_ZIG_ZON_TEMPLATE: &str = include_str!("./templates/build.zig.zon");
const ROOT_ZIG_TEMPLATE: &str = include_str!("./templates/root.zig"); const ROOT_ZIG_TEMPLATE: &str = include_str!("./templates/root.zig");
const TEST_ZIG_TEMPLATE: &str = include_str!("./templates/test.zig"); const TEST_ZIG_TEMPLATE: &str = include_str!("./templates/test.zig");
const TREE_SITTER_JSON_SCHEMA: &str = pub const TREE_SITTER_JSON_SCHEMA: &str =
"https://tree-sitter.github.io/tree-sitter/assets/schemas/config.schema.json"; "https://tree-sitter.github.io/tree-sitter/assets/schemas/config.schema.json";
#[derive(Serialize, Deserialize, Clone)] #[derive(Serialize, Deserialize, Clone)]
@ -356,7 +356,7 @@ pub fn generate_grammar_files(
"tree-sitter-cli":"#}, "tree-sitter-cli":"#},
); );
if !contents.contains("module") { if !contents.contains("module") {
warn!("Updating package.json"); info!("Migrating package.json to ESM");
contents = contents.replace( contents = contents.replace(
r#""repository":"#, r#""repository":"#,
indoc! {r#" indoc! {r#"
@ -378,6 +378,7 @@ pub fn generate_grammar_files(
|path| { |path| {
let mut contents = fs::read_to_string(path)?; let mut contents = fs::read_to_string(path)?;
if contents.contains("module.exports") { if contents.contains("module.exports") {
info!("Migrating grammars.js to ESM");
contents = contents.replace("module.exports =", "export default"); contents = contents.replace("module.exports =", "export default");
write_file(path, contents)?; write_file(path, contents)?;
} }
@ -393,10 +394,16 @@ pub fn generate_grammar_files(
allow_update, allow_update,
|path| generate_file(path, GITIGNORE_TEMPLATE, language_name, &generate_opts), |path| generate_file(path, GITIGNORE_TEMPLATE, language_name, &generate_opts),
|path| { |path| {
let contents = fs::read_to_string(path)?; let mut contents = fs::read_to_string(path)?;
if !contents.contains("Zig artifacts") { if !contents.contains("Zig artifacts") {
warn!("Replacing .gitignore"); info!("Adding zig entries to .gitignore");
generate_file(path, GITIGNORE_TEMPLATE, language_name, &generate_opts)?; contents.push('\n');
contents.push_str(indoc! {"
# Zig artifacts
.zig-cache/
zig-cache/
zig-out/
"});
} }
Ok(()) Ok(())
}, },
@ -409,8 +416,13 @@ pub fn generate_grammar_files(
|path| generate_file(path, GITATTRIBUTES_TEMPLATE, language_name, &generate_opts), |path| generate_file(path, GITATTRIBUTES_TEMPLATE, language_name, &generate_opts),
|path| { |path| {
let mut contents = fs::read_to_string(path)?; let mut contents = fs::read_to_string(path)?;
contents = contents.replace("bindings/c/* ", "bindings/c/** "); let c_bindings_entry = "bindings/c/* ";
if contents.contains(c_bindings_entry) {
info!("Updating c bindings entry in .gitattributes");
contents = contents.replace(c_bindings_entry, "bindings/c/** ");
}
if !contents.contains("Zig bindings") { if !contents.contains("Zig bindings") {
info!("Adding zig entries to .gitattributes");
contents.push('\n'); contents.push('\n');
contents.push_str(indoc! {" contents.push_str(indoc! {"
# Zig bindings # Zig bindings
@ -438,39 +450,40 @@ pub fn generate_grammar_files(
}, |path| { }, |path| {
let mut contents = fs::read_to_string(path)?; let mut contents = fs::read_to_string(path)?;
if !contents.contains("#[cfg(with_highlights_query)]") { if !contents.contains("#[cfg(with_highlights_query)]") {
let replacement = indoc! {r#" info!("Updating query constants in bindings/rust/lib.rs");
#[cfg(with_highlights_query)] let replacement = indoc! {r#"
/// The syntax highlighting query for this grammar. #[cfg(with_highlights_query)]
pub const HIGHLIGHTS_QUERY: &str = include_str!("../../HIGHLIGHTS_QUERY_PATH"); /// The syntax highlighting query for this grammar.
pub const HIGHLIGHTS_QUERY: &str = include_str!("../../HIGHLIGHTS_QUERY_PATH");
#[cfg(with_injections_query)] #[cfg(with_injections_query)]
/// The language injection query for this grammar. /// The language injection query for this grammar.
pub const INJECTIONS_QUERY: &str = include_str!("../../INJECTIONS_QUERY_PATH"); pub const INJECTIONS_QUERY: &str = include_str!("../../INJECTIONS_QUERY_PATH");
#[cfg(with_locals_query)] #[cfg(with_locals_query)]
/// The local variable query for this grammar. /// The local variable query for this grammar.
pub const LOCALS_QUERY: &str = include_str!("../../LOCALS_QUERY_PATH"); pub const LOCALS_QUERY: &str = include_str!("../../LOCALS_QUERY_PATH");
#[cfg(with_tags_query)] #[cfg(with_tags_query)]
/// The symbol tagging query for this grammar. /// The symbol tagging query for this grammar.
pub const TAGS_QUERY: &str = include_str!("../../TAGS_QUERY_PATH"); pub const TAGS_QUERY: &str = include_str!("../../TAGS_QUERY_PATH");
"#} "#}
.replace("HIGHLIGHTS_QUERY_PATH", generate_opts.highlights_query_path) .replace("HIGHLIGHTS_QUERY_PATH", generate_opts.highlights_query_path)
.replace("INJECTIONS_QUERY_PATH", generate_opts.injections_query_path) .replace("INJECTIONS_QUERY_PATH", generate_opts.injections_query_path)
.replace("LOCALS_QUERY_PATH", generate_opts.locals_query_path) .replace("LOCALS_QUERY_PATH", generate_opts.locals_query_path)
.replace("TAGS_QUERY_PATH", generate_opts.tags_query_path); .replace("TAGS_QUERY_PATH", generate_opts.tags_query_path);
contents = contents contents = contents
.replace( .replace(
indoc! {r#" indoc! {r#"
// NOTE: uncomment these to include any queries that this grammar contains: // NOTE: uncomment these to include any queries that this grammar contains:
// pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm"); // pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
// pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm"); // pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm");
// pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm"); // pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm");
// pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm"); // pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm");
"#}, "#},
&replacement, &replacement,
); );
} }
write_file(path, contents)?; write_file(path, contents)?;
Ok(()) Ok(())
@ -483,6 +496,7 @@ pub fn generate_grammar_files(
|path| { |path| {
let mut contents = fs::read_to_string(path)?; let mut contents = fs::read_to_string(path)?;
if !contents.contains("wasm32-unknown-unknown") { if !contents.contains("wasm32-unknown-unknown") {
info!("Adding wasm32-unknown-unknown target to bindings/rust/build.rs");
let replacement = indoc!{r#" let replacement = indoc!{r#"
c_config.flag("-utf-8"); c_config.flag("-utf-8");
@ -503,19 +517,18 @@ pub fn generate_grammar_files(
wasm_src.join("string.c"), wasm_src.join("string.c"),
]); ]);
} }
"#}; "#}
let indented_replacement = replacement
.lines() .lines()
.map(|line| if line.is_empty() { line.to_string() } else { format!(" {line}") }) .map(|line| if line.is_empty() { line.to_string() } else { format!(" {line}") })
.collect::<Vec<_>>() .collect::<Vec<_>>()
.join("\n"); .join("\n");
contents = contents.replace(r#" c_config.flag("-utf-8");"#, &indented_replacement); contents = contents.replace(r#" c_config.flag("-utf-8");"#, &replacement);
} }
// Introduce configuration variables for dynamic query inclusion // Introduce configuration variables for dynamic query inclusion
if !contents.contains("with_highlights_query") { if !contents.contains("with_highlights_query") {
info!("Adding support for dynamic query inclusion to bindings/rust/build.rs");
let replaced = indoc! {r#" let replaced = indoc! {r#"
c_config.compile("tree-sitter-KEBAB_PARSER_NAME"); c_config.compile("tree-sitter-KEBAB_PARSER_NAME");
}"#} }"#}
@ -572,6 +585,7 @@ pub fn generate_grammar_files(
|path| { |path| {
let contents = fs::read_to_string(path)?; let contents = fs::read_to_string(path)?;
if contents.contains("\"LICENSE\"") { if contents.contains("\"LICENSE\"") {
info!("Adding LICENSE entry to bindings/rust/Cargo.toml");
write_file(path, contents.replace("\"LICENSE\"", "\"/LICENSE\""))?; write_file(path, contents.replace("\"LICENSE\"", "\"/LICENSE\""))?;
} }
Ok(()) Ok(())
@ -592,7 +606,7 @@ pub fn generate_grammar_files(
|path| { |path| {
let contents = fs::read_to_string(path)?; let contents = fs::read_to_string(path)?;
if !contents.contains("Object.defineProperty") { if !contents.contains("Object.defineProperty") {
warn!("Replacing index.js"); info!("Replacing index.js");
generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts)?; generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts)?;
} }
Ok(()) Ok(())
@ -606,7 +620,7 @@ pub fn generate_grammar_files(
|path| { |path| {
let contents = fs::read_to_string(path)?; let contents = fs::read_to_string(path)?;
if !contents.contains("export default binding") { if !contents.contains("export default binding") {
warn!("Replacing index.d.ts"); info!("Replacing index.d.ts");
generate_file(path, INDEX_D_TS_TEMPLATE, language_name, &generate_opts)?; generate_file(path, INDEX_D_TS_TEMPLATE, language_name, &generate_opts)?;
} }
Ok(()) Ok(())
@ -627,7 +641,7 @@ pub fn generate_grammar_files(
|path| { |path| {
let contents = fs::read_to_string(path)?; let contents = fs::read_to_string(path)?;
if !contents.contains("import") { if !contents.contains("import") {
warn!("Replacing binding_test.js"); info!("Replacing binding_test.js");
generate_file( generate_file(
path, path,
BINDING_TEST_JS_TEMPLATE, BINDING_TEST_JS_TEMPLATE,
@ -650,6 +664,7 @@ pub fn generate_grammar_files(
|path| { |path| {
let contents = fs::read_to_string(path)?; let contents = fs::read_to_string(path)?;
if contents.contains("fs.exists(") { if contents.contains("fs.exists(") {
info!("Replacing `fs.exists` calls in binding.gyp");
write_file(path, contents.replace("fs.exists(", "fs.existsSync("))?; write_file(path, contents.replace("fs.exists(", "fs.existsSync("))?;
} }
Ok(()) Ok(())
@ -662,14 +677,17 @@ pub fn generate_grammar_files(
// Generate C bindings // Generate C bindings
if tree_sitter_config.bindings.c { if tree_sitter_config.bindings.c {
let kebab_case_name = language_name.to_kebab_case();
missing_path(bindings_dir.join("c"), create_dir)?.apply(|path| { missing_path(bindings_dir.join("c"), create_dir)?.apply(|path| {
let old_file = &path.join(format!("tree-sitter-{}.h", language_name.to_kebab_case())); let header_name = format!("tree-sitter-{kebab_case_name}.h");
let old_file = &path.join(&header_name);
if allow_update && fs::exists(old_file).unwrap_or(false) { if allow_update && fs::exists(old_file).unwrap_or(false) {
info!("Removing bindings/c/{header_name}");
fs::remove_file(old_file)?; fs::remove_file(old_file)?;
} }
missing_path(path.join("tree_sitter"), create_dir)?.apply(|include_path| { missing_path(path.join("tree_sitter"), create_dir)?.apply(|include_path| {
missing_path( missing_path(
include_path.join(format!("tree-sitter-{}.h", language_name.to_kebab_case())), include_path.join(&header_name),
|path| { |path| {
generate_file(path, PARSER_NAME_H_TEMPLATE, language_name, &generate_opts) generate_file(path, PARSER_NAME_H_TEMPLATE, language_name, &generate_opts)
}, },
@ -678,7 +696,7 @@ pub fn generate_grammar_files(
})?; })?;
missing_path( missing_path(
path.join(format!("tree-sitter-{}.pc.in", language_name.to_kebab_case())), path.join(format!("tree-sitter-{kebab_case_name}.pc.in")),
|path| { |path| {
generate_file( generate_file(
path, path,
@ -698,23 +716,27 @@ pub fn generate_grammar_files(
|path| { |path| {
let mut contents = fs::read_to_string(path)?; let mut contents = fs::read_to_string(path)?;
if !contents.contains("cd '$(DESTDIR)$(LIBDIR)' && ln -sf") { if !contents.contains("cd '$(DESTDIR)$(LIBDIR)' && ln -sf") {
warn!("Replacing Makefile"); info!("Replacing Makefile");
generate_file(path, MAKEFILE_TEMPLATE, language_name, &generate_opts)?; generate_file(path, MAKEFILE_TEMPLATE, language_name, &generate_opts)?;
} else { } else {
contents = contents let replaced = indoc! {r"
.replace( $(PARSER): $(SRC_DIR)/grammar.json
indoc! {r" $(TS) generate $^
$(PARSER): $(SRC_DIR)/grammar.json "};
$(TS) generate $^ if contents.contains(replaced) {
"}, info!("Adding --no-parser target to Makefile");
indoc! {r" contents = contents
$(SRC_DIR)/grammar.json: grammar.js .replace(
$(TS) generate --no-parser $^ replaced,
indoc! {r"
$(SRC_DIR)/grammar.json: grammar.js
$(TS) generate --no-parser $^
$(PARSER): $(SRC_DIR)/grammar.json $(PARSER): $(SRC_DIR)/grammar.json
$(TS) generate $^ $(TS) generate $^
"} "}
); );
}
write_file(path, contents)?; write_file(path, contents)?;
} }
Ok(()) Ok(())
@ -726,8 +748,8 @@ pub fn generate_grammar_files(
allow_update, allow_update,
|path| generate_file(path, CMAKELISTS_TXT_TEMPLATE, language_name, &generate_opts), |path| generate_file(path, CMAKELISTS_TXT_TEMPLATE, language_name, &generate_opts),
|path| { |path| {
let mut contents = fs::read_to_string(path)?; let contents = fs::read_to_string(path)?;
contents = contents let replaced_contents = contents
.replace("add_custom_target(test", "add_custom_target(ts-test") .replace("add_custom_target(test", "add_custom_target(ts-test")
.replace( .replace(
&formatdoc! {r#" &formatdoc! {r#"
@ -775,7 +797,10 @@ pub fn generate_grammar_files(
COMMENT "Generating parser.c") COMMENT "Generating parser.c")
"#} "#}
); );
write_file(path, contents)?; if !replaced_contents.eq(&contents) {
info!("Updating CMakeLists.txt");
write_file(path, replaced_contents)?;
}
Ok(()) Ok(())
}, },
)?; )?;
@ -811,7 +836,8 @@ pub fn generate_grammar_files(
// Generate Python bindings // Generate Python bindings
if tree_sitter_config.bindings.python { if tree_sitter_config.bindings.python {
missing_path(bindings_dir.join("python"), create_dir)?.apply(|path| { missing_path(bindings_dir.join("python"), create_dir)?.apply(|path| {
let lang_path = path.join(format!("tree_sitter_{}", language_name.to_snake_case())); let snake_case_grammar_name = format!("tree_sitter_{}", language_name.to_snake_case());
let lang_path = path.join(&snake_case_grammar_name);
missing_path(&lang_path, create_dir)?; missing_path(&lang_path, create_dir)?;
missing_path_else( missing_path_else(
@ -821,6 +847,7 @@ pub fn generate_grammar_files(
|path| { |path| {
let mut contents = fs::read_to_string(path)?; let mut contents = fs::read_to_string(path)?;
if !contents.contains("PyModuleDef_Init") { if !contents.contains("PyModuleDef_Init") {
info!("Updating bindings/python/{snake_case_grammar_name}/binding.c");
contents = contents contents = contents
.replace("PyModule_Create", "PyModuleDef_Init") .replace("PyModule_Create", "PyModuleDef_Init")
.replace( .replace(
@ -862,7 +889,7 @@ pub fn generate_grammar_files(
|path| { |path| {
let contents = fs::read_to_string(path)?; let contents = fs::read_to_string(path)?;
if !contents.contains("uncomment these to include any queries") { if !contents.contains("uncomment these to include any queries") {
warn!("Replacing __init__.py"); info!("Replacing __init__.py");
generate_file(path, INIT_PY_TEMPLATE, language_name, &generate_opts)?; generate_file(path, INIT_PY_TEMPLATE, language_name, &generate_opts)?;
} }
Ok(()) Ok(())
@ -876,9 +903,10 @@ pub fn generate_grammar_files(
|path| { |path| {
let mut contents = fs::read_to_string(path)?; let mut contents = fs::read_to_string(path)?;
if contents.contains("uncomment these to include any queries") { if contents.contains("uncomment these to include any queries") {
warn!("Replacing __init__.pyi"); info!("Replacing __init__.pyi");
generate_file(path, INIT_PYI_TEMPLATE, language_name, &generate_opts)?; generate_file(path, INIT_PYI_TEMPLATE, language_name, &generate_opts)?;
} else if !contents.contains("CapsuleType") { } else if !contents.contains("CapsuleType") {
info!("Updating __init__.pyi");
contents = contents contents = contents
.replace( .replace(
"from typing import Final", "from typing import Final",
@ -910,6 +938,7 @@ pub fn generate_grammar_files(
|path| { |path| {
let mut contents = fs::read_to_string(path)?; let mut contents = fs::read_to_string(path)?;
if !contents.contains("Parser(Language(") { if !contents.contains("Parser(Language(") {
info!("Updating Language function in bindings/python/tests/test_binding.py");
contents = contents contents = contents
.replace("tree_sitter.Language(", "Parser(Language(") .replace("tree_sitter.Language(", "Parser(Language(")
.replace(".language())\n", ".language()))\n") .replace(".language())\n", ".language()))\n")
@ -930,11 +959,19 @@ pub fn generate_grammar_files(
allow_update, allow_update,
|path| generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts), |path| generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts),
|path| { |path| {
let contents = fs::read_to_string(path)?; let mut contents = fs::read_to_string(path)?;
if !contents.contains("build_ext") { if !contents.contains("build_ext") {
warn!("Replacing setup.py"); info!("Replacing setup.py");
generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts)?; generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts)?;
} }
if !contents.contains(" and not get_config_var") {
info!("Updating Python free-threading support in setup.py");
contents = contents.replace(
r#"startswith("cp"):"#,
r#"startswith("cp") and not get_config_var("Py_GIL_DISABLED"):"#
);
write_file(path, contents)?;
}
Ok(()) Ok(())
}, },
)?; )?;
@ -953,6 +990,7 @@ pub fn generate_grammar_files(
|path| { |path| {
let mut contents = fs::read_to_string(path)?; let mut contents = fs::read_to_string(path)?;
if !contents.contains("cp310-*") { if !contents.contains("cp310-*") {
info!("Updating dependencies in pyproject.toml");
contents = contents contents = contents
.replace(r#"build = "cp39-*""#, r#"build = "cp310-*""#) .replace(r#"build = "cp39-*""#, r#"build = "cp310-*""#)
.replace(r#"python = ">=3.9""#, r#"python = ">=3.10""#) .replace(r#"python = ">=3.9""#, r#"python = ">=3.10""#)
@ -990,15 +1028,18 @@ pub fn generate_grammar_files(
allow_update, allow_update,
|path| generate_file(path, PACKAGE_SWIFT_TEMPLATE, language_name, &generate_opts), |path| generate_file(path, PACKAGE_SWIFT_TEMPLATE, language_name, &generate_opts),
|path| { |path| {
let mut contents = fs::read_to_string(path)?; let contents = fs::read_to_string(path)?;
contents = contents let replaced_contents = contents
.replace( .replace(
"https://github.com/ChimeHQ/SwiftTreeSitter", "https://github.com/ChimeHQ/SwiftTreeSitter",
"https://github.com/tree-sitter/swift-tree-sitter", "https://github.com/tree-sitter/swift-tree-sitter",
) )
.replace("version: \"0.8.0\")", "version: \"0.9.0\")") .replace("version: \"0.8.0\")", "version: \"0.9.0\")")
.replace("(url:", "(name: \"SwiftTreeSitter\", url:"); .replace("(url:", "(name: \"SwiftTreeSitter\", url:");
write_file(path, contents)?; if !replaced_contents.eq(&contents) {
info!("Updating tree-sitter dependency in Package.swift");
write_file(path, contents)?;
}
Ok(()) Ok(())
}, },
)?; )?;
@ -1016,7 +1057,7 @@ pub fn generate_grammar_files(
|path| { |path| {
let contents = fs::read_to_string(path)?; let contents = fs::read_to_string(path)?;
if !contents.contains("b.pkg_hash.len") { if !contents.contains("b.pkg_hash.len") {
warn!("Replacing build.zig"); info!("Replacing build.zig");
generate_file(path, BUILD_ZIG_TEMPLATE, language_name, &generate_opts) generate_file(path, BUILD_ZIG_TEMPLATE, language_name, &generate_opts)
} else { } else {
Ok(()) Ok(())
@ -1031,7 +1072,7 @@ pub fn generate_grammar_files(
|path| { |path| {
let contents = fs::read_to_string(path)?; let contents = fs::read_to_string(path)?;
if !contents.contains(".name = .tree_sitter_") { if !contents.contains(".name = .tree_sitter_") {
warn!("Replacing build.zig.zon"); info!("Replacing build.zig.zon");
generate_file(path, BUILD_ZIG_ZON_TEMPLATE, language_name, &generate_opts) generate_file(path, BUILD_ZIG_ZON_TEMPLATE, language_name, &generate_opts)
} else { } else {
Ok(()) Ok(())
@ -1047,7 +1088,7 @@ pub fn generate_grammar_files(
|path| { |path| {
let contents = fs::read_to_string(path)?; let contents = fs::read_to_string(path)?;
if contents.contains("ts.Language") { if contents.contains("ts.Language") {
warn!("Replacing root.zig"); info!("Replacing root.zig");
generate_file(path, ROOT_ZIG_TEMPLATE, language_name, &generate_opts) generate_file(path, ROOT_ZIG_TEMPLATE, language_name, &generate_opts)
} else { } else {
Ok(()) Ok(())

View file

@ -20,7 +20,7 @@ use tree_sitter_cli::{
LOG_GRAPH_ENABLED, START_SEED, LOG_GRAPH_ENABLED, START_SEED,
}, },
highlight::{self, HighlightOptions}, highlight::{self, HighlightOptions},
init::{generate_grammar_files, JsonConfigOpts}, init::{generate_grammar_files, JsonConfigOpts, TREE_SITTER_JSON_SCHEMA},
input::{get_input, get_tmp_source_file, CliInput}, input::{get_input, get_tmp_source_file, CliInput},
logger, logger,
parse::{self, ParseDebugType, ParseFileOptions, ParseOutput, ParseTheme}, parse::{self, ParseDebugType, ParseFileOptions, ParseOutput, ParseTheme},
@ -867,10 +867,26 @@ impl Init {
(opts.name.clone(), Some(opts)) (opts.name.clone(), Some(opts))
} else { } else {
let mut json = serde_json::from_str::<TreeSitterJSON>( let old_config = fs::read_to_string(current_dir.join("tree-sitter.json"))
&fs::read_to_string(current_dir.join("tree-sitter.json")) .with_context(|| "Failed to read tree-sitter.json")?;
.with_context(|| "Failed to read tree-sitter.json")?,
)?; let mut json = serde_json::from_str::<TreeSitterJSON>(&old_config)?;
if json.schema.is_none() {
json.schema = Some(TREE_SITTER_JSON_SCHEMA.to_string());
}
let new_config = format!("{}\n", serde_json::to_string_pretty(&json)?);
// Write the re-serialized config back, as newly added optional boolean fields
// will be included with explicit `false`s rather than implict `null`s
if self.update && !old_config.trim().eq(new_config.trim()) {
info!("Updating tree-sitter.json");
fs::write(
current_dir.join("tree-sitter.json"),
serde_json::to_string_pretty(&json)?,
)
.with_context(|| "Failed to write tree-sitter.json")?;
}
(json.grammars.swap_remove(0).name, None) (json.grammars.swap_remove(0).name, None)
}; };
@ -955,11 +971,21 @@ impl Build {
} else { } else {
let output_path = if let Some(ref path) = self.output { let output_path = if let Some(ref path) = self.output {
let path = Path::new(path); let path = Path::new(path);
if path.is_absolute() { let full_path = if path.is_absolute() {
path.to_path_buf() path.to_path_buf()
} else { } else {
current_dir.join(path) current_dir.join(path)
} };
let parent_path = full_path
.parent()
.context("Output path must have a parent")?;
let name = full_path
.file_name()
.context("Ouput path must have a filename")?;
fs::create_dir_all(parent_path).context("Failed to create output path")?;
let mut canon_path = parent_path.canonicalize().context("Invalid output path")?;
canon_path.push(name);
canon_path
} else { } else {
let file_name = grammar_path let file_name = grammar_path
.file_stem() .file_stem()
@ -984,7 +1010,7 @@ impl Build {
loader loader
.compile_parser_at_path(&grammar_path, output_path, flags) .compile_parser_at_path(&grammar_path, output_path, flags)
.unwrap(); .context("Failed to compile parser")?;
} }
Ok(()) Ok(())
} }
@ -1622,6 +1648,7 @@ impl Highlight {
let loader_config = config.get()?; let loader_config = config.get()?;
loader.find_all_languages(&loader_config)?; loader.find_all_languages(&loader_config)?;
loader.force_rebuild(self.rebuild || self.grammar_path.is_some()); loader.force_rebuild(self.rebuild || self.grammar_path.is_some());
let languages = loader.languages_at_path(current_dir)?;
let cancellation_flag = util::cancel_on_signal(); let cancellation_flag = util::cancel_on_signal();
@ -1702,7 +1729,6 @@ impl Highlight {
} => { } => {
let path = get_tmp_source_file(&contents)?; let path = get_tmp_source_file(&contents)?;
let languages = loader.languages_at_path(current_dir)?;
let language = languages let language = languages
.iter() .iter()
.find(|(_, n)| language_names.contains(&Box::from(n.as_str()))) .find(|(_, n)| language_names.contains(&Box::from(n.as_str())))
@ -1733,7 +1759,6 @@ impl Highlight {
if let (Some(l), Some(lc)) = (language.clone(), language_configuration) { if let (Some(l), Some(lc)) = (language.clone(), language_configuration) {
(l, lc) (l, lc)
} else { } else {
let languages = loader.languages_at_path(current_dir)?;
let language = languages let language = languages
.first() .first()
.map(|(l, _)| l.clone()) .map(|(l, _)| l.clone())

View file

@ -515,7 +515,6 @@ pub fn parse_file_at_path(
if opts.output == ParseOutput::Cst { if opts.output == ParseOutput::Cst {
render_cst(&source_code, &tree, &mut cursor, opts, &mut stdout)?; render_cst(&source_code, &tree, &mut cursor, opts, &mut stdout)?;
println!();
} }
if opts.output == ParseOutput::Xml { if opts.output == ParseOutput::Xml {
@ -785,7 +784,7 @@ pub fn render_cst<'a, 'b: 'a>(
.map(|(row, col)| (row as f64).log10() as usize + (col.len() as f64).log10() as usize + 1) .map(|(row, col)| (row as f64).log10() as usize + (col.len() as f64).log10() as usize + 1)
.max() .max()
.unwrap_or(1); .unwrap_or(1);
let mut indent_level = 1; let mut indent_level = usize::from(!opts.no_ranges);
let mut did_visit_children = false; let mut did_visit_children = false;
let mut in_error = false; let mut in_error = false;
loop { loop {
@ -883,35 +882,24 @@ fn write_node_text(
0 0
}; };
let formatted_line = render_line_feed(line, opts); let formatted_line = render_line_feed(line, opts);
if !opts.no_ranges { write!(
write!( out,
out, "{}{}{}{}{}{}",
"{}{}{}{}{}{}", if multiline { "\n" } else { " " },
if multiline { "\n" } else { "" }, if multiline && !opts.no_ranges {
if multiline { render_node_range(opts, cursor, is_named, true, total_width, node_range)
render_node_range(opts, cursor, is_named, true, total_width, node_range) } else {
} else { String::new()
String::new() },
}, if multiline {
if multiline { " ".repeat(indent_level + 1)
" ".repeat(indent_level + 1) } else {
} else { String::new()
String::new() },
}, paint(quote_color, &String::from(quote)),
paint(quote_color, &String::from(quote)), paint(color, &render_node_text(&formatted_line)),
&paint(color, &render_node_text(&formatted_line)), paint(quote_color, &String::from(quote)),
paint(quote_color, &String::from(quote)), )?;
)?;
} else {
write!(
out,
"\n{}{}{}{}",
" ".repeat(indent_level + 1),
paint(quote_color, &String::from(quote)),
&paint(color, &render_node_text(&formatted_line)),
paint(quote_color, &String::from(quote)),
)?;
}
} }
} }
@ -965,7 +953,7 @@ fn render_node_range(
fn cst_render_node( fn cst_render_node(
opts: &ParseFileOptions, opts: &ParseFileOptions,
cursor: &mut TreeCursor, cursor: &TreeCursor,
source_code: &[u8], source_code: &[u8],
out: &mut impl Write, out: &mut impl Write,
total_width: usize, total_width: usize,
@ -1011,10 +999,9 @@ fn cst_render_node(
} else { } else {
opts.parse_theme.node_kind opts.parse_theme.node_kind
}; };
write!(out, "{}", paint(kind_color, node.kind()),)?; write!(out, "{}", paint(kind_color, node.kind()))?;
if node.child_count() == 0 { if node.child_count() == 0 {
write!(out, " ")?;
// Node text from a pattern or external scanner // Node text from a pattern or external scanner
write_node_text( write_node_text(
opts, opts,

View file

@ -19,6 +19,7 @@
--light-scrollbar-track: #f1f1f1; --light-scrollbar-track: #f1f1f1;
--light-scrollbar-thumb: #c1c1c1; --light-scrollbar-thumb: #c1c1c1;
--light-scrollbar-thumb-hover: #a8a8a8; --light-scrollbar-thumb-hover: #a8a8a8;
--light-tree-row-bg: #e3f2fd;
--dark-bg: #1d1f21; --dark-bg: #1d1f21;
--dark-border: #2d2d2d; --dark-border: #2d2d2d;
@ -28,6 +29,7 @@
--dark-scrollbar-track: #25282c; --dark-scrollbar-track: #25282c;
--dark-scrollbar-thumb: #4a4d51; --dark-scrollbar-thumb: #4a4d51;
--dark-scrollbar-thumb-hover: #5a5d61; --dark-scrollbar-thumb-hover: #5a5d61;
--dark-tree-row-bg: #373737;
--primary-color: #0550ae; --primary-color: #0550ae;
--primary-color-alpha: rgba(5, 80, 174, 0.1); --primary-color-alpha: rgba(5, 80, 174, 0.1);
@ -42,6 +44,7 @@
--text-color: var(--dark-text); --text-color: var(--dark-text);
--panel-bg: var(--dark-panel-bg); --panel-bg: var(--dark-panel-bg);
--code-bg: var(--dark-code-bg); --code-bg: var(--dark-code-bg);
--tree-row-bg: var(--dark-tree-row-bg);
} }
[data-theme="light"] { [data-theme="light"] {
@ -50,6 +53,7 @@
--text-color: var(--light-text); --text-color: var(--light-text);
--panel-bg: white; --panel-bg: white;
--code-bg: white; --code-bg: white;
--tree-row-bg: var(--light-tree-row-bg);
} }
/* Base Styles */ /* Base Styles */
@ -275,7 +279,7 @@
} }
#output-container a.highlighted { #output-container a.highlighted {
background-color: #d9d9d9; background-color: #cae2ff;
color: red; color: red;
border-radius: 3px; border-radius: 3px;
text-decoration: underline; text-decoration: underline;
@ -346,7 +350,7 @@
} }
& #output-container a.highlighted { & #output-container a.highlighted {
background-color: #373b41; background-color: #656669;
color: red; color: red;
} }
@ -373,6 +377,9 @@
color: var(--dark-text); color: var(--dark-text);
} }
} }
.tree-row:has(.highlighted) {
background-color: var(--tree-row-bg);
}
</style> </style>
</head> </head>

View file

@ -32,7 +32,7 @@ class BuildExt(build_ext):
class BdistWheel(bdist_wheel): class BdistWheel(bdist_wheel):
def get_tag(self): def get_tag(self):
python, abi, platform = super().get_tag() python, abi, platform = super().get_tag()
if python.startswith("cp"): if python.startswith("cp") and not get_config_var("Py_GIL_DISABLED"):
python, abi = "cp310", "abi3" python, abi = "cp310", "abi3"
return python, abi, platform return python, abi, platform

View file

@ -595,6 +595,8 @@ impl std::fmt::Display for TestSummary {
render_assertion_results("queries", &self.query_results)?; render_assertion_results("queries", &self.query_results)?;
} }
write!(f, "{}", self.parse_stats)?;
Ok(()) Ok(())
} }
} }
@ -1066,7 +1068,6 @@ fn run_tests(
return Ok(true); return Ok(true);
} }
let failure_count = test_summary.parse_failures.len();
let mut ran_test_in_group = false; let mut ran_test_in_group = false;
let matches_filter = |name: &str, file_name: &Option<String>, opts: &TestOptions| { let matches_filter = |name: &str, file_name: &Option<String>, opts: &TestOptions| {
@ -1130,7 +1131,7 @@ fn run_tests(
test_summary.parse_results.pop_traversal(); test_summary.parse_results.pop_traversal();
if let Some(file_path) = file_path { if let Some(file_path) = file_path {
if opts.update && test_summary.parse_failures.len() - failure_count > 0 { if opts.update {
write_tests(&file_path, corrected_entries)?; write_tests(&file_path, corrected_entries)?;
} }
corrected_entries.clear(); corrected_entries.clear();

View file

@ -225,7 +225,7 @@ impl Pattern {
} }
// Find every matching combination of child patterns and child nodes. // Find every matching combination of child patterns and child nodes.
let mut finished_matches = Vec::<Match>::new(); let mut finished_matches = Vec::<Match<'_, 'tree>>::new();
if cursor.goto_first_child() { if cursor.goto_first_child() {
let mut match_states = vec![(0, mat)]; let mut match_states = vec![(0, mat)];
loop { loop {

View file

@ -33,7 +33,7 @@ log.workspace = true
pathdiff = { version = "0.2.3", optional = true } pathdiff = { version = "0.2.3", optional = true }
regex.workspace = true regex.workspace = true
regex-syntax.workspace = true regex-syntax.workspace = true
rquickjs = { version = "0.10.0", optional = true, features = [ rquickjs = { version = "0.11.0", optional = true, features = [
"bindgen", "bindgen",
"loader", "loader",
"macro", "macro",

View file

@ -70,12 +70,13 @@ impl InlinedProductionMapBuilder {
let production_map = production_indices_by_step_id let production_map = production_indices_by_step_id
.into_iter() .into_iter()
.map(|(step_id, production_indices)| { .map(|(step_id, production_indices)| {
let production = step_id.variable_index.map_or_else( let production =
|| &productions[step_id.production_index], core::ptr::from_ref::<Production>(step_id.variable_index.map_or_else(
|variable_index| { || &productions[step_id.production_index],
&grammar.variables[variable_index].productions[step_id.production_index] |variable_index| {
}, &grammar.variables[variable_index].productions[step_id.production_index]
) as *const Production; },
));
((production, step_id.step_index as u32), production_indices) ((production, step_id.step_index as u32), production_indices)
}) })
.collect(); .collect();

View file

@ -95,9 +95,27 @@ impl Console {
Type::Module => "module".to_string(), Type::Module => "module".to_string(),
Type::BigInt => v.get::<String>().unwrap_or_else(|_| "BigInt".to_string()), Type::BigInt => v.get::<String>().unwrap_or_else(|_| "BigInt".to_string()),
Type::Unknown => "unknown".to_string(), Type::Unknown => "unknown".to_string(),
Type::Array => {
let js_vals = v
.as_array()
.unwrap()
.iter::<Value<'_>>()
.filter_map(|x| x.ok())
.map(|x| {
if x.is_string() {
format!("'{}'", Self::format_args(&[x]))
} else {
Self::format_args(&[x])
}
})
.collect::<Vec<_>>()
.join(", ");
format!("[ {js_vals} ]")
}
Type::Symbol Type::Symbol
| Type::Object | Type::Object
| Type::Array | Type::Proxy
| Type::Function | Type::Function
| Type::Constructor | Type::Constructor
| Type::Promise | Type::Promise
@ -197,11 +215,11 @@ fn try_resolve_path(path: &Path) -> rquickjs::Result<PathBuf> {
} }
#[allow(clippy::needless_pass_by_value)] #[allow(clippy::needless_pass_by_value)]
fn require_from_module<'a>( fn require_from_module<'js>(
ctx: Ctx<'a>, ctx: Ctx<'js>,
module_path: String, module_path: String,
from_module: &str, from_module: &str,
) -> rquickjs::Result<Value<'a>> { ) -> rquickjs::Result<Value<'js>> {
let current_module = PathBuf::from(from_module); let current_module = PathBuf::from(from_module);
let current_dir = if current_module.is_file() { let current_dir = if current_module.is_file() {
current_module.parent().unwrap_or(Path::new(".")) current_module.parent().unwrap_or(Path::new("."))
@ -216,13 +234,13 @@ fn require_from_module<'a>(
load_module_from_content(&ctx, &resolved_path, &contents) load_module_from_content(&ctx, &resolved_path, &contents)
} }
fn load_module_from_content<'a>( fn load_module_from_content<'js>(
ctx: &Ctx<'a>, ctx: &Ctx<'js>,
path: &Path, path: &Path,
contents: &str, contents: &str,
) -> rquickjs::Result<Value<'a>> { ) -> rquickjs::Result<Value<'js>> {
if path.extension().is_some_and(|ext| ext == "json") { if path.extension().is_some_and(|ext| ext == "json") {
return ctx.eval::<Value, _>(format!("JSON.parse({contents:?})")); return ctx.eval::<Value<'js>, _>(format!("JSON.parse({contents:?})"));
} }
let exports = Object::new(ctx.clone())?; let exports = Object::new(ctx.clone())?;
@ -238,7 +256,7 @@ fn load_module_from_content<'a>(
let module_path = filename.clone(); let module_path = filename.clone();
let require = Function::new( let require = Function::new(
ctx.clone(), ctx.clone(),
move |ctx_inner: Ctx<'a>, target_path: String| -> rquickjs::Result<Value<'a>> { move |ctx_inner: Ctx<'js>, target_path: String| -> rquickjs::Result<Value<'js>> {
require_from_module(ctx_inner, target_path, &module_path) require_from_module(ctx_inner, target_path, &module_path)
}, },
)?; )?;
@ -246,8 +264,8 @@ fn load_module_from_content<'a>(
let wrapper = let wrapper =
format!("(function(exports, require, module, __filename, __dirname) {{ {contents} }})"); format!("(function(exports, require, module, __filename, __dirname) {{ {contents} }})");
let module_func = ctx.eval::<Function, _>(wrapper)?; let module_func = ctx.eval::<Function<'js>, _>(wrapper)?;
module_func.call::<_, Value>((exports, require, module_obj.clone(), filename, dirname))?; module_func.call::<_, Value<'js>>((exports, require, module_obj.clone(), filename, dirname))?;
module_obj.get("exports") module_obj.get("exports")
} }

View file

@ -189,7 +189,7 @@ struct HighlightIterLayer<'a> {
depth: usize, depth: usize,
} }
pub struct _QueryCaptures<'query, 'tree: 'query, T: TextProvider<I>, I: AsRef<[u8]>> { pub struct _QueryCaptures<'query, 'tree, T: TextProvider<I>, I: AsRef<[u8]>> {
ptr: *mut ffi::TSQueryCursor, ptr: *mut ffi::TSQueryCursor,
query: &'query Query, query: &'query Query,
text_provider: T, text_provider: T,
@ -225,7 +225,7 @@ impl<'tree> _QueryMatch<'_, 'tree> {
} }
} }
impl<'query, 'tree: 'query, T: TextProvider<I>, I: AsRef<[u8]>> Iterator impl<'query, 'tree, T: TextProvider<I>, I: AsRef<[u8]>> Iterator
for _QueryCaptures<'query, 'tree, T, I> for _QueryCaptures<'query, 'tree, T, I>
{ {
type Item = (QueryMatch<'query, 'tree>, usize); type Item = (QueryMatch<'query, 'tree>, usize);
@ -594,6 +594,7 @@ impl<'a> HighlightIterLayer<'a> {
} }
} }
// SAFETY:
// The `captures` iterator borrows the `Tree` and the `QueryCursor`, which // The `captures` iterator borrows the `Tree` and the `QueryCursor`, which
// prevents them from being moved. But both of these values are really just // prevents them from being moved. But both of these values are really just
// pointers, so it's actually ok to move them. // pointers, so it's actually ok to move them.

View file

@ -1,7 +1,7 @@
[package] [package]
name = "tree-sitter-language" name = "tree-sitter-language"
description = "The tree-sitter Language type, used by the library and by language implementations" description = "The tree-sitter Language type, used by the library and by language implementations"
version = "0.1.6" version = "0.1.7"
authors.workspace = true authors.workspace = true
edition.workspace = true edition.workspace = true
rust-version = "1.77" rust-version = "1.77"

View file

@ -23,9 +23,15 @@ typedef long unsigned int size_t;
typedef long unsigned int uintptr_t; typedef long unsigned int uintptr_t;
#define UINT16_MAX 65535 #define INT8_MAX 127
#define INT16_MAX 32767
#define INT32_MAX 2147483647L
#define INT64_MAX 9223372036854775807LL
#define UINT8_MAX 255
#define UINT16_MAX 65535
#define UINT32_MAX 4294967295U #define UINT32_MAX 4294967295U
#define UINT64_MAX 18446744073709551615ULL
#if defined(__wasm32__) #if defined(__wasm32__)

View file

@ -13,4 +13,6 @@ void *memset(void *dst, int value, size_t count);
int strncmp(const char *left, const char *right, size_t n); int strncmp(const char *left, const char *right, size_t n);
size_t strlen(const char *str);
#endif // TREE_SITTER_WASM_STRING_H_ #endif // TREE_SITTER_WASM_STRING_H_

View file

@ -1,4 +1,5 @@
#include <stdio.h> #include <stdio.h>
#include <string.h>
typedef struct { typedef struct {
bool left_justify; // - bool left_justify; // -
@ -105,12 +106,6 @@ static int ptr_to_str(void *ptr, char *buffer) {
return 2 + len; return 2 + len;
} }
size_t strlen(const char *str) {
const char *s = str;
while (*s) s++;
return s - str;
}
char *strncpy(char *dest, const char *src, size_t n) { char *strncpy(char *dest, const char *src, size_t n) {
char *d = dest; char *d = dest;
const char *s = src; const char *s = src;

View file

@ -58,3 +58,9 @@ int strncmp(const char *left, const char *right, size_t n) {
} }
return 0; return 0;
} }
size_t strlen(const char *str) {
const char *s = str;
while (*s) s++;
return s - str;
}

View file

@ -8,6 +8,7 @@ use std::sync::Mutex;
use std::{ use std::{
collections::HashMap, collections::HashMap,
env, fs, env, fs,
hash::{Hash as _, Hasher as _},
io::{BufRead, BufReader}, io::{BufRead, BufReader},
marker::PhantomData, marker::PhantomData,
mem, mem,
@ -764,7 +765,7 @@ impl Loader {
} }
#[must_use] #[must_use]
pub fn get_all_language_configurations(&self) -> Vec<(&LanguageConfiguration, &Path)> { pub fn get_all_language_configurations(&self) -> Vec<(&LanguageConfiguration<'static>, &Path)> {
self.language_configurations self.language_configurations
.iter() .iter()
.map(|c| (c, self.languages_by_id[c.language_id].0.as_ref())) .map(|c| (c, self.languages_by_id[c.language_id].0.as_ref()))
@ -774,7 +775,7 @@ impl Loader {
pub fn language_configuration_for_scope( pub fn language_configuration_for_scope(
&self, &self,
scope: &str, scope: &str,
) -> LoaderResult<Option<(Language, &LanguageConfiguration)>> { ) -> LoaderResult<Option<(Language, &LanguageConfiguration<'static>)>> {
for configuration in &self.language_configurations { for configuration in &self.language_configurations {
if configuration.scope.as_ref().is_some_and(|s| s == scope) { if configuration.scope.as_ref().is_some_and(|s| s == scope) {
let language = self.language_for_id(configuration.language_id)?; let language = self.language_for_id(configuration.language_id)?;
@ -787,7 +788,7 @@ impl Loader {
pub fn language_configuration_for_first_line_regex( pub fn language_configuration_for_first_line_regex(
&self, &self,
path: &Path, path: &Path,
) -> LoaderResult<Option<(Language, &LanguageConfiguration)>> { ) -> LoaderResult<Option<(Language, &LanguageConfiguration<'static>)>> {
self.language_configuration_ids_by_first_line_regex self.language_configuration_ids_by_first_line_regex
.iter() .iter()
.try_fold(None, |_, (regex, ids)| { .try_fold(None, |_, (regex, ids)| {
@ -816,7 +817,7 @@ impl Loader {
pub fn language_configuration_for_file_name( pub fn language_configuration_for_file_name(
&self, &self,
path: &Path, path: &Path,
) -> LoaderResult<Option<(Language, &LanguageConfiguration)>> { ) -> LoaderResult<Option<(Language, &LanguageConfiguration<'static>)>> {
// Find all the language configurations that match this file name // Find all the language configurations that match this file name
// or a suffix of the file name. // or a suffix of the file name.
let configuration_ids = path let configuration_ids = path
@ -888,7 +889,7 @@ impl Loader {
pub fn language_configuration_for_injection_string( pub fn language_configuration_for_injection_string(
&self, &self,
string: &str, string: &str,
) -> LoaderResult<Option<(Language, &LanguageConfiguration)>> { ) -> LoaderResult<Option<(Language, &LanguageConfiguration<'static>)>> {
let mut best_match_length = 0; let mut best_match_length = 0;
let mut best_match_position = None; let mut best_match_position = None;
for (i, configuration) in self.language_configurations.iter().enumerate() { for (i, configuration) in self.language_configurations.iter().enumerate() {
@ -1025,20 +1026,26 @@ impl Loader {
return Ok(wasm_store.load_language(&config.name, &wasm_bytes)?); return Ok(wasm_store.load_language(&config.name, &wasm_bytes)?);
} }
// Create a unique lock path based on the output path hash to prevent
// interference when multiple processes build the same grammar (by name)
// to different output locations
let lock_hash = {
let mut hasher = std::hash::DefaultHasher::new();
output_path.hash(&mut hasher);
format!("{:x}", hasher.finish())
};
let lock_path = if env::var("CROSS_RUNNER").is_ok() { let lock_path = if env::var("CROSS_RUNNER").is_ok() {
tempfile::tempdir() tempfile::tempdir()
.unwrap() .expect("create a temp dir")
.path() .path()
.join("tree-sitter") .to_path_buf()
.join("lock")
.join(format!("{}.lock", config.name))
} else { } else {
etcetera::choose_base_strategy()? etcetera::choose_base_strategy()?.cache_dir()
.cache_dir() }
.join("tree-sitter") .join("tree-sitter")
.join("lock") .join("lock")
.join(format!("{}.lock", config.name)) .join(format!("{}-{lock_hash}.lock", config.name));
};
if let Ok(lock_file) = fs::OpenOptions::new().write(true).open(&lock_path) { if let Ok(lock_file) = fs::OpenOptions::new().write(true).open(&lock_path) {
recompile = false; recompile = false;
@ -1089,6 +1096,26 @@ impl Loader {
} }
} }
// Ensure the dynamic library exists before trying to load it. This can
// happen in race conditions where we couldn't acquire the lock because
// another process was compiling but it still hasn't finished by the
// time we reach this point, so the output file still doesn't exist.
//
// Instead of allowing the `load_language` call below to fail, return a
// clearer error to the user here.
if !output_path.exists() {
let msg = format!(
"Dynamic library `{}` not found after build attempt. \
Are you running multiple processes building to the same output location?",
output_path.display()
);
Err(LoaderError::IO(IoError::new(
std::io::Error::new(std::io::ErrorKind::NotFound, msg),
Some(output_path.as_path()),
)))?;
}
Self::load_language(&output_path, &language_fn_name) Self::load_language(&output_path, &language_fn_name)
} }
@ -1278,6 +1305,11 @@ impl Loader {
})); }));
} }
} }
} else {
warn!(
"Failed to run `nm` to verify symbols in {}",
library_path.display()
);
} }
Ok(()) Ok(())
@ -1507,7 +1539,9 @@ impl Loader {
} }
#[must_use] #[must_use]
pub fn get_language_configuration_in_current_path(&self) -> Option<&LanguageConfiguration> { pub fn get_language_configuration_in_current_path(
&self,
) -> Option<&LanguageConfiguration<'static>> {
self.language_configuration_in_current_path self.language_configuration_in_current_path
.map(|i| &self.language_configurations[i]) .map(|i| &self.language_configurations[i])
} }
@ -1516,7 +1550,7 @@ impl Loader {
&mut self, &mut self,
parser_path: &Path, parser_path: &Path,
set_current_path_config: bool, set_current_path_config: bool,
) -> LoaderResult<&[LanguageConfiguration]> { ) -> LoaderResult<&[LanguageConfiguration<'static>]> {
let initial_language_configuration_count = self.language_configurations.len(); let initial_language_configuration_count = self.language_configurations.len();
match TreeSitterJSON::from_file(parser_path) { match TreeSitterJSON::from_file(parser_path) {

View file

@ -313,6 +313,7 @@ impl TagsContext {
) )
.ok_or(Error::Cancelled)?; .ok_or(Error::Cancelled)?;
// SAFETY:
// The `matches` iterator borrows the `Tree`, which prevents it from being // The `matches` iterator borrows the `Tree`, which prevents it from being
// moved. But the tree is really just a pointer, so it's actually ok to // moved. But the tree is really just a pointer, so it's actually ok to
// move it. // move it.

View file

@ -199,6 +199,7 @@ pub fn run_wasm(args: &BuildWasm) -> Result<()> {
"-D", "NDEBUG=", "-D", "NDEBUG=",
"-D", "_POSIX_C_SOURCE=200112L", "-D", "_POSIX_C_SOURCE=200112L",
"-D", "_DEFAULT_SOURCE=", "-D", "_DEFAULT_SOURCE=",
"-D", "_BSD_SOURCE=",
"-D", "_DARWIN_C_SOURCE=", "-D", "_DARWIN_C_SOURCE=",
"-I", "lib/src", "-I", "lib/src",
"-I", "lib/include", "-I", "lib/include",

View file

@ -73,9 +73,8 @@ The behaviors of these three files are described in the next section.
## Queries ## Queries
Tree-sitter's syntax highlighting system is based on *tree queries*, which are a general system for pattern-matching on Tree-sitter's Tree-sitter's syntax highlighting system is based on *tree queries*, which are a general system for pattern-matching on
syntax trees. See [this section][pattern matching] of the documentation for more information Tree-sitter's syntax trees. See [this section][pattern matching] of the documentation for more information about tree queries.
about tree queries.
Syntax highlighting is controlled by *three* different types of query files that are usually included in the `queries` folder. Syntax highlighting is controlled by *three* different types of query files that are usually included in the `queries` folder.
The default names for the query files use the `.scm` file. We chose this extension because it commonly used for files written The default names for the query files use the `.scm` file. We chose this extension because it commonly used for files written

View file

@ -3,7 +3,8 @@
Tree-sitter can be used in conjunction with its [query language][query language] as a part of code navigation systems. Tree-sitter can be used in conjunction with its [query language][query language] as a part of code navigation systems.
An example of such a system can be seen in the `tree-sitter tags` command, which emits a textual dump of the interesting An example of such a system can be seen in the `tree-sitter tags` command, which emits a textual dump of the interesting
syntactic nodes in its file argument. A notable application of this is GitHub's support for [search-based code navigation][gh search]. syntactic nodes in its file argument. A notable application of this is GitHub's support for [search-based code navigation][gh search].
This document exists to describe how to integrate with such systems, and how to extend this functionality to any language with a Tree-sitter grammar. This document exists to describe how to integrate with such systems, and how to extend this functionality to any language
with a Tree-sitter grammar.
## Tagging and captures ## Tagging and captures
@ -12,9 +13,9 @@ entities. Having found them, you use a syntax capture to label the entity and it
The essence of a given tag lies in two pieces of data: the _role_ of the entity that is matched The essence of a given tag lies in two pieces of data: the _role_ of the entity that is matched
(i.e. whether it is a definition or a reference) and the _kind_ of that entity, which describes how the entity is used (i.e. whether it is a definition or a reference) and the _kind_ of that entity, which describes how the entity is used
(i.e. whether it's a class definition, function call, variable reference, and so on). Our convention is to use a syntax capture (i.e. whether it's a class definition, function call, variable reference, and so on). Our convention is to use a syntax
following the `@role.kind` capture name format, and another inner capture, always called `@name`, that pulls out the name capture following the `@role.kind` capture name format, and another inner capture, always called `@name`, that pulls out
of a given identifier. the name of a given identifier.
You may optionally include a capture named `@doc` to bind a docstring. For convenience purposes, the tagging system provides You may optionally include a capture named `@doc` to bind a docstring. For convenience purposes, the tagging system provides
two built-in functions, `#select-adjacent!` and `#strip!` that are convenient for removing comment syntax from a docstring. two built-in functions, `#select-adjacent!` and `#strip!` that are convenient for removing comment syntax from a docstring.

View file

@ -51,7 +51,7 @@ cargo install --path crates/cli
If you're going to be in a fast iteration cycle and would like the CLI to build faster, you can use the `release-dev` profile: If you're going to be in a fast iteration cycle and would like the CLI to build faster, you can use the `release-dev` profile:
```sh ```sh
cargo build --release --profile release-dev cargo build --profile release-dev
# or # or
cargo install --path crates/cli --profile release-dev cargo install --path crates/cli --profile release-dev
``` ```
@ -93,7 +93,8 @@ cargo xtask build-wasm-stdlib
This command looks for the [Wasi SDK][wasi_sdk] indicated by the `TREE_SITTER_WASI_SDK_PATH` This command looks for the [Wasi SDK][wasi_sdk] indicated by the `TREE_SITTER_WASI_SDK_PATH`
environment variable. If you don't have the binary, it can be downloaded from wasi-sdk's [releases][wasi-sdk-releases] environment variable. If you don't have the binary, it can be downloaded from wasi-sdk's [releases][wasi-sdk-releases]
page. page. Note that any changes to `crates/language/wasm/**` requires rebuilding the tree-sitter Wasm stdlib via
`cargo xtask build-wasm-stdlib`.
### Debugging ### Debugging

View file

@ -61,7 +61,7 @@ function initializeCustomSelect({ initialValue = null, addListeners = false }) {
} }
window.initializePlayground = async (opts) => { window.initializePlayground = async (opts) => {
const { Parser, Language } = window.TreeSitter; const { Parser, Language, Query } = window.TreeSitter;
const { local } = opts; const { local } = opts;
if (local) { if (local) {
@ -357,11 +357,10 @@ window.initializePlayground = async (opts) => {
marks.forEach((m) => m.clear()); marks.forEach((m) => m.clear());
if (tree && query) { if (tree && query) {
const captures = query.captures( const captures = query.captures(tree.rootNode, {
tree.rootNode, startPosition: { row: startRow, column: 0 },
{ row: startRow, column: 0 }, endPosition: { row: endRow, column: 0 },
{ row: endRow, column: 0 }, });
);
let lastNodeId; let lastNodeId;
for (const { name, node } of captures) { for (const { name, node } of captures) {
if (node.id === lastNodeId) continue; if (node.id === lastNodeId) continue;
@ -410,7 +409,7 @@ window.initializePlayground = async (opts) => {
const queryText = queryEditor.getValue(); const queryText = queryEditor.getValue();
try { try {
query = parser.language.query(queryText); query = new Query(parser.language, queryText);
let match; let match;
let row = 0; let row = 0;

View file

@ -19,8 +19,8 @@ will attempt to build the parser in the current working directory.
### `-w/--wasm` ### `-w/--wasm`
Compile the parser as a Wasm module. This command looks for the [Wasi SDK][wasi_sdk] indicated by the `TREE_SITTER_WASI_SDK_PATH` Compile the parser as a Wasm module. This command looks for the [Wasi SDK][wasi_sdk] indicated by the `TREE_SITTER_WASI_SDK_PATH`
environment variable. If you don't have the binary, the CLI will attempt to download it for you to `<CACHE_DIR>/tree-sitter/wasi-sdk/`, where environment variable. If you don't have the binary, the CLI will attempt to download it for you to `<CACHE_DIR>/tree-sitter/wasi-sdk/`,
`<CACHE_DIR>` is resolved according to the [XDG base directory][XDG] or Window's [Known_Folder_Locations][Known_Folder]. where `<CACHE_DIR>` is resolved according to the [XDG base directory][XDG] or Window's [Known_Folder_Locations][Known_Folder].
### `-o/--output` ### `-o/--output`
@ -37,7 +37,8 @@ in the external scanner does so using their allocator.
### `-0/--debug` ### `-0/--debug`
Compile the parser with debug flags enabled. This is useful when debugging issues that require a debugger like `gdb` or `lldb`. Compile the parser with debug flags enabled. This is useful when debugging issues that require a debugger like `gdb` or
`lldb`.
[Known_Folder]: https://learn.microsoft.com/en-us/windows/win32/shell/knownfolderid [Known_Folder]: https://learn.microsoft.com/en-us/windows/win32/shell/knownfolderid
[wasi_sdk]: https://github.com/WebAssembly/wasi-sdk [wasi_sdk]: https://github.com/WebAssembly/wasi-sdk

View file

@ -1,6 +1,8 @@
# `tree-sitter dump-languages` # `tree-sitter dump-languages`
The `dump-languages` command prints out a list of all the languages that the CLI knows about. This can be useful for debugging purposes, or for scripting. The paths to search comes from the config file's [`parser-directories`][parser-directories] object. The `dump-languages` command prints out a list of all the languages that the CLI knows about. This can be useful for debugging
purposes, or for scripting. The paths to search comes from the config file's [`parser-directories`][parser-directories]
object.
```bash ```bash
tree-sitter dump-languages [OPTIONS] # Aliases: langs tree-sitter dump-languages [OPTIONS] # Aliases: langs
@ -10,6 +12,7 @@ tree-sitter dump-languages [OPTIONS] # Aliases: langs
### `--config-path` ### `--config-path`
The path to the configuration file. Ordinarily, the CLI will use the default location as explained in the [init-config](./init-config.md) command. This flag allows you to explicitly override that default, and use a config defined elsewhere. The path to the configuration file. Ordinarily, the CLI will use the default location as explained in the [init-config](./init-config.md)
command. This flag allows you to explicitly override that default, and use a config defined elsewhere.
[parser-directories]: ./init-config.md#parser-directories [parser-directories]: ./init-config.md#parser-directories

View file

@ -1,30 +1,39 @@
# `tree-sitter generate` # `tree-sitter generate`
The most important command you'll use is `tree-sitter generate`. This command reads the `grammar.js` file in your current The most important command for grammar development is `tree-sitter generate`, which reads the grammar in structured form
working directory and creates a file called `src/parser.c`, which implements the parser. After making changes to your grammar, and outputs C files that can be compiled into a shared or static library (e.g., using the [`build`](./build.md) command).
just run `tree-sitter generate` again.
```bash ```bash
tree-sitter generate [OPTIONS] [GRAMMAR_PATH] # Aliases: gen, g tree-sitter generate [OPTIONS] [GRAMMAR_PATH] # Aliases: gen, g
``` ```
The grammar path argument allows you to specify a path to a `grammar.js` JavaScript file, or `grammar.json` JSON file. The optional `GRAMMAR_PATH` argument should point to the structured grammar, in one of two forms:
In case your `grammar.js` file is in a non-standard path, you can specify it yourself. But, if you are using a parser - `grammar.js` a (ESM or CJS) JavaScript file; if the argument is omitted, it defaults to `./grammar.js`.
where `grammar.json` was already generated, or it was hand-written, you can tell the CLI to generate the parser *based* - `grammar.json` a structured representation of the grammar that is created as a byproduct of `generate`; this can be used
on this JSON file. This avoids relying on a JavaScript file and avoids the dependency on a JavaScript runtime. to regenerate a missing `parser.c` without requiring a JavaScript runtime (useful when distributing parsers to consumers).
If there is an ambiguity or *local ambiguity* in your grammar, Tree-sitter will detect it during parser generation, and If there is an ambiguity or *local ambiguity* in your grammar, Tree-sitter will detect it during parser generation, and
it will exit with a `Unresolved conflict` error message. To learn more about conflicts and how to handle them, check out it will exit with a `Unresolved conflict` error message. To learn more about conflicts and how to handle them, see
the section on [`Structuring Rules Well`](../creating-parsers/3-writing-the-grammar.md#structuring-rules-well) the section on [`Structuring Rules Well`](../creating-parsers/3-writing-the-grammar.md#structuring-rules-well)
in the user guide. in the user guide.
## Generated files
- `src/parser.c` implements the parser logic specified in the grammar.
- `src/tree_sitter/parser.h` provides basic C definitions that are used in the generated `parser.c` file.
- `src/tree_sitter/alloc.h` provides memory allocation macros that can be used in an external scanner.
- `src/tree_sitter/array.h` provides array macros that can be used in an external scanner.
- `src/grammar.json` contains a structured representation of the grammar; can be used to regenerate the parser without having
to re-evaluate the `grammar.js`.
- `src/node-types.json` provides type information about individual syntax nodes; see the section on [`Static Node Types`](../using-parsers/6-static-node-types.md).
## Options ## Options
### `-l/--log` ### `-l/--log`
Print the log of the parser generation process. This is really only useful if you know what you're doing, or are investigating Print the log of the parser generation process. This includes information such as what tokens are included in the error
a bug in the CLI itself. It logs info such as what tokens are included in the error recovery state, recovery state, what keywords were extracted, what states were split and why, and the entry point state.
what keywords were extracted, what states were split and why, and the entry point state.
### `--abi <VERSION>` ### `--abi <VERSION>`
@ -54,7 +63,8 @@ The path to the JavaScript runtime executable to use when generating the parser.
Note that you can also set this with `TREE_SITTER_JS_RUNTIME`. Starting from version 0.26, you can Note that you can also set this with `TREE_SITTER_JS_RUNTIME`. Starting from version 0.26, you can
also pass in `native` to use the experimental native QuickJS runtime that comes bundled with the CLI. also pass in `native` to use the experimental native QuickJS runtime that comes bundled with the CLI.
This avoids the dependency on a JavaScript runtime entirely. The native QuickJS runtime is compatible This avoids the dependency on a JavaScript runtime entirely. The native QuickJS runtime is compatible
with ESM as well as with CommonJS in strict mode. If your grammar depends on `npm` to install dependencies such as base grammars, the native runtime can be used *after* running `npm install`. with ESM as well as with CommonJS in strict mode. If your grammar depends on `npm` to install dependencies such as base
grammars, the native runtime can be used *after* running `npm install`.
### `--disable-optimization` ### `--disable-optimization`

View file

@ -52,7 +52,8 @@ The path to the directory containing the grammar.
### `--config-path <CONFIG_PATH>` ### `--config-path <CONFIG_PATH>`
The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more information. The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more
information.
### `-n/--test-number <TEST_NUMBER>` ### `-n/--test-number <TEST_NUMBER>`

View file

@ -1,4 +1,8 @@
# CLI Overview # CLI Overview
Let's go over all of the functionality of the `tree-sitter` command line interface. The `tree-sitter` command-line interface is used to create, manage, test, and build tree-sitter parsers. It is controlled
Once you feel that you have enough of a grasp on the CLI, you can move onto the grammar authoring section to learn more about writing your own parser. by
- a personal `tree-sitter/config.json` config file generated by [`tree-sitter init-config`](./init-config.md)
- a parser `tree-sitter.json` config file generated by [`tree-sitter init`](./init.md).

View file

@ -8,30 +8,94 @@ we recommend using git for version control of your grammar.
tree-sitter init [OPTIONS] # Aliases: i tree-sitter init [OPTIONS] # Aliases: i
``` ```
## Options ## Generated files
### `--update` ### Required files
Update outdated generated files, if needed. The following required files are always created if missing:
### `-p/--grammar-path <PATH>` - `tree-sitter.json` - The main configuration file that determines how `tree-sitter` interacts with the grammar. If missing,
the `init` command will prompt the user for the required fields. See [below](./init.md#structure-of-tree-sitterjson) for
the full documentation of the structure of this file.
- `package.json` - The `npm` manifest for the parser. This file is required for some `tree-sitter` subcommands, and if the
grammar has dependencies (e.g., another published base grammar that this grammar extends).
- `grammar.js` - An empty template for the main grammar file; see [the section on creating parsers](../2-creating-parser).
The path to the directory containing the grammar. ### Language bindings
Language bindings are files that allow your parser to be directly used by projects written in the respective language.
The following bindings are created if enabled in `tree-sitter.json`:
#### C/C++
- `Makefile` — This file tells [`make`][make] how to compile your language.
- `CMakeLists.txt` — This file tells [`cmake`][cmake] how to compile your language.
- `bindings/c/tree_sitter/tree-sitter-language.h` — This file provides the C interface of your language.
- `bindings/c/tree-sitter-language.pc` — This file provides [pkg-config][pkg-config] metadata about your language's C library.
#### Go
- `go.mod` — This file is the manifest of the Go module.
- `bindings/go/binding.go` — This file wraps your language in a Go module.
- `bindings/go/binding_test.go` — This file contains a test for the Go package.
#### Node
- `binding.gyp` — This file tells Node.js how to compile your language.
- `bindings/node/binding.cc` — This file wraps your language in a JavaScript module for Node.js.
- `bindings/node/index.js` — This is the file that Node.js initially loads when using your language.
- `bindings/node/index.d.ts` — This file provides type hints for your parser when used in TypeScript.
- `bindings/node/binding_test.js` — This file contains a test for the Node.js package.
#### Java
- `pom.xml` - This file is the manifest of the Maven package.
- `bindings/java/main/namespace/language/TreeSitterLanguage.java` - This file wraps your language in a Java class.
- `bindings/java/test/TreeSitterLanguageTest.java` - This file contains a test for the Java package.
#### Python
- `pyproject.toml` — This file is the manifest of the Python package.
- `setup.py` — This file tells Python how to compile your language.
- `bindings/python/tree_sitter_language/binding.c` — This file wraps your language in a Python module.
- `bindings/python/tree_sitter_language/__init__.py` — This file tells Python how to load your language.
- `bindings/python/tree_sitter_language/__init__.pyi` — This file provides type hints for your parser when used in Python.
- `bindings/python/tree_sitter_language/py.typed` — This file provides type hints for your parser when used in Python.
- `bindings/python/tests/test_binding.py` — This file contains a test for the Python package.
#### Rust
- `Cargo.toml` — This file is the manifest of the Rust package.
- `bindings/rust/build.rs` — This file tells Rust how to compile your language.
- `bindings/rust/lib.rs` — This file wraps your language in a Rust crate when used in Rust.
#### Swift
- `Package.swift` — This file tells Swift how to compile your language.
- `bindings/swift/TreeSitterLanguage/language.h` — This file wraps your language in a Swift module when used in Swift.
- `bindings/swift/TreeSitterLanguageTests/TreeSitterLanguageTests.swift` — This file contains a test for the Swift package.
#### Zig
- `build.zig` - This file tells Zig how to compile your language.
- `build.zig.zon` - This file is the manifest of the Zig package.
- `bindings/zig/root.zig` - This file wraps your language in a Zig module.
- `bindings/zig/test.zig` - This file contains a test for the Zig package.
### Additional files
In addition, the following files are created that aim to improve the development experience:
- `.editorconfig` — This file tells your editor how to format your code. More information about this file can be found [here][editorconfig].
- `.gitattributes` — This file tells Git how to handle line endings and tells GitHub which files are generated.
- `.gitignore` — This file tells Git which files to ignore when committing changes.
## Structure of `tree-sitter.json` ## Structure of `tree-sitter.json`
The main file of interest for users to configure is `tree-sitter.json`, which tells the CLI information about your grammar,
such as the location of queries.
### The `grammars` field ### The `grammars` field
This field is an array of objects, though you typically only need one object in this array unless your repo has This field is an array of objects, though you typically only need one object in this array unless your repo has
multiple grammars (for example, `Typescript` and `TSX`). multiple grammars (for example, `Typescript` and `TSX`), e.g.,
### Example
Typically, the objects in the `"tree-sitter"` array only needs to specify a few keys:
```json ```json
{ {
"tree-sitter": [ "tree-sitter": [
@ -49,7 +113,7 @@ Typically, the objects in the `"tree-sitter"` array only needs to specify a few
} }
``` ```
#### Basic Fields #### Basic fields
These keys specify basic information about the parser: These keys specify basic information about the parser:
@ -65,12 +129,12 @@ parser to files that should be checked for modifications during recompilation.
This is useful during development to have changes to other files besides scanner.c This is useful during development to have changes to other files besides scanner.c
be picked up by the cli. be picked up by the cli.
#### Language Detection #### Language detection
These keys help to decide whether the language applies to a given file: These keys help to decide whether the language applies to a given file:
- `file-types` — An array of filename suffix strings. The grammar will be used for files whose names end with one of - `file-types` — An array of filename suffix strings (not including the dot). The grammar will be used for files whose names
these suffixes. Note that the suffix may match an *entire* filename. end with one of these suffixes. Note that the suffix may match an *entire* filename.
- `first-line-regex` — A regex pattern that will be tested against the first line of a file - `first-line-regex` — A regex pattern that will be tested against the first line of a file
to determine whether this language applies to the file. If present, this regex will be used for any file whose to determine whether this language applies to the file. If present, this regex will be used for any file whose
@ -85,14 +149,14 @@ no `content-regex` will be preferred over this one.
should be used for a potential *language injection* site. should be used for a potential *language injection* site.
Language injection is described in more detail in [the relevant section](../3-syntax-highlighting.md#language-injection). Language injection is described in more detail in [the relevant section](../3-syntax-highlighting.md#language-injection).
#### Query Paths #### Query paths
These keys specify relative paths from the directory containing `tree-sitter.json` to the files that control syntax highlighting: These keys specify relative paths from the directory containing `tree-sitter.json` to the files that control syntax highlighting:
- `highlights` — Path to a *highlight query*. Default: `queries/highlights.scm` - `highlights` — Path to a *highlight query*. Default: `queries/highlights.scm`
- `locals` — Path to a *local variable query*. Default: `queries/locals.scm`. - `locals` — Path to a *local variable query*. Default: `queries/locals.scm`.
- `injections` — Path to an *injection query*. Default: `queries/injections.scm`. - `injections` — Path to an *injection query*. Default: `queries/injections.scm`.
- `tags` — Path to an *tag query*. Default: `queries/tags.scm`. - `tags` — Path to a *tag query*. Default: `queries/tags.scm`.
### The `metadata` field ### The `metadata` field
@ -121,81 +185,19 @@ Each key is a language name, and the value is a boolean.
- `swift` (default: `false`) - `swift` (default: `false`)
- `zig` (default: `false`) - `zig` (default: `false`)
## Binding Files ## Options
When you run `tree-sitter init`, the CLI will also generate a number of files in your repository that allow for your parser ### `-u/--update`
to be used from different language. Here is a list of these bindings files that are generated, and what their purpose is:
### C/C++ Update outdated generated files, if possible.
- `Makefile` — This file tells [`make`][make] how to compile your language. **Note:** Existing files that may have been edited manually are _not_ updated in general. To force an update to such files,
- `CMakeLists.txt` — This file tells [`cmake`][cmake] how to compile your language. remove them and call `tree-sitter init -u` again.
- `bindings/c/tree_sitter/tree-sitter-language.h` — This file provides the C interface of your language.
- `bindings/c/tree-sitter-language.pc` — This file provides [pkg-config][pkg-config] metadata about your language's C library.
- `src/tree_sitter/parser.h` — This file provides some basic C definitions that are used in your generated `parser.c` file.
- `src/tree_sitter/alloc.h` — This file provides some memory allocation macros that are to be used in your external scanner,
if you have one.
- `src/tree_sitter/array.h` — This file provides some array macros that are to be used in your external scanner,
if you have one.
### Go ### `-p/--grammar-path <PATH>`
- `go.mod` — This file is the manifest of the Go module. The path to the directory containing the grammar.
- `bindings/go/binding.go` — This file wraps your language in a Go module.
- `bindings/go/binding_test.go` — This file contains a test for the Go package.
### Node
- `binding.gyp` — This file tells Node.js how to compile your language.
- `package.json` — This file is the manifest of the Node.js package.
- `bindings/node/binding.cc` — This file wraps your language in a JavaScript module for Node.js.
- `bindings/node/index.js` — This is the file that Node.js initially loads when using your language.
- `bindings/node/index.d.ts` — This file provides type hints for your parser when used in TypeScript.
- `bindings/node/binding_test.js` — This file contains a test for the Node.js package.
### Java
- `pom.xml` - This file is the manifest of the Maven package.
- `bindings/java/main/namespace/language/TreeSitterLanguage.java` - This file wraps your language in a Java class.
- `bindings/java/test/TreeSitterLanguageTest.java` - This file contains a test for the Java package.
### Python
- `pyproject.toml` — This file is the manifest of the Python package.
- `setup.py` — This file tells Python how to compile your language.
- `bindings/python/tree_sitter_language/binding.c` — This file wraps your language in a Python module.
- `bindings/python/tree_sitter_language/__init__.py` — This file tells Python how to load your language.
`bindings/python/tree_sitter_language/__init__.pyi` — This file provides type hints for your parser when used in Python.
- `bindings/python/tree_sitter_language/py.typed` — This file provides type hints for your parser when used in Python.
- `bindings/python/tests/test_binding.py` — This file contains a test for the Python package.
### Rust
- `Cargo.toml` — This file is the manifest of the Rust package.
- `bindings/rust/lib.rs` — This file wraps your language in a Rust crate when used in Rust.
- `bindings/rust/build.rs` — This file wraps the building process for the Rust crate.
### Swift
- `Package.swift` — This file tells Swift how to compile your language.
- `bindings/swift/TreeSitterLanguage/language.h` — This file wraps your language in a Swift module when used in Swift.
- `bindings/swift/TreeSitterLanguageTests/TreeSitterLanguageTests.swift` — This file contains a test for the Swift package.
### Zig
- `build.zig` - This file tells Zig how to compile your language.
- `build.zig.zon` - This file is the manifest of the Zig package.
- `bindings/zig/root.zig` - This file wraps your language in a Zig module.
- `bindings/zig/test.zig` - This file contains a test for the Zig package.
### Additional Files
Additionally, there's a few other files that are generated when you run `tree-sitter init`,
that aim to improve the development experience:
- `.editorconfig` — This file tells your editor how to format your code. More information about this file can be found [here][editorconfig]
- `.gitattributes` — This file tells Git how to handle line endings, and tells GitHub what files are generated.
- `.gitignore` — This file tells Git what files to ignore when committing changes.
[cmake]: https://cmake.org/cmake/help/latest [cmake]: https://cmake.org/cmake/help/latest
[editorconfig]: https://editorconfig.org [editorconfig]: https://editorconfig.org

View file

@ -78,7 +78,8 @@ Suppress main output.
### `--edits <EDITS>...` ### `--edits <EDITS>...`
Apply edits after parsing the file. Edits are in the form of `row,col|position delcount insert_text` where row and col, or position are 0-indexed. Apply edits after parsing the file. Edits are in the form of `row,col|position delcount insert_text` where row and col,
or position are 0-indexed.
### `--encoding <ENCODING>` ### `--encoding <ENCODING>`
@ -95,7 +96,8 @@ Output parsing results in a JSON format.
### `--config-path <CONFIG_PATH>` ### `--config-path <CONFIG_PATH>`
The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more information. The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more
information.
### `-n/--test-number <TEST_NUMBER>` ### `-n/--test-number <TEST_NUMBER>`

View file

@ -7,8 +7,8 @@ tree-sitter playground [OPTIONS] # Aliases: play, pg, web-ui
``` ```
```admonish note ```admonish note
For this to work, you must have already built the parser as a Wasm module. This can be done with the [`build`](./build.md) subcommand For this to work, you must have already built the parser as a Wasm module. This can be done with the [`build`](./build.md)
(`tree-sitter build --wasm`). subcommand (`tree-sitter build --wasm`).
``` ```
## Options ## Options

View file

@ -47,8 +47,8 @@ The range of rows in which the query will be executed. The format is `start_row:
### `--containing-row-range <ROW_RANGE>` ### `--containing-row-range <ROW_RANGE>`
The range of rows in which the query will be executed. Only the matches that are fully contained within the provided row range The range of rows in which the query will be executed. Only the matches that are fully contained within the provided row
will be returned. range will be returned.
### `--scope <SCOPE>` ### `--scope <SCOPE>`
@ -64,7 +64,8 @@ Whether to run query tests or not.
### `--config-path <CONFIG_PATH>` ### `--config-path <CONFIG_PATH>`
The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more information. The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more
information.
### `-n/--test-number <TEST_NUMBER>` ### `-n/--test-number <TEST_NUMBER>`

View file

@ -31,7 +31,8 @@ The path to the directory containing the grammar.
### `--config-path <CONFIG_PATH>` ### `--config-path <CONFIG_PATH>`
The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more information. The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more
information.
### `-n/--test-number <TEST_NUMBER>` ### `-n/--test-number <TEST_NUMBER>`

View file

@ -63,7 +63,8 @@ When using the `--debug-graph` option, open the log file in the default browser.
### `--config-path <CONFIG_PATH>` ### `--config-path <CONFIG_PATH>`
The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more information. The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more
information.
### `--show-fields` ### `--show-fields`

View file

@ -25,11 +25,9 @@ tree-sitter version --bump minor # minor bump
tree-sitter version --bump major # major bump tree-sitter version --bump major # major bump
``` ```
As a grammar author, you should keep the version of your grammar in sync across As a grammar author, you should keep the version of your grammar in sync across different bindings. However, doing so manually
different bindings. However, doing so manually is error-prone and tedious, so is error-prone and tedious, so this command takes care of the burden. If you are using a version control system, it is recommended
this command takes care of the burden. If you are using a version control system, to commit the changes made by this command, and to tag the commit with the new version.
it is recommended to commit the changes made by this command, and to tag the
commit with the new version.
To print the current version without bumping it, use: To print the current version without bumping it, use:

View file

@ -17,8 +17,8 @@ DSL through the `RustRegex` class. Simply pass your regex pattern as a string:
``` ```
Unlike JavaScript's builtin `RegExp` class, which takes a pattern and flags as separate arguments, `RustRegex` only Unlike JavaScript's builtin `RegExp` class, which takes a pattern and flags as separate arguments, `RustRegex` only
accepts a single pattern string. While it doesn't support separate flags, you can use inline flags within the pattern itself. accepts a single pattern string. While it doesn't support separate flags, you can use inline flags within the pattern
For more details about Rust's regex syntax and capabilities, check out the [Rust regex documentation][rust regex]. itself. For more details about Rust's regex syntax and capabilities, check out the [Rust regex documentation][rust regex].
```admonish note ```admonish note
Only a subset of the Regex engine is actually supported. This is due to certain features like lookahead and lookaround Only a subset of the Regex engine is actually supported. This is due to certain features like lookahead and lookaround
@ -50,10 +50,10 @@ The previous `repeat` rule is implemented in `repeat1` but is included because i
- **Options : `optional(rule)`** — This function creates a rule that matches *zero or one* occurrence of a given rule. - **Options : `optional(rule)`** — This function creates a rule that matches *zero or one* occurrence of a given rule.
It is analogous to the `[x]` (square bracket) syntax in EBNF notation. It is analogous to the `[x]` (square bracket) syntax in EBNF notation.
- **Precedence : `prec(number, rule)`** — This function marks the given rule with a numerical precedence, which will be used - **Precedence : `prec(number, rule)`** — This function marks the given rule with a numerical precedence, which will be
to resolve [*LR(1) Conflicts*][lr-conflict] at parser-generation time. When two rules overlap in a way that represents either used to resolve [*LR(1) Conflicts*][lr-conflict] at parser-generation time. When two rules overlap in a way that represents
a true ambiguity or a *local* ambiguity given one token of lookahead, Tree-sitter will try to resolve the conflict by matching either a true ambiguity or a *local* ambiguity given one token of lookahead, Tree-sitter will try to resolve the conflict
the rule with the higher precedence. The default precedence of all rules is zero. This works similarly to the by matching the rule with the higher precedence. The default precedence of all rules is zero. This works similarly to the
[precedence directives][yacc-prec] in Yacc grammars. [precedence directives][yacc-prec] in Yacc grammars.
This function can also be used to assign lexical precedence to a given This function can also be used to assign lexical precedence to a given
@ -115,8 +115,8 @@ want to create syntax tree nodes at runtime.
- **`conflicts`** — an array of arrays of rule names. Each inner array represents a set of rules that's involved in an - **`conflicts`** — an array of arrays of rule names. Each inner array represents a set of rules that's involved in an
*LR(1) conflict* that is *intended to exist* in the grammar. When these conflicts occur at runtime, Tree-sitter will use *LR(1) conflict* that is *intended to exist* in the grammar. When these conflicts occur at runtime, Tree-sitter will use
the GLR algorithm to explore all the possible interpretations. If *multiple* parses end up succeeding, Tree-sitter will pick the GLR algorithm to explore all the possible interpretations. If *multiple* parses end up succeeding, Tree-sitter will
the subtree whose corresponding rule has the highest total *dynamic precedence*. pick the subtree whose corresponding rule has the highest total *dynamic precedence*.
- **`externals`** — an array of token names which can be returned by an - **`externals`** — an array of token names which can be returned by an
[*external scanner*][external-scanners]. External scanners allow you to write custom C code which runs during the lexing [*external scanner*][external-scanners]. External scanners allow you to write custom C code which runs during the lexing
@ -139,10 +139,10 @@ for more details.
array of reserved rules. The reserved rule in the array must be a terminal token meaning it must be a string, regex, token, array of reserved rules. The reserved rule in the array must be a terminal token meaning it must be a string, regex, token,
or terminal rule. The reserved rule must also exist and be used in the grammar, specifying arbitrary tokens will not work. or terminal rule. The reserved rule must also exist and be used in the grammar, specifying arbitrary tokens will not work.
The *first* reserved word set in the object is the global word set, meaning it applies to every rule in every parse state. The *first* reserved word set in the object is the global word set, meaning it applies to every rule in every parse state.
However, certain keywords are contextual, depending on the rule. For example, in JavaScript, keywords are typically not allowed However, certain keywords are contextual, depending on the rule. For example, in JavaScript, keywords are typically not
as ordinary variables, however, they *can* be used as a property name. In this situation, the `reserved` function would be used, allowed as ordinary variables, however, they *can* be used as a property name. In this situation, the `reserved` function
and the word set to pass in would be the name of the word set that is declared in the `reserved` object that corresponds to an would be used, and the word set to pass in would be the name of the word set that is declared in the `reserved` object that
empty array, signifying *no* keywords are reserved. corresponds to an empty array, signifying *no* keywords are reserved.
[bison-dprec]: https://www.gnu.org/software/bison/manual/html_node/Generalized-LR-Parsing.html [bison-dprec]: https://www.gnu.org/software/bison/manual/html_node/Generalized-LR-Parsing.html
[ebnf]: https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form [ebnf]: https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form

View file

@ -1,7 +1,7 @@
# Writing the Grammar # Writing the Grammar
Writing a grammar requires creativity. There are an infinite number of CFGs (context-free grammars) that can be used to describe Writing a grammar requires creativity. There are an infinite number of CFGs (context-free grammars) that can be used to
any given language. To produce a good Tree-sitter parser, you need to create a grammar with two important properties: describe any given language. To produce a good Tree-sitter parser, you need to create a grammar with two important properties:
1. **An intuitive structure** — Tree-sitter's output is a [concrete syntax tree][cst]; each node in the tree corresponds 1. **An intuitive structure** — Tree-sitter's output is a [concrete syntax tree][cst]; each node in the tree corresponds
directly to a [terminal or non-terminal symbol][non-terminal] in the grammar. So to produce an easy-to-analyze tree, there directly to a [terminal or non-terminal symbol][non-terminal] in the grammar. So to produce an easy-to-analyze tree, there
@ -139,8 +139,8 @@ instead. It's often useful to check your progress by trying to parse some real c
## Structuring Rules Well ## Structuring Rules Well
Imagine that you were just starting work on the [Tree-sitter JavaScript parser][tree-sitter-javascript]. Naively, you might Imagine that you were just starting work on the [Tree-sitter JavaScript parser][tree-sitter-javascript]. Naively, you might
try to directly mirror the structure of the [ECMAScript Language Spec][ecmascript-spec]. To illustrate the problem with this try to directly mirror the structure of the [ECMAScript Language Spec][ecmascript-spec]. To illustrate the problem with
approach, consider the following line of code: this approach, consider the following line of code:
```js ```js
return x + y; return x + y;
@ -181,16 +181,17 @@ which are unrelated to the actual code.
## Standard Rule Names ## Standard Rule Names
Tree-sitter places no restrictions on how to name the rules of your grammar. It can be helpful, however, to follow certain conventions Tree-sitter places no restrictions on how to name the rules of your grammar. It can be helpful, however, to follow certain
used by many other established grammars in the ecosystem. Some of these well-established patterns are listed below: conventions used by many other established grammars in the ecosystem. Some of these well-established patterns are listed
below:
- `source_file`: Represents an entire source file, this rule is commonly used as the root node for a grammar, - `source_file`: Represents an entire source file, this rule is commonly used as the root node for a grammar,
- `expression`/`statement`: Used to represent statements and expressions for a given language. Commonly defined as a choice between several - `expression`/`statement`: Used to represent statements and expressions for a given language. Commonly defined as a choice
more specific sub-expression/sub-statement rules. between several more specific sub-expression/sub-statement rules.
- `block`: Used as the parent node for block scopes, with its children representing the block's contents. - `block`: Used as the parent node for block scopes, with its children representing the block's contents.
- `type`: Represents the types of a language such as `int`, `char`, and `void`. - `type`: Represents the types of a language such as `int`, `char`, and `void`.
- `identifier`: Used for constructs like variable names, function arguments, and object fields; this rule is commonly used as the `word` - `identifier`: Used for constructs like variable names, function arguments, and object fields; this rule is commonly used
token in grammars. as the `word` token in grammars.
- `string`: Used to represent `"string literals"`. - `string`: Used to represent `"string literals"`.
- `comment`: Used to represent comments, this rule is commonly used as an `extra`. - `comment`: Used to represent comments, this rule is commonly used as an `extra`.
@ -308,9 +309,9 @@ This is where `prec.left` and `prec.right` come into use. We want to select the
## Using Conflicts ## Using Conflicts
Sometimes, conflicts are actually desirable. In our JavaScript grammar, expressions and patterns can create intentional ambiguity. Sometimes, conflicts are actually desirable. In our JavaScript grammar, expressions and patterns can create intentional
A construct like `[x, y]` could be legitimately parsed as both an array literal (like in `let a = [x, y]`) or as a destructuring ambiguity. A construct like `[x, y]` could be legitimately parsed as both an array literal (like in `let a = [x, y]`) or
pattern (like in `let [x, y] = arr`). as a destructuring pattern (like in `let [x, y] = arr`).
```js ```js
export default grammar({ export default grammar({
@ -564,8 +565,8 @@ as mentioned in the previous page, is `token(prec(N, ...))`.
## Keywords ## Keywords
Many languages have a set of _keyword_ tokens (e.g. `if`, `for`, `return`), as well as a more general token (e.g. `identifier`) Many languages have a set of _keyword_ tokens (e.g. `if`, `for`, `return`), as well as a more general token (e.g. `identifier`)
that matches any word, including many of the keyword strings. For example, JavaScript has a keyword `instanceof`, which is that matches any word, including many of the keyword strings. For example, JavaScript has a keyword `instanceof`, which
used as a binary operator, like this: is used as a binary operator, like this:
```js ```js
if (a instanceof Something) b(); if (a instanceof Something) b();

View file

@ -143,10 +143,10 @@ the second argument, the current character will be treated as whitespace; whites
associated with tokens emitted by the external scanner. associated with tokens emitted by the external scanner.
- **`void (*mark_end)(TSLexer *)`** — A function for marking the end of the recognized token. This allows matching tokens - **`void (*mark_end)(TSLexer *)`** — A function for marking the end of the recognized token. This allows matching tokens
that require multiple characters of lookahead. By default, (if you don't call `mark_end`), any character that you moved past that require multiple characters of lookahead. By default, (if you don't call `mark_end`), any character that you moved
using the `advance` function will be included in the size of the token. But once you call `mark_end`, then any later calls past using the `advance` function will be included in the size of the token. But once you call `mark_end`, then any later
to `advance` will _not_ increase the size of the returned token. You can call `mark_end` multiple times to increase the size calls to `advance` will _not_ increase the size of the returned token. You can call `mark_end` multiple times to increase
of the token. the size of the token.
- **`uint32_t (*get_column)(TSLexer *)`** — A function for querying the current column position of the lexer. It returns - **`uint32_t (*get_column)(TSLexer *)`** — A function for querying the current column position of the lexer. It returns
the number of codepoints since the start of the current line. The codepoint position is recalculated on every call to this the number of codepoints since the start of the current line. The codepoint position is recalculated on every call to this
@ -185,9 +185,9 @@ if (valid_symbols[INDENT] || valid_symbols[DEDENT]) {
### Allocator ### Allocator
Instead of using libc's `malloc`, `calloc`, `realloc`, and `free`, you should use the versions prefixed with `ts_` from `tree_sitter/alloc.h`. Instead of using libc's `malloc`, `calloc`, `realloc`, and `free`, you should use the versions prefixed with `ts_` from
These macros can allow a potential consumer to override the default allocator with their own implementation, but by default `tree_sitter/alloc.h`. These macros can allow a potential consumer to override the default allocator with their own implementation,
will use the libc functions. but by default will use the libc functions.
As a consumer of the tree-sitter core library as well as any parser libraries that might use allocations, you can enable As a consumer of the tree-sitter core library as well as any parser libraries that might use allocations, you can enable
overriding the default allocator and have it use the same one as the library allocator, of which you can set with `ts_set_allocator`. overriding the default allocator and have it use the same one as the library allocator, of which you can set with `ts_set_allocator`.
@ -195,7 +195,8 @@ To enable this overriding in scanners, you must compile them with the `TREE_SITT
the library must be linked into your final app dynamically, since it needs to resolve the internal functions at runtime. the library must be linked into your final app dynamically, since it needs to resolve the internal functions at runtime.
If you are compiling an executable binary that uses the core library, but want to load parsers dynamically at runtime, then If you are compiling an executable binary that uses the core library, but want to load parsers dynamically at runtime, then
you will have to use a special linker flag on Unix. For non-Darwin systems, that would be `--dynamic-list` and for Darwin you will have to use a special linker flag on Unix. For non-Darwin systems, that would be `--dynamic-list` and for Darwin
systems, that would be `-exported_symbols_list`. The CLI does exactly this, so you can use it as a reference (check out `cli/build.rs`). systems, that would be `-exported_symbols_list`. The CLI does exactly this, so you can use it as a reference (check out
`cli/build.rs`).
For example, assuming you wanted to allocate 100 bytes for your scanner, you'd do so like the following example: For example, assuming you wanted to allocate 100 bytes for your scanner, you'd do so like the following example:
@ -293,9 +294,10 @@ bool tree_sitter_my_language_external_scanner_scan(
## Other External Scanner Details ## Other External Scanner Details
External scanners have priority over Tree-sitter's normal lexing process. When a token listed in the externals array is valid External scanners have priority over Tree-sitter's normal lexing process. When a token listed in the externals array is
at a given position, the external scanner is called first. This makes external scanners a powerful way to override Tree-sitter's valid at a given position, the external scanner is called first. This makes external scanners a powerful way to override
default lexing behavior, especially for cases that can't be handled with regular lexical rules, parsing, or dynamic precedence. Tree-sitter's default lexing behavior, especially for cases that can't be handled with regular lexical rules, parsing, or
dynamic precedence.
During error recovery, Tree-sitter's first step is to call the external scanner's scan function with all tokens marked as During error recovery, Tree-sitter's first step is to call the external scanner's scan function with all tokens marked as
valid. Your scanner should detect and handle this case appropriately. One simple approach is to add an unused "sentinel" valid. Your scanner should detect and handle this case appropriately. One simple approach is to add an unused "sentinel"

View file

@ -39,8 +39,8 @@ It only shows the *named* nodes, as described in [this section][named-vs-anonymo
``` ```
The expected output section can also *optionally* show the [*field names*][node-field-names] associated with each child The expected output section can also *optionally* show the [*field names*][node-field-names] associated with each child
node. To include field names in your tests, you write a node's field name followed by a colon, before the node itself in node. To include field names in your tests, you write a node's field name followed by a colon, before the node itself
the S-expression: in the S-expression:
```query ```query
(source_file (source_file
@ -87,6 +87,11 @@ The recommendation is to be comprehensive in adding tests. If it's a visible nod
directory. It's typically a good idea to test all the permutations of each language construct. This increases test coverage, directory. It's typically a good idea to test all the permutations of each language construct. This increases test coverage,
but doubly acquaints readers with a way to examine expected outputs and understand the "edges" of a language. but doubly acquaints readers with a way to examine expected outputs and understand the "edges" of a language.
```admonish tip
After modifying the grammar, you can run `tree-sitter test -u`
to update all syntax trees in corpus files with current parser output.
```
## Attributes ## Attributes
Tests can be annotated with a few `attributes`. Attributes must be put in the header, below the test name, and start with Tests can be annotated with a few `attributes`. Attributes must be put in the header, below the test name, and start with
@ -99,8 +104,8 @@ you can repeat the attribute on a new line.
The following attributes are available: The following attributes are available:
* `:cst` - This attribute specifies that the expected output should be in the form of a CST instead of the normal S-expression. This * `:cst` - This attribute specifies that the expected output should be in the form of a CST instead of the normal S-expression.
CST matches the format given by `parse --cst`. This CST matches the format given by `parse --cst`.
* `:error` — This attribute will assert that the parse tree contains an error. It's useful to just validate that a certain * `:error` — This attribute will assert that the parse tree contains an error. It's useful to just validate that a certain
input is invalid without displaying the whole parse tree, as such you should omit the parse tree below the `---` line. input is invalid without displaying the whole parse tree, as such you should omit the parse tree below the `---` line.
* `:fail-fast` — This attribute will stop the testing of additional cases if the test marked with this attribute fails. * `:fail-fast` — This attribute will stop the testing of additional cases if the test marked with this attribute fails.

View file

@ -1,4 +1,4 @@
# Creating parsers # Creating parsers
Developing Tree-sitter grammars can have a difficult learning curve, but once you get the hang of it, it can be fun and even Developing Tree-sitter grammars can have a difficult learning curve, but once you get the hang of it, it can be fun and
zen-like. This document will help you to get started and to develop a useful mental model. even zen-like. This document will help you to get started and to develop a useful mental model.

View file

@ -10,7 +10,8 @@ file and efficiently update the syntax tree as the source file is edited. Tree-s
- **General** enough to parse any programming language - **General** enough to parse any programming language
- **Fast** enough to parse on every keystroke in a text editor - **Fast** enough to parse on every keystroke in a text editor
- **Robust** enough to provide useful results even in the presence of syntax errors - **Robust** enough to provide useful results even in the presence of syntax errors
- **Dependency-free** so that the runtime library (which is written in pure [C11](https://github.com/tree-sitter/tree-sitter/tree/master/lib)) can be embedded in any application - **Dependency-free** so that the runtime library (which is written in pure [C11](https://github.com/tree-sitter/tree-sitter/tree/master/lib))
can be embedded in any application
## Language Bindings ## Language Bindings

View file

@ -2,7 +2,8 @@
## Providing the Code ## Providing the Code
In the example on the previous page, we parsed source code stored in a simple string using the `ts_parser_parse_string` function: In the example on the previous page, we parsed source code stored in a simple string using the `ts_parser_parse_string`
function:
```c ```c
TSTree *ts_parser_parse_string( TSTree *ts_parser_parse_string(
@ -135,10 +136,10 @@ Consider a grammar rule like this:
if_statement: $ => seq("if", "(", $._expression, ")", $._statement); if_statement: $ => seq("if", "(", $._expression, ")", $._statement);
``` ```
A syntax node representing an `if_statement` in this language would have 5 children: the condition expression, the body statement, A syntax node representing an `if_statement` in this language would have 5 children: the condition expression, the body
as well as the `if`, `(`, and `)` tokens. The expression and the statement would be marked as _named_ nodes, because they statement, as well as the `if`, `(`, and `)` tokens. The expression and the statement would be marked as _named_ nodes,
have been given explicit names in the grammar. But the `if`, `(`, and `)` nodes would _not_ be named nodes, because they because they have been given explicit names in the grammar. But the `if`, `(`, and `)` nodes would _not_ be named nodes,
are represented in the grammar as simple strings. because they are represented in the grammar as simple strings.
You can check whether any given node is named: You can check whether any given node is named:

View file

@ -19,8 +19,8 @@ typedef struct {
void ts_tree_edit(TSTree *, const TSInputEdit *); void ts_tree_edit(TSTree *, const TSInputEdit *);
``` ```
Then, you can call `ts_parser_parse` again, passing in the old tree. This will create a new tree that internally shares structure Then, you can call `ts_parser_parse` again, passing in the old tree. This will create a new tree that internally shares
with the old tree. structure with the old tree.
When you edit a syntax tree, the positions of its nodes will change. If you have stored any `TSNode` instances outside of When you edit a syntax tree, the positions of its nodes will change. If you have stored any `TSNode` instances outside of
the `TSTree`, you must update their positions separately, using the same `TSInputEdit` value, in order to update their the `TSTree`, you must update their positions separately, using the same `TSInputEdit` value, in order to update their

View file

@ -108,9 +108,9 @@ In Tree-sitter grammars, there are usually certain rules that represent abstract
"type", "declaration"). In the `grammar.js` file, these are often written as [hidden rules][hidden rules] "type", "declaration"). In the `grammar.js` file, these are often written as [hidden rules][hidden rules]
whose definition is a simple [`choice`][grammar dsl] where each member is just a single symbol. whose definition is a simple [`choice`][grammar dsl] where each member is just a single symbol.
Normally, hidden rules are not mentioned in the node types file, since they don't appear in the syntax tree. But if you add Normally, hidden rules are not mentioned in the node types file, since they don't appear in the syntax tree. But if you
a hidden rule to the grammar's [`supertypes` list][grammar dsl], then it _will_ show up in the node add a hidden rule to the grammar's [`supertypes` list][grammar dsl], then it _will_ show up in the node types file, with
types file, with the following special entry: the following special entry:
- `"subtypes"` — An array of objects that specify the _types_ of nodes that this 'supertype' node can wrap. - `"subtypes"` — An array of objects that specify the _types_ of nodes that this 'supertype' node can wrap.

View file

@ -15,8 +15,11 @@ A given version of the tree-sitter library is only able to load parsers generate
| >=0.20.3, <=0.24 | 13 | 14 | | >=0.20.3, <=0.24 | 13 | 14 |
| >=0.25 | 13 | 15 | | >=0.25 | 13 | 15 |
By default, the tree-sitter CLI will generate parsers using the latest available ABI for that version, but an older ABI (supported by the CLI) can be selected by passing the [`--abi` option][abi_option] to the `generate` command. By default, the tree-sitter CLI will generate parsers using the latest available ABI for that version, but an older ABI
(supported by the CLI) can be selected by passing the [`--abi` option][abi_option] to the `generate` command.
Note that the ABI version range supported by the CLI can be smaller than for the library: When a new ABI version is released, older versions will be phased out over a deprecation period, which starts with no longer being able to generate parsers with the oldest ABI version. Note that the ABI version range supported by the CLI can be smaller than for the library: When a new ABI version is released,
older versions will be phased out over a deprecation period, which starts with no longer being able to generate parsers
with the oldest ABI version.
[abi_option]: ../cli/generate.md#--abi-version [abi_option]: ../cli/generate.md#--abi-version

View file

@ -6,8 +6,8 @@ the core concepts remain the same.
Tree-sitter's parsing functionality is implemented through its C API, with all functions documented in the [tree_sitter/api.h][api.h] Tree-sitter's parsing functionality is implemented through its C API, with all functions documented in the [tree_sitter/api.h][api.h]
header file, but if you're working in another language, you can use one of the following bindings found [here](../index.md#language-bindings), header file, but if you're working in another language, you can use one of the following bindings found [here](../index.md#language-bindings),
each providing idiomatic access to Tree-sitter's functionality. Of these bindings, the official ones have their own API docs each providing idiomatic access to Tree-sitter's functionality. Of these bindings, the official ones have their own API
hosted online at the following pages: doc hosted online at the following pages:
- [Go][go] - [Go][go]
- [Java] - [Java]

View file

@ -1,9 +1,9 @@
# Query Syntax # Query Syntax
A _query_ consists of one or more _patterns_, where each pattern is an [S-expression][s-exp] that matches a certain set of A _query_ consists of one or more _patterns_, where each pattern is an [S-expression][s-exp] that matches a certain set
nodes in a syntax tree. The expression to match a given node consists of a pair of parentheses containing two things: the of nodes in a syntax tree. The expression to match a given node consists of a pair of parentheses containing two things:
node's type, and optionally, a series of other S-expressions that match the node's children. For example, this pattern would the node's type, and optionally, a series of other S-expressions that match the node's children. For example, this pattern
match any `binary_expression` node whose children are both `number_literal` nodes: would match any `binary_expression` node whose children are both `number_literal` nodes:
```query ```query
(binary_expression (number_literal) (number_literal)) (binary_expression (number_literal) (number_literal))
@ -99,10 +99,10 @@ by `(ERROR)` queries. Specific missing node types can also be queried:
### Supertype Nodes ### Supertype Nodes
Some node types are marked as _supertypes_ in a grammar. A supertype is a node type that contains multiple Some node types are marked as _supertypes_ in a grammar. A supertype is a node type that contains multiple
subtypes. For example, in the [JavaScript grammar example][grammar], `expression` is a supertype that can represent any kind subtypes. For example, in the [JavaScript grammar example][grammar], `expression` is a supertype that can represent any
of expression, such as a `binary_expression`, `call_expression`, or `identifier`. You can use supertypes in queries to match kind of expression, such as a `binary_expression`, `call_expression`, or `identifier`. You can use supertypes in queries
any of their subtypes, rather than having to list out each subtype individually. For example, this pattern would match any to match any of their subtypes, rather than having to list out each subtype individually. For example, this pattern would
kind of expression, even though it's not a visible node in the syntax tree: match any kind of expression, even though it's not a visible node in the syntax tree:
```query ```query
(expression) @any-expression (expression) @any-expression

View file

@ -128,15 +128,15 @@ This pattern would match any builtin variable that is not a local variable, beca
# Directives # Directives
Similar to predicates, directives are a way to associate arbitrary metadata with a pattern. The only difference between predicates Similar to predicates, directives are a way to associate arbitrary metadata with a pattern. The only difference between
and directives is that directives end in a `!` character instead of `?` character. predicates and directives is that directives end in a `!` character instead of `?` character.
Tree-sitter's CLI supports the following directives by default: Tree-sitter's CLI supports the following directives by default:
## The `set!` directive ## The `set!` directive
This directive allows you to associate key-value pairs with a pattern. The key and value can be any arbitrary text that you This directive allows you to associate key-value pairs with a pattern. The key and value can be any arbitrary text that
see fit. you see fit.
```query ```query
((comment) @injection.content ((comment) @injection.content
@ -156,8 +156,8 @@ another capture are preserved. It takes two arguments, both of which are capture
### The `#strip!` directive ### The `#strip!` directive
The `#strip!` directive allows you to remove text from a capture. It takes two arguments: the first is the capture to strip The `#strip!` directive allows you to remove text from a capture. It takes two arguments: the first is the capture to strip
text from, and the second is a regular expression to match against the text. Any text matched by the regular expression will text from, and the second is a regular expression to match against the text. Any text matched by the regular expression
be removed from the text associated with the capture. will be removed from the text associated with the capture.
For an example on the `#select-adjacent!` and `#strip!` directives, For an example on the `#select-adjacent!` and `#strip!` directives,
view the [code navigation](../../4-code-navigation.md#examples) documentation. view the [code navigation](../../4-code-navigation.md#examples) documentation.

View file

@ -17,7 +17,7 @@
eachSystem = lib.genAttrs systems; eachSystem = lib.genAttrs systems;
pkgsFor = inputs.nixpkgs.legacyPackages; pkgsFor = inputs.nixpkgs.legacyPackages;
version = "0.26.3"; version = "0.27.0";
fs = lib.fileset; fs = lib.fileset;
src = fs.toSource { src = fs.toSource {

View file

@ -49,6 +49,7 @@ fn main() {
.include(&include_path) .include(&include_path)
.define("_POSIX_C_SOURCE", "200112L") .define("_POSIX_C_SOURCE", "200112L")
.define("_DEFAULT_SOURCE", None) .define("_DEFAULT_SOURCE", None)
.define("_BSD_SOURCE", None)
.define("_DARWIN_C_SOURCE", None) .define("_DARWIN_C_SOURCE", None)
.warnings(false) .warnings(false)
.file(src_path.join("lib.c")) .file(src_path.join("lib.c"))

View file

@ -317,7 +317,7 @@ pub trait Decode {
/// A stateful object for walking a syntax [`Tree`] efficiently. /// A stateful object for walking a syntax [`Tree`] efficiently.
#[doc(alias = "TSTreeCursor")] #[doc(alias = "TSTreeCursor")]
pub struct TreeCursor<'cursor>(ffi::TSTreeCursor, PhantomData<&'cursor ()>); pub struct TreeCursor<'tree>(ffi::TSTreeCursor, PhantomData<&'tree ()>);
/// A set of patterns that match nodes in a syntax tree. /// A set of patterns that match nodes in a syntax tree.
#[doc(alias = "TSQuery")] #[doc(alias = "TSQuery")]
@ -392,7 +392,7 @@ pub struct QueryMatch<'cursor, 'tree> {
} }
/// A sequence of [`QueryMatch`]es associated with a given [`QueryCursor`]. /// A sequence of [`QueryMatch`]es associated with a given [`QueryCursor`].
pub struct QueryMatches<'query, 'tree: 'query, T: TextProvider<I>, I: AsRef<[u8]>> { pub struct QueryMatches<'query, 'tree, T: TextProvider<I>, I: AsRef<[u8]>> {
ptr: *mut ffi::TSQueryCursor, ptr: *mut ffi::TSQueryCursor,
query: &'query Query, query: &'query Query,
text_provider: T, text_provider: T,
@ -407,7 +407,7 @@ pub struct QueryMatches<'query, 'tree: 'query, T: TextProvider<I>, I: AsRef<[u8]
/// ///
/// During iteration, each element contains a [`QueryMatch`] and index. The index can /// During iteration, each element contains a [`QueryMatch`] and index. The index can
/// be used to access the new capture inside of the [`QueryMatch::captures`]'s [`captures`]. /// be used to access the new capture inside of the [`QueryMatch::captures`]'s [`captures`].
pub struct QueryCaptures<'query, 'tree: 'query, T: TextProvider<I>, I: AsRef<[u8]>> { pub struct QueryCaptures<'query, 'tree, T: TextProvider<I>, I: AsRef<[u8]>> {
ptr: *mut ffi::TSQueryCursor, ptr: *mut ffi::TSQueryCursor,
query: &'query Query, query: &'query Query,
text_provider: T, text_provider: T,
@ -1581,7 +1581,7 @@ impl<'tree> Node<'tree> {
/// Get the [`Language`] that was used to parse this node's syntax tree. /// Get the [`Language`] that was used to parse this node's syntax tree.
#[doc(alias = "ts_node_language")] #[doc(alias = "ts_node_language")]
#[must_use] #[must_use]
pub fn language(&self) -> LanguageRef { pub fn language(&self) -> LanguageRef<'tree> {
LanguageRef(unsafe { ffi::ts_node_language(self.0) }, PhantomData) LanguageRef(unsafe { ffi::ts_node_language(self.0) }, PhantomData)
} }
@ -2082,11 +2082,11 @@ impl fmt::Display for Node<'_> {
} }
} }
impl<'cursor> TreeCursor<'cursor> { impl<'tree> TreeCursor<'tree> {
/// Get the tree cursor's current [`Node`]. /// Get the tree cursor's current [`Node`].
#[doc(alias = "ts_tree_cursor_current_node")] #[doc(alias = "ts_tree_cursor_current_node")]
#[must_use] #[must_use]
pub fn node(&self) -> Node<'cursor> { pub fn node(&self) -> Node<'tree> {
Node( Node(
unsafe { ffi::ts_tree_cursor_current_node(&self.0) }, unsafe { ffi::ts_tree_cursor_current_node(&self.0) },
PhantomData, PhantomData,
@ -2227,7 +2227,7 @@ impl<'cursor> TreeCursor<'cursor> {
/// Re-initialize this tree cursor to start at the original node that the /// Re-initialize this tree cursor to start at the original node that the
/// cursor was constructed with. /// cursor was constructed with.
#[doc(alias = "ts_tree_cursor_reset")] #[doc(alias = "ts_tree_cursor_reset")]
pub fn reset(&mut self, node: Node<'cursor>) { pub fn reset(&mut self, node: Node<'tree>) {
unsafe { ffi::ts_tree_cursor_reset(&mut self.0, node.0) }; unsafe { ffi::ts_tree_cursor_reset(&mut self.0, node.0) };
} }
@ -3404,7 +3404,7 @@ impl QueryProperty {
/// Provide a `StreamingIterator` instead of the traditional `Iterator`, as the /// Provide a `StreamingIterator` instead of the traditional `Iterator`, as the
/// underlying object in the C library gets updated on each iteration. Copies would /// underlying object in the C library gets updated on each iteration. Copies would
/// have their internal state overwritten, leading to Undefined Behavior /// have their internal state overwritten, leading to Undefined Behavior
impl<'query, 'tree: 'query, T: TextProvider<I>, I: AsRef<[u8]>> StreamingIterator impl<'query, 'tree, T: TextProvider<I>, I: AsRef<[u8]>> StreamingIterator
for QueryMatches<'query, 'tree, T, I> for QueryMatches<'query, 'tree, T, I>
{ {
type Item = QueryMatch<'query, 'tree>; type Item = QueryMatch<'query, 'tree>;
@ -3435,15 +3435,13 @@ impl<'query, 'tree: 'query, T: TextProvider<I>, I: AsRef<[u8]>> StreamingIterato
} }
} }
impl<'query, 'tree: 'query, T: TextProvider<I>, I: AsRef<[u8]>> StreamingIteratorMut impl<T: TextProvider<I>, I: AsRef<[u8]>> StreamingIteratorMut for QueryMatches<'_, '_, T, I> {
for QueryMatches<'query, 'tree, T, I>
{
fn get_mut(&mut self) -> Option<&mut Self::Item> { fn get_mut(&mut self) -> Option<&mut Self::Item> {
self.current_match.as_mut() self.current_match.as_mut()
} }
} }
impl<'query, 'tree: 'query, T: TextProvider<I>, I: AsRef<[u8]>> StreamingIterator impl<'query, 'tree, T: TextProvider<I>, I: AsRef<[u8]>> StreamingIterator
for QueryCaptures<'query, 'tree, T, I> for QueryCaptures<'query, 'tree, T, I>
{ {
type Item = (QueryMatch<'query, 'tree>, usize); type Item = (QueryMatch<'query, 'tree>, usize);
@ -3480,9 +3478,7 @@ impl<'query, 'tree: 'query, T: TextProvider<I>, I: AsRef<[u8]>> StreamingIterato
} }
} }
impl<'query, 'tree: 'query, T: TextProvider<I>, I: AsRef<[u8]>> StreamingIteratorMut impl<T: TextProvider<I>, I: AsRef<[u8]>> StreamingIteratorMut for QueryCaptures<'_, '_, T, I> {
for QueryCaptures<'query, 'tree, T, I>
{
fn get_mut(&mut self) -> Option<&mut Self::Item> { fn get_mut(&mut self) -> Option<&mut Self::Item> {
self.current_match.as_mut() self.current_match.as_mut()
} }
@ -3622,8 +3618,8 @@ impl From<ffi::TSRange> for Range {
} }
} }
impl From<&'_ InputEdit> for ffi::TSInputEdit { impl From<&InputEdit> for ffi::TSInputEdit {
fn from(val: &'_ InputEdit) -> Self { fn from(val: &InputEdit) -> Self {
Self { Self {
start_byte: val.start_byte as u32, start_byte: val.start_byte as u32,
old_end_byte: val.old_end_byte as u32, old_end_byte: val.old_end_byte as u32,

View file

@ -1,12 +1,12 @@
{ {
"name": "web-tree-sitter", "name": "web-tree-sitter",
"version": "0.26.3", "version": "0.27.0",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "web-tree-sitter", "name": "web-tree-sitter",
"version": "0.26.3", "version": "0.27.0",
"license": "MIT", "license": "MIT",
"devDependencies": { "devDependencies": {
"@eslint/js": "^9.39.1", "@eslint/js": "^9.39.1",

View file

@ -1,6 +1,6 @@
{ {
"name": "web-tree-sitter", "name": "web-tree-sitter",
"version": "0.26.3", "version": "0.27.0",
"description": "Tree-sitter bindings for the web", "description": "Tree-sitter bindings for the web",
"repository": { "repository": {
"type": "git", "type": "git",

View file

@ -0,0 +1,8 @@
export function newFinalizer<T>(handler: (value: T) => void): FinalizationRegistry<T> | undefined {
try {
return new FinalizationRegistry(handler);
} catch(e) {
console.error('Unsupported FinalizationRegistry:', e);
return;
}
}

View file

@ -1,5 +1,10 @@
import { C, Internal, assertInternal } from './constants'; import { C, Internal, assertInternal } from './constants';
import { Language } from './language'; import { Language } from './language';
import { newFinalizer } from './finalization_registry';
const finalizer = newFinalizer((address: number) => {
C._ts_lookahead_iterator_delete(address);
});
export class LookaheadIterator implements Iterable<string> { export class LookaheadIterator implements Iterable<string> {
/** @internal */ /** @internal */
@ -13,6 +18,7 @@ export class LookaheadIterator implements Iterable<string> {
assertInternal(internal); assertInternal(internal);
this[0] = address; this[0] = address;
this.language = language; this.language = language;
finalizer?.register(this, address, this);
} }
/** Get the current symbol of the lookahead iterator. */ /** Get the current symbol of the lookahead iterator. */
@ -27,6 +33,7 @@ export class LookaheadIterator implements Iterable<string> {
/** Delete the lookahead iterator, freeing its resources. */ /** Delete the lookahead iterator, freeing its resources. */
delete(): void { delete(): void {
finalizer?.unregister(this);
C._ts_lookahead_iterator_delete(this[0]); C._ts_lookahead_iterator_delete(this[0]);
this[0] = 0; this[0] = 0;
} }

View file

@ -3,6 +3,7 @@ import { Language } from './language';
import { marshalRange, unmarshalRange } from './marshal'; import { marshalRange, unmarshalRange } from './marshal';
import { checkModule, initializeBinding } from './bindings'; import { checkModule, initializeBinding } from './bindings';
import { Tree } from './tree'; import { Tree } from './tree';
import { newFinalizer } from './finalization_registry';
/** /**
* Options for parsing * Options for parsing
@ -82,6 +83,11 @@ export let LANGUAGE_VERSION: number;
*/ */
export let MIN_COMPATIBLE_VERSION: number; export let MIN_COMPATIBLE_VERSION: number;
const finalizer = newFinalizer((addresses: number[]) => {
C._ts_parser_delete(addresses[0]);
C._free(addresses[1]);
});
/** /**
* A stateful object that is used to produce a {@link Tree} based on some * A stateful object that is used to produce a {@link Tree} based on some
* source code. * source code.
@ -117,6 +123,7 @@ export class Parser {
*/ */
constructor() { constructor() {
this.initialize(); this.initialize();
finalizer?.register(this, [this[0], this[1]], this);
} }
/** @internal */ /** @internal */
@ -131,6 +138,7 @@ export class Parser {
/** Delete the parser, freeing its resources. */ /** Delete the parser, freeing its resources. */
delete() { delete() {
finalizer?.unregister(this);
C._ts_parser_delete(this[0]); C._ts_parser_delete(this[0]);
C._free(this[1]); C._free(this[1]);
this[0] = 0; this[0] = 0;

View file

@ -3,6 +3,7 @@ import { Node } from './node';
import { marshalNode, unmarshalCaptures } from './marshal'; import { marshalNode, unmarshalCaptures } from './marshal';
import { TRANSFER_BUFFER } from './parser'; import { TRANSFER_BUFFER } from './parser';
import { Language } from './language'; import { Language } from './language';
import { newFinalizer } from './finalization_registry';
const PREDICATE_STEP_TYPE_CAPTURE = 1; const PREDICATE_STEP_TYPE_CAPTURE = 1;
@ -506,6 +507,10 @@ function parsePattern(
} }
} }
const finalizer = newFinalizer((address: number) => {
C._ts_query_delete(address);
});
export class Query { export class Query {
/** @internal */ /** @internal */
private [0] = 0; // Internal handle for Wasm private [0] = 0; // Internal handle for Wasm
@ -687,10 +692,12 @@ export class Query {
this.assertedProperties = assertedProperties; this.assertedProperties = assertedProperties;
this.refutedProperties = refutedProperties; this.refutedProperties = refutedProperties;
this.exceededMatchLimit = false; this.exceededMatchLimit = false;
finalizer?.register(this, address, this);
} }
/** Delete the query, freeing its resources. */ /** Delete the query, freeing its resources. */
delete(): void { delete(): void {
finalizer?.unregister(this);
C._ts_query_delete(this[0]); C._ts_query_delete(this[0]);
this[0] = 0; this[0] = 0;
} }

View file

@ -5,6 +5,7 @@ import { TreeCursor } from './tree_cursor';
import { marshalEdit, marshalPoint, unmarshalNode, unmarshalRange } from './marshal'; import { marshalEdit, marshalPoint, unmarshalNode, unmarshalRange } from './marshal';
import { TRANSFER_BUFFER } from './parser'; import { TRANSFER_BUFFER } from './parser';
import { Edit } from './edit'; import { Edit } from './edit';
import { newFinalizer } from './finalization_registry';
/** @internal */ /** @internal */
export function getText(tree: Tree, startIndex: number, endIndex: number, startPosition: Point): string { export function getText(tree: Tree, startIndex: number, endIndex: number, startPosition: Point): string {
@ -28,6 +29,10 @@ export function getText(tree: Tree, startIndex: number, endIndex: number, startP
return result ?? ''; return result ?? '';
} }
const finalizer = newFinalizer((address: number) => {
C._ts_tree_delete(address);
});
/** A tree that represents the syntactic structure of a source code file. */ /** A tree that represents the syntactic structure of a source code file. */
export class Tree { export class Tree {
/** @internal */ /** @internal */
@ -45,6 +50,7 @@ export class Tree {
this[0] = address; this[0] = address;
this.language = language; this.language = language;
this.textCallback = textCallback; this.textCallback = textCallback;
finalizer?.register(this, address, this);
} }
/** Create a shallow copy of the syntax tree. This is very fast. */ /** Create a shallow copy of the syntax tree. This is very fast. */
@ -55,6 +61,7 @@ export class Tree {
/** Delete the syntax tree, freeing its resources. */ /** Delete the syntax tree, freeing its resources. */
delete(): void { delete(): void {
finalizer?.unregister(this);
C._ts_tree_delete(this[0]); C._ts_tree_delete(this[0]);
this[0] = 0; this[0] = 0;
} }

View file

@ -3,6 +3,11 @@ import { marshalNode, marshalPoint, marshalTreeCursor, unmarshalNode, unmarshalP
import { Node } from './node'; import { Node } from './node';
import { TRANSFER_BUFFER } from './parser'; import { TRANSFER_BUFFER } from './parser';
import { getText, Tree } from './tree'; import { getText, Tree } from './tree';
import { newFinalizer } from './finalization_registry';
const finalizer = newFinalizer((address: number) => {
C._ts_tree_cursor_delete_wasm(address);
});
/** A stateful object for walking a syntax {@link Tree} efficiently. */ /** A stateful object for walking a syntax {@link Tree} efficiently. */
export class TreeCursor { export class TreeCursor {
@ -30,6 +35,7 @@ export class TreeCursor {
assertInternal(internal); assertInternal(internal);
this.tree = tree; this.tree = tree;
unmarshalTreeCursor(this); unmarshalTreeCursor(this);
finalizer?.register(this, this.tree[0], this);
} }
/** Creates a deep copy of the tree cursor. This allocates new memory. */ /** Creates a deep copy of the tree cursor. This allocates new memory. */
@ -42,6 +48,7 @@ export class TreeCursor {
/** Delete the tree cursor, freeing its resources. */ /** Delete the tree cursor, freeing its resources. */
delete(): void { delete(): void {
finalizer?.unregister(this);
marshalTreeCursor(this); marshalTreeCursor(this);
C._ts_tree_cursor_delete_wasm(this.tree[0]); C._ts_tree_cursor_delete_wasm(this.tree[0]);
this[0] = this[1] = this[2] = 0; this[0] = this[1] = this[2] = 0;

View file

@ -0,0 +1,74 @@
import { describe, expect, it } from 'vitest';
import { gc, event, Finalizer } from './memory';
// hijack finalization registry before import web-tree-sitter
globalThis.FinalizationRegistry = Finalizer;
describe('Memory Management', () => {
describe('call .delete()', () => {
it('test free memory manually', async () => {
const timer = setInterval(() => {
gc();
}, 100);
let done = 0;
event.on('gc', () => {
done++;
});
await (async () => {
const { JavaScript } = await (await import('./helper')).default;
const { Parser, Query } = await import('../src');
const parser = new Parser();
parser.setLanguage(JavaScript);
const tree = parser.parse('1+1')!;
const copyTree = tree.copy();
const cursor = tree.walk();
const copyCursor = cursor.copy();
const lookaheadIterator = JavaScript.lookaheadIterator(cursor.currentNode.nextParseState)!;
const query = new Query(JavaScript, '(identifier) @element');
parser.delete();
tree.delete();
copyTree.delete();
cursor.delete();
copyCursor.delete();
lookaheadIterator.delete();
query.delete();
})();
// wait for gc
await new Promise((resolve) => setTimeout(resolve, 1000));
clearInterval(timer);
// expect no gc event fired
expect(done).toBe(0);
});
});
describe('do not call .delete()', () => {
it('test free memory automatically', async () => {
const timer = setInterval(() => {
gc();
}, 100);
let done = 0;
const promise = new Promise((resolve) => {
event.on('gc', () => {
if (++done === 7) {
resolve(undefined);
clearInterval(timer);
}
console.log('free memory times: ', done);
});
});
await (async () => {
const { JavaScript } = await (await import('./helper')).default;
const { Parser, Query } = await import('../src');
const parser = new Parser(); // 1
parser.setLanguage(JavaScript);
const tree = parser.parse('1+1')!; // 2
tree.copy(); // 3
const cursor = tree.walk(); // 4
cursor.copy(); // 5
JavaScript.lookaheadIterator(cursor.currentNode.nextParseState)!; // 6
new Query(JavaScript, '(identifier) @element'); // 7
})();
await promise;
});
});
});

View file

@ -0,0 +1,20 @@
import { EventEmitter } from 'events';
import { Session } from 'inspector';
const session = new Session();
session.connect();
export function gc() {
session.post('HeapProfiler.collectGarbage');
}
export const event = new EventEmitter();
export class Finalizer<T> extends FinalizationRegistry<T> {
constructor(handler: (value: T) => void) {
super((value) => {
handler(value);
event.emit('gc');
});
}
}

View file

@ -0,0 +1,25 @@
import { describe, it } from 'vitest';
describe('FinalizationRegistry is unsupported', () => {
it('test FinalizationRegistry is unsupported', async () => {
// @ts-expect-error: test FinalizationRegistry is not supported
globalThis.FinalizationRegistry = undefined;
const { JavaScript } = await (await import('./helper')).default;
const { Parser, Query } = await import('../src');
const parser = new Parser();
parser.setLanguage(JavaScript);
const tree = parser.parse('1+1')!;
const copyTree = tree.copy();
const cursor = tree.walk();
const copyCursor = cursor.copy();
const lookaheadIterator = JavaScript.lookaheadIterator(cursor.currentNode.nextParseState)!;
const query = new Query(JavaScript, '(identifier) @element');
parser.delete();
tree.delete();
copyTree.delete();
cursor.delete();
copyCursor.delete();
lookaheadIterator.delete();
query.delete();
});
});

File diff suppressed because it is too large Load diff