Compare commits
44 commits
ts-capture
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
6739742fb6 | ||
|
|
d251226a3c | ||
|
ae8184b8b9 | ||
|
470ecf8996 | ||
|
|
0cdb6bef7b | ||
|
cd603fa981 | ||
|
b12009a746 | ||
|
|
9f9a0bc410 | ||
|
5d290a2a75 | ||
|
|
5808350bfe | ||
|
|
e64e74d5ed | ||
|
|
1a88b26a10 | ||
|
|
6c05cdfb0c | ||
|
|
aefae11c0d | ||
|
630fa52717 | ||
|
|
eea85f4eff | ||
|
|
cd6672701b | ||
|
f4ca3d95ca | ||
|
17e3c7a5c5 | ||
|
dd60d5cff0 | ||
|
|
f1288ea5c9 | ||
|
|
47ae060966 | ||
|
|
a1893b4420 | ||
|
|
999e041d49 | ||
|
|
0d4d854809 | ||
|
|
93d793d249 | ||
|
|
82486d4b0a | ||
|
5d9605a91e | ||
|
|
5293dd683e | ||
|
|
62effdf128 | ||
|
|
8e4f21aba0 | ||
|
|
5208299bbb | ||
|
|
ba7350c7ee | ||
|
|
f96d518ebf | ||
|
|
d5b82fbbab | ||
|
a7d8c0cbb2 | ||
|
|
24007727d4 | ||
|
|
6aa63a7213 | ||
|
|
eacb95c85d | ||
|
|
6967640571 | ||
|
|
4ac2d5d276 | ||
|
|
642b56d9af | ||
|
|
0574fcf256 | ||
|
|
98de2bc1a8 |
79 changed files with 1907 additions and 1505 deletions
25
.github/scripts/wasm_stdlib.js
vendored
Normal file
25
.github/scripts/wasm_stdlib.js
vendored
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
module.exports = async ({ github, context, core }) => {
|
||||
if (context.eventName !== 'pull_request') return;
|
||||
|
||||
const prNumber = context.payload.pull_request.number;
|
||||
const owner = context.repo.owner;
|
||||
const repo = context.repo.repo;
|
||||
|
||||
const { data: files } = await github.rest.pulls.listFiles({
|
||||
owner,
|
||||
repo,
|
||||
pull_number: prNumber
|
||||
});
|
||||
|
||||
const changedFiles = files.map(file => file.filename);
|
||||
|
||||
const wasmStdLibSrc = 'crates/language/wasm/';
|
||||
const dirChanged = changedFiles.some(file => file.startsWith(wasmStdLibSrc));
|
||||
|
||||
if (!dirChanged) return;
|
||||
|
||||
const wasmStdLibHeader = 'lib/src/wasm/wasm-stdlib.h';
|
||||
const requiredChanged = changedFiles.includes(wasmStdLibHeader);
|
||||
|
||||
if (!requiredChanged) core.setFailed(`Changes detected in ${wasmStdLibSrc} but ${wasmStdLibHeader} was not modified.`);
|
||||
};
|
||||
2
.github/workflows/backport.yml
vendored
2
.github/workflows/backport.yml
vendored
|
|
@ -24,7 +24,7 @@ jobs:
|
|||
private-key: ${{ secrets.BACKPORT_KEY }}
|
||||
|
||||
- name: Create backport PR
|
||||
uses: korthout/backport-action@v3
|
||||
uses: korthout/backport-action@v4
|
||||
with:
|
||||
pull_title: "${pull_title}"
|
||||
label_pattern: "^ci:backport ([^ ]+)$"
|
||||
|
|
|
|||
4
.github/workflows/build.yml
vendored
4
.github/workflows/build.yml
vendored
|
|
@ -278,7 +278,7 @@ jobs:
|
|||
|
||||
- name: Upload CLI artifact
|
||||
if: "!matrix.no-run"
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: tree-sitter.${{ matrix.platform }}
|
||||
path: target/${{ matrix.target }}/release/tree-sitter${{ contains(matrix.target, 'windows') && '.exe' || '' }}
|
||||
|
|
@ -287,7 +287,7 @@ jobs:
|
|||
|
||||
- name: Upload Wasm artifacts
|
||||
if: matrix.platform == 'linux-x64'
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: tree-sitter.wasm
|
||||
path: |
|
||||
|
|
|
|||
3
.github/workflows/ci.yml
vendored
3
.github/workflows/ci.yml
vendored
|
|
@ -44,3 +44,6 @@ jobs:
|
|||
|
||||
build:
|
||||
uses: ./.github/workflows/build.yml
|
||||
|
||||
check-wasm-stdlib:
|
||||
uses: ./.github/workflows/wasm_stdlib.yml
|
||||
|
|
|
|||
2
.github/workflows/release.yml
vendored
2
.github/workflows/release.yml
vendored
|
|
@ -25,7 +25,7 @@ jobs:
|
|||
uses: actions/checkout@v6
|
||||
|
||||
- name: Download build artifacts
|
||||
uses: actions/download-artifact@v6
|
||||
uses: actions/download-artifact@v7
|
||||
with:
|
||||
path: artifacts
|
||||
|
||||
|
|
|
|||
19
.github/workflows/wasm_stdlib.yml
vendored
Normal file
19
.github/workflows/wasm_stdlib.yml
vendored
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
name: Check Wasm Stdlib build
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
|
||||
jobs:
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Check directory changes
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: |
|
||||
const scriptPath = `${process.env.GITHUB_WORKSPACE}/.github/scripts/wasm_stdlib.js`;
|
||||
const script = require(scriptPath);
|
||||
return script({ github, context, core });
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
cmake_minimum_required(VERSION 3.13)
|
||||
|
||||
project(tree-sitter
|
||||
VERSION "0.26.3"
|
||||
VERSION "0.27.0"
|
||||
DESCRIPTION "An incremental parsing system for programming tools"
|
||||
HOMEPAGE_URL "https://tree-sitter.github.io/tree-sitter/"
|
||||
LANGUAGES C)
|
||||
|
|
@ -81,7 +81,7 @@ set_target_properties(tree-sitter
|
|||
SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}"
|
||||
DEFINE_SYMBOL "")
|
||||
|
||||
target_compile_definitions(tree-sitter PRIVATE _POSIX_C_SOURCE=200112L _DEFAULT_SOURCE _DARWIN_C_SOURCE)
|
||||
target_compile_definitions(tree-sitter PRIVATE _POSIX_C_SOURCE=200112L _DEFAULT_SOURCE _BSD_SOURCE _DARWIN_C_SOURCE)
|
||||
|
||||
include(GNUInstallDirs)
|
||||
|
||||
|
|
|
|||
138
Cargo.lock
generated
138
Cargo.lock
generated
|
|
@ -166,9 +166,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.19.0"
|
||||
version = "3.19.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43"
|
||||
checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510"
|
||||
dependencies = [
|
||||
"allocator-api2",
|
||||
]
|
||||
|
|
@ -187,9 +187,9 @@ checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3"
|
|||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.49"
|
||||
version = "1.2.53"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "90583009037521a116abf44494efecd645ba48b6622457080f080b85544e2215"
|
||||
checksum = "755d2fce177175ffca841e9a06afdb2c4ab0f593d53b4dee48147dfaade85932"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"shlex",
|
||||
|
|
@ -241,9 +241,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.5.53"
|
||||
version = "4.5.54"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8"
|
||||
checksum = "c6e6ff9dcd79cff5cd969a17a545d79e84ab086e444102a591e288a8aa3ce394"
|
||||
dependencies = [
|
||||
"clap_builder",
|
||||
"clap_derive",
|
||||
|
|
@ -251,9 +251,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "clap_builder"
|
||||
version = "4.5.53"
|
||||
version = "4.5.54"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00"
|
||||
checksum = "fa42cf4d2b7a41bc8f663a7cab4031ebafa1bf3875705bfaf8466dc60ab52c00"
|
||||
dependencies = [
|
||||
"anstream",
|
||||
"anstyle",
|
||||
|
|
@ -263,9 +263,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "clap_complete"
|
||||
version = "4.5.61"
|
||||
version = "4.5.65"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39615915e2ece2550c0149addac32fb5bd312c657f43845bb9088cb9c8a7c992"
|
||||
checksum = "430b4dc2b5e3861848de79627b2bedc9f3342c7da5173a14eaa5d0f8dc18ae5d"
|
||||
dependencies = [
|
||||
"clap",
|
||||
]
|
||||
|
|
@ -338,9 +338,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "convert_case"
|
||||
version = "0.8.0"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baaaa0ecca5b51987b9423ccdc971514dd8b0bb7b4060b983d3664dad3f1f89f"
|
||||
checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9"
|
||||
dependencies = [
|
||||
"unicode-segmentation",
|
||||
]
|
||||
|
|
@ -664,9 +664,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "find-msvc-tools"
|
||||
version = "0.1.5"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844"
|
||||
checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db"
|
||||
|
||||
[[package]]
|
||||
name = "fnv"
|
||||
|
|
@ -980,9 +980,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.15"
|
||||
version = "1.0.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
|
||||
checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
|
||||
|
||||
[[package]]
|
||||
name = "jni"
|
||||
|
|
@ -1119,7 +1119,7 @@ version = "0.6.5"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ad38eb12aea514a0466ea40a80fd8cc83637065948eb4a426e4aa46261175227"
|
||||
dependencies = [
|
||||
"rustix 1.1.2",
|
||||
"rustix 1.1.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1371,9 +1371,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.103"
|
||||
version = "1.0.104"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8"
|
||||
checksum = "9695f8df41bb4f3d222c95a67532365f569318332d03d5f3f67f37b20e6ebdf0"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
|
@ -1527,9 +1527,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "rquickjs"
|
||||
version = "0.10.0"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a135375fbac5ba723bb6a48f432a72f81539cedde422f0121a86c7c4e96d8e0d"
|
||||
checksum = "c50dc6d6c587c339edb4769cf705867497a2baf0eca8b4645fa6ecd22f02c77a"
|
||||
dependencies = [
|
||||
"rquickjs-core",
|
||||
"rquickjs-macro",
|
||||
|
|
@ -1537,9 +1537,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "rquickjs-core"
|
||||
version = "0.10.0"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bccb7121a123865c8ace4dea42e7ed84d78b90cbaf4ca32c59849d8d210c9672"
|
||||
checksum = "b8bf7840285c321c3ab20e752a9afb95548c75cd7f4632a0627cea3507e310c1"
|
||||
dependencies = [
|
||||
"hashbrown 0.16.1",
|
||||
"phf",
|
||||
|
|
@ -1549,9 +1549,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "rquickjs-macro"
|
||||
version = "0.10.0"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "89f93602cc3112c7f30bf5f29e722784232138692c7df4c52ebbac7e035d900d"
|
||||
checksum = "7106215ff41a5677b104906a13e1a440b880f4b6362b5dc4f3978c267fad2b80"
|
||||
dependencies = [
|
||||
"convert_case",
|
||||
"fnv",
|
||||
|
|
@ -1568,9 +1568,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "rquickjs-sys"
|
||||
version = "0.10.0"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57b1b6528590d4d65dc86b5159eae2d0219709546644c66408b2441696d1d725"
|
||||
checksum = "27344601ef27460e82d6a4e1ecb9e7e99f518122095f3c51296da8e9be2b9d83"
|
||||
dependencies = [
|
||||
"bindgen",
|
||||
"cc",
|
||||
|
|
@ -1597,9 +1597,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "1.1.2"
|
||||
version = "1.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e"
|
||||
checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34"
|
||||
dependencies = [
|
||||
"bitflags 2.10.0",
|
||||
"errno",
|
||||
|
|
@ -1614,12 +1614,6 @@ version = "1.0.22"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
|
||||
|
||||
[[package]]
|
||||
name = "ryu"
|
||||
version = "1.0.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
|
||||
|
||||
[[package]]
|
||||
name = "same-file"
|
||||
version = "1.0.6"
|
||||
|
|
@ -1631,9 +1625,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "schemars"
|
||||
version = "1.1.0"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9558e172d4e8533736ba97870c4b2cd63f84b382a3d6eb063da41b91cce17289"
|
||||
checksum = "54e910108742c57a770f492731f99be216a52fadd361b06c8fb59d74ccc267d2"
|
||||
dependencies = [
|
||||
"dyn-clone",
|
||||
"ref-cast",
|
||||
|
|
@ -1644,9 +1638,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "schemars_derive"
|
||||
version = "1.1.0"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "301858a4023d78debd2353c7426dc486001bddc91ae31a76fb1f55132f7e2633"
|
||||
checksum = "4908ad288c5035a8eb12cfdf0d49270def0a268ee162b75eeee0f85d155a7c45"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -1707,16 +1701,16 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.145"
|
||||
version = "1.0.149"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c"
|
||||
checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
|
||||
dependencies = [
|
||||
"indexmap",
|
||||
"itoa",
|
||||
"memchr",
|
||||
"ryu",
|
||||
"serde",
|
||||
"serde_core",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1784,9 +1778,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
|||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.111"
|
||||
version = "2.0.112"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87"
|
||||
checksum = "21f182278bf2d2bcb3c88b1b08a37df029d71ce3d3ae26168e3c653b213b99d4"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -1806,20 +1800,20 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "target-lexicon"
|
||||
version = "0.13.3"
|
||||
version = "0.13.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df7f62577c25e07834649fc3b39fafdc597c0a3527dc1c60129201ccfcbaa50c"
|
||||
checksum = "b1dd07eb858a2067e2f3c7155d54e929265c264e6f37efe3ee7a8d1b5a1dd0ba"
|
||||
|
||||
[[package]]
|
||||
name = "tempfile"
|
||||
version = "3.23.0"
|
||||
version = "3.24.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16"
|
||||
checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.3.4",
|
||||
"once_cell",
|
||||
"rustix 1.1.2",
|
||||
"rustix 1.1.3",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
|
|
@ -1905,18 +1899,18 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "toml_datetime"
|
||||
version = "0.7.3"
|
||||
version = "0.7.5+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2cdb639ebbc97961c51720f858597f7f24c4fc295327923af55b74c3c724533"
|
||||
checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml_edit"
|
||||
version = "0.23.9"
|
||||
version = "0.23.10+spec-1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5d7cbc3b4b49633d57a0509303158ca50de80ae32c265093b24c414705807832"
|
||||
checksum = "84c8b9f757e028cee9fa244aea147aab2a9ec09d5325a9b01e0a49730c2b5269"
|
||||
dependencies = [
|
||||
"indexmap",
|
||||
"toml_datetime",
|
||||
|
|
@ -1926,9 +1920,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "toml_parser"
|
||||
version = "1.0.4"
|
||||
version = "1.0.6+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c0cbe268d35bdb4bb5a56a2de88d0ad0eb70af5384a99d648cd4b3d04039800e"
|
||||
checksum = "a3198b4b0a8e11f09dd03e133c0280504d0801269e9afa46362ffde1cbeebf44"
|
||||
dependencies = [
|
||||
"winnow",
|
||||
]
|
||||
|
|
@ -1941,9 +1935,9 @@ checksum = "ea68304e134ecd095ac6c3574494fc62b909f416c4fca77e440530221e549d3d"
|
|||
|
||||
[[package]]
|
||||
name = "tracing"
|
||||
version = "0.1.43"
|
||||
version = "0.1.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647"
|
||||
checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"
|
||||
dependencies = [
|
||||
"pin-project-lite",
|
||||
"tracing-attributes",
|
||||
|
|
@ -1963,16 +1957,16 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "tracing-core"
|
||||
version = "0.1.35"
|
||||
version = "0.1.36"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7a04e24fab5c89c6a36eb8558c9656f30d81de51dfa4d3b45f26b21d61fa0a6c"
|
||||
checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter"
|
||||
version = "0.26.3"
|
||||
version = "0.27.0"
|
||||
dependencies = [
|
||||
"bindgen",
|
||||
"cc",
|
||||
|
|
@ -1986,7 +1980,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "tree-sitter-cli"
|
||||
version = "0.26.3"
|
||||
version = "0.27.0"
|
||||
dependencies = [
|
||||
"ansi_colours",
|
||||
"anstyle",
|
||||
|
|
@ -2034,7 +2028,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "tree-sitter-config"
|
||||
version = "0.26.3"
|
||||
version = "0.27.0"
|
||||
dependencies = [
|
||||
"etcetera",
|
||||
"log",
|
||||
|
|
@ -2045,7 +2039,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "tree-sitter-generate"
|
||||
version = "0.26.3"
|
||||
version = "0.27.0"
|
||||
dependencies = [
|
||||
"bitflags 2.10.0",
|
||||
"dunce",
|
||||
|
|
@ -2068,7 +2062,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "tree-sitter-highlight"
|
||||
version = "0.26.3"
|
||||
version = "0.27.0"
|
||||
dependencies = [
|
||||
"regex",
|
||||
"streaming-iterator",
|
||||
|
|
@ -2078,11 +2072,11 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "tree-sitter-language"
|
||||
version = "0.1.6"
|
||||
version = "0.1.7"
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-loader"
|
||||
version = "0.26.3"
|
||||
version = "0.27.0"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"etcetera",
|
||||
|
|
@ -2104,7 +2098,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "tree-sitter-tags"
|
||||
version = "0.26.3"
|
||||
version = "0.27.0"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
"regex",
|
||||
|
|
@ -2316,7 +2310,7 @@ dependencies = [
|
|||
"postcard",
|
||||
"psm",
|
||||
"pulley-interpreter",
|
||||
"rustix 1.1.2",
|
||||
"rustix 1.1.3",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"smallvec",
|
||||
|
|
@ -2425,7 +2419,7 @@ dependencies = [
|
|||
"anyhow",
|
||||
"cc",
|
||||
"cfg-if",
|
||||
"rustix 1.1.2",
|
||||
"rustix 1.1.3",
|
||||
"wasmtime-asm-macros",
|
||||
"wasmtime-versioned-export-macros",
|
||||
"windows-sys 0.59.0",
|
||||
|
|
@ -2932,3 +2926,9 @@ dependencies = [
|
|||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e9747e91771f56fd7893e1164abd78febd14a670ceec257caad15e051de35f06"
|
||||
|
|
|
|||
24
Cargo.toml
24
Cargo.toml
|
|
@ -14,13 +14,13 @@ members = [
|
|||
resolver = "2"
|
||||
|
||||
[workspace.package]
|
||||
version = "0.26.3"
|
||||
version = "0.27.0"
|
||||
authors = [
|
||||
"Max Brunsfeld <maxbrunsfeld@gmail.com>",
|
||||
"Amaan Qureshi <amaanq12@gmail.com>",
|
||||
]
|
||||
edition = "2021"
|
||||
rust-version = "1.84"
|
||||
rust-version = "1.85"
|
||||
homepage = "https://tree-sitter.github.io/tree-sitter"
|
||||
repository = "https://github.com/tree-sitter/tree-sitter"
|
||||
license = "MIT"
|
||||
|
|
@ -106,8 +106,8 @@ ansi_colours = "1.2.3"
|
|||
anstyle = "1.0.13"
|
||||
anyhow = "1.0.100"
|
||||
bstr = "1.12.0"
|
||||
cc = "1.2.48"
|
||||
clap = { version = "4.5.53", features = [
|
||||
cc = "1.2.53"
|
||||
clap = { version = "4.5.54", features = [
|
||||
"cargo",
|
||||
"derive",
|
||||
"env",
|
||||
|
|
@ -115,7 +115,7 @@ clap = { version = "4.5.53", features = [
|
|||
"string",
|
||||
"unstable-styles",
|
||||
] }
|
||||
clap_complete = "4.5.61"
|
||||
clap_complete = "4.5.65"
|
||||
clap_complete_nushell = "4.5.10"
|
||||
crc32fast = "1.5.0"
|
||||
ctor = "0.2.9"
|
||||
|
|
@ -140,7 +140,7 @@ rustc-hash = "2.1.1"
|
|||
schemars = "1.0.5"
|
||||
semver = { version = "1.0.27", features = ["serde"] }
|
||||
serde = { version = "1.0.219", features = ["derive"] }
|
||||
serde_json = { version = "1.0.145", features = ["preserve_order"] }
|
||||
serde_json = { version = "1.0.149", features = ["preserve_order"] }
|
||||
similar = "2.7.0"
|
||||
smallbitvec = "2.6.0"
|
||||
streaming-iterator = "0.1.9"
|
||||
|
|
@ -153,11 +153,11 @@ walkdir = "2.5.0"
|
|||
wasmparser = "0.243.0"
|
||||
webbrowser = "1.0.5"
|
||||
|
||||
tree-sitter = { version = "0.26.3", path = "./lib" }
|
||||
tree-sitter-generate = { version = "0.26.3", path = "./crates/generate" }
|
||||
tree-sitter-loader = { version = "0.26.3", path = "./crates/loader" }
|
||||
tree-sitter-config = { version = "0.26.3", path = "./crates/config" }
|
||||
tree-sitter-highlight = { version = "0.26.3", path = "./crates/highlight" }
|
||||
tree-sitter-tags = { version = "0.26.3", path = "./crates/tags" }
|
||||
tree-sitter = { version = "0.27.0", path = "./lib" }
|
||||
tree-sitter-generate = { version = "0.27.0", path = "./crates/generate" }
|
||||
tree-sitter-loader = { version = "0.27.0", path = "./crates/loader" }
|
||||
tree-sitter-config = { version = "0.27.0", path = "./crates/config" }
|
||||
tree-sitter-highlight = { version = "0.27.0", path = "./crates/highlight" }
|
||||
tree-sitter-tags = { version = "0.27.0", path = "./crates/tags" }
|
||||
|
||||
tree-sitter-language = { version = "0.1", path = "./crates/language" }
|
||||
|
|
|
|||
4
Makefile
4
Makefile
|
|
@ -1,4 +1,4 @@
|
|||
VERSION := 0.26.3
|
||||
VERSION := 0.27.0
|
||||
DESCRIPTION := An incremental parsing system for programming tools
|
||||
HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/
|
||||
|
||||
|
|
@ -24,7 +24,7 @@ OBJ := $(SRC:.c=.o)
|
|||
ARFLAGS := rcs
|
||||
CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types
|
||||
override CFLAGS += -std=c11 -fPIC -fvisibility=hidden
|
||||
override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE -D_DARWIN_C_SOURCE
|
||||
override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE -D_BSD_SOURCE -D_DARWIN_C_SOURCE
|
||||
override CFLAGS += -Ilib/src -Ilib/src/wasm -Ilib/include
|
||||
|
||||
# ABI versioning
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ let package = Package(
|
|||
.headerSearchPath("src"),
|
||||
.define("_POSIX_C_SOURCE", to: "200112L"),
|
||||
.define("_DEFAULT_SOURCE"),
|
||||
.define("_BSD_SOURCE"),
|
||||
.define("_DARWIN_C_SOURCE"),
|
||||
]),
|
||||
],
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ pub fn build(b: *std.Build) !void {
|
|||
|
||||
lib.root_module.addCMacro("_POSIX_C_SOURCE", "200112L");
|
||||
lib.root_module.addCMacro("_DEFAULT_SOURCE", "");
|
||||
lib.root_module.addCMacro("_BSD_SOURCE", "");
|
||||
lib.root_module.addCMacro("_DARWIN_C_SOURCE", "");
|
||||
|
||||
if (wasm) {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
.{
|
||||
.name = .tree_sitter,
|
||||
.fingerprint = 0x841224b447ac0d4f,
|
||||
.version = "0.26.3",
|
||||
.version = "0.27.0",
|
||||
.minimum_zig_version = "0.14.1",
|
||||
.paths = .{
|
||||
"build.zig",
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@
|
|||
[npmjs.com]: https://www.npmjs.org/package/tree-sitter-cli
|
||||
[npmjs.com badge]: https://img.shields.io/npm/v/tree-sitter-cli.svg?color=%23BF4A4A
|
||||
|
||||
The Tree-sitter CLI allows you to develop, test, and use Tree-sitter grammars from the command line. It works on `MacOS`, `Linux`, and `Windows`.
|
||||
The Tree-sitter CLI allows you to develop, test, and use Tree-sitter grammars from the command line. It works on `MacOS`,
|
||||
`Linux`, and `Windows`.
|
||||
|
||||
### Installation
|
||||
|
||||
|
|
@ -34,9 +35,11 @@ The `tree-sitter` binary itself has no dependencies, but specific commands have
|
|||
|
||||
### Commands
|
||||
|
||||
* `generate` - The `tree-sitter generate` command will generate a Tree-sitter parser based on the grammar in the current working directory. See [the documentation] for more information.
|
||||
* `generate` - The `tree-sitter generate` command will generate a Tree-sitter parser based on the grammar in the current
|
||||
working directory. See [the documentation] for more information.
|
||||
|
||||
* `test` - The `tree-sitter test` command will run the unit tests for the Tree-sitter parser in the current working directory. See [the documentation] for more information.
|
||||
* `test` - The `tree-sitter test` command will run the unit tests for the Tree-sitter parser in the current working directory.
|
||||
See [the documentation] for more information.
|
||||
|
||||
* `parse` - The `tree-sitter parse` command will parse a file (or list of files) using Tree-sitter parsers.
|
||||
|
||||
|
|
|
|||
4
crates/cli/npm/package-lock.json
generated
4
crates/cli/npm/package-lock.json
generated
|
|
@ -1,12 +1,12 @@
|
|||
{
|
||||
"name": "tree-sitter-cli",
|
||||
"version": "0.26.3",
|
||||
"version": "0.27.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "tree-sitter-cli",
|
||||
"version": "0.26.3",
|
||||
"version": "0.27.0",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "tree-sitter-cli",
|
||||
"version": "0.26.3",
|
||||
"version": "0.27.0",
|
||||
"author": {
|
||||
"name": "Max Brunsfeld",
|
||||
"email": "maxbrunsfeld@gmail.com"
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ use anyhow::{anyhow, Context, Result};
|
|||
use crc32fast::hash as crc32;
|
||||
use heck::{ToKebabCase, ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase};
|
||||
use indoc::{formatdoc, indoc};
|
||||
use log::warn;
|
||||
use log::info;
|
||||
use rand::{thread_rng, Rng};
|
||||
use semver::Version;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
|
@ -123,7 +123,7 @@ const BUILD_ZIG_ZON_TEMPLATE: &str = include_str!("./templates/build.zig.zon");
|
|||
const ROOT_ZIG_TEMPLATE: &str = include_str!("./templates/root.zig");
|
||||
const TEST_ZIG_TEMPLATE: &str = include_str!("./templates/test.zig");
|
||||
|
||||
const TREE_SITTER_JSON_SCHEMA: &str =
|
||||
pub const TREE_SITTER_JSON_SCHEMA: &str =
|
||||
"https://tree-sitter.github.io/tree-sitter/assets/schemas/config.schema.json";
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone)]
|
||||
|
|
@ -356,7 +356,7 @@ pub fn generate_grammar_files(
|
|||
"tree-sitter-cli":"#},
|
||||
);
|
||||
if !contents.contains("module") {
|
||||
warn!("Updating package.json");
|
||||
info!("Migrating package.json to ESM");
|
||||
contents = contents.replace(
|
||||
r#""repository":"#,
|
||||
indoc! {r#"
|
||||
|
|
@ -378,6 +378,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if contents.contains("module.exports") {
|
||||
info!("Migrating grammars.js to ESM");
|
||||
contents = contents.replace("module.exports =", "export default");
|
||||
write_file(path, contents)?;
|
||||
}
|
||||
|
|
@ -393,10 +394,16 @@ pub fn generate_grammar_files(
|
|||
allow_update,
|
||||
|path| generate_file(path, GITIGNORE_TEMPLATE, language_name, &generate_opts),
|
||||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("Zig artifacts") {
|
||||
warn!("Replacing .gitignore");
|
||||
generate_file(path, GITIGNORE_TEMPLATE, language_name, &generate_opts)?;
|
||||
info!("Adding zig entries to .gitignore");
|
||||
contents.push('\n');
|
||||
contents.push_str(indoc! {"
|
||||
# Zig artifacts
|
||||
.zig-cache/
|
||||
zig-cache/
|
||||
zig-out/
|
||||
"});
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
|
|
@ -409,8 +416,13 @@ pub fn generate_grammar_files(
|
|||
|path| generate_file(path, GITATTRIBUTES_TEMPLATE, language_name, &generate_opts),
|
||||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
contents = contents.replace("bindings/c/* ", "bindings/c/** ");
|
||||
let c_bindings_entry = "bindings/c/* ";
|
||||
if contents.contains(c_bindings_entry) {
|
||||
info!("Updating c bindings entry in .gitattributes");
|
||||
contents = contents.replace(c_bindings_entry, "bindings/c/** ");
|
||||
}
|
||||
if !contents.contains("Zig bindings") {
|
||||
info!("Adding zig entries to .gitattributes");
|
||||
contents.push('\n');
|
||||
contents.push_str(indoc! {"
|
||||
# Zig bindings
|
||||
|
|
@ -438,39 +450,40 @@ pub fn generate_grammar_files(
|
|||
}, |path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("#[cfg(with_highlights_query)]") {
|
||||
let replacement = indoc! {r#"
|
||||
#[cfg(with_highlights_query)]
|
||||
/// The syntax highlighting query for this grammar.
|
||||
pub const HIGHLIGHTS_QUERY: &str = include_str!("../../HIGHLIGHTS_QUERY_PATH");
|
||||
info!("Updating query constants in bindings/rust/lib.rs");
|
||||
let replacement = indoc! {r#"
|
||||
#[cfg(with_highlights_query)]
|
||||
/// The syntax highlighting query for this grammar.
|
||||
pub const HIGHLIGHTS_QUERY: &str = include_str!("../../HIGHLIGHTS_QUERY_PATH");
|
||||
|
||||
#[cfg(with_injections_query)]
|
||||
/// The language injection query for this grammar.
|
||||
pub const INJECTIONS_QUERY: &str = include_str!("../../INJECTIONS_QUERY_PATH");
|
||||
#[cfg(with_injections_query)]
|
||||
/// The language injection query for this grammar.
|
||||
pub const INJECTIONS_QUERY: &str = include_str!("../../INJECTIONS_QUERY_PATH");
|
||||
|
||||
#[cfg(with_locals_query)]
|
||||
/// The local variable query for this grammar.
|
||||
pub const LOCALS_QUERY: &str = include_str!("../../LOCALS_QUERY_PATH");
|
||||
#[cfg(with_locals_query)]
|
||||
/// The local variable query for this grammar.
|
||||
pub const LOCALS_QUERY: &str = include_str!("../../LOCALS_QUERY_PATH");
|
||||
|
||||
#[cfg(with_tags_query)]
|
||||
/// The symbol tagging query for this grammar.
|
||||
pub const TAGS_QUERY: &str = include_str!("../../TAGS_QUERY_PATH");
|
||||
"#}
|
||||
.replace("HIGHLIGHTS_QUERY_PATH", generate_opts.highlights_query_path)
|
||||
.replace("INJECTIONS_QUERY_PATH", generate_opts.injections_query_path)
|
||||
.replace("LOCALS_QUERY_PATH", generate_opts.locals_query_path)
|
||||
.replace("TAGS_QUERY_PATH", generate_opts.tags_query_path);
|
||||
contents = contents
|
||||
.replace(
|
||||
indoc! {r#"
|
||||
// NOTE: uncomment these to include any queries that this grammar contains:
|
||||
#[cfg(with_tags_query)]
|
||||
/// The symbol tagging query for this grammar.
|
||||
pub const TAGS_QUERY: &str = include_str!("../../TAGS_QUERY_PATH");
|
||||
"#}
|
||||
.replace("HIGHLIGHTS_QUERY_PATH", generate_opts.highlights_query_path)
|
||||
.replace("INJECTIONS_QUERY_PATH", generate_opts.injections_query_path)
|
||||
.replace("LOCALS_QUERY_PATH", generate_opts.locals_query_path)
|
||||
.replace("TAGS_QUERY_PATH", generate_opts.tags_query_path);
|
||||
contents = contents
|
||||
.replace(
|
||||
indoc! {r#"
|
||||
// NOTE: uncomment these to include any queries that this grammar contains:
|
||||
|
||||
// pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
|
||||
// pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm");
|
||||
// pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm");
|
||||
// pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm");
|
||||
"#},
|
||||
&replacement,
|
||||
);
|
||||
// pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
|
||||
// pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm");
|
||||
// pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm");
|
||||
// pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm");
|
||||
"#},
|
||||
&replacement,
|
||||
);
|
||||
}
|
||||
write_file(path, contents)?;
|
||||
Ok(())
|
||||
|
|
@ -483,6 +496,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("wasm32-unknown-unknown") {
|
||||
info!("Adding wasm32-unknown-unknown target to bindings/rust/build.rs");
|
||||
let replacement = indoc!{r#"
|
||||
c_config.flag("-utf-8");
|
||||
|
||||
|
|
@ -503,19 +517,18 @@ pub fn generate_grammar_files(
|
|||
wasm_src.join("string.c"),
|
||||
]);
|
||||
}
|
||||
"#};
|
||||
|
||||
let indented_replacement = replacement
|
||||
"#}
|
||||
.lines()
|
||||
.map(|line| if line.is_empty() { line.to_string() } else { format!(" {line}") })
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
|
||||
contents = contents.replace(r#" c_config.flag("-utf-8");"#, &indented_replacement);
|
||||
contents = contents.replace(r#" c_config.flag("-utf-8");"#, &replacement);
|
||||
}
|
||||
|
||||
// Introduce configuration variables for dynamic query inclusion
|
||||
if !contents.contains("with_highlights_query") {
|
||||
info!("Adding support for dynamic query inclusion to bindings/rust/build.rs");
|
||||
let replaced = indoc! {r#"
|
||||
c_config.compile("tree-sitter-KEBAB_PARSER_NAME");
|
||||
}"#}
|
||||
|
|
@ -572,6 +585,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
if contents.contains("\"LICENSE\"") {
|
||||
info!("Adding LICENSE entry to bindings/rust/Cargo.toml");
|
||||
write_file(path, contents.replace("\"LICENSE\"", "\"/LICENSE\""))?;
|
||||
}
|
||||
Ok(())
|
||||
|
|
@ -592,7 +606,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("Object.defineProperty") {
|
||||
warn!("Replacing index.js");
|
||||
info!("Replacing index.js");
|
||||
generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts)?;
|
||||
}
|
||||
Ok(())
|
||||
|
|
@ -606,7 +620,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("export default binding") {
|
||||
warn!("Replacing index.d.ts");
|
||||
info!("Replacing index.d.ts");
|
||||
generate_file(path, INDEX_D_TS_TEMPLATE, language_name, &generate_opts)?;
|
||||
}
|
||||
Ok(())
|
||||
|
|
@ -627,7 +641,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("import") {
|
||||
warn!("Replacing binding_test.js");
|
||||
info!("Replacing binding_test.js");
|
||||
generate_file(
|
||||
path,
|
||||
BINDING_TEST_JS_TEMPLATE,
|
||||
|
|
@ -650,6 +664,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
if contents.contains("fs.exists(") {
|
||||
info!("Replacing `fs.exists` calls in binding.gyp");
|
||||
write_file(path, contents.replace("fs.exists(", "fs.existsSync("))?;
|
||||
}
|
||||
Ok(())
|
||||
|
|
@ -662,14 +677,17 @@ pub fn generate_grammar_files(
|
|||
|
||||
// Generate C bindings
|
||||
if tree_sitter_config.bindings.c {
|
||||
let kebab_case_name = language_name.to_kebab_case();
|
||||
missing_path(bindings_dir.join("c"), create_dir)?.apply(|path| {
|
||||
let old_file = &path.join(format!("tree-sitter-{}.h", language_name.to_kebab_case()));
|
||||
let header_name = format!("tree-sitter-{kebab_case_name}.h");
|
||||
let old_file = &path.join(&header_name);
|
||||
if allow_update && fs::exists(old_file).unwrap_or(false) {
|
||||
info!("Removing bindings/c/{header_name}");
|
||||
fs::remove_file(old_file)?;
|
||||
}
|
||||
missing_path(path.join("tree_sitter"), create_dir)?.apply(|include_path| {
|
||||
missing_path(
|
||||
include_path.join(format!("tree-sitter-{}.h", language_name.to_kebab_case())),
|
||||
include_path.join(&header_name),
|
||||
|path| {
|
||||
generate_file(path, PARSER_NAME_H_TEMPLATE, language_name, &generate_opts)
|
||||
},
|
||||
|
|
@ -678,7 +696,7 @@ pub fn generate_grammar_files(
|
|||
})?;
|
||||
|
||||
missing_path(
|
||||
path.join(format!("tree-sitter-{}.pc.in", language_name.to_kebab_case())),
|
||||
path.join(format!("tree-sitter-{kebab_case_name}.pc.in")),
|
||||
|path| {
|
||||
generate_file(
|
||||
path,
|
||||
|
|
@ -698,23 +716,27 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("cd '$(DESTDIR)$(LIBDIR)' && ln -sf") {
|
||||
warn!("Replacing Makefile");
|
||||
info!("Replacing Makefile");
|
||||
generate_file(path, MAKEFILE_TEMPLATE, language_name, &generate_opts)?;
|
||||
} else {
|
||||
contents = contents
|
||||
.replace(
|
||||
indoc! {r"
|
||||
$(PARSER): $(SRC_DIR)/grammar.json
|
||||
$(TS) generate $^
|
||||
"},
|
||||
indoc! {r"
|
||||
$(SRC_DIR)/grammar.json: grammar.js
|
||||
$(TS) generate --no-parser $^
|
||||
let replaced = indoc! {r"
|
||||
$(PARSER): $(SRC_DIR)/grammar.json
|
||||
$(TS) generate $^
|
||||
"};
|
||||
if contents.contains(replaced) {
|
||||
info!("Adding --no-parser target to Makefile");
|
||||
contents = contents
|
||||
.replace(
|
||||
replaced,
|
||||
indoc! {r"
|
||||
$(SRC_DIR)/grammar.json: grammar.js
|
||||
$(TS) generate --no-parser $^
|
||||
|
||||
$(PARSER): $(SRC_DIR)/grammar.json
|
||||
$(TS) generate $^
|
||||
"}
|
||||
);
|
||||
$(PARSER): $(SRC_DIR)/grammar.json
|
||||
$(TS) generate $^
|
||||
"}
|
||||
);
|
||||
}
|
||||
write_file(path, contents)?;
|
||||
}
|
||||
Ok(())
|
||||
|
|
@ -726,8 +748,8 @@ pub fn generate_grammar_files(
|
|||
allow_update,
|
||||
|path| generate_file(path, CMAKELISTS_TXT_TEMPLATE, language_name, &generate_opts),
|
||||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
contents = contents
|
||||
let contents = fs::read_to_string(path)?;
|
||||
let replaced_contents = contents
|
||||
.replace("add_custom_target(test", "add_custom_target(ts-test")
|
||||
.replace(
|
||||
&formatdoc! {r#"
|
||||
|
|
@ -775,7 +797,10 @@ pub fn generate_grammar_files(
|
|||
COMMENT "Generating parser.c")
|
||||
"#}
|
||||
);
|
||||
write_file(path, contents)?;
|
||||
if !replaced_contents.eq(&contents) {
|
||||
info!("Updating CMakeLists.txt");
|
||||
write_file(path, replaced_contents)?;
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
)?;
|
||||
|
|
@ -811,7 +836,8 @@ pub fn generate_grammar_files(
|
|||
// Generate Python bindings
|
||||
if tree_sitter_config.bindings.python {
|
||||
missing_path(bindings_dir.join("python"), create_dir)?.apply(|path| {
|
||||
let lang_path = path.join(format!("tree_sitter_{}", language_name.to_snake_case()));
|
||||
let snake_case_grammar_name = format!("tree_sitter_{}", language_name.to_snake_case());
|
||||
let lang_path = path.join(&snake_case_grammar_name);
|
||||
missing_path(&lang_path, create_dir)?;
|
||||
|
||||
missing_path_else(
|
||||
|
|
@ -821,6 +847,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("PyModuleDef_Init") {
|
||||
info!("Updating bindings/python/{snake_case_grammar_name}/binding.c");
|
||||
contents = contents
|
||||
.replace("PyModule_Create", "PyModuleDef_Init")
|
||||
.replace(
|
||||
|
|
@ -862,7 +889,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("uncomment these to include any queries") {
|
||||
warn!("Replacing __init__.py");
|
||||
info!("Replacing __init__.py");
|
||||
generate_file(path, INIT_PY_TEMPLATE, language_name, &generate_opts)?;
|
||||
}
|
||||
Ok(())
|
||||
|
|
@ -876,9 +903,10 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if contents.contains("uncomment these to include any queries") {
|
||||
warn!("Replacing __init__.pyi");
|
||||
info!("Replacing __init__.pyi");
|
||||
generate_file(path, INIT_PYI_TEMPLATE, language_name, &generate_opts)?;
|
||||
} else if !contents.contains("CapsuleType") {
|
||||
info!("Updating __init__.pyi");
|
||||
contents = contents
|
||||
.replace(
|
||||
"from typing import Final",
|
||||
|
|
@ -910,6 +938,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("Parser(Language(") {
|
||||
info!("Updating Language function in bindings/python/tests/test_binding.py");
|
||||
contents = contents
|
||||
.replace("tree_sitter.Language(", "Parser(Language(")
|
||||
.replace(".language())\n", ".language()))\n")
|
||||
|
|
@ -930,11 +959,19 @@ pub fn generate_grammar_files(
|
|||
allow_update,
|
||||
|path| generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts),
|
||||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("build_ext") {
|
||||
warn!("Replacing setup.py");
|
||||
info!("Replacing setup.py");
|
||||
generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts)?;
|
||||
}
|
||||
if !contents.contains(" and not get_config_var") {
|
||||
info!("Updating Python free-threading support in setup.py");
|
||||
contents = contents.replace(
|
||||
r#"startswith("cp"):"#,
|
||||
r#"startswith("cp") and not get_config_var("Py_GIL_DISABLED"):"#
|
||||
);
|
||||
write_file(path, contents)?;
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
)?;
|
||||
|
|
@ -953,6 +990,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("cp310-*") {
|
||||
info!("Updating dependencies in pyproject.toml");
|
||||
contents = contents
|
||||
.replace(r#"build = "cp39-*""#, r#"build = "cp310-*""#)
|
||||
.replace(r#"python = ">=3.9""#, r#"python = ">=3.10""#)
|
||||
|
|
@ -990,15 +1028,18 @@ pub fn generate_grammar_files(
|
|||
allow_update,
|
||||
|path| generate_file(path, PACKAGE_SWIFT_TEMPLATE, language_name, &generate_opts),
|
||||
|path| {
|
||||
let mut contents = fs::read_to_string(path)?;
|
||||
contents = contents
|
||||
let contents = fs::read_to_string(path)?;
|
||||
let replaced_contents = contents
|
||||
.replace(
|
||||
"https://github.com/ChimeHQ/SwiftTreeSitter",
|
||||
"https://github.com/tree-sitter/swift-tree-sitter",
|
||||
)
|
||||
.replace("version: \"0.8.0\")", "version: \"0.9.0\")")
|
||||
.replace("(url:", "(name: \"SwiftTreeSitter\", url:");
|
||||
write_file(path, contents)?;
|
||||
if !replaced_contents.eq(&contents) {
|
||||
info!("Updating tree-sitter dependency in Package.swift");
|
||||
write_file(path, contents)?;
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
)?;
|
||||
|
|
@ -1016,7 +1057,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
if !contents.contains("b.pkg_hash.len") {
|
||||
warn!("Replacing build.zig");
|
||||
info!("Replacing build.zig");
|
||||
generate_file(path, BUILD_ZIG_TEMPLATE, language_name, &generate_opts)
|
||||
} else {
|
||||
Ok(())
|
||||
|
|
@ -1031,7 +1072,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
if !contents.contains(".name = .tree_sitter_") {
|
||||
warn!("Replacing build.zig.zon");
|
||||
info!("Replacing build.zig.zon");
|
||||
generate_file(path, BUILD_ZIG_ZON_TEMPLATE, language_name, &generate_opts)
|
||||
} else {
|
||||
Ok(())
|
||||
|
|
@ -1047,7 +1088,7 @@ pub fn generate_grammar_files(
|
|||
|path| {
|
||||
let contents = fs::read_to_string(path)?;
|
||||
if contents.contains("ts.Language") {
|
||||
warn!("Replacing root.zig");
|
||||
info!("Replacing root.zig");
|
||||
generate_file(path, ROOT_ZIG_TEMPLATE, language_name, &generate_opts)
|
||||
} else {
|
||||
Ok(())
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ use tree_sitter_cli::{
|
|||
LOG_GRAPH_ENABLED, START_SEED,
|
||||
},
|
||||
highlight::{self, HighlightOptions},
|
||||
init::{generate_grammar_files, JsonConfigOpts},
|
||||
init::{generate_grammar_files, JsonConfigOpts, TREE_SITTER_JSON_SCHEMA},
|
||||
input::{get_input, get_tmp_source_file, CliInput},
|
||||
logger,
|
||||
parse::{self, ParseDebugType, ParseFileOptions, ParseOutput, ParseTheme},
|
||||
|
|
@ -867,10 +867,26 @@ impl Init {
|
|||
|
||||
(opts.name.clone(), Some(opts))
|
||||
} else {
|
||||
let mut json = serde_json::from_str::<TreeSitterJSON>(
|
||||
&fs::read_to_string(current_dir.join("tree-sitter.json"))
|
||||
.with_context(|| "Failed to read tree-sitter.json")?,
|
||||
)?;
|
||||
let old_config = fs::read_to_string(current_dir.join("tree-sitter.json"))
|
||||
.with_context(|| "Failed to read tree-sitter.json")?;
|
||||
|
||||
let mut json = serde_json::from_str::<TreeSitterJSON>(&old_config)?;
|
||||
if json.schema.is_none() {
|
||||
json.schema = Some(TREE_SITTER_JSON_SCHEMA.to_string());
|
||||
}
|
||||
|
||||
let new_config = format!("{}\n", serde_json::to_string_pretty(&json)?);
|
||||
// Write the re-serialized config back, as newly added optional boolean fields
|
||||
// will be included with explicit `false`s rather than implict `null`s
|
||||
if self.update && !old_config.trim().eq(new_config.trim()) {
|
||||
info!("Updating tree-sitter.json");
|
||||
fs::write(
|
||||
current_dir.join("tree-sitter.json"),
|
||||
serde_json::to_string_pretty(&json)?,
|
||||
)
|
||||
.with_context(|| "Failed to write tree-sitter.json")?;
|
||||
}
|
||||
|
||||
(json.grammars.swap_remove(0).name, None)
|
||||
};
|
||||
|
||||
|
|
@ -955,11 +971,21 @@ impl Build {
|
|||
} else {
|
||||
let output_path = if let Some(ref path) = self.output {
|
||||
let path = Path::new(path);
|
||||
if path.is_absolute() {
|
||||
let full_path = if path.is_absolute() {
|
||||
path.to_path_buf()
|
||||
} else {
|
||||
current_dir.join(path)
|
||||
}
|
||||
};
|
||||
let parent_path = full_path
|
||||
.parent()
|
||||
.context("Output path must have a parent")?;
|
||||
let name = full_path
|
||||
.file_name()
|
||||
.context("Ouput path must have a filename")?;
|
||||
fs::create_dir_all(parent_path).context("Failed to create output path")?;
|
||||
let mut canon_path = parent_path.canonicalize().context("Invalid output path")?;
|
||||
canon_path.push(name);
|
||||
canon_path
|
||||
} else {
|
||||
let file_name = grammar_path
|
||||
.file_stem()
|
||||
|
|
@ -984,7 +1010,7 @@ impl Build {
|
|||
|
||||
loader
|
||||
.compile_parser_at_path(&grammar_path, output_path, flags)
|
||||
.unwrap();
|
||||
.context("Failed to compile parser")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -1622,6 +1648,7 @@ impl Highlight {
|
|||
let loader_config = config.get()?;
|
||||
loader.find_all_languages(&loader_config)?;
|
||||
loader.force_rebuild(self.rebuild || self.grammar_path.is_some());
|
||||
let languages = loader.languages_at_path(current_dir)?;
|
||||
|
||||
let cancellation_flag = util::cancel_on_signal();
|
||||
|
||||
|
|
@ -1702,7 +1729,6 @@ impl Highlight {
|
|||
} => {
|
||||
let path = get_tmp_source_file(&contents)?;
|
||||
|
||||
let languages = loader.languages_at_path(current_dir)?;
|
||||
let language = languages
|
||||
.iter()
|
||||
.find(|(_, n)| language_names.contains(&Box::from(n.as_str())))
|
||||
|
|
@ -1733,7 +1759,6 @@ impl Highlight {
|
|||
if let (Some(l), Some(lc)) = (language.clone(), language_configuration) {
|
||||
(l, lc)
|
||||
} else {
|
||||
let languages = loader.languages_at_path(current_dir)?;
|
||||
let language = languages
|
||||
.first()
|
||||
.map(|(l, _)| l.clone())
|
||||
|
|
|
|||
|
|
@ -515,7 +515,6 @@ pub fn parse_file_at_path(
|
|||
|
||||
if opts.output == ParseOutput::Cst {
|
||||
render_cst(&source_code, &tree, &mut cursor, opts, &mut stdout)?;
|
||||
println!();
|
||||
}
|
||||
|
||||
if opts.output == ParseOutput::Xml {
|
||||
|
|
@ -785,7 +784,7 @@ pub fn render_cst<'a, 'b: 'a>(
|
|||
.map(|(row, col)| (row as f64).log10() as usize + (col.len() as f64).log10() as usize + 1)
|
||||
.max()
|
||||
.unwrap_or(1);
|
||||
let mut indent_level = 1;
|
||||
let mut indent_level = usize::from(!opts.no_ranges);
|
||||
let mut did_visit_children = false;
|
||||
let mut in_error = false;
|
||||
loop {
|
||||
|
|
@ -883,35 +882,24 @@ fn write_node_text(
|
|||
0
|
||||
};
|
||||
let formatted_line = render_line_feed(line, opts);
|
||||
if !opts.no_ranges {
|
||||
write!(
|
||||
out,
|
||||
"{}{}{}{}{}{}",
|
||||
if multiline { "\n" } else { "" },
|
||||
if multiline {
|
||||
render_node_range(opts, cursor, is_named, true, total_width, node_range)
|
||||
} else {
|
||||
String::new()
|
||||
},
|
||||
if multiline {
|
||||
" ".repeat(indent_level + 1)
|
||||
} else {
|
||||
String::new()
|
||||
},
|
||||
paint(quote_color, &String::from(quote)),
|
||||
&paint(color, &render_node_text(&formatted_line)),
|
||||
paint(quote_color, &String::from(quote)),
|
||||
)?;
|
||||
} else {
|
||||
write!(
|
||||
out,
|
||||
"\n{}{}{}{}",
|
||||
" ".repeat(indent_level + 1),
|
||||
paint(quote_color, &String::from(quote)),
|
||||
&paint(color, &render_node_text(&formatted_line)),
|
||||
paint(quote_color, &String::from(quote)),
|
||||
)?;
|
||||
}
|
||||
write!(
|
||||
out,
|
||||
"{}{}{}{}{}{}",
|
||||
if multiline { "\n" } else { " " },
|
||||
if multiline && !opts.no_ranges {
|
||||
render_node_range(opts, cursor, is_named, true, total_width, node_range)
|
||||
} else {
|
||||
String::new()
|
||||
},
|
||||
if multiline {
|
||||
" ".repeat(indent_level + 1)
|
||||
} else {
|
||||
String::new()
|
||||
},
|
||||
paint(quote_color, &String::from(quote)),
|
||||
paint(color, &render_node_text(&formatted_line)),
|
||||
paint(quote_color, &String::from(quote)),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -965,7 +953,7 @@ fn render_node_range(
|
|||
|
||||
fn cst_render_node(
|
||||
opts: &ParseFileOptions,
|
||||
cursor: &mut TreeCursor,
|
||||
cursor: &TreeCursor,
|
||||
source_code: &[u8],
|
||||
out: &mut impl Write,
|
||||
total_width: usize,
|
||||
|
|
@ -1011,10 +999,9 @@ fn cst_render_node(
|
|||
} else {
|
||||
opts.parse_theme.node_kind
|
||||
};
|
||||
write!(out, "{}", paint(kind_color, node.kind()),)?;
|
||||
write!(out, "{}", paint(kind_color, node.kind()))?;
|
||||
|
||||
if node.child_count() == 0 {
|
||||
write!(out, " ")?;
|
||||
// Node text from a pattern or external scanner
|
||||
write_node_text(
|
||||
opts,
|
||||
|
|
|
|||
|
|
@ -19,7 +19,8 @@
|
|||
--light-scrollbar-track: #f1f1f1;
|
||||
--light-scrollbar-thumb: #c1c1c1;
|
||||
--light-scrollbar-thumb-hover: #a8a8a8;
|
||||
|
||||
--light-tree-row-bg: #e3f2fd;
|
||||
|
||||
--dark-bg: #1d1f21;
|
||||
--dark-border: #2d2d2d;
|
||||
--dark-text: #c5c8c6;
|
||||
|
|
@ -28,6 +29,7 @@
|
|||
--dark-scrollbar-track: #25282c;
|
||||
--dark-scrollbar-thumb: #4a4d51;
|
||||
--dark-scrollbar-thumb-hover: #5a5d61;
|
||||
--dark-tree-row-bg: #373737;
|
||||
|
||||
--primary-color: #0550ae;
|
||||
--primary-color-alpha: rgba(5, 80, 174, 0.1);
|
||||
|
|
@ -42,6 +44,7 @@
|
|||
--text-color: var(--dark-text);
|
||||
--panel-bg: var(--dark-panel-bg);
|
||||
--code-bg: var(--dark-code-bg);
|
||||
--tree-row-bg: var(--dark-tree-row-bg);
|
||||
}
|
||||
|
||||
[data-theme="light"] {
|
||||
|
|
@ -50,6 +53,7 @@
|
|||
--text-color: var(--light-text);
|
||||
--panel-bg: white;
|
||||
--code-bg: white;
|
||||
--tree-row-bg: var(--light-tree-row-bg);
|
||||
}
|
||||
|
||||
/* Base Styles */
|
||||
|
|
@ -275,7 +279,7 @@
|
|||
}
|
||||
|
||||
#output-container a.highlighted {
|
||||
background-color: #d9d9d9;
|
||||
background-color: #cae2ff;
|
||||
color: red;
|
||||
border-radius: 3px;
|
||||
text-decoration: underline;
|
||||
|
|
@ -346,7 +350,7 @@
|
|||
}
|
||||
|
||||
& #output-container a.highlighted {
|
||||
background-color: #373b41;
|
||||
background-color: #656669;
|
||||
color: red;
|
||||
}
|
||||
|
||||
|
|
@ -373,6 +377,9 @@
|
|||
color: var(--dark-text);
|
||||
}
|
||||
}
|
||||
.tree-row:has(.highlighted) {
|
||||
background-color: var(--tree-row-bg);
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ class BuildExt(build_ext):
|
|||
class BdistWheel(bdist_wheel):
|
||||
def get_tag(self):
|
||||
python, abi, platform = super().get_tag()
|
||||
if python.startswith("cp"):
|
||||
if python.startswith("cp") and not get_config_var("Py_GIL_DISABLED"):
|
||||
python, abi = "cp310", "abi3"
|
||||
return python, abi, platform
|
||||
|
||||
|
|
|
|||
|
|
@ -595,6 +595,8 @@ impl std::fmt::Display for TestSummary {
|
|||
render_assertion_results("queries", &self.query_results)?;
|
||||
}
|
||||
|
||||
write!(f, "{}", self.parse_stats)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
@ -1066,7 +1068,6 @@ fn run_tests(
|
|||
return Ok(true);
|
||||
}
|
||||
|
||||
let failure_count = test_summary.parse_failures.len();
|
||||
let mut ran_test_in_group = false;
|
||||
|
||||
let matches_filter = |name: &str, file_name: &Option<String>, opts: &TestOptions| {
|
||||
|
|
@ -1130,7 +1131,7 @@ fn run_tests(
|
|||
test_summary.parse_results.pop_traversal();
|
||||
|
||||
if let Some(file_path) = file_path {
|
||||
if opts.update && test_summary.parse_failures.len() - failure_count > 0 {
|
||||
if opts.update {
|
||||
write_tests(&file_path, corrected_entries)?;
|
||||
}
|
||||
corrected_entries.clear();
|
||||
|
|
|
|||
|
|
@ -225,7 +225,7 @@ impl Pattern {
|
|||
}
|
||||
|
||||
// Find every matching combination of child patterns and child nodes.
|
||||
let mut finished_matches = Vec::<Match>::new();
|
||||
let mut finished_matches = Vec::<Match<'_, 'tree>>::new();
|
||||
if cursor.goto_first_child() {
|
||||
let mut match_states = vec![(0, mat)];
|
||||
loop {
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ log.workspace = true
|
|||
pathdiff = { version = "0.2.3", optional = true }
|
||||
regex.workspace = true
|
||||
regex-syntax.workspace = true
|
||||
rquickjs = { version = "0.10.0", optional = true, features = [
|
||||
rquickjs = { version = "0.11.0", optional = true, features = [
|
||||
"bindgen",
|
||||
"loader",
|
||||
"macro",
|
||||
|
|
|
|||
|
|
@ -70,12 +70,13 @@ impl InlinedProductionMapBuilder {
|
|||
let production_map = production_indices_by_step_id
|
||||
.into_iter()
|
||||
.map(|(step_id, production_indices)| {
|
||||
let production = step_id.variable_index.map_or_else(
|
||||
|| &productions[step_id.production_index],
|
||||
|variable_index| {
|
||||
&grammar.variables[variable_index].productions[step_id.production_index]
|
||||
},
|
||||
) as *const Production;
|
||||
let production =
|
||||
core::ptr::from_ref::<Production>(step_id.variable_index.map_or_else(
|
||||
|| &productions[step_id.production_index],
|
||||
|variable_index| {
|
||||
&grammar.variables[variable_index].productions[step_id.production_index]
|
||||
},
|
||||
));
|
||||
((production, step_id.step_index as u32), production_indices)
|
||||
})
|
||||
.collect();
|
||||
|
|
|
|||
|
|
@ -95,9 +95,27 @@ impl Console {
|
|||
Type::Module => "module".to_string(),
|
||||
Type::BigInt => v.get::<String>().unwrap_or_else(|_| "BigInt".to_string()),
|
||||
Type::Unknown => "unknown".to_string(),
|
||||
Type::Array => {
|
||||
let js_vals = v
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter::<Value<'_>>()
|
||||
.filter_map(|x| x.ok())
|
||||
.map(|x| {
|
||||
if x.is_string() {
|
||||
format!("'{}'", Self::format_args(&[x]))
|
||||
} else {
|
||||
Self::format_args(&[x])
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
|
||||
format!("[ {js_vals} ]")
|
||||
}
|
||||
Type::Symbol
|
||||
| Type::Object
|
||||
| Type::Array
|
||||
| Type::Proxy
|
||||
| Type::Function
|
||||
| Type::Constructor
|
||||
| Type::Promise
|
||||
|
|
@ -197,11 +215,11 @@ fn try_resolve_path(path: &Path) -> rquickjs::Result<PathBuf> {
|
|||
}
|
||||
|
||||
#[allow(clippy::needless_pass_by_value)]
|
||||
fn require_from_module<'a>(
|
||||
ctx: Ctx<'a>,
|
||||
fn require_from_module<'js>(
|
||||
ctx: Ctx<'js>,
|
||||
module_path: String,
|
||||
from_module: &str,
|
||||
) -> rquickjs::Result<Value<'a>> {
|
||||
) -> rquickjs::Result<Value<'js>> {
|
||||
let current_module = PathBuf::from(from_module);
|
||||
let current_dir = if current_module.is_file() {
|
||||
current_module.parent().unwrap_or(Path::new("."))
|
||||
|
|
@ -216,13 +234,13 @@ fn require_from_module<'a>(
|
|||
load_module_from_content(&ctx, &resolved_path, &contents)
|
||||
}
|
||||
|
||||
fn load_module_from_content<'a>(
|
||||
ctx: &Ctx<'a>,
|
||||
fn load_module_from_content<'js>(
|
||||
ctx: &Ctx<'js>,
|
||||
path: &Path,
|
||||
contents: &str,
|
||||
) -> rquickjs::Result<Value<'a>> {
|
||||
) -> rquickjs::Result<Value<'js>> {
|
||||
if path.extension().is_some_and(|ext| ext == "json") {
|
||||
return ctx.eval::<Value, _>(format!("JSON.parse({contents:?})"));
|
||||
return ctx.eval::<Value<'js>, _>(format!("JSON.parse({contents:?})"));
|
||||
}
|
||||
|
||||
let exports = Object::new(ctx.clone())?;
|
||||
|
|
@ -238,7 +256,7 @@ fn load_module_from_content<'a>(
|
|||
let module_path = filename.clone();
|
||||
let require = Function::new(
|
||||
ctx.clone(),
|
||||
move |ctx_inner: Ctx<'a>, target_path: String| -> rquickjs::Result<Value<'a>> {
|
||||
move |ctx_inner: Ctx<'js>, target_path: String| -> rquickjs::Result<Value<'js>> {
|
||||
require_from_module(ctx_inner, target_path, &module_path)
|
||||
},
|
||||
)?;
|
||||
|
|
@ -246,8 +264,8 @@ fn load_module_from_content<'a>(
|
|||
let wrapper =
|
||||
format!("(function(exports, require, module, __filename, __dirname) {{ {contents} }})");
|
||||
|
||||
let module_func = ctx.eval::<Function, _>(wrapper)?;
|
||||
module_func.call::<_, Value>((exports, require, module_obj.clone(), filename, dirname))?;
|
||||
let module_func = ctx.eval::<Function<'js>, _>(wrapper)?;
|
||||
module_func.call::<_, Value<'js>>((exports, require, module_obj.clone(), filename, dirname))?;
|
||||
|
||||
module_obj.get("exports")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -189,7 +189,7 @@ struct HighlightIterLayer<'a> {
|
|||
depth: usize,
|
||||
}
|
||||
|
||||
pub struct _QueryCaptures<'query, 'tree: 'query, T: TextProvider<I>, I: AsRef<[u8]>> {
|
||||
pub struct _QueryCaptures<'query, 'tree, T: TextProvider<I>, I: AsRef<[u8]>> {
|
||||
ptr: *mut ffi::TSQueryCursor,
|
||||
query: &'query Query,
|
||||
text_provider: T,
|
||||
|
|
@ -225,7 +225,7 @@ impl<'tree> _QueryMatch<'_, 'tree> {
|
|||
}
|
||||
}
|
||||
|
||||
impl<'query, 'tree: 'query, T: TextProvider<I>, I: AsRef<[u8]>> Iterator
|
||||
impl<'query, 'tree, T: TextProvider<I>, I: AsRef<[u8]>> Iterator
|
||||
for _QueryCaptures<'query, 'tree, T, I>
|
||||
{
|
||||
type Item = (QueryMatch<'query, 'tree>, usize);
|
||||
|
|
@ -594,6 +594,7 @@ impl<'a> HighlightIterLayer<'a> {
|
|||
}
|
||||
}
|
||||
|
||||
// SAFETY:
|
||||
// The `captures` iterator borrows the `Tree` and the `QueryCursor`, which
|
||||
// prevents them from being moved. But both of these values are really just
|
||||
// pointers, so it's actually ok to move them.
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
[package]
|
||||
name = "tree-sitter-language"
|
||||
description = "The tree-sitter Language type, used by the library and by language implementations"
|
||||
version = "0.1.6"
|
||||
version = "0.1.7"
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version = "1.77"
|
||||
|
|
|
|||
|
|
@ -23,9 +23,15 @@ typedef long unsigned int size_t;
|
|||
|
||||
typedef long unsigned int uintptr_t;
|
||||
|
||||
#define UINT16_MAX 65535
|
||||
#define INT8_MAX 127
|
||||
#define INT16_MAX 32767
|
||||
#define INT32_MAX 2147483647L
|
||||
#define INT64_MAX 9223372036854775807LL
|
||||
|
||||
#define UINT8_MAX 255
|
||||
#define UINT16_MAX 65535
|
||||
#define UINT32_MAX 4294967295U
|
||||
#define UINT64_MAX 18446744073709551615ULL
|
||||
|
||||
#if defined(__wasm32__)
|
||||
|
||||
|
|
|
|||
|
|
@ -13,4 +13,6 @@ void *memset(void *dst, int value, size_t count);
|
|||
|
||||
int strncmp(const char *left, const char *right, size_t n);
|
||||
|
||||
size_t strlen(const char *str);
|
||||
|
||||
#endif // TREE_SITTER_WASM_STRING_H_
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
typedef struct {
|
||||
bool left_justify; // -
|
||||
|
|
@ -105,12 +106,6 @@ static int ptr_to_str(void *ptr, char *buffer) {
|
|||
return 2 + len;
|
||||
}
|
||||
|
||||
size_t strlen(const char *str) {
|
||||
const char *s = str;
|
||||
while (*s) s++;
|
||||
return s - str;
|
||||
}
|
||||
|
||||
char *strncpy(char *dest, const char *src, size_t n) {
|
||||
char *d = dest;
|
||||
const char *s = src;
|
||||
|
|
|
|||
|
|
@ -58,3 +58,9 @@ int strncmp(const char *left, const char *right, size_t n) {
|
|||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t strlen(const char *str) {
|
||||
const char *s = str;
|
||||
while (*s) s++;
|
||||
return s - str;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ use std::sync::Mutex;
|
|||
use std::{
|
||||
collections::HashMap,
|
||||
env, fs,
|
||||
hash::{Hash as _, Hasher as _},
|
||||
io::{BufRead, BufReader},
|
||||
marker::PhantomData,
|
||||
mem,
|
||||
|
|
@ -764,7 +765,7 @@ impl Loader {
|
|||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn get_all_language_configurations(&self) -> Vec<(&LanguageConfiguration, &Path)> {
|
||||
pub fn get_all_language_configurations(&self) -> Vec<(&LanguageConfiguration<'static>, &Path)> {
|
||||
self.language_configurations
|
||||
.iter()
|
||||
.map(|c| (c, self.languages_by_id[c.language_id].0.as_ref()))
|
||||
|
|
@ -774,7 +775,7 @@ impl Loader {
|
|||
pub fn language_configuration_for_scope(
|
||||
&self,
|
||||
scope: &str,
|
||||
) -> LoaderResult<Option<(Language, &LanguageConfiguration)>> {
|
||||
) -> LoaderResult<Option<(Language, &LanguageConfiguration<'static>)>> {
|
||||
for configuration in &self.language_configurations {
|
||||
if configuration.scope.as_ref().is_some_and(|s| s == scope) {
|
||||
let language = self.language_for_id(configuration.language_id)?;
|
||||
|
|
@ -787,7 +788,7 @@ impl Loader {
|
|||
pub fn language_configuration_for_first_line_regex(
|
||||
&self,
|
||||
path: &Path,
|
||||
) -> LoaderResult<Option<(Language, &LanguageConfiguration)>> {
|
||||
) -> LoaderResult<Option<(Language, &LanguageConfiguration<'static>)>> {
|
||||
self.language_configuration_ids_by_first_line_regex
|
||||
.iter()
|
||||
.try_fold(None, |_, (regex, ids)| {
|
||||
|
|
@ -816,7 +817,7 @@ impl Loader {
|
|||
pub fn language_configuration_for_file_name(
|
||||
&self,
|
||||
path: &Path,
|
||||
) -> LoaderResult<Option<(Language, &LanguageConfiguration)>> {
|
||||
) -> LoaderResult<Option<(Language, &LanguageConfiguration<'static>)>> {
|
||||
// Find all the language configurations that match this file name
|
||||
// or a suffix of the file name.
|
||||
let configuration_ids = path
|
||||
|
|
@ -888,7 +889,7 @@ impl Loader {
|
|||
pub fn language_configuration_for_injection_string(
|
||||
&self,
|
||||
string: &str,
|
||||
) -> LoaderResult<Option<(Language, &LanguageConfiguration)>> {
|
||||
) -> LoaderResult<Option<(Language, &LanguageConfiguration<'static>)>> {
|
||||
let mut best_match_length = 0;
|
||||
let mut best_match_position = None;
|
||||
for (i, configuration) in self.language_configurations.iter().enumerate() {
|
||||
|
|
@ -1025,20 +1026,26 @@ impl Loader {
|
|||
return Ok(wasm_store.load_language(&config.name, &wasm_bytes)?);
|
||||
}
|
||||
|
||||
// Create a unique lock path based on the output path hash to prevent
|
||||
// interference when multiple processes build the same grammar (by name)
|
||||
// to different output locations
|
||||
let lock_hash = {
|
||||
let mut hasher = std::hash::DefaultHasher::new();
|
||||
output_path.hash(&mut hasher);
|
||||
format!("{:x}", hasher.finish())
|
||||
};
|
||||
|
||||
let lock_path = if env::var("CROSS_RUNNER").is_ok() {
|
||||
tempfile::tempdir()
|
||||
.unwrap()
|
||||
.expect("create a temp dir")
|
||||
.path()
|
||||
.join("tree-sitter")
|
||||
.join("lock")
|
||||
.join(format!("{}.lock", config.name))
|
||||
.to_path_buf()
|
||||
} else {
|
||||
etcetera::choose_base_strategy()?
|
||||
.cache_dir()
|
||||
.join("tree-sitter")
|
||||
.join("lock")
|
||||
.join(format!("{}.lock", config.name))
|
||||
};
|
||||
etcetera::choose_base_strategy()?.cache_dir()
|
||||
}
|
||||
.join("tree-sitter")
|
||||
.join("lock")
|
||||
.join(format!("{}-{lock_hash}.lock", config.name));
|
||||
|
||||
if let Ok(lock_file) = fs::OpenOptions::new().write(true).open(&lock_path) {
|
||||
recompile = false;
|
||||
|
|
@ -1089,6 +1096,26 @@ impl Loader {
|
|||
}
|
||||
}
|
||||
|
||||
// Ensure the dynamic library exists before trying to load it. This can
|
||||
// happen in race conditions where we couldn't acquire the lock because
|
||||
// another process was compiling but it still hasn't finished by the
|
||||
// time we reach this point, so the output file still doesn't exist.
|
||||
//
|
||||
// Instead of allowing the `load_language` call below to fail, return a
|
||||
// clearer error to the user here.
|
||||
if !output_path.exists() {
|
||||
let msg = format!(
|
||||
"Dynamic library `{}` not found after build attempt. \
|
||||
Are you running multiple processes building to the same output location?",
|
||||
output_path.display()
|
||||
);
|
||||
|
||||
Err(LoaderError::IO(IoError::new(
|
||||
std::io::Error::new(std::io::ErrorKind::NotFound, msg),
|
||||
Some(output_path.as_path()),
|
||||
)))?;
|
||||
}
|
||||
|
||||
Self::load_language(&output_path, &language_fn_name)
|
||||
}
|
||||
|
||||
|
|
@ -1278,6 +1305,11 @@ impl Loader {
|
|||
}));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
warn!(
|
||||
"Failed to run `nm` to verify symbols in {}",
|
||||
library_path.display()
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
|
@ -1507,7 +1539,9 @@ impl Loader {
|
|||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn get_language_configuration_in_current_path(&self) -> Option<&LanguageConfiguration> {
|
||||
pub fn get_language_configuration_in_current_path(
|
||||
&self,
|
||||
) -> Option<&LanguageConfiguration<'static>> {
|
||||
self.language_configuration_in_current_path
|
||||
.map(|i| &self.language_configurations[i])
|
||||
}
|
||||
|
|
@ -1516,7 +1550,7 @@ impl Loader {
|
|||
&mut self,
|
||||
parser_path: &Path,
|
||||
set_current_path_config: bool,
|
||||
) -> LoaderResult<&[LanguageConfiguration]> {
|
||||
) -> LoaderResult<&[LanguageConfiguration<'static>]> {
|
||||
let initial_language_configuration_count = self.language_configurations.len();
|
||||
|
||||
match TreeSitterJSON::from_file(parser_path) {
|
||||
|
|
|
|||
|
|
@ -313,6 +313,7 @@ impl TagsContext {
|
|||
)
|
||||
.ok_or(Error::Cancelled)?;
|
||||
|
||||
// SAFETY:
|
||||
// The `matches` iterator borrows the `Tree`, which prevents it from being
|
||||
// moved. But the tree is really just a pointer, so it's actually ok to
|
||||
// move it.
|
||||
|
|
|
|||
|
|
@ -199,6 +199,7 @@ pub fn run_wasm(args: &BuildWasm) -> Result<()> {
|
|||
"-D", "NDEBUG=",
|
||||
"-D", "_POSIX_C_SOURCE=200112L",
|
||||
"-D", "_DEFAULT_SOURCE=",
|
||||
"-D", "_BSD_SOURCE=",
|
||||
"-D", "_DARWIN_C_SOURCE=",
|
||||
"-I", "lib/src",
|
||||
"-I", "lib/include",
|
||||
|
|
|
|||
|
|
@ -73,9 +73,8 @@ The behaviors of these three files are described in the next section.
|
|||
|
||||
## Queries
|
||||
|
||||
Tree-sitter's syntax highlighting system is based on *tree queries*, which are a general system for pattern-matching on Tree-sitter's
|
||||
syntax trees. See [this section][pattern matching] of the documentation for more information
|
||||
about tree queries.
|
||||
Tree-sitter's syntax highlighting system is based on *tree queries*, which are a general system for pattern-matching on
|
||||
Tree-sitter's syntax trees. See [this section][pattern matching] of the documentation for more information about tree queries.
|
||||
|
||||
Syntax highlighting is controlled by *three* different types of query files that are usually included in the `queries` folder.
|
||||
The default names for the query files use the `.scm` file. We chose this extension because it commonly used for files written
|
||||
|
|
|
|||
|
|
@ -3,7 +3,8 @@
|
|||
Tree-sitter can be used in conjunction with its [query language][query language] as a part of code navigation systems.
|
||||
An example of such a system can be seen in the `tree-sitter tags` command, which emits a textual dump of the interesting
|
||||
syntactic nodes in its file argument. A notable application of this is GitHub's support for [search-based code navigation][gh search].
|
||||
This document exists to describe how to integrate with such systems, and how to extend this functionality to any language with a Tree-sitter grammar.
|
||||
This document exists to describe how to integrate with such systems, and how to extend this functionality to any language
|
||||
with a Tree-sitter grammar.
|
||||
|
||||
## Tagging and captures
|
||||
|
||||
|
|
@ -12,9 +13,9 @@ entities. Having found them, you use a syntax capture to label the entity and it
|
|||
|
||||
The essence of a given tag lies in two pieces of data: the _role_ of the entity that is matched
|
||||
(i.e. whether it is a definition or a reference) and the _kind_ of that entity, which describes how the entity is used
|
||||
(i.e. whether it's a class definition, function call, variable reference, and so on). Our convention is to use a syntax capture
|
||||
following the `@role.kind` capture name format, and another inner capture, always called `@name`, that pulls out the name
|
||||
of a given identifier.
|
||||
(i.e. whether it's a class definition, function call, variable reference, and so on). Our convention is to use a syntax
|
||||
capture following the `@role.kind` capture name format, and another inner capture, always called `@name`, that pulls out
|
||||
the name of a given identifier.
|
||||
|
||||
You may optionally include a capture named `@doc` to bind a docstring. For convenience purposes, the tagging system provides
|
||||
two built-in functions, `#select-adjacent!` and `#strip!` that are convenient for removing comment syntax from a docstring.
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ cargo install --path crates/cli
|
|||
If you're going to be in a fast iteration cycle and would like the CLI to build faster, you can use the `release-dev` profile:
|
||||
|
||||
```sh
|
||||
cargo build --release --profile release-dev
|
||||
cargo build --profile release-dev
|
||||
# or
|
||||
cargo install --path crates/cli --profile release-dev
|
||||
```
|
||||
|
|
@ -93,7 +93,8 @@ cargo xtask build-wasm-stdlib
|
|||
|
||||
This command looks for the [Wasi SDK][wasi_sdk] indicated by the `TREE_SITTER_WASI_SDK_PATH`
|
||||
environment variable. If you don't have the binary, it can be downloaded from wasi-sdk's [releases][wasi-sdk-releases]
|
||||
page.
|
||||
page. Note that any changes to `crates/language/wasm/**` requires rebuilding the tree-sitter Wasm stdlib via
|
||||
`cargo xtask build-wasm-stdlib`.
|
||||
|
||||
### Debugging
|
||||
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ function initializeCustomSelect({ initialValue = null, addListeners = false }) {
|
|||
}
|
||||
|
||||
window.initializePlayground = async (opts) => {
|
||||
const { Parser, Language } = window.TreeSitter;
|
||||
const { Parser, Language, Query } = window.TreeSitter;
|
||||
|
||||
const { local } = opts;
|
||||
if (local) {
|
||||
|
|
@ -357,11 +357,10 @@ window.initializePlayground = async (opts) => {
|
|||
marks.forEach((m) => m.clear());
|
||||
|
||||
if (tree && query) {
|
||||
const captures = query.captures(
|
||||
tree.rootNode,
|
||||
{ row: startRow, column: 0 },
|
||||
{ row: endRow, column: 0 },
|
||||
);
|
||||
const captures = query.captures(tree.rootNode, {
|
||||
startPosition: { row: startRow, column: 0 },
|
||||
endPosition: { row: endRow, column: 0 },
|
||||
});
|
||||
let lastNodeId;
|
||||
for (const { name, node } of captures) {
|
||||
if (node.id === lastNodeId) continue;
|
||||
|
|
@ -410,7 +409,7 @@ window.initializePlayground = async (opts) => {
|
|||
const queryText = queryEditor.getValue();
|
||||
|
||||
try {
|
||||
query = parser.language.query(queryText);
|
||||
query = new Query(parser.language, queryText);
|
||||
let match;
|
||||
|
||||
let row = 0;
|
||||
|
|
|
|||
|
|
@ -19,8 +19,8 @@ will attempt to build the parser in the current working directory.
|
|||
### `-w/--wasm`
|
||||
|
||||
Compile the parser as a Wasm module. This command looks for the [Wasi SDK][wasi_sdk] indicated by the `TREE_SITTER_WASI_SDK_PATH`
|
||||
environment variable. If you don't have the binary, the CLI will attempt to download it for you to `<CACHE_DIR>/tree-sitter/wasi-sdk/`, where
|
||||
`<CACHE_DIR>` is resolved according to the [XDG base directory][XDG] or Window's [Known_Folder_Locations][Known_Folder].
|
||||
environment variable. If you don't have the binary, the CLI will attempt to download it for you to `<CACHE_DIR>/tree-sitter/wasi-sdk/`,
|
||||
where `<CACHE_DIR>` is resolved according to the [XDG base directory][XDG] or Window's [Known_Folder_Locations][Known_Folder].
|
||||
|
||||
### `-o/--output`
|
||||
|
||||
|
|
@ -37,7 +37,8 @@ in the external scanner does so using their allocator.
|
|||
|
||||
### `-0/--debug`
|
||||
|
||||
Compile the parser with debug flags enabled. This is useful when debugging issues that require a debugger like `gdb` or `lldb`.
|
||||
Compile the parser with debug flags enabled. This is useful when debugging issues that require a debugger like `gdb` or
|
||||
`lldb`.
|
||||
|
||||
[Known_Folder]: https://learn.microsoft.com/en-us/windows/win32/shell/knownfolderid
|
||||
[wasi_sdk]: https://github.com/WebAssembly/wasi-sdk
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
# `tree-sitter dump-languages`
|
||||
|
||||
The `dump-languages` command prints out a list of all the languages that the CLI knows about. This can be useful for debugging purposes, or for scripting. The paths to search comes from the config file's [`parser-directories`][parser-directories] object.
|
||||
The `dump-languages` command prints out a list of all the languages that the CLI knows about. This can be useful for debugging
|
||||
purposes, or for scripting. The paths to search comes from the config file's [`parser-directories`][parser-directories]
|
||||
object.
|
||||
|
||||
```bash
|
||||
tree-sitter dump-languages [OPTIONS] # Aliases: langs
|
||||
|
|
@ -10,6 +12,7 @@ tree-sitter dump-languages [OPTIONS] # Aliases: langs
|
|||
|
||||
### `--config-path`
|
||||
|
||||
The path to the configuration file. Ordinarily, the CLI will use the default location as explained in the [init-config](./init-config.md) command. This flag allows you to explicitly override that default, and use a config defined elsewhere.
|
||||
The path to the configuration file. Ordinarily, the CLI will use the default location as explained in the [init-config](./init-config.md)
|
||||
command. This flag allows you to explicitly override that default, and use a config defined elsewhere.
|
||||
|
||||
[parser-directories]: ./init-config.md#parser-directories
|
||||
|
|
|
|||
|
|
@ -1,30 +1,39 @@
|
|||
# `tree-sitter generate`
|
||||
|
||||
The most important command you'll use is `tree-sitter generate`. This command reads the `grammar.js` file in your current
|
||||
working directory and creates a file called `src/parser.c`, which implements the parser. After making changes to your grammar,
|
||||
just run `tree-sitter generate` again.
|
||||
The most important command for grammar development is `tree-sitter generate`, which reads the grammar in structured form
|
||||
and outputs C files that can be compiled into a shared or static library (e.g., using the [`build`](./build.md) command).
|
||||
|
||||
```bash
|
||||
tree-sitter generate [OPTIONS] [GRAMMAR_PATH] # Aliases: gen, g
|
||||
```
|
||||
|
||||
The grammar path argument allows you to specify a path to a `grammar.js` JavaScript file, or `grammar.json` JSON file.
|
||||
In case your `grammar.js` file is in a non-standard path, you can specify it yourself. But, if you are using a parser
|
||||
where `grammar.json` was already generated, or it was hand-written, you can tell the CLI to generate the parser *based*
|
||||
on this JSON file. This avoids relying on a JavaScript file and avoids the dependency on a JavaScript runtime.
|
||||
The optional `GRAMMAR_PATH` argument should point to the structured grammar, in one of two forms:
|
||||
- `grammar.js` a (ESM or CJS) JavaScript file; if the argument is omitted, it defaults to `./grammar.js`.
|
||||
- `grammar.json` a structured representation of the grammar that is created as a byproduct of `generate`; this can be used
|
||||
to regenerate a missing `parser.c` without requiring a JavaScript runtime (useful when distributing parsers to consumers).
|
||||
|
||||
If there is an ambiguity or *local ambiguity* in your grammar, Tree-sitter will detect it during parser generation, and
|
||||
it will exit with a `Unresolved conflict` error message. To learn more about conflicts and how to handle them, check out
|
||||
it will exit with a `Unresolved conflict` error message. To learn more about conflicts and how to handle them, see
|
||||
the section on [`Structuring Rules Well`](../creating-parsers/3-writing-the-grammar.md#structuring-rules-well)
|
||||
in the user guide.
|
||||
|
||||
## Generated files
|
||||
|
||||
- `src/parser.c` implements the parser logic specified in the grammar.
|
||||
- `src/tree_sitter/parser.h` provides basic C definitions that are used in the generated `parser.c` file.
|
||||
- `src/tree_sitter/alloc.h` provides memory allocation macros that can be used in an external scanner.
|
||||
- `src/tree_sitter/array.h` provides array macros that can be used in an external scanner.
|
||||
- `src/grammar.json` contains a structured representation of the grammar; can be used to regenerate the parser without having
|
||||
to re-evaluate the `grammar.js`.
|
||||
- `src/node-types.json` provides type information about individual syntax nodes; see the section on [`Static Node Types`](../using-parsers/6-static-node-types.md).
|
||||
|
||||
|
||||
## Options
|
||||
|
||||
### `-l/--log`
|
||||
|
||||
Print the log of the parser generation process. This is really only useful if you know what you're doing, or are investigating
|
||||
a bug in the CLI itself. It logs info such as what tokens are included in the error recovery state,
|
||||
what keywords were extracted, what states were split and why, and the entry point state.
|
||||
Print the log of the parser generation process. This includes information such as what tokens are included in the error
|
||||
recovery state, what keywords were extracted, what states were split and why, and the entry point state.
|
||||
|
||||
### `--abi <VERSION>`
|
||||
|
||||
|
|
@ -54,7 +63,8 @@ The path to the JavaScript runtime executable to use when generating the parser.
|
|||
Note that you can also set this with `TREE_SITTER_JS_RUNTIME`. Starting from version 0.26, you can
|
||||
also pass in `native` to use the experimental native QuickJS runtime that comes bundled with the CLI.
|
||||
This avoids the dependency on a JavaScript runtime entirely. The native QuickJS runtime is compatible
|
||||
with ESM as well as with CommonJS in strict mode. If your grammar depends on `npm` to install dependencies such as base grammars, the native runtime can be used *after* running `npm install`.
|
||||
with ESM as well as with CommonJS in strict mode. If your grammar depends on `npm` to install dependencies such as base
|
||||
grammars, the native runtime can be used *after* running `npm install`.
|
||||
|
||||
### `--disable-optimization`
|
||||
|
||||
|
|
|
|||
|
|
@ -52,7 +52,8 @@ The path to the directory containing the grammar.
|
|||
|
||||
### `--config-path <CONFIG_PATH>`
|
||||
|
||||
The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more information.
|
||||
The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more
|
||||
information.
|
||||
|
||||
### `-n/--test-number <TEST_NUMBER>`
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,8 @@
|
|||
# CLI Overview
|
||||
|
||||
Let's go over all of the functionality of the `tree-sitter` command line interface.
|
||||
Once you feel that you have enough of a grasp on the CLI, you can move onto the grammar authoring section to learn more about writing your own parser.
|
||||
The `tree-sitter` command-line interface is used to create, manage, test, and build tree-sitter parsers. It is controlled
|
||||
by
|
||||
|
||||
- a personal `tree-sitter/config.json` config file generated by [`tree-sitter init-config`](./init-config.md)
|
||||
- a parser `tree-sitter.json` config file generated by [`tree-sitter init`](./init.md).
|
||||
|
||||
|
|
|
|||
|
|
@ -8,30 +8,94 @@ we recommend using git for version control of your grammar.
|
|||
tree-sitter init [OPTIONS] # Aliases: i
|
||||
```
|
||||
|
||||
## Options
|
||||
## Generated files
|
||||
|
||||
### `--update`
|
||||
### Required files
|
||||
|
||||
Update outdated generated files, if needed.
|
||||
The following required files are always created if missing:
|
||||
|
||||
### `-p/--grammar-path <PATH>`
|
||||
- `tree-sitter.json` - The main configuration file that determines how `tree-sitter` interacts with the grammar. If missing,
|
||||
the `init` command will prompt the user for the required fields. See [below](./init.md#structure-of-tree-sitterjson) for
|
||||
the full documentation of the structure of this file.
|
||||
- `package.json` - The `npm` manifest for the parser. This file is required for some `tree-sitter` subcommands, and if the
|
||||
grammar has dependencies (e.g., another published base grammar that this grammar extends).
|
||||
- `grammar.js` - An empty template for the main grammar file; see [the section on creating parsers](../2-creating-parser).
|
||||
|
||||
The path to the directory containing the grammar.
|
||||
### Language bindings
|
||||
|
||||
Language bindings are files that allow your parser to be directly used by projects written in the respective language.
|
||||
The following bindings are created if enabled in `tree-sitter.json`:
|
||||
|
||||
#### C/C++
|
||||
|
||||
- `Makefile` — This file tells [`make`][make] how to compile your language.
|
||||
- `CMakeLists.txt` — This file tells [`cmake`][cmake] how to compile your language.
|
||||
- `bindings/c/tree_sitter/tree-sitter-language.h` — This file provides the C interface of your language.
|
||||
- `bindings/c/tree-sitter-language.pc` — This file provides [pkg-config][pkg-config] metadata about your language's C library.
|
||||
|
||||
#### Go
|
||||
|
||||
- `go.mod` — This file is the manifest of the Go module.
|
||||
- `bindings/go/binding.go` — This file wraps your language in a Go module.
|
||||
- `bindings/go/binding_test.go` — This file contains a test for the Go package.
|
||||
|
||||
#### Node
|
||||
|
||||
- `binding.gyp` — This file tells Node.js how to compile your language.
|
||||
- `bindings/node/binding.cc` — This file wraps your language in a JavaScript module for Node.js.
|
||||
- `bindings/node/index.js` — This is the file that Node.js initially loads when using your language.
|
||||
- `bindings/node/index.d.ts` — This file provides type hints for your parser when used in TypeScript.
|
||||
- `bindings/node/binding_test.js` — This file contains a test for the Node.js package.
|
||||
|
||||
#### Java
|
||||
|
||||
- `pom.xml` - This file is the manifest of the Maven package.
|
||||
- `bindings/java/main/namespace/language/TreeSitterLanguage.java` - This file wraps your language in a Java class.
|
||||
- `bindings/java/test/TreeSitterLanguageTest.java` - This file contains a test for the Java package.
|
||||
|
||||
#### Python
|
||||
|
||||
- `pyproject.toml` — This file is the manifest of the Python package.
|
||||
- `setup.py` — This file tells Python how to compile your language.
|
||||
- `bindings/python/tree_sitter_language/binding.c` — This file wraps your language in a Python module.
|
||||
- `bindings/python/tree_sitter_language/__init__.py` — This file tells Python how to load your language.
|
||||
- `bindings/python/tree_sitter_language/__init__.pyi` — This file provides type hints for your parser when used in Python.
|
||||
- `bindings/python/tree_sitter_language/py.typed` — This file provides type hints for your parser when used in Python.
|
||||
- `bindings/python/tests/test_binding.py` — This file contains a test for the Python package.
|
||||
|
||||
#### Rust
|
||||
|
||||
- `Cargo.toml` — This file is the manifest of the Rust package.
|
||||
- `bindings/rust/build.rs` — This file tells Rust how to compile your language.
|
||||
- `bindings/rust/lib.rs` — This file wraps your language in a Rust crate when used in Rust.
|
||||
|
||||
#### Swift
|
||||
|
||||
- `Package.swift` — This file tells Swift how to compile your language.
|
||||
- `bindings/swift/TreeSitterLanguage/language.h` — This file wraps your language in a Swift module when used in Swift.
|
||||
- `bindings/swift/TreeSitterLanguageTests/TreeSitterLanguageTests.swift` — This file contains a test for the Swift package.
|
||||
|
||||
#### Zig
|
||||
|
||||
- `build.zig` - This file tells Zig how to compile your language.
|
||||
- `build.zig.zon` - This file is the manifest of the Zig package.
|
||||
- `bindings/zig/root.zig` - This file wraps your language in a Zig module.
|
||||
- `bindings/zig/test.zig` - This file contains a test for the Zig package.
|
||||
|
||||
### Additional files
|
||||
|
||||
In addition, the following files are created that aim to improve the development experience:
|
||||
|
||||
- `.editorconfig` — This file tells your editor how to format your code. More information about this file can be found [here][editorconfig].
|
||||
- `.gitattributes` — This file tells Git how to handle line endings and tells GitHub which files are generated.
|
||||
- `.gitignore` — This file tells Git which files to ignore when committing changes.
|
||||
|
||||
## Structure of `tree-sitter.json`
|
||||
|
||||
The main file of interest for users to configure is `tree-sitter.json`, which tells the CLI information about your grammar,
|
||||
such as the location of queries.
|
||||
|
||||
### The `grammars` field
|
||||
|
||||
This field is an array of objects, though you typically only need one object in this array unless your repo has
|
||||
multiple grammars (for example, `Typescript` and `TSX`).
|
||||
|
||||
### Example
|
||||
|
||||
Typically, the objects in the `"tree-sitter"` array only needs to specify a few keys:
|
||||
|
||||
multiple grammars (for example, `Typescript` and `TSX`), e.g.,
|
||||
```json
|
||||
{
|
||||
"tree-sitter": [
|
||||
|
|
@ -49,7 +113,7 @@ Typically, the objects in the `"tree-sitter"` array only needs to specify a few
|
|||
}
|
||||
```
|
||||
|
||||
#### Basic Fields
|
||||
#### Basic fields
|
||||
|
||||
These keys specify basic information about the parser:
|
||||
|
||||
|
|
@ -65,12 +129,12 @@ parser to files that should be checked for modifications during recompilation.
|
|||
This is useful during development to have changes to other files besides scanner.c
|
||||
be picked up by the cli.
|
||||
|
||||
#### Language Detection
|
||||
#### Language detection
|
||||
|
||||
These keys help to decide whether the language applies to a given file:
|
||||
|
||||
- `file-types` — An array of filename suffix strings. The grammar will be used for files whose names end with one of
|
||||
these suffixes. Note that the suffix may match an *entire* filename.
|
||||
- `file-types` — An array of filename suffix strings (not including the dot). The grammar will be used for files whose names
|
||||
end with one of these suffixes. Note that the suffix may match an *entire* filename.
|
||||
|
||||
- `first-line-regex` — A regex pattern that will be tested against the first line of a file
|
||||
to determine whether this language applies to the file. If present, this regex will be used for any file whose
|
||||
|
|
@ -85,14 +149,14 @@ no `content-regex` will be preferred over this one.
|
|||
should be used for a potential *language injection* site.
|
||||
Language injection is described in more detail in [the relevant section](../3-syntax-highlighting.md#language-injection).
|
||||
|
||||
#### Query Paths
|
||||
#### Query paths
|
||||
|
||||
These keys specify relative paths from the directory containing `tree-sitter.json` to the files that control syntax highlighting:
|
||||
|
||||
- `highlights` — Path to a *highlight query*. Default: `queries/highlights.scm`
|
||||
- `locals` — Path to a *local variable query*. Default: `queries/locals.scm`.
|
||||
- `injections` — Path to an *injection query*. Default: `queries/injections.scm`.
|
||||
- `tags` — Path to an *tag query*. Default: `queries/tags.scm`.
|
||||
- `tags` — Path to a *tag query*. Default: `queries/tags.scm`.
|
||||
|
||||
### The `metadata` field
|
||||
|
||||
|
|
@ -121,81 +185,19 @@ Each key is a language name, and the value is a boolean.
|
|||
- `swift` (default: `false`)
|
||||
- `zig` (default: `false`)
|
||||
|
||||
## Binding Files
|
||||
## Options
|
||||
|
||||
When you run `tree-sitter init`, the CLI will also generate a number of files in your repository that allow for your parser
|
||||
to be used from different language. Here is a list of these bindings files that are generated, and what their purpose is:
|
||||
### `-u/--update`
|
||||
|
||||
### C/C++
|
||||
Update outdated generated files, if possible.
|
||||
|
||||
- `Makefile` — This file tells [`make`][make] how to compile your language.
|
||||
- `CMakeLists.txt` — This file tells [`cmake`][cmake] how to compile your language.
|
||||
- `bindings/c/tree_sitter/tree-sitter-language.h` — This file provides the C interface of your language.
|
||||
- `bindings/c/tree-sitter-language.pc` — This file provides [pkg-config][pkg-config] metadata about your language's C library.
|
||||
- `src/tree_sitter/parser.h` — This file provides some basic C definitions that are used in your generated `parser.c` file.
|
||||
- `src/tree_sitter/alloc.h` — This file provides some memory allocation macros that are to be used in your external scanner,
|
||||
if you have one.
|
||||
- `src/tree_sitter/array.h` — This file provides some array macros that are to be used in your external scanner,
|
||||
if you have one.
|
||||
**Note:** Existing files that may have been edited manually are _not_ updated in general. To force an update to such files,
|
||||
remove them and call `tree-sitter init -u` again.
|
||||
|
||||
### Go
|
||||
### `-p/--grammar-path <PATH>`
|
||||
|
||||
- `go.mod` — This file is the manifest of the Go module.
|
||||
- `bindings/go/binding.go` — This file wraps your language in a Go module.
|
||||
- `bindings/go/binding_test.go` — This file contains a test for the Go package.
|
||||
The path to the directory containing the grammar.
|
||||
|
||||
### Node
|
||||
|
||||
- `binding.gyp` — This file tells Node.js how to compile your language.
|
||||
- `package.json` — This file is the manifest of the Node.js package.
|
||||
- `bindings/node/binding.cc` — This file wraps your language in a JavaScript module for Node.js.
|
||||
- `bindings/node/index.js` — This is the file that Node.js initially loads when using your language.
|
||||
- `bindings/node/index.d.ts` — This file provides type hints for your parser when used in TypeScript.
|
||||
- `bindings/node/binding_test.js` — This file contains a test for the Node.js package.
|
||||
|
||||
### Java
|
||||
|
||||
- `pom.xml` - This file is the manifest of the Maven package.
|
||||
- `bindings/java/main/namespace/language/TreeSitterLanguage.java` - This file wraps your language in a Java class.
|
||||
- `bindings/java/test/TreeSitterLanguageTest.java` - This file contains a test for the Java package.
|
||||
|
||||
### Python
|
||||
|
||||
- `pyproject.toml` — This file is the manifest of the Python package.
|
||||
- `setup.py` — This file tells Python how to compile your language.
|
||||
- `bindings/python/tree_sitter_language/binding.c` — This file wraps your language in a Python module.
|
||||
- `bindings/python/tree_sitter_language/__init__.py` — This file tells Python how to load your language.
|
||||
`bindings/python/tree_sitter_language/__init__.pyi` — This file provides type hints for your parser when used in Python.
|
||||
- `bindings/python/tree_sitter_language/py.typed` — This file provides type hints for your parser when used in Python.
|
||||
- `bindings/python/tests/test_binding.py` — This file contains a test for the Python package.
|
||||
|
||||
### Rust
|
||||
|
||||
- `Cargo.toml` — This file is the manifest of the Rust package.
|
||||
- `bindings/rust/lib.rs` — This file wraps your language in a Rust crate when used in Rust.
|
||||
- `bindings/rust/build.rs` — This file wraps the building process for the Rust crate.
|
||||
|
||||
### Swift
|
||||
|
||||
- `Package.swift` — This file tells Swift how to compile your language.
|
||||
- `bindings/swift/TreeSitterLanguage/language.h` — This file wraps your language in a Swift module when used in Swift.
|
||||
- `bindings/swift/TreeSitterLanguageTests/TreeSitterLanguageTests.swift` — This file contains a test for the Swift package.
|
||||
|
||||
### Zig
|
||||
|
||||
- `build.zig` - This file tells Zig how to compile your language.
|
||||
- `build.zig.zon` - This file is the manifest of the Zig package.
|
||||
- `bindings/zig/root.zig` - This file wraps your language in a Zig module.
|
||||
- `bindings/zig/test.zig` - This file contains a test for the Zig package.
|
||||
|
||||
### Additional Files
|
||||
|
||||
Additionally, there's a few other files that are generated when you run `tree-sitter init`,
|
||||
that aim to improve the development experience:
|
||||
|
||||
- `.editorconfig` — This file tells your editor how to format your code. More information about this file can be found [here][editorconfig]
|
||||
- `.gitattributes` — This file tells Git how to handle line endings, and tells GitHub what files are generated.
|
||||
- `.gitignore` — This file tells Git what files to ignore when committing changes.
|
||||
|
||||
[cmake]: https://cmake.org/cmake/help/latest
|
||||
[editorconfig]: https://editorconfig.org
|
||||
|
|
|
|||
|
|
@ -78,7 +78,8 @@ Suppress main output.
|
|||
|
||||
### `--edits <EDITS>...`
|
||||
|
||||
Apply edits after parsing the file. Edits are in the form of `row,col|position delcount insert_text` where row and col, or position are 0-indexed.
|
||||
Apply edits after parsing the file. Edits are in the form of `row,col|position delcount insert_text` where row and col,
|
||||
or position are 0-indexed.
|
||||
|
||||
### `--encoding <ENCODING>`
|
||||
|
||||
|
|
@ -95,7 +96,8 @@ Output parsing results in a JSON format.
|
|||
|
||||
### `--config-path <CONFIG_PATH>`
|
||||
|
||||
The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more information.
|
||||
The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more
|
||||
information.
|
||||
|
||||
### `-n/--test-number <TEST_NUMBER>`
|
||||
|
||||
|
|
|
|||
|
|
@ -7,8 +7,8 @@ tree-sitter playground [OPTIONS] # Aliases: play, pg, web-ui
|
|||
```
|
||||
|
||||
```admonish note
|
||||
For this to work, you must have already built the parser as a Wasm module. This can be done with the [`build`](./build.md) subcommand
|
||||
(`tree-sitter build --wasm`).
|
||||
For this to work, you must have already built the parser as a Wasm module. This can be done with the [`build`](./build.md)
|
||||
subcommand (`tree-sitter build --wasm`).
|
||||
```
|
||||
|
||||
## Options
|
||||
|
|
|
|||
|
|
@ -47,8 +47,8 @@ The range of rows in which the query will be executed. The format is `start_row:
|
|||
|
||||
### `--containing-row-range <ROW_RANGE>`
|
||||
|
||||
The range of rows in which the query will be executed. Only the matches that are fully contained within the provided row range
|
||||
will be returned.
|
||||
The range of rows in which the query will be executed. Only the matches that are fully contained within the provided row
|
||||
range will be returned.
|
||||
|
||||
### `--scope <SCOPE>`
|
||||
|
||||
|
|
@ -64,7 +64,8 @@ Whether to run query tests or not.
|
|||
|
||||
### `--config-path <CONFIG_PATH>`
|
||||
|
||||
The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more information.
|
||||
The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more
|
||||
information.
|
||||
|
||||
### `-n/--test-number <TEST_NUMBER>`
|
||||
|
||||
|
|
|
|||
|
|
@ -31,7 +31,8 @@ The path to the directory containing the grammar.
|
|||
|
||||
### `--config-path <CONFIG_PATH>`
|
||||
|
||||
The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more information.
|
||||
The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more
|
||||
information.
|
||||
|
||||
### `-n/--test-number <TEST_NUMBER>`
|
||||
|
||||
|
|
|
|||
|
|
@ -63,7 +63,8 @@ When using the `--debug-graph` option, open the log file in the default browser.
|
|||
|
||||
### `--config-path <CONFIG_PATH>`
|
||||
|
||||
The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more information.
|
||||
The path to an alternative configuration (`config.json`) file. See [the init-config command](./init-config.md) for more
|
||||
information.
|
||||
|
||||
### `--show-fields`
|
||||
|
||||
|
|
|
|||
|
|
@ -25,11 +25,9 @@ tree-sitter version --bump minor # minor bump
|
|||
tree-sitter version --bump major # major bump
|
||||
```
|
||||
|
||||
As a grammar author, you should keep the version of your grammar in sync across
|
||||
different bindings. However, doing so manually is error-prone and tedious, so
|
||||
this command takes care of the burden. If you are using a version control system,
|
||||
it is recommended to commit the changes made by this command, and to tag the
|
||||
commit with the new version.
|
||||
As a grammar author, you should keep the version of your grammar in sync across different bindings. However, doing so manually
|
||||
is error-prone and tedious, so this command takes care of the burden. If you are using a version control system, it is recommended
|
||||
to commit the changes made by this command, and to tag the commit with the new version.
|
||||
|
||||
To print the current version without bumping it, use:
|
||||
|
||||
|
|
|
|||
|
|
@ -17,8 +17,8 @@ DSL through the `RustRegex` class. Simply pass your regex pattern as a string:
|
|||
```
|
||||
|
||||
Unlike JavaScript's builtin `RegExp` class, which takes a pattern and flags as separate arguments, `RustRegex` only
|
||||
accepts a single pattern string. While it doesn't support separate flags, you can use inline flags within the pattern itself.
|
||||
For more details about Rust's regex syntax and capabilities, check out the [Rust regex documentation][rust regex].
|
||||
accepts a single pattern string. While it doesn't support separate flags, you can use inline flags within the pattern
|
||||
itself. For more details about Rust's regex syntax and capabilities, check out the [Rust regex documentation][rust regex].
|
||||
|
||||
```admonish note
|
||||
Only a subset of the Regex engine is actually supported. This is due to certain features like lookahead and lookaround
|
||||
|
|
@ -50,10 +50,10 @@ The previous `repeat` rule is implemented in `repeat1` but is included because i
|
|||
- **Options : `optional(rule)`** — This function creates a rule that matches *zero or one* occurrence of a given rule.
|
||||
It is analogous to the `[x]` (square bracket) syntax in EBNF notation.
|
||||
|
||||
- **Precedence : `prec(number, rule)`** — This function marks the given rule with a numerical precedence, which will be used
|
||||
to resolve [*LR(1) Conflicts*][lr-conflict] at parser-generation time. When two rules overlap in a way that represents either
|
||||
a true ambiguity or a *local* ambiguity given one token of lookahead, Tree-sitter will try to resolve the conflict by matching
|
||||
the rule with the higher precedence. The default precedence of all rules is zero. This works similarly to the
|
||||
- **Precedence : `prec(number, rule)`** — This function marks the given rule with a numerical precedence, which will be
|
||||
used to resolve [*LR(1) Conflicts*][lr-conflict] at parser-generation time. When two rules overlap in a way that represents
|
||||
either a true ambiguity or a *local* ambiguity given one token of lookahead, Tree-sitter will try to resolve the conflict
|
||||
by matching the rule with the higher precedence. The default precedence of all rules is zero. This works similarly to the
|
||||
[precedence directives][yacc-prec] in Yacc grammars.
|
||||
|
||||
This function can also be used to assign lexical precedence to a given
|
||||
|
|
@ -115,8 +115,8 @@ want to create syntax tree nodes at runtime.
|
|||
|
||||
- **`conflicts`** — an array of arrays of rule names. Each inner array represents a set of rules that's involved in an
|
||||
*LR(1) conflict* that is *intended to exist* in the grammar. When these conflicts occur at runtime, Tree-sitter will use
|
||||
the GLR algorithm to explore all the possible interpretations. If *multiple* parses end up succeeding, Tree-sitter will pick
|
||||
the subtree whose corresponding rule has the highest total *dynamic precedence*.
|
||||
the GLR algorithm to explore all the possible interpretations. If *multiple* parses end up succeeding, Tree-sitter will
|
||||
pick the subtree whose corresponding rule has the highest total *dynamic precedence*.
|
||||
|
||||
- **`externals`** — an array of token names which can be returned by an
|
||||
[*external scanner*][external-scanners]. External scanners allow you to write custom C code which runs during the lexing
|
||||
|
|
@ -139,10 +139,10 @@ for more details.
|
|||
array of reserved rules. The reserved rule in the array must be a terminal token meaning it must be a string, regex, token,
|
||||
or terminal rule. The reserved rule must also exist and be used in the grammar, specifying arbitrary tokens will not work.
|
||||
The *first* reserved word set in the object is the global word set, meaning it applies to every rule in every parse state.
|
||||
However, certain keywords are contextual, depending on the rule. For example, in JavaScript, keywords are typically not allowed
|
||||
as ordinary variables, however, they *can* be used as a property name. In this situation, the `reserved` function would be used,
|
||||
and the word set to pass in would be the name of the word set that is declared in the `reserved` object that corresponds to an
|
||||
empty array, signifying *no* keywords are reserved.
|
||||
However, certain keywords are contextual, depending on the rule. For example, in JavaScript, keywords are typically not
|
||||
allowed as ordinary variables, however, they *can* be used as a property name. In this situation, the `reserved` function
|
||||
would be used, and the word set to pass in would be the name of the word set that is declared in the `reserved` object that
|
||||
corresponds to an empty array, signifying *no* keywords are reserved.
|
||||
|
||||
[bison-dprec]: https://www.gnu.org/software/bison/manual/html_node/Generalized-LR-Parsing.html
|
||||
[ebnf]: https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# Writing the Grammar
|
||||
|
||||
Writing a grammar requires creativity. There are an infinite number of CFGs (context-free grammars) that can be used to describe
|
||||
any given language. To produce a good Tree-sitter parser, you need to create a grammar with two important properties:
|
||||
Writing a grammar requires creativity. There are an infinite number of CFGs (context-free grammars) that can be used to
|
||||
describe any given language. To produce a good Tree-sitter parser, you need to create a grammar with two important properties:
|
||||
|
||||
1. **An intuitive structure** — Tree-sitter's output is a [concrete syntax tree][cst]; each node in the tree corresponds
|
||||
directly to a [terminal or non-terminal symbol][non-terminal] in the grammar. So to produce an easy-to-analyze tree, there
|
||||
|
|
@ -139,8 +139,8 @@ instead. It's often useful to check your progress by trying to parse some real c
|
|||
## Structuring Rules Well
|
||||
|
||||
Imagine that you were just starting work on the [Tree-sitter JavaScript parser][tree-sitter-javascript]. Naively, you might
|
||||
try to directly mirror the structure of the [ECMAScript Language Spec][ecmascript-spec]. To illustrate the problem with this
|
||||
approach, consider the following line of code:
|
||||
try to directly mirror the structure of the [ECMAScript Language Spec][ecmascript-spec]. To illustrate the problem with
|
||||
this approach, consider the following line of code:
|
||||
|
||||
```js
|
||||
return x + y;
|
||||
|
|
@ -181,16 +181,17 @@ which are unrelated to the actual code.
|
|||
|
||||
## Standard Rule Names
|
||||
|
||||
Tree-sitter places no restrictions on how to name the rules of your grammar. It can be helpful, however, to follow certain conventions
|
||||
used by many other established grammars in the ecosystem. Some of these well-established patterns are listed below:
|
||||
Tree-sitter places no restrictions on how to name the rules of your grammar. It can be helpful, however, to follow certain
|
||||
conventions used by many other established grammars in the ecosystem. Some of these well-established patterns are listed
|
||||
below:
|
||||
|
||||
- `source_file`: Represents an entire source file, this rule is commonly used as the root node for a grammar,
|
||||
- `expression`/`statement`: Used to represent statements and expressions for a given language. Commonly defined as a choice between several
|
||||
more specific sub-expression/sub-statement rules.
|
||||
- `expression`/`statement`: Used to represent statements and expressions for a given language. Commonly defined as a choice
|
||||
between several more specific sub-expression/sub-statement rules.
|
||||
- `block`: Used as the parent node for block scopes, with its children representing the block's contents.
|
||||
- `type`: Represents the types of a language such as `int`, `char`, and `void`.
|
||||
- `identifier`: Used for constructs like variable names, function arguments, and object fields; this rule is commonly used as the `word`
|
||||
token in grammars.
|
||||
- `identifier`: Used for constructs like variable names, function arguments, and object fields; this rule is commonly used
|
||||
as the `word` token in grammars.
|
||||
- `string`: Used to represent `"string literals"`.
|
||||
- `comment`: Used to represent comments, this rule is commonly used as an `extra`.
|
||||
|
||||
|
|
@ -308,9 +309,9 @@ This is where `prec.left` and `prec.right` come into use. We want to select the
|
|||
|
||||
## Using Conflicts
|
||||
|
||||
Sometimes, conflicts are actually desirable. In our JavaScript grammar, expressions and patterns can create intentional ambiguity.
|
||||
A construct like `[x, y]` could be legitimately parsed as both an array literal (like in `let a = [x, y]`) or as a destructuring
|
||||
pattern (like in `let [x, y] = arr`).
|
||||
Sometimes, conflicts are actually desirable. In our JavaScript grammar, expressions and patterns can create intentional
|
||||
ambiguity. A construct like `[x, y]` could be legitimately parsed as both an array literal (like in `let a = [x, y]`) or
|
||||
as a destructuring pattern (like in `let [x, y] = arr`).
|
||||
|
||||
```js
|
||||
export default grammar({
|
||||
|
|
@ -564,8 +565,8 @@ as mentioned in the previous page, is `token(prec(N, ...))`.
|
|||
## Keywords
|
||||
|
||||
Many languages have a set of _keyword_ tokens (e.g. `if`, `for`, `return`), as well as a more general token (e.g. `identifier`)
|
||||
that matches any word, including many of the keyword strings. For example, JavaScript has a keyword `instanceof`, which is
|
||||
used as a binary operator, like this:
|
||||
that matches any word, including many of the keyword strings. For example, JavaScript has a keyword `instanceof`, which
|
||||
is used as a binary operator, like this:
|
||||
|
||||
```js
|
||||
if (a instanceof Something) b();
|
||||
|
|
|
|||
|
|
@ -143,10 +143,10 @@ the second argument, the current character will be treated as whitespace; whites
|
|||
associated with tokens emitted by the external scanner.
|
||||
|
||||
- **`void (*mark_end)(TSLexer *)`** — A function for marking the end of the recognized token. This allows matching tokens
|
||||
that require multiple characters of lookahead. By default, (if you don't call `mark_end`), any character that you moved past
|
||||
using the `advance` function will be included in the size of the token. But once you call `mark_end`, then any later calls
|
||||
to `advance` will _not_ increase the size of the returned token. You can call `mark_end` multiple times to increase the size
|
||||
of the token.
|
||||
that require multiple characters of lookahead. By default, (if you don't call `mark_end`), any character that you moved
|
||||
past using the `advance` function will be included in the size of the token. But once you call `mark_end`, then any later
|
||||
calls to `advance` will _not_ increase the size of the returned token. You can call `mark_end` multiple times to increase
|
||||
the size of the token.
|
||||
|
||||
- **`uint32_t (*get_column)(TSLexer *)`** — A function for querying the current column position of the lexer. It returns
|
||||
the number of codepoints since the start of the current line. The codepoint position is recalculated on every call to this
|
||||
|
|
@ -185,9 +185,9 @@ if (valid_symbols[INDENT] || valid_symbols[DEDENT]) {
|
|||
|
||||
### Allocator
|
||||
|
||||
Instead of using libc's `malloc`, `calloc`, `realloc`, and `free`, you should use the versions prefixed with `ts_` from `tree_sitter/alloc.h`.
|
||||
These macros can allow a potential consumer to override the default allocator with their own implementation, but by default
|
||||
will use the libc functions.
|
||||
Instead of using libc's `malloc`, `calloc`, `realloc`, and `free`, you should use the versions prefixed with `ts_` from
|
||||
`tree_sitter/alloc.h`. These macros can allow a potential consumer to override the default allocator with their own implementation,
|
||||
but by default will use the libc functions.
|
||||
|
||||
As a consumer of the tree-sitter core library as well as any parser libraries that might use allocations, you can enable
|
||||
overriding the default allocator and have it use the same one as the library allocator, of which you can set with `ts_set_allocator`.
|
||||
|
|
@ -195,7 +195,8 @@ To enable this overriding in scanners, you must compile them with the `TREE_SITT
|
|||
the library must be linked into your final app dynamically, since it needs to resolve the internal functions at runtime.
|
||||
If you are compiling an executable binary that uses the core library, but want to load parsers dynamically at runtime, then
|
||||
you will have to use a special linker flag on Unix. For non-Darwin systems, that would be `--dynamic-list` and for Darwin
|
||||
systems, that would be `-exported_symbols_list`. The CLI does exactly this, so you can use it as a reference (check out `cli/build.rs`).
|
||||
systems, that would be `-exported_symbols_list`. The CLI does exactly this, so you can use it as a reference (check out
|
||||
`cli/build.rs`).
|
||||
|
||||
For example, assuming you wanted to allocate 100 bytes for your scanner, you'd do so like the following example:
|
||||
|
||||
|
|
@ -293,9 +294,10 @@ bool tree_sitter_my_language_external_scanner_scan(
|
|||
|
||||
## Other External Scanner Details
|
||||
|
||||
External scanners have priority over Tree-sitter's normal lexing process. When a token listed in the externals array is valid
|
||||
at a given position, the external scanner is called first. This makes external scanners a powerful way to override Tree-sitter's
|
||||
default lexing behavior, especially for cases that can't be handled with regular lexical rules, parsing, or dynamic precedence.
|
||||
External scanners have priority over Tree-sitter's normal lexing process. When a token listed in the externals array is
|
||||
valid at a given position, the external scanner is called first. This makes external scanners a powerful way to override
|
||||
Tree-sitter's default lexing behavior, especially for cases that can't be handled with regular lexical rules, parsing, or
|
||||
dynamic precedence.
|
||||
|
||||
During error recovery, Tree-sitter's first step is to call the external scanner's scan function with all tokens marked as
|
||||
valid. Your scanner should detect and handle this case appropriately. One simple approach is to add an unused "sentinel"
|
||||
|
|
|
|||
|
|
@ -39,8 +39,8 @@ It only shows the *named* nodes, as described in [this section][named-vs-anonymo
|
|||
```
|
||||
|
||||
The expected output section can also *optionally* show the [*field names*][node-field-names] associated with each child
|
||||
node. To include field names in your tests, you write a node's field name followed by a colon, before the node itself in
|
||||
the S-expression:
|
||||
node. To include field names in your tests, you write a node's field name followed by a colon, before the node itself
|
||||
in the S-expression:
|
||||
|
||||
```query
|
||||
(source_file
|
||||
|
|
@ -87,6 +87,11 @@ The recommendation is to be comprehensive in adding tests. If it's a visible nod
|
|||
directory. It's typically a good idea to test all the permutations of each language construct. This increases test coverage,
|
||||
but doubly acquaints readers with a way to examine expected outputs and understand the "edges" of a language.
|
||||
|
||||
```admonish tip
|
||||
After modifying the grammar, you can run `tree-sitter test -u`
|
||||
to update all syntax trees in corpus files with current parser output.
|
||||
```
|
||||
|
||||
## Attributes
|
||||
|
||||
Tests can be annotated with a few `attributes`. Attributes must be put in the header, below the test name, and start with
|
||||
|
|
@ -99,8 +104,8 @@ you can repeat the attribute on a new line.
|
|||
|
||||
The following attributes are available:
|
||||
|
||||
* `:cst` - This attribute specifies that the expected output should be in the form of a CST instead of the normal S-expression. This
|
||||
CST matches the format given by `parse --cst`.
|
||||
* `:cst` - This attribute specifies that the expected output should be in the form of a CST instead of the normal S-expression.
|
||||
This CST matches the format given by `parse --cst`.
|
||||
* `:error` — This attribute will assert that the parse tree contains an error. It's useful to just validate that a certain
|
||||
input is invalid without displaying the whole parse tree, as such you should omit the parse tree below the `---` line.
|
||||
* `:fail-fast` — This attribute will stop the testing of additional cases if the test marked with this attribute fails.
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# Creating parsers
|
||||
|
||||
Developing Tree-sitter grammars can have a difficult learning curve, but once you get the hang of it, it can be fun and even
|
||||
zen-like. This document will help you to get started and to develop a useful mental model.
|
||||
Developing Tree-sitter grammars can have a difficult learning curve, but once you get the hang of it, it can be fun and
|
||||
even zen-like. This document will help you to get started and to develop a useful mental model.
|
||||
|
|
|
|||
|
|
@ -10,7 +10,8 @@ file and efficiently update the syntax tree as the source file is edited. Tree-s
|
|||
- **General** enough to parse any programming language
|
||||
- **Fast** enough to parse on every keystroke in a text editor
|
||||
- **Robust** enough to provide useful results even in the presence of syntax errors
|
||||
- **Dependency-free** so that the runtime library (which is written in pure [C11](https://github.com/tree-sitter/tree-sitter/tree/master/lib)) can be embedded in any application
|
||||
- **Dependency-free** so that the runtime library (which is written in pure [C11](https://github.com/tree-sitter/tree-sitter/tree/master/lib))
|
||||
can be embedded in any application
|
||||
|
||||
## Language Bindings
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,8 @@
|
|||
|
||||
## Providing the Code
|
||||
|
||||
In the example on the previous page, we parsed source code stored in a simple string using the `ts_parser_parse_string` function:
|
||||
In the example on the previous page, we parsed source code stored in a simple string using the `ts_parser_parse_string`
|
||||
function:
|
||||
|
||||
```c
|
||||
TSTree *ts_parser_parse_string(
|
||||
|
|
@ -135,10 +136,10 @@ Consider a grammar rule like this:
|
|||
if_statement: $ => seq("if", "(", $._expression, ")", $._statement);
|
||||
```
|
||||
|
||||
A syntax node representing an `if_statement` in this language would have 5 children: the condition expression, the body statement,
|
||||
as well as the `if`, `(`, and `)` tokens. The expression and the statement would be marked as _named_ nodes, because they
|
||||
have been given explicit names in the grammar. But the `if`, `(`, and `)` nodes would _not_ be named nodes, because they
|
||||
are represented in the grammar as simple strings.
|
||||
A syntax node representing an `if_statement` in this language would have 5 children: the condition expression, the body
|
||||
statement, as well as the `if`, `(`, and `)` tokens. The expression and the statement would be marked as _named_ nodes,
|
||||
because they have been given explicit names in the grammar. But the `if`, `(`, and `)` nodes would _not_ be named nodes,
|
||||
because they are represented in the grammar as simple strings.
|
||||
|
||||
You can check whether any given node is named:
|
||||
|
||||
|
|
|
|||
|
|
@ -19,8 +19,8 @@ typedef struct {
|
|||
void ts_tree_edit(TSTree *, const TSInputEdit *);
|
||||
```
|
||||
|
||||
Then, you can call `ts_parser_parse` again, passing in the old tree. This will create a new tree that internally shares structure
|
||||
with the old tree.
|
||||
Then, you can call `ts_parser_parse` again, passing in the old tree. This will create a new tree that internally shares
|
||||
structure with the old tree.
|
||||
|
||||
When you edit a syntax tree, the positions of its nodes will change. If you have stored any `TSNode` instances outside of
|
||||
the `TSTree`, you must update their positions separately, using the same `TSInputEdit` value, in order to update their
|
||||
|
|
|
|||
|
|
@ -108,9 +108,9 @@ In Tree-sitter grammars, there are usually certain rules that represent abstract
|
|||
"type", "declaration"). In the `grammar.js` file, these are often written as [hidden rules][hidden rules]
|
||||
whose definition is a simple [`choice`][grammar dsl] where each member is just a single symbol.
|
||||
|
||||
Normally, hidden rules are not mentioned in the node types file, since they don't appear in the syntax tree. But if you add
|
||||
a hidden rule to the grammar's [`supertypes` list][grammar dsl], then it _will_ show up in the node
|
||||
types file, with the following special entry:
|
||||
Normally, hidden rules are not mentioned in the node types file, since they don't appear in the syntax tree. But if you
|
||||
add a hidden rule to the grammar's [`supertypes` list][grammar dsl], then it _will_ show up in the node types file, with
|
||||
the following special entry:
|
||||
|
||||
- `"subtypes"` — An array of objects that specify the _types_ of nodes that this 'supertype' node can wrap.
|
||||
|
||||
|
|
|
|||
|
|
@ -15,8 +15,11 @@ A given version of the tree-sitter library is only able to load parsers generate
|
|||
| >=0.20.3, <=0.24 | 13 | 14 |
|
||||
| >=0.25 | 13 | 15 |
|
||||
|
||||
By default, the tree-sitter CLI will generate parsers using the latest available ABI for that version, but an older ABI (supported by the CLI) can be selected by passing the [`--abi` option][abi_option] to the `generate` command.
|
||||
By default, the tree-sitter CLI will generate parsers using the latest available ABI for that version, but an older ABI
|
||||
(supported by the CLI) can be selected by passing the [`--abi` option][abi_option] to the `generate` command.
|
||||
|
||||
Note that the ABI version range supported by the CLI can be smaller than for the library: When a new ABI version is released, older versions will be phased out over a deprecation period, which starts with no longer being able to generate parsers with the oldest ABI version.
|
||||
Note that the ABI version range supported by the CLI can be smaller than for the library: When a new ABI version is released,
|
||||
older versions will be phased out over a deprecation period, which starts with no longer being able to generate parsers
|
||||
with the oldest ABI version.
|
||||
|
||||
[abi_option]: ../cli/generate.md#--abi-version
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@ the core concepts remain the same.
|
|||
|
||||
Tree-sitter's parsing functionality is implemented through its C API, with all functions documented in the [tree_sitter/api.h][api.h]
|
||||
header file, but if you're working in another language, you can use one of the following bindings found [here](../index.md#language-bindings),
|
||||
each providing idiomatic access to Tree-sitter's functionality. Of these bindings, the official ones have their own API docs
|
||||
hosted online at the following pages:
|
||||
each providing idiomatic access to Tree-sitter's functionality. Of these bindings, the official ones have their own API
|
||||
doc hosted online at the following pages:
|
||||
|
||||
- [Go][go]
|
||||
- [Java]
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
# Query Syntax
|
||||
|
||||
A _query_ consists of one or more _patterns_, where each pattern is an [S-expression][s-exp] that matches a certain set of
|
||||
nodes in a syntax tree. The expression to match a given node consists of a pair of parentheses containing two things: the
|
||||
node's type, and optionally, a series of other S-expressions that match the node's children. For example, this pattern would
|
||||
match any `binary_expression` node whose children are both `number_literal` nodes:
|
||||
A _query_ consists of one or more _patterns_, where each pattern is an [S-expression][s-exp] that matches a certain set
|
||||
of nodes in a syntax tree. The expression to match a given node consists of a pair of parentheses containing two things:
|
||||
the node's type, and optionally, a series of other S-expressions that match the node's children. For example, this pattern
|
||||
would match any `binary_expression` node whose children are both `number_literal` nodes:
|
||||
|
||||
```query
|
||||
(binary_expression (number_literal) (number_literal))
|
||||
|
|
@ -99,10 +99,10 @@ by `(ERROR)` queries. Specific missing node types can also be queried:
|
|||
### Supertype Nodes
|
||||
|
||||
Some node types are marked as _supertypes_ in a grammar. A supertype is a node type that contains multiple
|
||||
subtypes. For example, in the [JavaScript grammar example][grammar], `expression` is a supertype that can represent any kind
|
||||
of expression, such as a `binary_expression`, `call_expression`, or `identifier`. You can use supertypes in queries to match
|
||||
any of their subtypes, rather than having to list out each subtype individually. For example, this pattern would match any
|
||||
kind of expression, even though it's not a visible node in the syntax tree:
|
||||
subtypes. For example, in the [JavaScript grammar example][grammar], `expression` is a supertype that can represent any
|
||||
kind of expression, such as a `binary_expression`, `call_expression`, or `identifier`. You can use supertypes in queries
|
||||
to match any of their subtypes, rather than having to list out each subtype individually. For example, this pattern would
|
||||
match any kind of expression, even though it's not a visible node in the syntax tree:
|
||||
|
||||
```query
|
||||
(expression) @any-expression
|
||||
|
|
|
|||
|
|
@ -128,15 +128,15 @@ This pattern would match any builtin variable that is not a local variable, beca
|
|||
|
||||
# Directives
|
||||
|
||||
Similar to predicates, directives are a way to associate arbitrary metadata with a pattern. The only difference between predicates
|
||||
and directives is that directives end in a `!` character instead of `?` character.
|
||||
Similar to predicates, directives are a way to associate arbitrary metadata with a pattern. The only difference between
|
||||
predicates and directives is that directives end in a `!` character instead of `?` character.
|
||||
|
||||
Tree-sitter's CLI supports the following directives by default:
|
||||
|
||||
## The `set!` directive
|
||||
|
||||
This directive allows you to associate key-value pairs with a pattern. The key and value can be any arbitrary text that you
|
||||
see fit.
|
||||
This directive allows you to associate key-value pairs with a pattern. The key and value can be any arbitrary text that
|
||||
you see fit.
|
||||
|
||||
```query
|
||||
((comment) @injection.content
|
||||
|
|
@ -156,8 +156,8 @@ another capture are preserved. It takes two arguments, both of which are capture
|
|||
### The `#strip!` directive
|
||||
|
||||
The `#strip!` directive allows you to remove text from a capture. It takes two arguments: the first is the capture to strip
|
||||
text from, and the second is a regular expression to match against the text. Any text matched by the regular expression will
|
||||
be removed from the text associated with the capture.
|
||||
text from, and the second is a regular expression to match against the text. Any text matched by the regular expression
|
||||
will be removed from the text associated with the capture.
|
||||
|
||||
For an example on the `#select-adjacent!` and `#strip!` directives,
|
||||
view the [code navigation](../../4-code-navigation.md#examples) documentation.
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
eachSystem = lib.genAttrs systems;
|
||||
pkgsFor = inputs.nixpkgs.legacyPackages;
|
||||
|
||||
version = "0.26.3";
|
||||
version = "0.27.0";
|
||||
|
||||
fs = lib.fileset;
|
||||
src = fs.toSource {
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ fn main() {
|
|||
.include(&include_path)
|
||||
.define("_POSIX_C_SOURCE", "200112L")
|
||||
.define("_DEFAULT_SOURCE", None)
|
||||
.define("_BSD_SOURCE", None)
|
||||
.define("_DARWIN_C_SOURCE", None)
|
||||
.warnings(false)
|
||||
.file(src_path.join("lib.c"))
|
||||
|
|
|
|||
|
|
@ -317,7 +317,7 @@ pub trait Decode {
|
|||
|
||||
/// A stateful object for walking a syntax [`Tree`] efficiently.
|
||||
#[doc(alias = "TSTreeCursor")]
|
||||
pub struct TreeCursor<'cursor>(ffi::TSTreeCursor, PhantomData<&'cursor ()>);
|
||||
pub struct TreeCursor<'tree>(ffi::TSTreeCursor, PhantomData<&'tree ()>);
|
||||
|
||||
/// A set of patterns that match nodes in a syntax tree.
|
||||
#[doc(alias = "TSQuery")]
|
||||
|
|
@ -392,7 +392,7 @@ pub struct QueryMatch<'cursor, 'tree> {
|
|||
}
|
||||
|
||||
/// A sequence of [`QueryMatch`]es associated with a given [`QueryCursor`].
|
||||
pub struct QueryMatches<'query, 'tree: 'query, T: TextProvider<I>, I: AsRef<[u8]>> {
|
||||
pub struct QueryMatches<'query, 'tree, T: TextProvider<I>, I: AsRef<[u8]>> {
|
||||
ptr: *mut ffi::TSQueryCursor,
|
||||
query: &'query Query,
|
||||
text_provider: T,
|
||||
|
|
@ -407,7 +407,7 @@ pub struct QueryMatches<'query, 'tree: 'query, T: TextProvider<I>, I: AsRef<[u8]
|
|||
///
|
||||
/// During iteration, each element contains a [`QueryMatch`] and index. The index can
|
||||
/// be used to access the new capture inside of the [`QueryMatch::captures`]'s [`captures`].
|
||||
pub struct QueryCaptures<'query, 'tree: 'query, T: TextProvider<I>, I: AsRef<[u8]>> {
|
||||
pub struct QueryCaptures<'query, 'tree, T: TextProvider<I>, I: AsRef<[u8]>> {
|
||||
ptr: *mut ffi::TSQueryCursor,
|
||||
query: &'query Query,
|
||||
text_provider: T,
|
||||
|
|
@ -1581,7 +1581,7 @@ impl<'tree> Node<'tree> {
|
|||
/// Get the [`Language`] that was used to parse this node's syntax tree.
|
||||
#[doc(alias = "ts_node_language")]
|
||||
#[must_use]
|
||||
pub fn language(&self) -> LanguageRef {
|
||||
pub fn language(&self) -> LanguageRef<'tree> {
|
||||
LanguageRef(unsafe { ffi::ts_node_language(self.0) }, PhantomData)
|
||||
}
|
||||
|
||||
|
|
@ -2082,11 +2082,11 @@ impl fmt::Display for Node<'_> {
|
|||
}
|
||||
}
|
||||
|
||||
impl<'cursor> TreeCursor<'cursor> {
|
||||
impl<'tree> TreeCursor<'tree> {
|
||||
/// Get the tree cursor's current [`Node`].
|
||||
#[doc(alias = "ts_tree_cursor_current_node")]
|
||||
#[must_use]
|
||||
pub fn node(&self) -> Node<'cursor> {
|
||||
pub fn node(&self) -> Node<'tree> {
|
||||
Node(
|
||||
unsafe { ffi::ts_tree_cursor_current_node(&self.0) },
|
||||
PhantomData,
|
||||
|
|
@ -2227,7 +2227,7 @@ impl<'cursor> TreeCursor<'cursor> {
|
|||
/// Re-initialize this tree cursor to start at the original node that the
|
||||
/// cursor was constructed with.
|
||||
#[doc(alias = "ts_tree_cursor_reset")]
|
||||
pub fn reset(&mut self, node: Node<'cursor>) {
|
||||
pub fn reset(&mut self, node: Node<'tree>) {
|
||||
unsafe { ffi::ts_tree_cursor_reset(&mut self.0, node.0) };
|
||||
}
|
||||
|
||||
|
|
@ -3404,7 +3404,7 @@ impl QueryProperty {
|
|||
/// Provide a `StreamingIterator` instead of the traditional `Iterator`, as the
|
||||
/// underlying object in the C library gets updated on each iteration. Copies would
|
||||
/// have their internal state overwritten, leading to Undefined Behavior
|
||||
impl<'query, 'tree: 'query, T: TextProvider<I>, I: AsRef<[u8]>> StreamingIterator
|
||||
impl<'query, 'tree, T: TextProvider<I>, I: AsRef<[u8]>> StreamingIterator
|
||||
for QueryMatches<'query, 'tree, T, I>
|
||||
{
|
||||
type Item = QueryMatch<'query, 'tree>;
|
||||
|
|
@ -3435,15 +3435,13 @@ impl<'query, 'tree: 'query, T: TextProvider<I>, I: AsRef<[u8]>> StreamingIterato
|
|||
}
|
||||
}
|
||||
|
||||
impl<'query, 'tree: 'query, T: TextProvider<I>, I: AsRef<[u8]>> StreamingIteratorMut
|
||||
for QueryMatches<'query, 'tree, T, I>
|
||||
{
|
||||
impl<T: TextProvider<I>, I: AsRef<[u8]>> StreamingIteratorMut for QueryMatches<'_, '_, T, I> {
|
||||
fn get_mut(&mut self) -> Option<&mut Self::Item> {
|
||||
self.current_match.as_mut()
|
||||
}
|
||||
}
|
||||
|
||||
impl<'query, 'tree: 'query, T: TextProvider<I>, I: AsRef<[u8]>> StreamingIterator
|
||||
impl<'query, 'tree, T: TextProvider<I>, I: AsRef<[u8]>> StreamingIterator
|
||||
for QueryCaptures<'query, 'tree, T, I>
|
||||
{
|
||||
type Item = (QueryMatch<'query, 'tree>, usize);
|
||||
|
|
@ -3480,9 +3478,7 @@ impl<'query, 'tree: 'query, T: TextProvider<I>, I: AsRef<[u8]>> StreamingIterato
|
|||
}
|
||||
}
|
||||
|
||||
impl<'query, 'tree: 'query, T: TextProvider<I>, I: AsRef<[u8]>> StreamingIteratorMut
|
||||
for QueryCaptures<'query, 'tree, T, I>
|
||||
{
|
||||
impl<T: TextProvider<I>, I: AsRef<[u8]>> StreamingIteratorMut for QueryCaptures<'_, '_, T, I> {
|
||||
fn get_mut(&mut self) -> Option<&mut Self::Item> {
|
||||
self.current_match.as_mut()
|
||||
}
|
||||
|
|
@ -3622,8 +3618,8 @@ impl From<ffi::TSRange> for Range {
|
|||
}
|
||||
}
|
||||
|
||||
impl From<&'_ InputEdit> for ffi::TSInputEdit {
|
||||
fn from(val: &'_ InputEdit) -> Self {
|
||||
impl From<&InputEdit> for ffi::TSInputEdit {
|
||||
fn from(val: &InputEdit) -> Self {
|
||||
Self {
|
||||
start_byte: val.start_byte as u32,
|
||||
old_end_byte: val.old_end_byte as u32,
|
||||
|
|
|
|||
4
lib/binding_web/package-lock.json
generated
4
lib/binding_web/package-lock.json
generated
|
|
@ -1,12 +1,12 @@
|
|||
{
|
||||
"name": "web-tree-sitter",
|
||||
"version": "0.26.3",
|
||||
"version": "0.27.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "web-tree-sitter",
|
||||
"version": "0.26.3",
|
||||
"version": "0.27.0",
|
||||
"license": "MIT",
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.39.1",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "web-tree-sitter",
|
||||
"version": "0.26.3",
|
||||
"version": "0.27.0",
|
||||
"description": "Tree-sitter bindings for the web",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
|
|
|
|||
8
lib/binding_web/src/finalization_registry.ts
Normal file
8
lib/binding_web/src/finalization_registry.ts
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
export function newFinalizer<T>(handler: (value: T) => void): FinalizationRegistry<T> | undefined {
|
||||
try {
|
||||
return new FinalizationRegistry(handler);
|
||||
} catch(e) {
|
||||
console.error('Unsupported FinalizationRegistry:', e);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,10 @@
|
|||
import { C, Internal, assertInternal } from './constants';
|
||||
import { Language } from './language';
|
||||
import { newFinalizer } from './finalization_registry';
|
||||
|
||||
const finalizer = newFinalizer((address: number) => {
|
||||
C._ts_lookahead_iterator_delete(address);
|
||||
});
|
||||
|
||||
export class LookaheadIterator implements Iterable<string> {
|
||||
/** @internal */
|
||||
|
|
@ -13,6 +18,7 @@ export class LookaheadIterator implements Iterable<string> {
|
|||
assertInternal(internal);
|
||||
this[0] = address;
|
||||
this.language = language;
|
||||
finalizer?.register(this, address, this);
|
||||
}
|
||||
|
||||
/** Get the current symbol of the lookahead iterator. */
|
||||
|
|
@ -27,6 +33,7 @@ export class LookaheadIterator implements Iterable<string> {
|
|||
|
||||
/** Delete the lookahead iterator, freeing its resources. */
|
||||
delete(): void {
|
||||
finalizer?.unregister(this);
|
||||
C._ts_lookahead_iterator_delete(this[0]);
|
||||
this[0] = 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import { Language } from './language';
|
|||
import { marshalRange, unmarshalRange } from './marshal';
|
||||
import { checkModule, initializeBinding } from './bindings';
|
||||
import { Tree } from './tree';
|
||||
import { newFinalizer } from './finalization_registry';
|
||||
|
||||
/**
|
||||
* Options for parsing
|
||||
|
|
@ -82,6 +83,11 @@ export let LANGUAGE_VERSION: number;
|
|||
*/
|
||||
export let MIN_COMPATIBLE_VERSION: number;
|
||||
|
||||
const finalizer = newFinalizer((addresses: number[]) => {
|
||||
C._ts_parser_delete(addresses[0]);
|
||||
C._free(addresses[1]);
|
||||
});
|
||||
|
||||
/**
|
||||
* A stateful object that is used to produce a {@link Tree} based on some
|
||||
* source code.
|
||||
|
|
@ -117,6 +123,7 @@ export class Parser {
|
|||
*/
|
||||
constructor() {
|
||||
this.initialize();
|
||||
finalizer?.register(this, [this[0], this[1]], this);
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
|
|
@ -131,6 +138,7 @@ export class Parser {
|
|||
|
||||
/** Delete the parser, freeing its resources. */
|
||||
delete() {
|
||||
finalizer?.unregister(this);
|
||||
C._ts_parser_delete(this[0]);
|
||||
C._free(this[1]);
|
||||
this[0] = 0;
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import { Node } from './node';
|
|||
import { marshalNode, unmarshalCaptures } from './marshal';
|
||||
import { TRANSFER_BUFFER } from './parser';
|
||||
import { Language } from './language';
|
||||
import { newFinalizer } from './finalization_registry';
|
||||
|
||||
const PREDICATE_STEP_TYPE_CAPTURE = 1;
|
||||
|
||||
|
|
@ -506,6 +507,10 @@ function parsePattern(
|
|||
}
|
||||
}
|
||||
|
||||
const finalizer = newFinalizer((address: number) => {
|
||||
C._ts_query_delete(address);
|
||||
});
|
||||
|
||||
export class Query {
|
||||
/** @internal */
|
||||
private [0] = 0; // Internal handle for Wasm
|
||||
|
|
@ -687,10 +692,12 @@ export class Query {
|
|||
this.assertedProperties = assertedProperties;
|
||||
this.refutedProperties = refutedProperties;
|
||||
this.exceededMatchLimit = false;
|
||||
finalizer?.register(this, address, this);
|
||||
}
|
||||
|
||||
/** Delete the query, freeing its resources. */
|
||||
delete(): void {
|
||||
finalizer?.unregister(this);
|
||||
C._ts_query_delete(this[0]);
|
||||
this[0] = 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import { TreeCursor } from './tree_cursor';
|
|||
import { marshalEdit, marshalPoint, unmarshalNode, unmarshalRange } from './marshal';
|
||||
import { TRANSFER_BUFFER } from './parser';
|
||||
import { Edit } from './edit';
|
||||
import { newFinalizer } from './finalization_registry';
|
||||
|
||||
/** @internal */
|
||||
export function getText(tree: Tree, startIndex: number, endIndex: number, startPosition: Point): string {
|
||||
|
|
@ -28,6 +29,10 @@ export function getText(tree: Tree, startIndex: number, endIndex: number, startP
|
|||
return result ?? '';
|
||||
}
|
||||
|
||||
const finalizer = newFinalizer((address: number) => {
|
||||
C._ts_tree_delete(address);
|
||||
});
|
||||
|
||||
/** A tree that represents the syntactic structure of a source code file. */
|
||||
export class Tree {
|
||||
/** @internal */
|
||||
|
|
@ -45,6 +50,7 @@ export class Tree {
|
|||
this[0] = address;
|
||||
this.language = language;
|
||||
this.textCallback = textCallback;
|
||||
finalizer?.register(this, address, this);
|
||||
}
|
||||
|
||||
/** Create a shallow copy of the syntax tree. This is very fast. */
|
||||
|
|
@ -55,6 +61,7 @@ export class Tree {
|
|||
|
||||
/** Delete the syntax tree, freeing its resources. */
|
||||
delete(): void {
|
||||
finalizer?.unregister(this);
|
||||
C._ts_tree_delete(this[0]);
|
||||
this[0] = 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,11 @@ import { marshalNode, marshalPoint, marshalTreeCursor, unmarshalNode, unmarshalP
|
|||
import { Node } from './node';
|
||||
import { TRANSFER_BUFFER } from './parser';
|
||||
import { getText, Tree } from './tree';
|
||||
import { newFinalizer } from './finalization_registry';
|
||||
|
||||
const finalizer = newFinalizer((address: number) => {
|
||||
C._ts_tree_cursor_delete_wasm(address);
|
||||
});
|
||||
|
||||
/** A stateful object for walking a syntax {@link Tree} efficiently. */
|
||||
export class TreeCursor {
|
||||
|
|
@ -30,6 +35,7 @@ export class TreeCursor {
|
|||
assertInternal(internal);
|
||||
this.tree = tree;
|
||||
unmarshalTreeCursor(this);
|
||||
finalizer?.register(this, this.tree[0], this);
|
||||
}
|
||||
|
||||
/** Creates a deep copy of the tree cursor. This allocates new memory. */
|
||||
|
|
@ -42,6 +48,7 @@ export class TreeCursor {
|
|||
|
||||
/** Delete the tree cursor, freeing its resources. */
|
||||
delete(): void {
|
||||
finalizer?.unregister(this);
|
||||
marshalTreeCursor(this);
|
||||
C._ts_tree_cursor_delete_wasm(this.tree[0]);
|
||||
this[0] = this[1] = this[2] = 0;
|
||||
|
|
|
|||
74
lib/binding_web/test/memory.test.ts
Normal file
74
lib/binding_web/test/memory.test.ts
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { gc, event, Finalizer } from './memory';
|
||||
|
||||
// hijack finalization registry before import web-tree-sitter
|
||||
globalThis.FinalizationRegistry = Finalizer;
|
||||
|
||||
describe('Memory Management', () => {
|
||||
describe('call .delete()', () => {
|
||||
it('test free memory manually', async () => {
|
||||
const timer = setInterval(() => {
|
||||
gc();
|
||||
}, 100);
|
||||
let done = 0;
|
||||
event.on('gc', () => {
|
||||
done++;
|
||||
});
|
||||
await (async () => {
|
||||
const { JavaScript } = await (await import('./helper')).default;
|
||||
const { Parser, Query } = await import('../src');
|
||||
const parser = new Parser();
|
||||
parser.setLanguage(JavaScript);
|
||||
const tree = parser.parse('1+1')!;
|
||||
const copyTree = tree.copy();
|
||||
const cursor = tree.walk();
|
||||
const copyCursor = cursor.copy();
|
||||
const lookaheadIterator = JavaScript.lookaheadIterator(cursor.currentNode.nextParseState)!;
|
||||
const query = new Query(JavaScript, '(identifier) @element');
|
||||
parser.delete();
|
||||
tree.delete();
|
||||
copyTree.delete();
|
||||
cursor.delete();
|
||||
copyCursor.delete();
|
||||
lookaheadIterator.delete();
|
||||
query.delete();
|
||||
})();
|
||||
// wait for gc
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
clearInterval(timer);
|
||||
// expect no gc event fired
|
||||
expect(done).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('do not call .delete()', () => {
|
||||
it('test free memory automatically', async () => {
|
||||
const timer = setInterval(() => {
|
||||
gc();
|
||||
}, 100);
|
||||
let done = 0;
|
||||
const promise = new Promise((resolve) => {
|
||||
event.on('gc', () => {
|
||||
if (++done === 7) {
|
||||
resolve(undefined);
|
||||
clearInterval(timer);
|
||||
}
|
||||
console.log('free memory times: ', done);
|
||||
});
|
||||
});
|
||||
await (async () => {
|
||||
const { JavaScript } = await (await import('./helper')).default;
|
||||
const { Parser, Query } = await import('../src');
|
||||
const parser = new Parser(); // 1
|
||||
parser.setLanguage(JavaScript);
|
||||
const tree = parser.parse('1+1')!; // 2
|
||||
tree.copy(); // 3
|
||||
const cursor = tree.walk(); // 4
|
||||
cursor.copy(); // 5
|
||||
JavaScript.lookaheadIterator(cursor.currentNode.nextParseState)!; // 6
|
||||
new Query(JavaScript, '(identifier) @element'); // 7
|
||||
})();
|
||||
await promise;
|
||||
});
|
||||
});
|
||||
});
|
||||
20
lib/binding_web/test/memory.ts
Normal file
20
lib/binding_web/test/memory.ts
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
import { EventEmitter } from 'events';
|
||||
import { Session } from 'inspector';
|
||||
|
||||
const session = new Session();
|
||||
session.connect();
|
||||
|
||||
export function gc() {
|
||||
session.post('HeapProfiler.collectGarbage');
|
||||
}
|
||||
|
||||
export const event = new EventEmitter();
|
||||
|
||||
export class Finalizer<T> extends FinalizationRegistry<T> {
|
||||
constructor(handler: (value: T) => void) {
|
||||
super((value) => {
|
||||
handler(value);
|
||||
event.emit('gc');
|
||||
});
|
||||
}
|
||||
}
|
||||
25
lib/binding_web/test/memory_unsupported.test.ts
Normal file
25
lib/binding_web/test/memory_unsupported.test.ts
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
import { describe, it } from 'vitest';
|
||||
|
||||
describe('FinalizationRegistry is unsupported', () => {
|
||||
it('test FinalizationRegistry is unsupported', async () => {
|
||||
// @ts-expect-error: test FinalizationRegistry is not supported
|
||||
globalThis.FinalizationRegistry = undefined;
|
||||
const { JavaScript } = await (await import('./helper')).default;
|
||||
const { Parser, Query } = await import('../src');
|
||||
const parser = new Parser();
|
||||
parser.setLanguage(JavaScript);
|
||||
const tree = parser.parse('1+1')!;
|
||||
const copyTree = tree.copy();
|
||||
const cursor = tree.walk();
|
||||
const copyCursor = cursor.copy();
|
||||
const lookaheadIterator = JavaScript.lookaheadIterator(cursor.currentNode.nextParseState)!;
|
||||
const query = new Query(JavaScript, '(identifier) @element');
|
||||
parser.delete();
|
||||
tree.delete();
|
||||
copyTree.delete();
|
||||
cursor.delete();
|
||||
copyCursor.delete();
|
||||
lookaheadIterator.delete();
|
||||
query.delete();
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large
Load diff
Loading…
Add table
Add a link
Reference in a new issue