traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into include-symbol_id-in-node-types-json

Browse source
This commit is contained in:
bglgwyng 2025-11-19 21:53:39 +09:00
commit f1f11bde00
37 changed files with 1090 additions and 478 deletions
Download patch file Download diff file Expand all files Collapse all files

65
Cargo.lock generated
View file

@ -67,22 +67,22 @@ dependencies = [
[[package]]
name = "anstyle-query"
version = "1.1.4"
version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2"
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
dependencies = [
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
name = "anstyle-wincon"
version = "3.0.10"
version = "3.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a"
checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
dependencies = [
"anstyle",
"once_cell_polyfill",
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@ -187,9 +187,9 @@ checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a"
[[package]]
name = "cc"
version = "1.2.44"
version = "1.2.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37521ac7aabe3d13122dc382493e20c9416f299d2ccd5b3a5340a2570cdeb0f3"
checksum = "b97463e1064cb1b1c1384ad0a0b9c8abd0988e2a91f52606c80ef14aadb63e36"
dependencies = [
"find-msvc-tools",
"shlex",
@ -236,7 +236,7 @@ checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
dependencies = [
"glob",
"libc",
"libloading",
"libloading 0.8.9",
]
[[package]]
@ -664,9 +664,9 @@ dependencies = [
[[package]]
name = "find-msvc-tools"
version = "0.1.4"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127"
checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844"
[[package]]
name = "fnv"
@ -1058,6 +1058,16 @@ dependencies = [
"windows-link",
]
[[package]]
name = "libloading"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60"
dependencies = [
"cfg-if",
"windows-link",
]
[[package]]
name = "libm"
version = "0.2.15"
@ -1391,9 +1401,9 @@ dependencies = [
[[package]]
name = "quote"
version = "1.0.41"
version = "1.0.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1"
checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f"
dependencies = [
"proc-macro2",
]
@ -1621,9 +1631,9 @@ dependencies = [
[[package]]
name = "schemars"
version = "1.0.4"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82d20c4491bc164fa2f6c5d44565947a52ad80b9505d8e36f8d54c27c739fcd0"
checksum = "9558e172d4e8533736ba97870c4b2cd63f84b382a3d6eb063da41b91cce17289"
dependencies = [
"dyn-clone",
"ref-cast",
@ -1634,9 +1644,9 @@ dependencies = [
[[package]]
name = "schemars_derive"
version = "1.0.4"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33d020396d1d138dc19f1165df7545479dcd58d93810dc5d646a16e55abefa80"
checksum = "301858a4023d78debd2353c7426dc486001bddc91ae31a76fb1f55132f7e2633"
dependencies = [
"proc-macro2",
"quote",
@ -1774,9 +1784,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]]
name = "syn"
version = "2.0.108"
version = "2.0.110"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da58917d35242480a05c2897064da0a80589a2a0476c9a3f2fdc83b53502e917"
checksum = "a99801b5bd34ede4cf3fc688c5919368fea4e4814a4664359503e6015b280aea"
dependencies = [
"proc-macro2",
"quote",
@ -2016,7 +2026,7 @@ dependencies = [
"tree-sitter-tests-proc-macro",
"unindent",
"walkdir",
"wasmparser 0.240.0",
"wasmparser 0.241.2",
"webbrowser",
"widestring",
]
@ -2025,18 +2035,17 @@ dependencies = [
name = "tree-sitter-config"
version = "0.26.0"
dependencies = [
"anyhow",
"etcetera",
"log",
"serde",
"serde_json",
"thiserror 2.0.17",
]
[[package]]
name = "tree-sitter-generate"
version = "0.26.0"
dependencies = [
"anyhow",
"bitflags 2.10.0",
"dunce",
"indexmap",
@ -2074,12 +2083,11 @@ version = "0.1.5"
name = "tree-sitter-loader"
version = "0.26.0"
dependencies = [
"anyhow",
"cc",
"etcetera",
"fs4",
"indoc",
"libloading",
"libloading 0.9.0",
"log",
"once_cell",
"regex",
@ -2087,6 +2095,7 @@ dependencies = [
"serde",
"serde_json",
"tempfile",
"thiserror 2.0.17",
"tree-sitter",
"tree-sitter-highlight",
"tree-sitter-tags",
@ -2114,9 +2123,9 @@ dependencies = [
[[package]]
name = "unicode-ident"
version = "1.0.20"
version = "1.0.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "462eeb75aeb73aea900253ce739c8e18a67423fadf006037cd3ff27e82748a06"
checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5"
[[package]]
name = "unicode-segmentation"
@ -2261,9 +2270,9 @@ dependencies = [
[[package]]
name = "wasmparser"
version = "0.240.0"
version = "0.241.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b722dcf61e0ea47440b53ff83ccb5df8efec57a69d150e4f24882e4eba7e24a4"
checksum = "46d90019b1afd4b808c263e428de644f3003691f243387d30d673211ee0cb8e8"
dependencies = [
"bitflags 2.10.0",
"hashbrown 0.15.5",

8
Cargo.toml
View file

@ -106,7 +106,7 @@ ansi_colours = "1.2.3"
anstyle = "1.0.13"
anyhow = "1.0.100"
bstr = "1.12.0"
cc = "1.2.44"
cc = "1.2.46"
clap = { version = "4.5.51", features = [
"cargo",
"derive",
@ -128,7 +128,7 @@ heck = "0.5.0"
html-escape = "0.2.13"
indexmap = "2.11.4"
indoc = "2.0.6"
libloading = "0.8.9"
libloading = "0.9.0"
log = { version = "0.4.28", features = ["std"] }
memchr = "2.7.6"
once_cell = "1.21.3"
@ -137,7 +137,7 @@ rand = "0.8.5"
regex = "1.11.3"
regex-syntax = "0.8.6"
rustc-hash = "2.1.1"
schemars = "1.0.4"
schemars = "1.0.5"
semver = { version = "1.0.27", features = ["serde"] }
serde = { version = "1.0.219", features = ["derive"] }
serde_json = { version = "1.0.145", features = ["preserve_order"] }
@ -150,7 +150,7 @@ tiny_http = "0.12.0"
topological-sort = "0.2.2"
unindent = "0.2.4"
walkdir = "2.5.0"
wasmparser = "0.240.0"
wasmparser = "0.241.2"
webbrowser = "1.0.5"
tree-sitter = { version = "0.26.0", path = "./lib" }

1
crates/cli/npm/dsl.d.ts vendored
View file

@ -29,6 +29,7 @@ type Rule =
| PrecRule
| Repeat1Rule
| RepeatRule
| ReservedRule
| SeqRule
| StringRule
| SymbolRule<string>

224
crates/cli/src/init.rs
View file

@ -14,7 +14,11 @@ use semver::Version;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use tree_sitter_generate::write_file;
use tree_sitter_loader::{Author, Bindings, Grammar, Links, Metadata, PathsJSON, TreeSitterJSON};
use tree_sitter_loader::{
Author, Bindings, Grammar, Links, Metadata, PathsJSON, TreeSitterJSON,
DEFAULT_HIGHLIGHTS_QUERY_FILE_NAME, DEFAULT_INJECTIONS_QUERY_FILE_NAME,
DEFAULT_LOCALS_QUERY_FILE_NAME, DEFAULT_TAGS_QUERY_FILE_NAME,
};
const CLI_VERSION: &str = env!("CARGO_PKG_VERSION");
const CLI_VERSION_PLACEHOLDER: &str = "CLI_VERSION";
@ -60,6 +64,11 @@ const AUTHOR_EMAIL_PLACEHOLDER_GRAMMAR: &str = " PARSER_AUTHOR_EMAIL";
const FUNDING_URL_PLACEHOLDER: &str = "FUNDING_URL";
const HIGHLIGHTS_QUERY_PATH_PLACEHOLDER: &str = "HIGHLIGHTS_QUERY_PATH";
const INJECTIONS_QUERY_PATH_PLACEHOLDER: &str = "INJECTIONS_QUERY_PATH";
const LOCALS_QUERY_PATH_PLACEHOLDER: &str = "LOCALS_QUERY_PATH";
const TAGS_QUERY_PATH_PLACEHOLDER: &str = "TAGS_QUERY_PATH";
const GRAMMAR_JS_TEMPLATE: &str = include_str!("./templates/grammar.js");
const PACKAGE_JSON_TEMPLATE: &str = include_str!("./templates/package.json");
const GITIGNORE_TEMPLATE: &str = include_str!("./templates/gitignore");
@ -205,6 +214,10 @@ struct GenerateOpts<'a> {
camel_parser_name: &'a str,
title_parser_name: &'a str,
class_name: &'a str,
highlights_query_path: &'a str,
injections_query_path: &'a str,
locals_query_path: &'a str,
tags_query_path: &'a str,
}
pub fn generate_grammar_files(
@ -255,6 +268,11 @@ pub fn generate_grammar_files(
.clone()
.unwrap_or_else(|| format!("TreeSitter{}", language_name.to_upper_camel_case()));
let default_highlights_path = Path::new("queries").join(DEFAULT_HIGHLIGHTS_QUERY_FILE_NAME);
let default_injections_path = Path::new("queries").join(DEFAULT_INJECTIONS_QUERY_FILE_NAME);
let default_locals_path = Path::new("queries").join(DEFAULT_LOCALS_QUERY_FILE_NAME);
let default_tags_path = Path::new("queries").join(DEFAULT_TAGS_QUERY_FILE_NAME);
let generate_opts = GenerateOpts {
author_name: authors
.map(|a| a.first().map(|a| a.name.as_str()))
@ -281,6 +299,18 @@ pub fn generate_grammar_files(
camel_parser_name: &camel_name,
title_parser_name: &title_name,
class_name: &class_name,
highlights_query_path: tree_sitter_config.grammars[0]
.highlights
.to_variable_value(&default_highlights_path),
injections_query_path: tree_sitter_config.grammars[0]
.injections
.to_variable_value(&default_injections_path),
locals_query_path: tree_sitter_config.grammars[0]
.locals
.to_variable_value(&default_locals_path),
tags_query_path: tree_sitter_config.grammars[0]
.tags
.to_variable_value(&default_tags_path),
};
// Create package.json
@ -388,8 +418,47 @@ pub fn generate_grammar_files(
// Generate Rust bindings
if tree_sitter_config.bindings.rust {
missing_path(bindings_dir.join("rust"), create_dir)?.apply(|path| {
missing_path(path.join("lib.rs"), |path| {
missing_path_else(path.join("lib.rs"), allow_update, |path| {
generate_file(path, LIB_RS_TEMPLATE, language_name, &generate_opts)
}, |path| {
let mut contents = fs::read_to_string(path)?;
if !contents.contains("#[cfg(with_highlights_query)]") {
let replacement = indoc! {r#"
#[cfg(with_highlights_query)]
/// The syntax highlighting query for this grammar.
pub const HIGHLIGHTS_QUERY: &str = include_str!("../../HIGHLIGHTS_QUERY_PATH");
#[cfg(with_injections_query)]
/// The language injection query for this grammar.
pub const INJECTIONS_QUERY: &str = include_str!("../../INJECTIONS_QUERY_PATH");
#[cfg(with_locals_query)]
/// The local variable query for this grammar.
pub const LOCALS_QUERY: &str = include_str!("../../LOCALS_QUERY_PATH");
#[cfg(with_tags_query)]
/// The symbol tagging query for this grammar.
pub const TAGS_QUERY: &str = include_str!("../../TAGS_QUERY_PATH");
"#}
.replace("HIGHLIGHTS_QUERY_PATH", generate_opts.highlights_query_path)
.replace("INJECTIONS_QUERY_PATH", generate_opts.injections_query_path)
.replace("LOCALS_QUERY_PATH", generate_opts.locals_query_path)
.replace("TAGS_QUERY_PATH", generate_opts.tags_query_path);
contents = contents
.replace(
indoc! {r#"
// NOTE: uncomment these to include any queries that this grammar contains:
// pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
// pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm");
// pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm");
// pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm");
"#},
&replacement,
);
}
write_file(path, contents)?;
Ok(())
})?;
missing_path_else(
@ -397,39 +466,78 @@ pub fn generate_grammar_files(
allow_update,
|path| generate_file(path, BUILD_RS_TEMPLATE, language_name, &generate_opts),
|path| {
let replacement = indoc!{r#"
c_config.flag("-utf-8");
if std::env::var("TARGET").unwrap() == "wasm32-unknown-unknown" {
let Ok(wasm_headers) = std::env::var("DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS") else {
panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS must be set by the language crate");
};
let Ok(wasm_src) =
std::env::var("DEP_TREE_SITTER_LANGUAGE_WASM_SRC").map(std::path::PathBuf::from)
else {
panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_SRC must be set by the language crate");
};
c_config.include(&wasm_headers);
c_config.files([
wasm_src.join("stdio.c"),
wasm_src.join("stdlib.c"),
wasm_src.join("string.c"),
]);
}
"#};
let indented_replacement = replacement
.lines()
.map(|line| if line.is_empty() { line.to_string() } else { format!(" {line}") })
.collect::<Vec<_>>()
.join("\n");
let mut contents = fs::read_to_string(path)?;
if !contents.contains("wasm32-unknown-unknown") {
let replacement = indoc!{r#"
c_config.flag("-utf-8");
if std::env::var("TARGET").unwrap() == "wasm32-unknown-unknown" {
let Ok(wasm_headers) = std::env::var("DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS") else {
panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_HEADERS must be set by the language crate");
};
let Ok(wasm_src) =
std::env::var("DEP_TREE_SITTER_LANGUAGE_WASM_SRC").map(std::path::PathBuf::from)
else {
panic!("Environment variable DEP_TREE_SITTER_LANGUAGE_WASM_SRC must be set by the language crate");
};
c_config.include(&wasm_headers);
c_config.files([
wasm_src.join("stdio.c"),
wasm_src.join("stdlib.c"),
wasm_src.join("string.c"),
]);
}
"#};
let indented_replacement = replacement
.lines()
.map(|line| if line.is_empty() { line.to_string() } else { format!(" {line}") })
.collect::<Vec<_>>()
.join("\n");
contents = contents.replace(r#" c_config.flag("-utf-8");"#, &indented_replacement);
}
// Introduce configuration variables for dynamic query inclusion
if !contents.contains("with_highlights_query") {
let replaced = indoc! {r#"
c_config.compile("tree-sitter-KEBAB_PARSER_NAME");
}"#}
.replace("KEBAB_PARSER_NAME", &language_name.to_kebab_case());
let replacement = indoc! {r#"
c_config.compile("tree-sitter-KEBAB_PARSER_NAME");
println!("cargo:rustc-check-cfg=cfg(with_highlights_query)");
if !"HIGHLIGHTS_QUERY_PATH".is_empty() && std::path::Path::new("HIGHLIGHTS_QUERY_PATH").exists() {
println!("cargo:rustc-cfg=with_highlights_query");
}
println!("cargo:rustc-check-cfg=cfg(with_injections_query)");
if !"INJECTIONS_QUERY_PATH".is_empty() && std::path::Path::new("INJECTIONS_QUERY_PATH").exists() {
println!("cargo:rustc-cfg=with_injections_query");
}
println!("cargo:rustc-check-cfg=cfg(with_locals_query)");
if !"LOCALS_QUERY_PATH".is_empty() && std::path::Path::new("LOCALS_QUERY_PATH").exists() {
println!("cargo:rustc-cfg=with_locals_query");
}
println!("cargo:rustc-check-cfg=cfg(with_tags_query)");
if !"TAGS_QUERY_PATH".is_empty() && std::path::Path::new("TAGS_QUERY_PATH").exists() {
println!("cargo:rustc-cfg=with_tags_query");
}
}"#}
.replace("KEBAB_PARSER_NAME", &language_name.to_kebab_case())
.replace("HIGHLIGHTS_QUERY_PATH", generate_opts.highlights_query_path)
.replace("INJECTIONS_QUERY_PATH", generate_opts.injections_query_path)
.replace("LOCALS_QUERY_PATH", generate_opts.locals_query_path)
.replace("TAGS_QUERY_PATH", generate_opts.tags_query_path);
contents = contents.replace(
&replaced,
&replacement,
);
}
write_file(path, contents)?;
Ok(())
},
@ -468,7 +576,7 @@ pub fn generate_grammar_files(
|path| generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts),
|path| {
let contents = fs::read_to_string(path)?;
if !contents.contains("new URL") {
if !contents.contains("Object.defineProperty") {
warn!("Replacing index.js");
generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts)?;
}
@ -476,9 +584,19 @@ pub fn generate_grammar_files(
},
)?;
missing_path(path.join("index.d.ts"), |path| {
generate_file(path, INDEX_D_TS_TEMPLATE, language_name, &generate_opts)
})?;
missing_path_else(
path.join("index.d.ts"),
allow_update,
|path| generate_file(path, INDEX_D_TS_TEMPLATE, language_name, &generate_opts),
|path| {
let contents = fs::read_to_string(path)?;
if !contents.contains("export default binding") {
warn!("Replacing index.d.ts");
generate_file(path, INDEX_D_TS_TEMPLATE, language_name, &generate_opts)?;
}
Ok(())
},
)?;
missing_path_else(
path.join("binding_test.js"),
@ -717,9 +835,21 @@ pub fn generate_grammar_files(
},
)?;
missing_path(lang_path.join("__init__.py"), |path| {
generate_file(path, INIT_PY_TEMPLATE, language_name, &generate_opts)
})?;
missing_path_else(
lang_path.join("__init__.py"),
allow_update,
|path| {
generate_file(path, INIT_PY_TEMPLATE, language_name, &generate_opts)
},
|path| {
let contents = fs::read_to_string(path)?;
if !contents.contains("uncomment these to include any queries") {
warn!("Replacing __init__.py");
generate_file(path, INIT_PY_TEMPLATE, language_name, &generate_opts)?;
}
Ok(())
},
)?;
missing_path_else(
lang_path.join("__init__.pyi"),
@ -727,7 +857,10 @@ pub fn generate_grammar_files(
|path| generate_file(path, INIT_PYI_TEMPLATE, language_name, &generate_opts),
|path| {
let mut contents = fs::read_to_string(path)?;
if !contents.contains("CapsuleType") {
if contents.contains("uncomment these to include any queries") {
warn!("Replacing __init__.pyi");
generate_file(path, INIT_PYI_TEMPLATE, language_name, &generate_opts)?;
} else if !contents.contains("CapsuleType") {
contents = contents
.replace(
"from typing import Final",
@ -990,7 +1123,20 @@ fn generate_file(
PARSER_VERSION_PLACEHOLDER,
&generate_opts.version.to_string(),
)
.replace(PARSER_CLASS_NAME_PLACEHOLDER, generate_opts.class_name);
.replace(PARSER_CLASS_NAME_PLACEHOLDER, generate_opts.class_name)
.replace(
HIGHLIGHTS_QUERY_PATH_PLACEHOLDER,
generate_opts.highlights_query_path,
)
.replace(
INJECTIONS_QUERY_PATH_PLACEHOLDER,
generate_opts.injections_query_path,
)
.replace(
LOCALS_QUERY_PATH_PLACEHOLDER,
generate_opts.locals_query_path,
)
.replace(TAGS_QUERY_PATH_PLACEHOLDER, generate_opts.tags_query_path);
if let Some(name) = generate_opts.author_name {
replacement = replacement.replace(AUTHOR_NAME_PLACEHOLDER, name);

18
crates/cli/src/main.rs
View file

@ -114,13 +114,13 @@ struct Generate {
/// Only generate `grammar.json` and `node-types.json`
#[arg(long)]
pub no_parser: bool,
/// Compile all defined languages in the current dir
/// Deprecated: use the `build` command
#[arg(long, short = 'b')]
pub build: bool,
/// Compile a parser in debug mode
/// Deprecated: use the `build` command
#[arg(long, short = '0')]
pub debug_build: bool,
/// The path to the directory containing the parser library
/// Deprecated: use the `build` command
#[arg(long, value_name = "PATH")]
pub libdir: Option<PathBuf>,
/// The path to output the generated source files
@ -261,15 +261,10 @@ struct Parse {
#[arg(long)]
pub open_log: bool,
/// Deprecated: use --json-summary
#[arg(
long,
short = 'j',
conflicts_with = "json_summary",
conflicts_with = "stat"
)]
#[arg(long, conflicts_with = "json_summary", conflicts_with = "stat")]
pub json: bool,
/// Output parsing results in a JSON format
#[arg(long, conflicts_with = "json", conflicts_with = "stat")]
#[arg(long, short = 'j', conflicts_with = "json", conflicts_with = "stat")]
pub json_summary: bool,
/// The path to an alternative config.json file
#[arg(long)]
@ -348,7 +343,7 @@ struct Test {
/// Show only the pass-fail overview tree
#[arg(long)]
pub overview_only: bool,
/// Output the test summary in a JSON output
/// Output the test summary in a JSON format
#[arg(long)]
pub json_summary: bool,
}
@ -905,6 +900,7 @@ impl Generate {
}
}
if self.build {
warn!("--build is deprecated, use the `build` command");
if let Some(path) = self.libdir {
loader = loader::Loader::with_parser_lib_path(path);
}

2
crates/cli/src/templates/.editorconfig
View file

@ -7,7 +7,7 @@ charset = utf-8
indent_style = space
indent_size = 2
[*.js]
[*.{js,ts}]
indent_style = space
indent_size = 2

33
crates/cli/src/templates/__init__.py
View file

@ -6,32 +6,33 @@ from ._binding import language
def _get_query(name, file):
query = _files(f"{__package__}.queries") / file
globals()[name] = query.read_text()
try:
query = _files(f"{__package__}") / file
globals()[name] = query.read_text()
except FileNotFoundError:
globals()[name] = None
return globals()[name]
def __getattr__(name):
# NOTE: uncomment these to include any queries that this grammar contains:
# if name == "HIGHLIGHTS_QUERY":
# return _get_query("HIGHLIGHTS_QUERY", "highlights.scm")
# if name == "INJECTIONS_QUERY":
# return _get_query("INJECTIONS_QUERY", "injections.scm")
# if name == "LOCALS_QUERY":
# return _get_query("LOCALS_QUERY", "locals.scm")
# if name == "TAGS_QUERY":
# return _get_query("TAGS_QUERY", "tags.scm")
if name == "HIGHLIGHTS_QUERY":
return _get_query("HIGHLIGHTS_QUERY", "HIGHLIGHTS_QUERY_PATH")
if name == "INJECTIONS_QUERY":
return _get_query("INJECTIONS_QUERY", "INJECTIONS_QUERY_PATH")
if name == "LOCALS_QUERY":
return _get_query("LOCALS_QUERY", "LOCALS_QUERY_PATH")
if name == "TAGS_QUERY":
return _get_query("TAGS_QUERY", "TAGS_QUERY_PATH")
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
__all__ = [
"language",
# "HIGHLIGHTS_QUERY",
# "INJECTIONS_QUERY",
# "LOCALS_QUERY",
# "TAGS_QUERY",
"HIGHLIGHTS_QUERY",
"INJECTIONS_QUERY",
"LOCALS_QUERY",
"TAGS_QUERY",
]

18
crates/cli/src/templates/__init__.pyi
View file

@ -1,11 +1,17 @@
from typing import Final
from typing_extensions import CapsuleType
# NOTE: uncomment these to include any queries that this grammar contains:
HIGHLIGHTS_QUERY: Final[str] | None
"""The syntax highlighting query for this grammar."""
# HIGHLIGHTS_QUERY: Final[str]
# INJECTIONS_QUERY: Final[str]
# LOCALS_QUERY: Final[str]
# TAGS_QUERY: Final[str]
INJECTIONS_QUERY: Final[str] | None
"""The language injection query for this grammar."""
def language() -> CapsuleType: ...
LOCALS_QUERY: Final[str] | None
"""The local variable query for this grammar."""
TAGS_QUERY: Final[str] | None
"""The symbol tagging query for this grammar."""
def language() -> CapsuleType:
"""The tree-sitter language function for this grammar."""

17
crates/cli/src/templates/build.rs
View file

@ -36,4 +36,21 @@ fn main() {
}
c_config.compile("tree-sitter-KEBAB_PARSER_NAME");
println!("cargo:rustc-check-cfg=cfg(with_highlights_query)");
if !"HIGHLIGHTS_QUERY_PATH".is_empty() && std::path::Path::new("HIGHLIGHTS_QUERY_PATH").exists() {
println!("cargo:rustc-cfg=with_highlights_query");
}
println!("cargo:rustc-check-cfg=cfg(with_injections_query)");
if !"INJECTIONS_QUERY_PATH".is_empty() && std::path::Path::new("INJECTIONS_QUERY_PATH").exists() {
println!("cargo:rustc-cfg=with_injections_query");
}
println!("cargo:rustc-check-cfg=cfg(with_locals_query)");
if !"LOCALS_QUERY_PATH".is_empty() && std::path::Path::new("LOCALS_QUERY_PATH").exists() {
println!("cargo:rustc-cfg=with_locals_query");
}
println!("cargo:rustc-check-cfg=cfg(with_tags_query)");
if !"TAGS_QUERY_PATH".is_empty() && std::path::Path::new("TAGS_QUERY_PATH").exists() {
println!("cargo:rustc-cfg=with_tags_query");
}
}

39
crates/cli/src/templates/index.d.ts vendored
View file

@ -18,10 +18,43 @@ type NodeInfo =
children: ChildNode[];
});
type Language = {
/**
* The tree-sitter language object for this grammar.
*
* @see {@linkcode https://tree-sitter.github.io/node-tree-sitter/interfaces/Parser.Language.html Parser.Language}
*
* @example
* import Parser from "tree-sitter";
* import CAMEL_PARSER_NAME from "tree-sitter-KEBAB_PARSER_NAME";
*
* const parser = new Parser();
* parser.setLanguage(CAMEL_PARSER_NAME);
*/
declare const binding: {
/**
* The inner language object.
* @private
*/
language: unknown;
/**
* The content of the `node-types.json` file for this grammar.
*
* @see {@linkplain https://tree-sitter.github.io/tree-sitter/using-parsers/6-static-node-types Static Node Types}
*/
nodeTypeInfo: NodeInfo[];
/** The syntax highlighting query for this grammar. */
HIGHLIGHTS_QUERY?: string;
/** The language injection query for this grammar. */
INJECTIONS_QUERY?: string;
/** The local variable query for this grammar. */
LOCALS_QUERY?: string;
/** The symbol tagging query for this grammar. */
TAGS_QUERY?: string;
};
declare const language: Language;
export = language;
export default binding;

26
crates/cli/src/templates/index.js
View file

@ -1,3 +1,4 @@
import { readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
const root = fileURLToPath(new URL("../..", import.meta.url));
@ -8,8 +9,29 @@ const binding = typeof process.versions.bun === "string"
: (await import("node-gyp-build")).default(root);
try {
const nodeTypes = await import(`${root}/src/node-types.json`, {with: {type: "json"}});
const nodeTypes = await import(`${root}/src/node-types.json`, { with: { type: "json" } });
binding.nodeTypeInfo = nodeTypes.default;
} catch (_) {}
} catch { }
const queries = [
["HIGHLIGHTS_QUERY", `${root}/HIGHLIGHTS_QUERY_PATH`],
["INJECTIONS_QUERY", `${root}/INJECTIONS_QUERY_PATH`],
["LOCALS_QUERY", `${root}/LOCALS_QUERY_PATH`],
["TAGS_QUERY", `${root}/TAGS_QUERY_PATH`],
];
for (const [prop, path] of queries) {
Object.defineProperty(binding, prop, {
configurable: true,
enumerable: true,
get() {
delete binding[prop];
try {
binding[prop] = readFileSync(path, "utf8");
} catch { }
return binding[prop];
}
});
}
export default binding;

19
crates/cli/src/templates/lib.rs
View file

@ -32,12 +32,21 @@ pub const LANGUAGE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_PARSE
/// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers/6-static-node-types
pub const NODE_TYPES: &str = include_str!("../../src/node-types.json");
// NOTE: uncomment these to include any queries that this grammar contains:
#[cfg(with_highlights_query)]
/// The syntax highlighting query for this grammar.
pub const HIGHLIGHTS_QUERY: &str = include_str!("../../HIGHLIGHTS_QUERY_PATH");
// pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
// pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm");
// pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm");
// pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm");
#[cfg(with_injections_query)]
/// The language injection query for this grammar.
pub const INJECTIONS_QUERY: &str = include_str!("../../INJECTIONS_QUERY_PATH");
#[cfg(with_locals_query)]
/// The local variable query for this grammar.
pub const LOCALS_QUERY: &str = include_str!("../../LOCALS_QUERY_PATH");
#[cfg(with_tags_query)]
/// The symbol tagging query for this grammar.
pub const TAGS_QUERY: &str = include_str!("../../TAGS_QUERY_PATH");
#[cfg(test)]
mod tests {

2
crates/config/Cargo.toml
View file

@ -20,8 +20,8 @@ path = "src/tree_sitter_config.rs"
workspace = true
[dependencies]
anyhow.workspace = true
etcetera.workspace = true
log.workspace = true
serde.workspace = true
serde_json.workspace = true
thiserror.workspace = true

79
crates/config/src/tree_sitter_config.rs
View file

@ -1,12 +1,54 @@
#![cfg_attr(not(any(test, doctest)), doc = include_str!("../README.md"))]
use std::{env, fs, path::PathBuf};
use std::{
env, fs,
path::{Path, PathBuf},
};
use anyhow::{Context, Result};
use etcetera::BaseStrategy as _;
use log::warn;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use thiserror::Error;
pub type ConfigResult<T> = Result<T, ConfigError>;
#[derive(Debug, Error)]
pub enum ConfigError {
#[error("Bad JSON config {0} -- {1}")]
ConfigRead(String, serde_json::Error),
#[error(transparent)]
HomeDir(#[from] etcetera::HomeDirError),
#[error(transparent)]
IO(IoError),
#[error(transparent)]
Serialization(#[from] serde_json::Error),
}
#[derive(Debug, Error)]
pub struct IoError {
pub error: std::io::Error,
pub path: Option<String>,
}
impl IoError {
fn new(error: std::io::Error, path: Option<&Path>) -> Self {
Self {
error,
path: path.map(|p| p.to_string_lossy().to_string()),
}
}
}
impl std::fmt::Display for IoError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.error)?;
if let Some(ref path) = self.path {
write!(f, " ({path})")?;
}
Ok(())
}
}
/// Holds the contents of tree-sitter's configuration file.
///
@ -23,7 +65,7 @@ pub struct Config {
}
impl Config {
pub fn find_config_file() -> Result<Option<PathBuf>> {
pub fn find_config_file() -> ConfigResult<Option<PathBuf>> {
if let Ok(path) = env::var("TREE_SITTER_DIR") {
let mut path = PathBuf::from(path);
path.push("config.json");
@ -46,8 +88,12 @@ impl Config {
.join("tree-sitter")
.join("config.json");
if legacy_apple_path.is_file() {
fs::create_dir_all(xdg_path.parent().unwrap())?;
fs::rename(&legacy_apple_path, &xdg_path)?;
let xdg_dir = xdg_path.parent().unwrap();
fs::create_dir_all(xdg_dir)
.map_err(|e| ConfigError::IO(IoError::new(e, Some(xdg_dir))))?;
fs::rename(&legacy_apple_path, &xdg_path).map_err(|e| {
ConfigError::IO(IoError::new(e, Some(legacy_apple_path.as_path())))
})?;
warn!(
"Your config.json file has been automatically migrated from \"{}\" to \"{}\"",
legacy_apple_path.display(),
@ -67,7 +113,7 @@ impl Config {
Ok(None)
}
fn xdg_config_file() -> Result<PathBuf> {
fn xdg_config_file() -> ConfigResult<PathBuf> {
let xdg_path = etcetera::choose_base_strategy()?
.config_dir()
.join("tree-sitter")
@ -84,7 +130,7 @@ impl Config {
/// [`etcetera::choose_base_strategy`](https://docs.rs/etcetera/*/etcetera/#basestrategy)
/// - `$HOME/.tree-sitter/config.json` as a fallback from where tree-sitter _used_ to store
/// its configuration
pub fn load(path: Option<PathBuf>) -> Result<Self> {
pub fn load(path: Option<PathBuf>) -> ConfigResult<Self> {
let location = if let Some(path) = path {
path
} else if let Some(path) = Self::find_config_file()? {
@ -94,9 +140,9 @@ impl Config {
};
let content = fs::read_to_string(&location)
.with_context(|| format!("Failed to read {}", location.to_string_lossy()))?;
.map_err(|e| ConfigError::IO(IoError::new(e, Some(location.as_path()))))?;
let config = serde_json::from_str(&content)
.with_context(|| format!("Bad JSON config {}", location.to_string_lossy()))?;
.map_err(|e| ConfigError::ConfigRead(location.to_string_lossy().to_string(), e))?;
Ok(Self { location, config })
}
@ -106,7 +152,7 @@ impl Config {
/// disk.
///
/// (Note that this is typically only done by the `tree-sitter init-config` command.)
pub fn initial() -> Result<Self> {
pub fn initial() -> ConfigResult<Self> {
let location = if let Ok(path) = env::var("TREE_SITTER_DIR") {
let mut path = PathBuf::from(path);
path.push("config.json");
@ -119,17 +165,20 @@ impl Config {
}
/// Saves this configuration to the file that it was originally loaded from.
pub fn save(&self) -> Result<()> {
pub fn save(&self) -> ConfigResult<()> {
let json = serde_json::to_string_pretty(&self.config)?;
fs::create_dir_all(self.location.parent().unwrap())?;
fs::write(&self.location, json)?;
let config_dir = self.location.parent().unwrap();
fs::create_dir_all(config_dir)
.map_err(|e| ConfigError::IO(IoError::new(e, Some(config_dir))))?;
fs::write(&self.location, json)
.map_err(|e| ConfigError::IO(IoError::new(e, Some(self.location.as_path()))))?;
Ok(())
}
/// Parses a component-specific configuration from the configuration file. The type `C` must
/// be [deserializable](https://docs.rs/serde/*/serde/trait.Deserialize.html) from a JSON
/// object, and must only include the fields relevant to that component.
pub fn get<C>(&self) -> Result<C>
pub fn get<C>(&self) -> ConfigResult<C>
where
C: for<'de> Deserialize<'de>,
{
@ -140,7 +189,7 @@ impl Config {
/// Adds a component-specific configuration to the configuration file. The type `C` must be
/// [serializable](https://docs.rs/serde/*/serde/trait.Serialize.html) into a JSON object, and
/// must only include the fields relevant to that component.
pub fn add<C>(&mut self, config: C) -> Result<()>
pub fn add<C>(&mut self, config: C) -> ConfigResult<()>
where
C: Serialize,
{

1
crates/generate/Cargo.toml
View file

@ -25,7 +25,6 @@ load = ["dep:semver"]
qjs-rt = ["load", "rquickjs", "pathdiff"]
[dependencies]
anyhow.workspace = true
bitflags = "2.9.4"
dunce = "1.0.5"
indexmap.workspace = true

139
crates/generate/src/generate.rs
View file

@ -7,7 +7,6 @@ use std::{
process::{Command, Stdio},
};
use anyhow::Result;
use bitflags::bitflags;
use log::warn;
use regex::{Regex, RegexBuilder};
@ -62,8 +61,8 @@ pub type GenerateResult<T> = Result<T, GenerateError>;
pub enum GenerateError {
#[error("Error with specified path -- {0}")]
GrammarPath(String),
#[error("{0}")]
IO(String),
#[error(transparent)]
IO(IoError),
#[cfg(feature = "load")]
#[error(transparent)]
LoadGrammarFile(#[from] LoadGrammarError),
@ -82,9 +81,28 @@ pub enum GenerateError {
SuperTypeCycle(#[from] SuperTypeCycleError),
}
impl From<std::io::Error> for GenerateError {
fn from(value: std::io::Error) -> Self {
Self::IO(value.to_string())
#[derive(Debug, Error, Serialize)]
pub struct IoError {
pub error: String,
pub path: Option<String>,
}
impl IoError {
fn new(error: &std::io::Error, path: Option<&Path>) -> Self {
Self {
error: error.to_string(),
path: path.map(|p| p.to_string_lossy().to_string()),
}
}
}
impl std::fmt::Display for IoError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.error)?;
if let Some(ref path) = self.path {
write!(f, " ({path})")?;
}
Ok(())
}
}
@ -99,18 +117,11 @@ pub enum LoadGrammarError {
#[error("Failed to load grammar.js -- {0}")]
LoadJSGrammarFile(#[from] JSError),
#[error("Failed to load grammar.json -- {0}")]
IO(String),
IO(IoError),
#[error("Unknown grammar file extension: {0:?}")]
FileExtension(PathBuf),
}
#[cfg(feature = "load")]
impl From<std::io::Error> for LoadGrammarError {
fn from(value: std::io::Error) -> Self {
Self::IO(value.to_string())
}
}
#[cfg(feature = "load")]
#[derive(Debug, Error, Serialize)]
pub enum ParseVersionError {
@ -118,8 +129,8 @@ pub enum ParseVersionError {
Version(String),
#[error("{0}")]
JSON(String),
#[error("{0}")]
IO(String),
#[error(transparent)]
IO(IoError),
}
#[cfg(feature = "load")]
@ -134,8 +145,21 @@ pub enum JSError {
JSRuntimeUtf8 { runtime: String, error: String },
#[error("`{runtime}` process exited with status {code}")]
JSRuntimeExit { runtime: String, code: i32 },
#[error("{0}")]
IO(String),
#[error("Failed to open stdin for `{runtime}`")]
JSRuntimeStdin { runtime: String },
#[error("Failed to write {item} to `{runtime}`'s stdin -- {error}")]
JSRuntimeWrite {
runtime: String,
item: String,
error: String,
},
#[error("Failed to read output from `{runtime}` -- {error}")]
JSRuntimeRead { runtime: String, error: String },
#[error(transparent)]
IO(IoError),
#[cfg(feature = "qjs-rt")]
#[error("Failed to get relative path")]
RelativePath,
#[error("Could not parse this package's version as semver -- {0}")]
Semver(String),
#[error("Failed to serialze grammar JSON -- {0}")]
@ -145,13 +169,6 @@ pub enum JSError {
QuickJS(String),
}
#[cfg(feature = "load")]
impl From<std::io::Error> for JSError {
fn from(value: std::io::Error) -> Self {
Self::IO(value.to_string())
}
}
#[cfg(feature = "load")]
impl From<serde_json::Error> for JSError {
fn from(value: serde_json::Error) -> Self {
@ -212,7 +229,8 @@ where
.try_exists()
.map_err(|e| GenerateError::GrammarPath(e.to_string()))?
{
fs::create_dir_all(&path_buf)?;
fs::create_dir_all(&path_buf)
.map_err(|e| GenerateError::IO(IoError::new(&e, Some(path_buf.as_path()))))?;
repo_path = path_buf;
repo_path.join("grammar.js")
} else {
@ -229,15 +247,12 @@ where
let header_path = src_path.join("tree_sitter");
// Ensure that the output directory exists
fs::create_dir_all(&src_path)?;
fs::create_dir_all(&src_path)
.map_err(|e| GenerateError::IO(IoError::new(&e, Some(src_path.as_path()))))?;
if grammar_path.file_name().unwrap() != "grammar.json" {
fs::write(src_path.join("grammar.json"), &grammar_json).map_err(|e| {
GenerateError::IO(format!(
"Failed to write grammar.json to {} -- {e}",
src_path.display()
))
})?;
fs::write(src_path.join("grammar.json"), &grammar_json)
.map_err(|e| GenerateError::IO(IoError::new(&e, Some(src_path.as_path()))))?;
}
// If our job is only to generate `grammar.json` and not `parser.c`, stop here.
@ -306,7 +321,8 @@ where
);
write_file(&src_path.join("parser.c"), c_code)?;
fs::create_dir_all(&header_path)?;
fs::create_dir_all(&header_path)
.map_err(|e| GenerateError::IO(IoError::new(&e, Some(header_path.as_path()))))?;
write_file(&header_path.join("alloc.h"), ALLOC_HEADER)?;
write_file(&header_path.join("array.h"), ARRAY_HEADER)?;
write_file(&header_path.join("parser.h"), PARSER_HEADER)?;
@ -373,9 +389,8 @@ fn read_grammar_version(repo_path: &Path) -> Result<Option<Version>, ParseVersio
let json = path
.exists()
.then(|| {
let contents = fs::read_to_string(path.as_path()).map_err(|e| {
ParseVersionError::IO(format!("Failed to read `{}` -- {e}", path.display()))
})?;
let contents = fs::read_to_string(path.as_path())
.map_err(|e| ParseVersionError::IO(IoError::new(&e, Some(path.as_path()))))?;
serde_json::from_str::<TreeSitterJson>(&contents).map_err(|e| {
ParseVersionError::JSON(format!("Failed to parse `{}` -- {e}", path.display()))
})
@ -409,14 +424,16 @@ pub fn load_grammar_file(
}
match grammar_path.extension().and_then(|e| e.to_str()) {
Some("js") => Ok(load_js_grammar_file(grammar_path, js_runtime)?),
Some("json") => Ok(fs::read_to_string(grammar_path)?),
Some("json") => Ok(fs::read_to_string(grammar_path)
.map_err(|e| LoadGrammarError::IO(IoError::new(&e, Some(grammar_path))))?),
_ => Err(LoadGrammarError::FileExtension(grammar_path.to_owned()))?,
}
}
#[cfg(feature = "load")]
fn load_js_grammar_file(grammar_path: &Path, js_runtime: Option<&str>) -> JSResult<String> {
let grammar_path = dunce::canonicalize(grammar_path)?;
let grammar_path = dunce::canonicalize(grammar_path)
.map_err(|e| JSError::IO(IoError::new(&e, Some(grammar_path))))?;
#[cfg(feature = "qjs-rt")]
if js_runtime == Some("native") {
@ -457,7 +474,9 @@ fn load_js_grammar_file(grammar_path: &Path, js_runtime: Option<&str>) -> JSResu
let mut js_stdin = js_process
.stdin
.take()
.ok_or_else(|| JSError::IO(format!("Failed to open stdin for `{js_runtime}`")))?;
.ok_or_else(|| JSError::JSRuntimeStdin {
runtime: js_runtime.to_string(),
})?;
let cli_version = Version::parse(env!("CARGO_PKG_VERSION"))?;
write!(
@ -467,21 +486,26 @@ fn load_js_grammar_file(grammar_path: &Path, js_runtime: Option<&str>) -> JSResu
globalThis.TREE_SITTER_CLI_VERSION_PATCH = {};",
cli_version.major, cli_version.minor, cli_version.patch,
)
.map_err(|e| {
JSError::IO(format!(
"Failed to write tree-sitter version to `{js_runtime}`'s stdin -- {e}"
))
})?;
js_stdin.write(include_bytes!("./dsl.js")).map_err(|e| {
JSError::IO(format!(
"Failed to write grammar dsl to `{js_runtime}`'s stdin -- {e}"
))
.map_err(|e| JSError::JSRuntimeWrite {
runtime: js_runtime.to_string(),
item: "tree-sitter version".to_string(),
error: e.to_string(),
})?;
js_stdin
.write(include_bytes!("./dsl.js"))
.map_err(|e| JSError::JSRuntimeWrite {
runtime: js_runtime.to_string(),
item: "grammar dsl".to_string(),
error: e.to_string(),
})?;
drop(js_stdin);
let output = js_process
.wait_with_output()
.map_err(|e| JSError::IO(format!("Failed to read output from `{js_runtime}` -- {e}")))?;
.map_err(|e| JSError::JSRuntimeRead {
runtime: js_runtime.to_string(),
error: e.to_string(),
})?;
match output.status.code() {
Some(0) => {
let stdout = String::from_utf8(output.stdout).map_err(|e| JSError::JSRuntimeUtf8 {
@ -497,9 +521,15 @@ fn load_js_grammar_file(grammar_path: &Path, js_runtime: Option<&str>) -> JSResu
grammar_json = &stdout[pos + 1..];
let mut stdout = std::io::stdout().lock();
stdout.write_all(node_output.as_bytes())?;
stdout.write_all(b"\n")?;
stdout.flush()?;
stdout
.write_all(node_output.as_bytes())
.map_err(|e| JSError::IO(IoError::new(&e, None)))?;
stdout
.write_all(b"\n")
.map_err(|e| JSError::IO(IoError::new(&e, None)))?;
stdout
.flush()
.map_err(|e| JSError::IO(IoError::new(&e, None)))?;
}
Ok(serde_json::to_string_pretty(&serde_json::from_str::<
@ -519,8 +549,7 @@ fn load_js_grammar_file(grammar_path: &Path, js_runtime: Option<&str>) -> JSResu
#[cfg(feature = "load")]
pub fn write_file(path: &Path, body: impl AsRef<[u8]>) -> GenerateResult<()> {
fs::write(path, body)
.map_err(|e| GenerateError::IO(format!("Failed to write {:?} -- {e}", path.file_name())))
fs::write(path, body).map_err(|e| GenerateError::IO(IoError::new(&e, Some(path))))
}
#[cfg(test)]

1
crates/generate/src/node_types.rs
View file

@ -1,6 +1,5 @@
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
use anyhow::Result;
use serde::Serialize;
use thiserror::Error;

1
crates/generate/src/parse_grammar.rs
View file

@ -1,6 +1,5 @@
use std::collections::HashSet;
use anyhow::Result;
use log::warn;
use regex::Regex;
use serde::{Deserialize, Serialize};

1
crates/generate/src/prepare_grammar.rs
View file

@ -12,7 +12,6 @@ use std::{
mem,
};
use anyhow::Result;
pub use expand_tokens::ExpandTokensError;
pub use extract_tokens::ExtractTokensError;
pub use flatten_grammar::FlattenGrammarError;

1
crates/generate/src/prepare_grammar/expand_tokens.rs
View file

@ -1,4 +1,3 @@
use anyhow::Result;
use regex_syntax::{
hir::{Class, Hir, HirKind},
ParserBuilder,

1
crates/generate/src/prepare_grammar/extract_tokens.rs
View file

@ -1,6 +1,5 @@
use std::collections::HashMap;
use anyhow::Result;
use serde::Serialize;
use thiserror::Error;

1
crates/generate/src/prepare_grammar/flatten_grammar.rs
View file

@ -1,6 +1,5 @@
use std::collections::HashMap;
use anyhow::Result;
use serde::Serialize;
use thiserror::Error;

1
crates/generate/src/prepare_grammar/intern_symbols.rs
View file

@ -1,4 +1,3 @@
use anyhow::Result;
use log::warn;
use serde::Serialize;
use thiserror::Error;

1
crates/generate/src/prepare_grammar/process_inlines.rs
View file

@ -1,6 +1,5 @@
use std::collections::HashMap;
use anyhow::Result;
use serde::Serialize;
use thiserror::Error;

6
crates/generate/src/quickjs.rs
View file

@ -10,7 +10,7 @@ use rquickjs::{
Context, Ctx, Function, Module, Object, Runtime, Type, Value,
};
use super::{JSError, JSResult};
use super::{IoError, JSError, JSResult};
const DSL: &[u8] = include_bytes!("dsl.js");
@ -266,10 +266,10 @@ pub fn execute_native_runtime(grammar_path: &Path) -> JSResult<String> {
let loader = ScriptLoader::default().with_extension("mjs");
runtime.set_loader(resolver, loader);
let cwd = std::env::current_dir()?;
let cwd = std::env::current_dir().map_err(|e| JSError::IO(IoError::new(&e, None)))?;
let relative_path = pathdiff::diff_paths(grammar_path, &cwd)
.map(|p| p.to_string_lossy().to_string())
.ok_or_else(|| JSError::IO("Failed to get relative path".to_string()))?;
.ok_or(JSError::RelativePath)?;
context.with(|ctx| -> JSResult<String> {
let globals = ctx.globals();

2
crates/loader/Cargo.toml
View file

@ -28,7 +28,6 @@ wasm = ["tree-sitter/wasm"]
default = ["tree-sitter-highlight", "tree-sitter-tags"]
[dependencies]
anyhow.workspace = true
cc.workspace = true
etcetera.workspace = true
fs4.workspace = true
@ -41,6 +40,7 @@ semver.workspace = true
serde.workspace = true
serde_json.workspace = true
tempfile.workspace = true
thiserror.workspace = true
tree-sitter = { workspace = true }
tree-sitter-highlight = { workspace = true, optional = true }

765
crates/loader/src/loader.rs
View file

File diff suppressed because it is too large Load diff

5
docs/src/assets/js/playground.js
View file

@ -146,8 +146,9 @@ window.initializePlayground = async (opts) => {
});
queryEditor.on('keydown', (_, event) => {
if (event.key === 'ArrowLeft' || event.key === 'ArrowRight') {
event.stopPropagation(); // Prevent mdBook from going back/forward
const key = event.key;
if (key === 'ArrowLeft' || key === 'ArrowRight' || key === '?') {
event.stopPropagation(); // Prevent mdBook from going back/forward, or showing help
}
});

12
docs/src/cli/fuzz.md
View file

@ -17,6 +17,18 @@ A list of test names to skip fuzzing.
The directory containing the parser. This is primarily useful in multi-language repositories.
### `-p/--grammar-path`
The path to the directory containing the grammar.
### `--lib-path`
The path to the parser's dynamic library. This is used instead of the cached or automatically generated dynamic library.
### `--lang-name`
If `--lib-path` is used, the name of the language used to extract the library's language function
### `--edits <EDITS>`
The maximum number of edits to perform. The default is 3.

17
docs/src/cli/generate.md
View file

@ -34,17 +34,6 @@ The ABI to use for parser generation. The default is ABI 15, with ABI 14 being a
Only generate `grammar.json` and `node-types.json`
### `-0/--debug-build`
Compile the parser with debug flags enabled. This is useful when debugging issues that require a debugger like `gdb` or `lldb`.
### `--libdir <PATH>`
The directory to place the compiled parser(s) in.
On Unix systems, the default path is `$XDG_CACHE_HOME/tree-sitter` if `$XDG_CACHE_HOME` is set,
otherwise `$HOME/.config/tree-sitter` is used. On Windows, the default path is `%LOCALAPPDATA%\tree-sitter` if available,
otherwise `$HOME\AppData\Local\tree-sitter` is used.
### `-o/--output`
The directory to place the generated parser in. The default is `src/` in the current directory.
@ -55,7 +44,7 @@ Print the overview of states from the given rule. This is useful for debugging a
item sets for all given states in a given rule. To solely view state count numbers for rules, pass in `-` for the rule argument.
To view the overview of states for every rule, pass in `*` for the rule argument.
### `--json`
### `--json-summary`
Report conflicts in a JSON format.
@ -65,3 +54,7 @@ The path to the JavaScript runtime executable to use when generating the parser.
Note that you can also set this with `TREE_SITTER_JS_RUNTIME`. Starting from version 0.26.0, you can
also pass in `native` to use the native QuickJS runtime that comes bundled with the CLI. This avoids
the dependency on a JavaScript runtime entirely.
### `--disable-optimization`
Disable optimizations when generating the parser. Currently, this only affects the merging of compatible parse states.

4
docs/src/cli/highlight.md
View file

@ -57,3 +57,7 @@ The path to an alternative configuration (`config.json`) file. See [the init-con
### `-n/--test-number <TEST_NUMBER>`
Highlight the contents of a specific test.
### `-r/--rebuild`
Force a rebuild of the parser before running the fuzzer.

14
docs/src/cli/parse.md
View file

@ -1,8 +1,8 @@
# `tree-sitter parse`
The `parse` command parses source files using a Tree-sitter parser. You can pass any number of file paths and glob patterns
to `tree-sitter parse`, and it will parse all the given files. The command will exit with a non-zero status code if any
parse errors occurred.
to `tree-sitter parse`, and it will parse all the given files. If no paths are provided, input will be parsed from stdin.
The command will exit with a non-zero status code if any parse errors occurred.
```bash
tree-sitter parse [OPTIONS] [PATHS]... # Aliases: p
@ -18,6 +18,14 @@ The path to a file that contains paths to source files to parse.
The path to the directory containing the grammar.
### `-l/--lib-path`
The path to the parser's dynamic library. This is used instead of the cached or automatically generated dynamic library.
### `--lang-name`
If `--lib-path` is used, the name of the language used to extract the library's language function
### `--scope <SCOPE>`
The language scope to use for parsing. This is useful when the language is ambiguous.
@ -81,7 +89,7 @@ in `UTF-16BE` or `UTF-16LE`. If no `BOM` is present, `UTF-8` is the default. One
When using the `--debug-graph` option, open the log file in the default browser.
### `-j/--json`
### `-j/--json-summary`
Output parsing results in a JSON format.

8
docs/src/cli/playground.md
View file

@ -13,10 +13,6 @@ For this to work, you must have already built the parser as a Wasm module. This
## Options
### `-e/--export <EXPORT_PATH>`
Export static playground files to the specified directory instead of serving them.
### `-q/--quiet`
Don't automatically open the playground in the default browser.
@ -24,3 +20,7 @@ Don't automatically open the playground in the default browser.
### `--grammar-path <GRAMMAR_PATH>`
The path to the directory containing the grammar and wasm files.
### `-e/--export <EXPORT_PATH>`
Export static playground files to the specified directory instead of serving them.

12
docs/src/cli/query.md
View file

@ -12,6 +12,14 @@ tree-sitter query [OPTIONS] <QUERY_PATH> [PATHS]... # Aliases: q
The path to the directory containing the grammar.
### `--lib-path`
The path to the parser's dynamic library. This is used instead of the cached or automatically generated dynamic library.
### `--lang-name`
If `--lib-path` is used, the name of the language used to extract the library's language function
### `-t/--time`
Print the time taken to execute the query on the file.
@ -51,3 +59,7 @@ The path to an alternative configuration (`config.json`) file. See [the init-con
### `-n/--test-number <TEST_NUMBER>`
Query the contents of a specific test.
### `-r/--rebuild`
Force a rebuild of the parser before executing the query.

4
docs/src/cli/tags.md
View file

@ -36,3 +36,7 @@ The path to an alternative configuration (`config.json`) file. See [the init-con
### `-n/--test-number <TEST_NUMBER>`
Generate tags from the contents of a specific test.
### `-r/--rebuild`
Force a rebuild of the parser before running the tags.

12
docs/src/cli/test.md
View file

@ -24,6 +24,14 @@ Only run tests from the given filename in the corpus.
The path to the directory containing the grammar.
### `--lib-path`
The path to the parser's dynamic library. This is used instead of the cached or automatically generated dynamic library.
### `--lang-name`
If `--lib-path` is used, the name of the language used to extract the library's language function
### `-u/--update`
Update the expected output of tests.
@ -78,3 +86,7 @@ Force a rebuild of the parser before running tests.
### `--overview-only`
Only show the overview of the test results, and not the diff.
### `--json-summary`
Output the test summary in a JSON format.

8
docs/src/cli/version.md
View file

@ -42,3 +42,11 @@ tree-sitter version
### `-p/--grammar-path <PATH>`
The path to the directory containing the grammar.
### `--bump`
Automatically bump the version. Possible values are:
- `patch`: Bump the patch version.
- `minor`: Bump the minor version.
- `major`: Bump the major version.