Add the source to the closure

the parser being built

(cherry picked from commit 0cdb6bef7b)

CMakeLists.txt

@@ -81,7 +81,7 @@ set_target_properties(tree-sitter
                       SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}"
                       DEFINE_SYMBOL "")
 
-target_compile_definitions(tree-sitter PRIVATE _POSIX_C_SOURCE=200112L _DEFAULT_SOURCE _DARWIN_C_SOURCE)
+target_compile_definitions(tree-sitter PRIVATE _POSIX_C_SOURCE=200112L _DEFAULT_SOURCE _BSD_SOURCE _DARWIN_C_SOURCE)
 
 include(GNUInstallDirs)
 

Cargo.toml

@@ -1,15 +1,6 @@
 [workspace]
-default-members = ["crates/cli"]
 members = [
-  "crates/cli",
-  "crates/config",
-  "crates/generate",
   "crates/highlight",
-  "crates/loader",
-  "crates/tags",
-  "crates/xtask",
-  "crates/language",
-  "lib",
 ]
 resolver = "2"
 

Makefile

@@ -24,7 +24,7 @@ OBJ := $(SRC:.c=.o)
 ARFLAGS := rcs
 CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types
 override CFLAGS += -std=c11 -fPIC -fvisibility=hidden
-override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE -D_DARWIN_C_SOURCE
+override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE -D_BSD_SOURCE -D_DARWIN_C_SOURCE
 override CFLAGS += -Ilib/src -Ilib/src/wasm -Ilib/include
 
 # ABI versioning

Package.swift

@@ -27,6 +27,7 @@ let package = Package(
                         .headerSearchPath("src"),
                         .define("_POSIX_C_SOURCE", to: "200112L"),
                         .define("_DEFAULT_SOURCE"),
+                        .define("_BSD_SOURCE"),
                         .define("_DARWIN_C_SOURCE"),
                 ]),
     ],

build.zig

@@ -40,6 +40,7 @@ pub fn build(b: *std.Build) !void {
 
     lib.root_module.addCMacro("_POSIX_C_SOURCE", "200112L");
     lib.root_module.addCMacro("_DEFAULT_SOURCE", "");
+    lib.root_module.addCMacro("_BSD_SOURCE", "");
     lib.root_module.addCMacro("_DARWIN_C_SOURCE", "");
 
     if (wasm) {

crates/cli/src/init.rs

@@ -8,7 +8,7 @@ use anyhow::{anyhow, Context, Result};
 use crc32fast::hash as crc32;
 use heck::{ToKebabCase, ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase};
 use indoc::{formatdoc, indoc};
-use log::warn;
+use log::info;
 use rand::{thread_rng, Rng};
 use semver::Version;
 use serde::{Deserialize, Serialize};
@@ -123,7 +123,7 @@ const BUILD_ZIG_ZON_TEMPLATE: &str = include_str!("./templates/build.zig.zon");
 const ROOT_ZIG_TEMPLATE: &str = include_str!("./templates/root.zig");
 const TEST_ZIG_TEMPLATE: &str = include_str!("./templates/test.zig");
 
-const TREE_SITTER_JSON_SCHEMA: &str =
+pub const TREE_SITTER_JSON_SCHEMA: &str =
     "https://tree-sitter.github.io/tree-sitter/assets/schemas/config.schema.json";
 
 #[derive(Serialize, Deserialize, Clone)]
@@ -356,7 +356,7 @@ pub fn generate_grammar_files(
                     "tree-sitter-cli":"#},
                 );
             if !contents.contains("module") {
-                warn!("Updating package.json");
+                info!("Migrating package.json to ESM");
                 contents = contents.replace(
                     r#""repository":"#,
                     indoc! {r#"
@@ -378,6 +378,7 @@ pub fn generate_grammar_files(
             |path| {
                 let mut contents = fs::read_to_string(path)?;
                 if contents.contains("module.exports") {
+                    info!("Migrating grammars.js to ESM");
                     contents = contents.replace("module.exports =", "export default");
                     write_file(path, contents)?;
                 }
@@ -393,10 +394,16 @@ pub fn generate_grammar_files(
         allow_update,
         |path| generate_file(path, GITIGNORE_TEMPLATE, language_name, &generate_opts),
         |path| {
-            let contents = fs::read_to_string(path)?;
+            let mut contents = fs::read_to_string(path)?;
             if !contents.contains("Zig artifacts") {
-                warn!("Replacing .gitignore");
+                info!("Adding zig entries to .gitignore");
-                generate_file(path, GITIGNORE_TEMPLATE, language_name, &generate_opts)?;
+                contents.push('\n');
+                contents.push_str(indoc! {"
+                # Zig artifacts
+                .zig-cache/
+                zig-cache/
+                zig-out/
+                "});
             }
             Ok(())
         },
@@ -409,8 +416,13 @@ pub fn generate_grammar_files(
         |path| generate_file(path, GITATTRIBUTES_TEMPLATE, language_name, &generate_opts),
         |path| {
             let mut contents = fs::read_to_string(path)?;
-            contents = contents.replace("bindings/c/* ", "bindings/c/** ");
+            let c_bindings_entry = "bindings/c/* ";
+            if contents.contains(c_bindings_entry) {
+                info!("Updating c bindings entry in .gitattributes");
+                contents = contents.replace(c_bindings_entry, "bindings/c/** ");
+            }
             if !contents.contains("Zig bindings") {
+                info!("Adding zig entries to .gitattributes");
                 contents.push('\n');
                 contents.push_str(indoc! {"
                 # Zig bindings
@@ -438,39 +450,40 @@ pub fn generate_grammar_files(
             }, |path| {
                 let mut contents = fs::read_to_string(path)?;
                 if !contents.contains("#[cfg(with_highlights_query)]") {
-                let replacement = indoc! {r#"
+                    info!("Updating query constants in bindings/rust/lib.rs");
-                    #[cfg(with_highlights_query)]
+                    let replacement = indoc! {r#"
-                    /// The syntax highlighting query for this grammar.
+                        #[cfg(with_highlights_query)]
-                    pub const HIGHLIGHTS_QUERY: &str = include_str!("../../HIGHLIGHTS_QUERY_PATH");
+                        /// The syntax highlighting query for this grammar.
+                        pub const HIGHLIGHTS_QUERY: &str = include_str!("../../HIGHLIGHTS_QUERY_PATH");
 
-                    #[cfg(with_injections_query)]
+                        #[cfg(with_injections_query)]
-                    /// The language injection query for this grammar.
+                        /// The language injection query for this grammar.
-                    pub const INJECTIONS_QUERY: &str = include_str!("../../INJECTIONS_QUERY_PATH");
+                        pub const INJECTIONS_QUERY: &str = include_str!("../../INJECTIONS_QUERY_PATH");
 
-                    #[cfg(with_locals_query)]
+                        #[cfg(with_locals_query)]
-                    /// The local variable query for this grammar.
+                        /// The local variable query for this grammar.
-                    pub const LOCALS_QUERY: &str = include_str!("../../LOCALS_QUERY_PATH");
+                        pub const LOCALS_QUERY: &str = include_str!("../../LOCALS_QUERY_PATH");
 
-                    #[cfg(with_tags_query)]
+                        #[cfg(with_tags_query)]
-                    /// The symbol tagging query for this grammar.
+                        /// The symbol tagging query for this grammar.
-                    pub const TAGS_QUERY: &str = include_str!("../../TAGS_QUERY_PATH");
+                        pub const TAGS_QUERY: &str = include_str!("../../TAGS_QUERY_PATH");
-                    "#}
+                        "#}
-                    .replace("HIGHLIGHTS_QUERY_PATH", generate_opts.highlights_query_path)
+                        .replace("HIGHLIGHTS_QUERY_PATH", generate_opts.highlights_query_path)
-                    .replace("INJECTIONS_QUERY_PATH", generate_opts.injections_query_path)
+                        .replace("INJECTIONS_QUERY_PATH", generate_opts.injections_query_path)
-                    .replace("LOCALS_QUERY_PATH", generate_opts.locals_query_path)
+                        .replace("LOCALS_QUERY_PATH", generate_opts.locals_query_path)
-                    .replace("TAGS_QUERY_PATH", generate_opts.tags_query_path);
+                        .replace("TAGS_QUERY_PATH", generate_opts.tags_query_path);
-                contents = contents
+                    contents = contents
-                    .replace(
+                        .replace(
-                        indoc! {r#"
+                            indoc! {r#"
-                        // NOTE: uncomment these to include any queries that this grammar contains:
+                            // NOTE: uncomment these to include any queries that this grammar contains:
 
-                        // pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
+                            // pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
-                        // pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm");
+                            // pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm");
-                        // pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm");
+                            // pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm");
-                        // pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm");
+                            // pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm");
-                        "#},
+                            "#},
-                        &replacement,
+                            &replacement,
-                    );
+                        );
                 }
                 write_file(path, contents)?;
                 Ok(())
@@ -483,6 +496,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let mut contents = fs::read_to_string(path)?;
                     if !contents.contains("wasm32-unknown-unknown") {
+                        info!("Adding wasm32-unknown-unknown target to bindings/rust/build.rs");
                         let replacement = indoc!{r#"
                             c_config.flag("-utf-8");
 
@@ -503,19 +517,18 @@ pub fn generate_grammar_files(
                                     wasm_src.join("string.c"),
                                 ]);
                             }
-                        "#};
+                        "#}
-
-                        let indented_replacement = replacement
                             .lines()
                             .map(|line| if line.is_empty() { line.to_string() } else { format!("    {line}") })
                             .collect::<Vec<_>>()
                             .join("\n");
 
-                        contents = contents.replace(r#"    c_config.flag("-utf-8");"#, &indented_replacement);
+                        contents = contents.replace(r#"    c_config.flag("-utf-8");"#, &replacement);
                     }
 
                     // Introduce configuration variables for dynamic query inclusion
                     if !contents.contains("with_highlights_query") {
+                        info!("Adding support for dynamic query inclusion to bindings/rust/build.rs");
                         let replaced = indoc! {r#"
                                 c_config.compile("tree-sitter-KEBAB_PARSER_NAME");
                             }"#}
@@ -572,6 +585,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let contents = fs::read_to_string(path)?;
                     if contents.contains("\"LICENSE\"") {
+                        info!("Adding LICENSE entry to bindings/rust/Cargo.toml");
                         write_file(path, contents.replace("\"LICENSE\"", "\"/LICENSE\""))?;
                     }
                     Ok(())
@@ -592,7 +606,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let contents = fs::read_to_string(path)?;
                     if !contents.contains("Object.defineProperty") {
-                        warn!("Replacing index.js");
+                        info!("Replacing index.js");
                         generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts)?;
                     }
                     Ok(())
@@ -606,7 +620,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let contents = fs::read_to_string(path)?;
                     if !contents.contains("export default binding") {
-                        warn!("Replacing index.d.ts");
+                        info!("Replacing index.d.ts");
                         generate_file(path, INDEX_D_TS_TEMPLATE, language_name, &generate_opts)?;
                     }
                     Ok(())
@@ -627,7 +641,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let contents = fs::read_to_string(path)?;
                     if !contents.contains("import") {
-                        warn!("Replacing binding_test.js");
+                        info!("Replacing binding_test.js");
                         generate_file(
                             path,
                             BINDING_TEST_JS_TEMPLATE,
@@ -650,6 +664,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let contents = fs::read_to_string(path)?;
                     if contents.contains("fs.exists(") {
+                        info!("Replacing `fs.exists` calls in binding.gyp");
                         write_file(path, contents.replace("fs.exists(", "fs.existsSync("))?;
                     }
                     Ok(())
@@ -662,14 +677,17 @@ pub fn generate_grammar_files(
 
     // Generate C bindings
     if tree_sitter_config.bindings.c {
+        let kebab_case_name = language_name.to_kebab_case();
         missing_path(bindings_dir.join("c"), create_dir)?.apply(|path| {
-            let old_file = &path.join(format!("tree-sitter-{}.h", language_name.to_kebab_case()));
+            let header_name = format!("tree-sitter-{kebab_case_name}.h");
+            let old_file = &path.join(&header_name);
             if allow_update && fs::exists(old_file).unwrap_or(false) {
+                info!("Removing bindings/c/{header_name}");
                 fs::remove_file(old_file)?;
             }
             missing_path(path.join("tree_sitter"), create_dir)?.apply(|include_path| {
                 missing_path(
-                    include_path.join(format!("tree-sitter-{}.h", language_name.to_kebab_case())),
+                    include_path.join(&header_name),
                     |path| {
                         generate_file(path, PARSER_NAME_H_TEMPLATE, language_name, &generate_opts)
                     },
@@ -678,7 +696,7 @@ pub fn generate_grammar_files(
             })?;
 
             missing_path(
-                path.join(format!("tree-sitter-{}.pc.in", language_name.to_kebab_case())),
+                path.join(format!("tree-sitter-{kebab_case_name}.pc.in")),
                 |path| {
                     generate_file(
                         path,
@@ -698,23 +716,27 @@ pub fn generate_grammar_files(
                 |path| {
                     let mut contents = fs::read_to_string(path)?;
                     if !contents.contains("cd '$(DESTDIR)$(LIBDIR)' && ln -sf") {
-                        warn!("Replacing Makefile");
+                        info!("Replacing Makefile");
                         generate_file(path, MAKEFILE_TEMPLATE, language_name, &generate_opts)?;
                     } else {
-                        contents = contents
+                        let replaced = indoc! {r"
-                            .replace(
+                            $(PARSER): $(SRC_DIR)/grammar.json
-                                indoc! {r"
+                                    $(TS) generate $^
-                                $(PARSER): $(SRC_DIR)/grammar.json
+                            "};
-                                        $(TS) generate $^
+                        if contents.contains(replaced) {
-                                "},
+                            info!("Adding --no-parser target to Makefile");
-                                indoc! {r"
+                            contents = contents
-                                $(SRC_DIR)/grammar.json: grammar.js
+                                .replace(
-                                        $(TS) generate --no-parser $^
+                                    replaced,
+                                    indoc! {r"
+                                    $(SRC_DIR)/grammar.json: grammar.js
+                                            $(TS) generate --no-parser $^
 
-                                $(PARSER): $(SRC_DIR)/grammar.json
+                                    $(PARSER): $(SRC_DIR)/grammar.json
-                                        $(TS) generate $^
+                                            $(TS) generate $^
-                                "}
+                                    "}
-                            );
+                                );
+                        }
                         write_file(path, contents)?;
                     }
                     Ok(())
@@ -726,8 +748,8 @@ pub fn generate_grammar_files(
                 allow_update,
                 |path| generate_file(path, CMAKELISTS_TXT_TEMPLATE, language_name, &generate_opts),
                 |path| {
-                    let mut contents = fs::read_to_string(path)?;
+                    let contents = fs::read_to_string(path)?;
-                    contents = contents
+                    let replaced_contents = contents
                         .replace("add_custom_target(test", "add_custom_target(ts-test")
                         .replace(
                             &formatdoc! {r#"
@@ -775,7 +797,10 @@ pub fn generate_grammar_files(
                                                COMMENT "Generating parser.c")
                             "#}
                         );
-                    write_file(path, contents)?;
+                    if !replaced_contents.eq(&contents) {
+                        info!("Updating CMakeLists.txt");
+                        write_file(path, replaced_contents)?;
+                    }
                     Ok(())
                 },
             )?;
@@ -811,7 +836,8 @@ pub fn generate_grammar_files(
     // Generate Python bindings
     if tree_sitter_config.bindings.python {
         missing_path(bindings_dir.join("python"), create_dir)?.apply(|path| {
-            let lang_path = path.join(format!("tree_sitter_{}", language_name.to_snake_case()));
+            let snake_case_grammar_name = format!("tree_sitter_{}", language_name.to_snake_case());
+            let lang_path = path.join(&snake_case_grammar_name);
             missing_path(&lang_path, create_dir)?;
 
             missing_path_else(
@@ -821,6 +847,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let mut contents = fs::read_to_string(path)?;
                     if !contents.contains("PyModuleDef_Init") {
+                        info!("Updating bindings/python/{snake_case_grammar_name}/binding.c");
                         contents = contents
                             .replace("PyModule_Create", "PyModuleDef_Init")
                             .replace(
@@ -862,7 +889,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let contents = fs::read_to_string(path)?;
                     if !contents.contains("uncomment these to include any queries") {
-                        warn!("Replacing __init__.py");
+                        info!("Replacing __init__.py");
                         generate_file(path, INIT_PY_TEMPLATE, language_name, &generate_opts)?;
                     }
                     Ok(())
@@ -876,9 +903,10 @@ pub fn generate_grammar_files(
                 |path| {
                     let mut contents = fs::read_to_string(path)?;
                     if contents.contains("uncomment these to include any queries") {
-                        warn!("Replacing __init__.pyi");
+                        info!("Replacing __init__.pyi");
                         generate_file(path, INIT_PYI_TEMPLATE, language_name, &generate_opts)?;
                     } else if !contents.contains("CapsuleType") {
+                        info!("Updating __init__.pyi");
                         contents = contents
                             .replace(
                                 "from typing import Final",
@@ -910,6 +938,7 @@ pub fn generate_grammar_files(
                     |path| {
                         let mut contents = fs::read_to_string(path)?;
                         if !contents.contains("Parser(Language(") {
+                            info!("Updating Language function in bindings/python/tests/test_binding.py");
                             contents = contents
                                 .replace("tree_sitter.Language(", "Parser(Language(")
                                 .replace(".language())\n", ".language()))\n")
@@ -930,11 +959,19 @@ pub fn generate_grammar_files(
                 allow_update,
                 |path| generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts),
                 |path| {
-                    let contents = fs::read_to_string(path)?;
+                    let mut contents = fs::read_to_string(path)?;
                     if !contents.contains("build_ext") {
-                        warn!("Replacing setup.py");
+                        info!("Replacing setup.py");
                         generate_file(path, SETUP_PY_TEMPLATE, language_name, &generate_opts)?;
                     }
+                    if !contents.contains(" and not get_config_var") {
+                        info!("Updating Python free-threading support in setup.py");
+                        contents = contents.replace(
+                            r#"startswith("cp"):"#,
+                            r#"startswith("cp") and not get_config_var("Py_GIL_DISABLED"):"#
+                        );
+                        write_file(path, contents)?;
+                    }
                     Ok(())
                 },
             )?;
@@ -953,6 +990,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let mut contents = fs::read_to_string(path)?;
                     if !contents.contains("cp310-*") {
+                        info!("Updating dependencies in pyproject.toml");
                         contents = contents
                             .replace(r#"build = "cp39-*""#, r#"build = "cp310-*""#)
                             .replace(r#"python = ">=3.9""#, r#"python = ">=3.10""#)
@@ -990,15 +1028,18 @@ pub fn generate_grammar_files(
                 allow_update,
                 |path| generate_file(path, PACKAGE_SWIFT_TEMPLATE, language_name, &generate_opts),
                 |path| {
-                    let mut contents = fs::read_to_string(path)?;
+                    let contents = fs::read_to_string(path)?;
-                    contents = contents
+                    let replaced_contents = contents
                         .replace(
                             "https://github.com/ChimeHQ/SwiftTreeSitter",
                             "https://github.com/tree-sitter/swift-tree-sitter",
                         )
                         .replace("version: \"0.8.0\")", "version: \"0.9.0\")")
                         .replace("(url:", "(name: \"SwiftTreeSitter\", url:");
-                    write_file(path, contents)?;
+                    if !replaced_contents.eq(&contents) {
+                        info!("Updating tree-sitter dependency in Package.swift");
+                        write_file(path, contents)?;
+                    }
                     Ok(())
                 },
             )?;
@@ -1016,7 +1057,7 @@ pub fn generate_grammar_files(
             |path| {
                 let contents = fs::read_to_string(path)?;
                 if !contents.contains("b.pkg_hash.len") {
-                    warn!("Replacing build.zig");
+                    info!("Replacing build.zig");
                     generate_file(path, BUILD_ZIG_TEMPLATE, language_name, &generate_opts)
                 } else {
                     Ok(())
@@ -1031,7 +1072,7 @@ pub fn generate_grammar_files(
             |path| {
                 let contents = fs::read_to_string(path)?;
                 if !contents.contains(".name = .tree_sitter_") {
-                    warn!("Replacing build.zig.zon");
+                    info!("Replacing build.zig.zon");
                     generate_file(path, BUILD_ZIG_ZON_TEMPLATE, language_name, &generate_opts)
                 } else {
                     Ok(())
@@ -1047,7 +1088,7 @@ pub fn generate_grammar_files(
                 |path| {
                     let contents = fs::read_to_string(path)?;
                     if contents.contains("ts.Language") {
-                        warn!("Replacing root.zig");
+                        info!("Replacing root.zig");
                         generate_file(path, ROOT_ZIG_TEMPLATE, language_name, &generate_opts)
                     } else {
                         Ok(())

crates/cli/src/main.rs

@@ -20,7 +20,7 @@ use tree_sitter_cli::{
         LOG_GRAPH_ENABLED, START_SEED,
     },
     highlight::{self, HighlightOptions},
-    init::{generate_grammar_files, JsonConfigOpts},
+    init::{generate_grammar_files, JsonConfigOpts, TREE_SITTER_JSON_SCHEMA},
     input::{get_input, get_tmp_source_file, CliInput},
     logger,
     parse::{self, ParseDebugType, ParseFileOptions, ParseOutput, ParseTheme},
@@ -867,10 +867,26 @@ impl Init {
 
             (opts.name.clone(), Some(opts))
         } else {
-            let mut json = serde_json::from_str::<TreeSitterJSON>(
+            let old_config = fs::read_to_string(current_dir.join("tree-sitter.json"))
-                &fs::read_to_string(current_dir.join("tree-sitter.json"))
+                .with_context(|| "Failed to read tree-sitter.json")?;
-                    .with_context(|| "Failed to read tree-sitter.json")?,
+
-            )?;
+            let mut json = serde_json::from_str::<TreeSitterJSON>(&old_config)?;
+            if json.schema.is_none() {
+                json.schema = Some(TREE_SITTER_JSON_SCHEMA.to_string());
+            }
+
+            let new_config = format!("{}\n", serde_json::to_string_pretty(&json)?);
+            // Write the re-serialized config back, as newly added optional boolean fields
+            // will be included with explicit `false`s rather than implict `null`s
+            if self.update && !old_config.trim().eq(new_config.trim()) {
+                info!("Updating tree-sitter.json");
+                fs::write(
+                    current_dir.join("tree-sitter.json"),
+                    serde_json::to_string_pretty(&json)?,
+                )
+                .with_context(|| "Failed to write tree-sitter.json")?;
+            }
+
             (json.grammars.swap_remove(0).name, None)
         };
 
@@ -955,11 +971,21 @@ impl Build {
         } else {
             let output_path = if let Some(ref path) = self.output {
                 let path = Path::new(path);
-                if path.is_absolute() {
+                let full_path = if path.is_absolute() {
                     path.to_path_buf()
                 } else {
                     current_dir.join(path)
-                }
+                };
+                let parent_path = full_path
+                    .parent()
+                    .context("Output path must have a parent")?;
+                let name = full_path
+                    .file_name()
+                    .context("Ouput path must have a filename")?;
+                fs::create_dir_all(parent_path).context("Failed to create output path")?;
+                let mut canon_path = parent_path.canonicalize().context("Invalid output path")?;
+                canon_path.push(name);
+                canon_path
             } else {
                 let file_name = grammar_path
                     .file_stem()
@@ -984,7 +1010,7 @@ impl Build {
 
             loader
                 .compile_parser_at_path(&grammar_path, output_path, flags)
-                .unwrap();
+                .context("Failed to compile parser")?;
         }
         Ok(())
     }
@@ -1622,6 +1648,7 @@ impl Highlight {
         let loader_config = config.get()?;
         loader.find_all_languages(&loader_config)?;
         loader.force_rebuild(self.rebuild || self.grammar_path.is_some());
+        let languages = loader.languages_at_path(current_dir)?;
 
         let cancellation_flag = util::cancel_on_signal();
 
@@ -1702,7 +1729,6 @@ impl Highlight {
             } => {
                 let path = get_tmp_source_file(&contents)?;
 
-                let languages = loader.languages_at_path(current_dir)?;
                 let language = languages
                     .iter()
                     .find(|(_, n)| language_names.contains(&Box::from(n.as_str())))
@@ -1733,7 +1759,6 @@ impl Highlight {
                     if let (Some(l), Some(lc)) = (language.clone(), language_configuration) {
                         (l, lc)
                     } else {
-                        let languages = loader.languages_at_path(current_dir)?;
                         let language = languages
                             .first()
                             .map(|(l, _)| l.clone())

crates/cli/src/parse.rs

@@ -515,7 +515,6 @@ pub fn parse_file_at_path(
 
         if opts.output == ParseOutput::Cst {
             render_cst(&source_code, &tree, &mut cursor, opts, &mut stdout)?;
-            println!();
         }
 
         if opts.output == ParseOutput::Xml {
@@ -785,7 +784,7 @@ pub fn render_cst<'a, 'b: 'a>(
         .map(|(row, col)| (row as f64).log10() as usize + (col.len() as f64).log10() as usize + 1)
         .max()
         .unwrap_or(1);
-    let mut indent_level = 1;
+    let mut indent_level = usize::from(!opts.no_ranges);
     let mut did_visit_children = false;
     let mut in_error = false;
     loop {
@@ -883,35 +882,24 @@ fn write_node_text(
                     0
                 };
             let formatted_line = render_line_feed(line, opts);
-            if !opts.no_ranges {
+            write!(
-                write!(
+                out,
-                    out,
+                "{}{}{}{}{}{}",
-                    "{}{}{}{}{}{}",
+                if multiline { "\n" } else { " " },
-                    if multiline { "\n" } else { "" },
+                if multiline && !opts.no_ranges {
-                    if multiline {
+                    render_node_range(opts, cursor, is_named, true, total_width, node_range)
-                        render_node_range(opts, cursor, is_named, true, total_width, node_range)
+                } else {
-                    } else {
+                    String::new()
-                        String::new()
+                },
-                    },
+                if multiline {
-                    if multiline {
+                    "  ".repeat(indent_level + 1)
-                        "  ".repeat(indent_level + 1)
+                } else {
-                    } else {
+                    String::new()
-                        String::new()
+                },
-                    },
+                paint(quote_color, &String::from(quote)),
-                    paint(quote_color, &String::from(quote)),
+                paint(color, &render_node_text(&formatted_line)),
-                    &paint(color, &render_node_text(&formatted_line)),
+                paint(quote_color, &String::from(quote)),
-                    paint(quote_color, &String::from(quote)),
+            )?;
-                )?;
-            } else {
-                write!(
-                    out,
-                    "\n{}{}{}{}",
-                    "  ".repeat(indent_level + 1),
-                    paint(quote_color, &String::from(quote)),
-                    &paint(color, &render_node_text(&formatted_line)),
-                    paint(quote_color, &String::from(quote)),
-                )?;
-            }
         }
     }
 
@@ -1011,10 +999,9 @@ fn cst_render_node(
         } else {
             opts.parse_theme.node_kind
         };
-        write!(out, "{}", paint(kind_color, node.kind()),)?;
+        write!(out, "{}", paint(kind_color, node.kind()))?;
 
         if node.child_count() == 0 {
-            write!(out, " ")?;
             // Node text from a pattern or external scanner
             write_node_text(
                 opts,

crates/cli/src/templates/setup.py

@@ -32,7 +32,7 @@ class BuildExt(build_ext):
 class BdistWheel(bdist_wheel):
     def get_tag(self):
         python, abi, platform = super().get_tag()
-        if python.startswith("cp"):
+        if python.startswith("cp") and not get_config_var("Py_GIL_DISABLED"):
             python, abi = "cp310", "abi3"
         return python, abi, platform
 

crates/cli/src/test.rs

@@ -595,6 +595,8 @@ impl std::fmt::Display for TestSummary {
             render_assertion_results("queries", &self.query_results)?;
         }
 
+        write!(f, "{}", self.parse_stats)?;
+
         Ok(())
     }
 }

crates/generate/src/prepare_grammar/process_inlines.rs

@@ -70,12 +70,13 @@ impl InlinedProductionMapBuilder {
         let production_map = production_indices_by_step_id
             .into_iter()
             .map(|(step_id, production_indices)| {
-                let production = step_id.variable_index.map_or_else(
+                let production =
-                    || &productions[step_id.production_index],
+                    core::ptr::from_ref::<Production>(step_id.variable_index.map_or_else(
-                    |variable_index| {
+                        || &productions[step_id.production_index],
-                        &grammar.variables[variable_index].productions[step_id.production_index]
+                        |variable_index| {
-                    },
+                            &grammar.variables[variable_index].productions[step_id.production_index]
-                ) as *const Production;
+                        },
+                    ));
                 ((production, step_id.step_index as u32), production_indices)
             })
             .collect();

crates/highlight/Cargo.toml

@@ -28,4 +28,4 @@ regex.workspace = true
 thiserror.workspace = true
 streaming-iterator.workspace = true
 
-tree-sitter.workspace = true
+tree-sitter = "0.26"

crates/highlight/src/c_lib.rs

@@ -297,6 +297,7 @@ impl TSHighlighter {
                     })
                 })
             },
+            &|_, _, _| true,
         );
 
         if let Ok(highlights) = highlights {

crates/highlight/src/highlight.rs

@@ -162,15 +162,17 @@ struct LocalScope<'a> {
     local_defs: Vec<LocalDef<'a>>,
 }
 
-struct HighlightIter<'a, F>
+struct HighlightIter<'a, F, G>
 where
     F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
+    G: Fn(&QueryMatch, &Query, &[u8]) -> bool + 'a,
 {
     source: &'a [u8],
     language_name: &'a str,
     byte_offset: usize,
     highlighter: &'a mut Highlighter,
     injection_callback: F,
+    capture_filter: &'a G,
     cancellation_flag: Option<&'a AtomicUsize>,
     layers: Vec<HighlightIterLayer<'a>>,
     iter_count: usize,
@@ -181,7 +183,7 @@ where
 struct HighlightIterLayer<'a> {
     _tree: Tree,
     cursor: QueryCursor,
-    captures: iter::Peekable<_QueryCaptures<'a, 'a, &'a [u8], &'a [u8]>>,
+    captures: iter::Peekable<Box<dyn Iterator<Item = (QueryMatch<'a, 'a>, usize)> + 'a>>,
     config: &'a HighlightConfiguration,
     highlight_end_stack: Vec<usize>,
     scope_stack: Vec<LocalScope<'a>>,
@@ -244,6 +246,7 @@ impl<'query, 'tree: 'query, T: TextProvider<I>, I: AsRef<[u8]>> Iterator
                         &m.assume_init(),
                         self.ptr,
                     ));
+
                     if result.satisfies_text_predicates(
                         self.query,
                         &mut self.buffer1,
@@ -252,6 +255,7 @@ impl<'query, 'tree: 'query, T: TextProvider<I>, I: AsRef<[u8]>> Iterator
                     ) {
                         return Some((result, capture_index as usize));
                     }
+
                     result.remove();
                 } else {
                     return None;
@@ -287,6 +291,7 @@ impl Highlighter {
         source: &'a [u8],
         cancellation_flag: Option<&'a AtomicUsize>,
         mut injection_callback: impl FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
+        query_filter: &'a impl Fn(&QueryMatch, &Query, &[u8]) -> bool,
     ) -> Result<impl Iterator<Item = Result<HighlightEvent, Error>> + 'a, Error> {
         let layers = HighlightIterLayer::new(
             source,
@@ -294,6 +299,7 @@ impl Highlighter {
             self,
             cancellation_flag,
             &mut injection_callback,
+            query_filter,
             config,
             0,
             vec![Range {
@@ -309,6 +315,7 @@ impl Highlighter {
             language_name: &config.language_name,
             byte_offset: 0,
             injection_callback,
+            capture_filter: query_filter,
             cancellation_flag,
             highlighter: self,
             iter_count: 0,
@@ -509,12 +516,16 @@ impl<'a> HighlightIterLayer<'a> {
     /// disjoint ranges are parsed as one syntax tree), these will be eagerly processed and
     /// added to the returned vector.
     #[allow(clippy::too_many_arguments)]
-    fn new<F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a>(
+    fn new<
+        F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
+        G: Fn(&QueryMatch, &Query, &[u8]) -> bool,
+    >(
         source: &'a [u8],
         parent_name: Option<&str>,
         highlighter: &mut Highlighter,
         cancellation_flag: Option<&'a AtomicUsize>,
         injection_callback: &mut F,
+        query_filter: &'a G,
         mut config: &'a HighlightConfiguration,
         mut depth: usize,
         mut ranges: Vec<Range>,
@@ -601,12 +612,22 @@ impl<'a> HighlightIterLayer<'a> {
                 let cursor_ref = unsafe {
                     mem::transmute::<&mut QueryCursor, &'static mut QueryCursor>(&mut cursor)
                 };
+
                 let captures = unsafe {
-                    std::mem::transmute::<QueryCaptures<_, _>, _QueryCaptures<_, _>>(
+                    std::mem::transmute::<
-                        cursor_ref.captures(&config.query, tree_ref.root_node(), source),
+                        QueryCaptures<_, _>,
-                    )
+                        _QueryCaptures<'a, 'a, &'a [u8], &'a [u8]>,
-                }
+                    >(cursor_ref.captures(
-                .peekable();
+                        &config.query,
+                        tree_ref.root_node(),
+                        source,
+                    ))
+                };
+
+                let captures: Box<dyn Iterator<Item = _>> =
+                    Box::new(captures.filter(|(result, _): &(_, _)| {
+                        query_filter(result, &config.query, source)
+                    }));
 
                 result.push(HighlightIterLayer {
                     highlight_end_stack: Vec::new(),
@@ -618,7 +639,7 @@ impl<'a> HighlightIterLayer<'a> {
                     cursor,
                     depth,
                     _tree: tree,
-                    captures,
+                    captures: captures.peekable(),
                     config,
                     ranges,
                 });
@@ -756,9 +777,10 @@ impl<'a> HighlightIterLayer<'a> {
     }
 }
 
-impl<'a, F> HighlightIter<'a, F>
+impl<'a, F, G> HighlightIter<'a, F, G>
 where
     F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
+    G: Fn(&QueryMatch, &Query, &[u8]) -> bool,
 {
     fn emit_event(
         &mut self,
@@ -822,9 +844,10 @@ where
     }
 }
 
-impl<'a, F> Iterator for HighlightIter<'a, F>
+impl<'a, F, G> Iterator for HighlightIter<'a, F, G>
 where
     F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
+    G: Fn(&QueryMatch, &Query, &[u8]) -> bool,
 {
     type Item = Result<HighlightEvent, Error>;
 
@@ -921,6 +944,7 @@ where
                                 self.highlighter,
                                 self.cancellation_flag,
                                 &mut self.injection_callback,
+                                self.capture_filter,
                                 config,
                                 self.layers[0].depth + 1,
                                 ranges,

crates/language/wasm/include/stdint.h

@@ -23,9 +23,15 @@ typedef long unsigned int size_t;
 
 typedef long unsigned int uintptr_t;
 
-#define UINT16_MAX 65535
+#define INT8_MAX 127
+#define INT16_MAX 32767
+#define INT32_MAX 2147483647L
+#define INT64_MAX 9223372036854775807LL
 
+#define UINT8_MAX 255
+#define UINT16_MAX 65535
 #define UINT32_MAX 4294967295U
+#define UINT64_MAX 18446744073709551615ULL
 
 #if defined(__wasm32__)
 

crates/language/wasm/include/string.h

@@ -13,4 +13,6 @@ void *memset(void *dst, int value, size_t count);
 
 int strncmp(const char *left, const char *right, size_t n);
 
+size_t strlen(const char *str);
+
 #endif // TREE_SITTER_WASM_STRING_H_

crates/language/wasm/src/stdio.c

@@ -1,4 +1,5 @@
 #include <stdio.h>
+#include <string.h>
 
 typedef struct {
   bool left_justify;   // -
@@ -105,12 +106,6 @@ static int ptr_to_str(void *ptr, char *buffer) {
   return 2 + len;
 }
 
-size_t strlen(const char *str) {
-  const char *s = str;
-  while (*s) s++;
-  return s - str;
-}
-
 char *strncpy(char *dest, const char *src, size_t n) {
   char *d = dest;
   const char *s = src;

crates/language/wasm/src/string.c

@@ -58,3 +58,9 @@ int strncmp(const char *left, const char *right, size_t n) {
   }
   return 0;
 }
+
+size_t strlen(const char *str) {
+  const char *s = str;
+  while (*s) s++;
+  return s - str;
+}

crates/loader/src/loader.rs

@@ -8,6 +8,7 @@ use std::sync::Mutex;
 use std::{
     collections::HashMap,
     env, fs,
+    hash::{Hash as _, Hasher as _},
     io::{BufRead, BufReader},
     marker::PhantomData,
     mem,
@@ -1025,20 +1026,26 @@ impl Loader {
             return Ok(wasm_store.load_language(&config.name, &wasm_bytes)?);
         }
 
+        // Create a unique lock path based on the output path hash to prevent
+        // interference when multiple processes build the same grammar (by name)
+        // to different output locations
+        let lock_hash = {
+            let mut hasher = std::hash::DefaultHasher::new();
+            output_path.hash(&mut hasher);
+            format!("{:x}", hasher.finish())
+        };
+
         let lock_path = if env::var("CROSS_RUNNER").is_ok() {
             tempfile::tempdir()
-                .unwrap()
+                .expect("create a temp dir")
                 .path()
-                .join("tree-sitter")
+                .to_path_buf()
-                .join("lock")
-                .join(format!("{}.lock", config.name))
         } else {
-            etcetera::choose_base_strategy()?
+            etcetera::choose_base_strategy()?.cache_dir()
-                .cache_dir()
+        }
-                .join("tree-sitter")
+        .join("tree-sitter")
-                .join("lock")
+        .join("lock")
-                .join(format!("{}.lock", config.name))
+        .join(format!("{}-{lock_hash}.lock", config.name));
-        };
 
         if let Ok(lock_file) = fs::OpenOptions::new().write(true).open(&lock_path) {
             recompile = false;
@@ -1089,6 +1096,26 @@ impl Loader {
             }
         }
 
+        // Ensure the dynamic library exists before trying to load it. This can
+        // happen in race conditions where we couldn't acquire the lock because
+        // another process was compiling but it still hasn't finished by the
+        // time we reach this point, so the output file still doesn't exist.
+        //
+        // Instead of allowing the `load_language` call below to fail, return a
+        // clearer error to the user here.
+        if !output_path.exists() {
+            let msg = format!(
+                "Dynamic library `{}` not found after build attempt. \
+                Are you running multiple processes building to the same output location?",
+                output_path.display()
+            );
+
+            Err(LoaderError::IO(IoError::new(
+                std::io::Error::new(std::io::ErrorKind::NotFound, msg),
+                Some(output_path.as_path()),
+            )))?;
+        }
+
         Self::load_language(&output_path, &language_fn_name)
     }
 
@@ -1278,6 +1305,11 @@ impl Loader {
                     }));
                 }
             }
+        } else {
+            warn!(
+                "Failed to run `nm` to verify symbols in {}",
+                library_path.display()
+            );
         }
 
         Ok(())

crates/xtask/src/build_wasm.rs

@@ -199,6 +199,7 @@ pub fn run_wasm(args: &BuildWasm) -> Result<()> {
         "-D", "NDEBUG=",
         "-D", "_POSIX_C_SOURCE=200112L",
         "-D", "_DEFAULT_SOURCE=",
+        "-D", "_BSD_SOURCE=",
         "-D", "_DARWIN_C_SOURCE=",
         "-I", "lib/src",
         "-I", "lib/include",

docs/src/assets/js/playground.js

@@ -61,7 +61,7 @@ function initializeCustomSelect({ initialValue = null, addListeners = false }) {
 }
 
 window.initializePlayground = async (opts) => {
-  const { Parser, Language } = window.TreeSitter;
+  const { Parser, Language, Query } = window.TreeSitter;
 
   const { local } = opts;
   if (local) {
@@ -357,11 +357,10 @@ window.initializePlayground = async (opts) => {
       marks.forEach((m) => m.clear());
 
       if (tree && query) {
-        const captures = query.captures(
+        const captures = query.captures(tree.rootNode, {
-          tree.rootNode,
+          startPosition: { row: startRow, column: 0 },
-          { row: startRow, column: 0 },
+          endPosition: { row: endRow, column: 0 },
-          { row: endRow, column: 0 },
+        });
-        );
         let lastNodeId;
         for (const { name, node } of captures) {
           if (node.id === lastNodeId) continue;
@@ -410,7 +409,7 @@ window.initializePlayground = async (opts) => {
       const queryText = queryEditor.getValue();
 
       try {
-        query = parser.language.query(queryText);
+        query = new Query(parser.language, queryText);
         let match;
 
         let row = 0;

lib/binding_rust/build.rs

@@ -49,6 +49,7 @@ fn main() {
         .include(&include_path)
         .define("_POSIX_C_SOURCE", "200112L")
         .define("_DEFAULT_SOURCE", None)
+        .define("_BSD_SOURCE", None)
         .define("_DARWIN_C_SOURCE", None)
         .warnings(false)
         .file(src_path.join("lib.c"))