From c21db9fea9d3a22946131f2f7915194160ef63fe Mon Sep 17 00:00:00 2001 From: Carlo Cabrera <30379873+carlocab@users.noreply.github.com> Date: Fri, 4 Oct 2024 13:09:17 +0800 Subject: [PATCH 01/61] build(cmake): support amalgamated build This mirrors passing `AMALGAMATED=1` to `make` when using the `Makefile`. It can be enabled by passing `-DAMALGAMATED=ON` to `cmake`. (cherry picked from commit 666db18c28ea2da32ee0c124e4ce8a00976ecd45) --- lib/CMakeLists.txt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 7d465419..ce93620e 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -8,9 +8,14 @@ project(tree-sitter option(BUILD_SHARED_LIBS "Build using shared libraries" ON) option(TREE_SITTER_FEATURE_WASM "Enable the Wasm feature" OFF) +option(AMALGAMATED "Build using an amalgamated source" OFF) -file(GLOB TS_SOURCE_FILES src/*.c) -list(REMOVE_ITEM TS_SOURCE_FILES "${PROJECT_SOURCE_DIR}/src/lib.c") +if(AMALGAMATED) + set(TS_SOURCE_FILES "${PROJECT_SOURCE_DIR}/src/lib.c") +else() + file(GLOB TS_SOURCE_FILES src/*.c) + list(REMOVE_ITEM TS_SOURCE_FILES "${PROJECT_SOURCE_DIR}/src/lib.c") +endif() add_library(tree-sitter ${TS_SOURCE_FILES}) From 75d2915f48fd73f17530963ce2778b7e335a54c6 Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Sat, 5 Oct 2024 18:17:01 +0300 Subject: [PATCH 02/61] build(cmake): correct library scopes (cherry picked from commit edfd47e1c69250903a417c729aa4a850d403fa53) --- lib/CMakeLists.txt | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index ce93620e..5426552e 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -25,41 +25,45 @@ if(NOT MSVC) target_compile_options(tree-sitter PRIVATE -Wall -Wextra -Wshadow -Wno-unused-parameter -pedantic) endif() -if(NOT BUILD_SHARED_LIBS) - if(WIN32) - set(CMAKE_FIND_LIBRARY_SUFFIXES .lib .a) - else() - set(CMAKE_FIND_LIBRARY_SUFFIXES .a) - endif() -endif() - if(TREE_SITTER_FEATURE_WASM) if(NOT DEFINED CACHE{WASMTIME_INCLUDE_DIR}) message(CHECK_START "Looking for wasmtime headers") find_path(WASMTIME_INCLUDE_DIR wasmtime.h - PATHS ENV DEP_WASMTIME_C_API_INCLUDE - REQUIRED) + PATHS ENV DEP_WASMTIME_C_API_INCLUDE) + if(NOT ${WASMTIME_INCLUDE_DIR}) + unset(WASMTIME_INCLUDE_DIR CACHE) + message(FATAL_ERROR "Could not find wasmtime headers.\nDid you forget to set CMAKE_INCLUDE_PATH?") + endif() message(CHECK_PASS "found") endif() if(NOT DEFINED CACHE{WASMTIME_LIBRARY}) message(CHECK_START "Looking for wasmtime library") - find_library(WASMTIME_LIBRARY wasmtime - REQUIRED) + if(BUILD_SHARED_LIBS) + find_library(WASMTIME_LIBRARY wasmtime) + elseif(MSVC) + find_library(WASMTIME_LIBRARY wasmtime.lib) + else() + find_library(WASMTIME_LIBRARY libwasmtime.a) + endif() + if(NOT ${WASMTIME_LIBRARY}) + unset(WASMTIME_LIBRARY CACHE) + message(FATAL_ERROR "Could not find wasmtime library.\nDid you forget to set CMAKE_LIBRARY_PATH?") + endif() message(CHECK_PASS "found") endif() target_compile_definitions(tree-sitter PUBLIC TREE_SITTER_FEATURE_WASM) target_include_directories(tree-sitter SYSTEM PRIVATE "${WASMTIME_INCLUDE_DIR}") - target_link_libraries(tree-sitter PRIVATE "${WASMTIME_LIBRARY}") + target_link_libraries(tree-sitter PUBLIC "${WASMTIME_LIBRARY}") set_property(TARGET tree-sitter PROPERTY C_STANDARD_REQUIRED ON) if(NOT BUILD_SHARED_LIBS) if(WIN32) target_compile_definitions(tree-sitter PRIVATE WASM_API_EXTERN= WASI_API_EXTERN=) - target_link_libraries(tree-sitter PRIVATE ws2_32 advapi32 userenv ntdll shell32 ole32 bcrypt) + target_link_libraries(tree-sitter INTERFACE ws2_32 advapi32 userenv ntdll shell32 ole32 bcrypt) elseif(NOT APPLE) - target_link_libraries(tree-sitter PRIVATE pthread dl m) + target_link_libraries(tree-sitter INTERFACE pthread dl m) endif() endif() endif() @@ -69,7 +73,8 @@ set_target_properties(tree-sitter C_STANDARD 11 C_VISIBILITY_PRESET hidden POSITION_INDEPENDENT_CODE ON - SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}") + SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}" + DEFINE_SYMBOL "") configure_file(tree-sitter.pc.in "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter.pc" @ONLY) From c639d547f9db4ecdfebe71a007fe5ff4e2134674 Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Fri, 4 Oct 2024 17:23:37 +0300 Subject: [PATCH 03/61] feat: drop legacy binding updates (cherry picked from commit 1d76ec3a1c7089a97c50960636bd94f83d865494) --- cli/src/init.rs | 349 +++++++----------------------------------------- 1 file changed, 45 insertions(+), 304 deletions(-) diff --git a/cli/src/init.rs b/cli/src/init.rs index 4cab1aa8..5e203cd6 100644 --- a/cli/src/init.rs +++ b/cli/src/init.rs @@ -7,11 +7,10 @@ use std::{ use anyhow::{anyhow, Context, Result}; use heck::{ToKebabCase, ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase}; -use indoc::indoc; use regex::Regex; use semver::Version; use serde::{Deserialize, Serialize}; -use serde_json::{json, Map, Value}; +use serde_json::{Map, Value}; use tree_sitter_generate::write_file; use tree_sitter_loader::{ Author, Bindings, Grammar, Links, Metadata, PackageJSON, PackageJSONAuthor, @@ -112,22 +111,6 @@ pub fn path_in_ignore(repo_path: &Path) -> bool { .any(|dir| repo_path.ends_with(dir)) } -fn insert_after( - map: Map, - after: &str, - key: &str, - value: Value, -) -> Map { - let mut entries = map.into_iter().collect::>(); - let after_index = entries - .iter() - .position(|(k, _)| k == after) - .unwrap_or(entries.len() - 1) - + 1; - entries.insert(after_index, (key.to_string(), value)); - entries.into_iter().collect() -} - #[derive(Serialize, Deserialize, Clone)] pub struct JsonConfigOpts { pub name: String, @@ -378,13 +361,11 @@ pub fn migrate_package_json(repo_path: &Path) -> Result { pub fn generate_grammar_files( repo_path: &Path, language_name: &str, - allow_update: bool, + _allow_update: bool, opts: Option, ) -> Result<()> { let dashed_language_name = language_name.to_kebab_case(); - // TODO: remove legacy code updates in v0.24.0 - let tree_sitter_config = missing_path_else( repo_path.join("tree-sitter.json"), true, @@ -434,124 +415,15 @@ pub fn generate_grammar_files( version: &tree_sitter_config.metadata.version, }; - // Create or update package.json - missing_path_else( - repo_path.join("package.json"), - allow_update, - |path| { - generate_file( - path, - PACKAGE_JSON_TEMPLATE, - dashed_language_name.as_str(), - &generate_opts, - ) - }, - |path| { - let package_json_str = - fs::read_to_string(path).with_context(|| "Failed to read package.json")?; - let mut package_json = serde_json::from_str::>(&package_json_str) - .with_context(|| "Failed to parse package.json")?; - let mut updated = false; - - let dependencies = package_json - .entry("dependencies".to_string()) - .or_insert_with(|| Value::Object(Map::new())) - .as_object_mut() - .unwrap(); - if dependencies.remove("nan").is_some() { - eprintln!("Replacing nan dependency with node-addon-api in package.json"); - dependencies.insert("node-addon-api".to_string(), "^8.0.0".into()); - updated = true; - } - if !dependencies.contains_key("node-gyp-build") { - eprintln!("Adding node-gyp-build dependency to package.json"); - dependencies.insert("node-gyp-build".to_string(), "^4.8.1".into()); - updated = true; - } - - let dev_dependencies = package_json - .entry("devDependencies".to_string()) - .or_insert_with(|| Value::Object(Map::new())) - .as_object_mut() - .unwrap(); - if !dev_dependencies.contains_key("prebuildify") { - eprintln!("Adding prebuildify devDependency to package.json"); - dev_dependencies.insert("prebuildify".to_string(), "^6.0.1".into()); - updated = true; - } - - let node_test = "node --test bindings/node/*_test.js"; - let scripts = package_json - .entry("scripts".to_string()) - .or_insert_with(|| Value::Object(Map::new())) - .as_object_mut() - .unwrap(); - if !scripts.get("test").is_some_and(|v| v == node_test) { - eprintln!("Updating package.json scripts"); - *scripts = Map::from_iter([ - ("install".to_string(), "node-gyp-build".into()), - ("prestart".to_string(), "tree-sitter build --wasm".into()), - ("start".to_string(), "tree-sitter playground".into()), - ("test".to_string(), node_test.into()), - ]); - updated = true; - } - - // insert `peerDependencies` after `dependencies` - if !package_json.contains_key("peerDependencies") { - eprintln!("Adding peerDependencies to package.json"); - package_json = insert_after( - package_json, - "dependencies", - "peerDependencies", - json!({"tree-sitter": "^0.21.1"}), - ); - - package_json = insert_after( - package_json, - "peerDependencies", - "peerDependenciesMeta", - json!({"tree_sitter": {"optional": true}}), - ); - updated = true; - } - - // insert `types` right after `main` - if !package_json.contains_key("types") { - eprintln!("Adding types to package.json"); - package_json = insert_after(package_json, "main", "types", "bindings/node".into()); - updated = true; - } - - // insert `files` right after `keywords` - if !package_json.contains_key("files") { - eprintln!("Adding files to package.json"); - package_json = insert_after( - package_json, - "keywords", - "files", - json!([ - "grammar.js", - "binding.gyp", - "prebuilds/**", - "bindings/node/*", - "queries/*", - "src/**", - "*.wasm" - ]), - ); - updated = true; - } - - if updated { - let mut package_json_str = serde_json::to_string_pretty(&package_json)?; - package_json_str.push('\n'); - write_file(path, package_json_str)?; - } - - Ok(()) - }, - )?; + // Create package.json + missing_path(repo_path.join("package.json"), |path| { + generate_file( + path, + PACKAGE_JSON_TEMPLATE, + dashed_language_name.as_str(), + &generate_opts, + ) + })?; // Do not create a grammar.js file in a repo with multiple language configs if !tree_sitter_config.has_multiple_language_configs() { @@ -580,83 +452,22 @@ pub fn generate_grammar_files( // Generate Rust bindings if tree_sitter_config.bindings.rust { missing_path(bindings_dir.join("rust"), create_dir)?.apply(|path| { - missing_path_else( - path.join("lib.rs"), - allow_update, - |path| generate_file(path, LIB_RS_TEMPLATE, language_name, &generate_opts), - |path| { - let lib_rs = - fs::read_to_string(path).with_context(|| "Failed to read lib.rs")?; - if !lib_rs.contains("tree_sitter_language") { - generate_file(path, LIB_RS_TEMPLATE, language_name, &generate_opts)?; - eprintln!("Updated lib.rs with `tree_sitter_language` dependency"); - } - Ok(()) - }, - )?; + missing_path(path.join("lib.rs"), |path| { + generate_file(path, LIB_RS_TEMPLATE, language_name, &generate_opts) + })?; - missing_path_else( - path.join("build.rs"), - allow_update, - |path| generate_file(path, BUILD_RS_TEMPLATE, language_name, &generate_opts), - |path| { - let build_rs = - fs::read_to_string(path).with_context(|| "Failed to read build.rs")?; - if !build_rs.contains("-utf-8") { - let index = build_rs - .find(" let parser_path = src_dir.join(\"parser.c\")") - .ok_or_else(|| anyhow!(indoc!{ - "Failed to auto-update build.rs with the `/utf-8` flag for windows. - To fix this, remove `bindings/rust/build.rs` and re-run `tree-sitter generate`"}))?; + missing_path(path.join("build.rs"), |path| { + generate_file(path, BUILD_RS_TEMPLATE, language_name, &generate_opts) + })?; - let build_rs = format!( - "{}{}{}\n{}", - &build_rs[..index], - " #[cfg(target_env = \"msvc\")]\n", - " c_config.flag(\"-utf-8\");\n", - &build_rs[index..] - ); - - write_file(path, build_rs)?; - eprintln!("Updated build.rs with the /utf-8 flag for Windows compilation"); - } - Ok(()) - }, - )?; - - missing_path_else( - repo_path.join("Cargo.toml"), - allow_update, - |path| generate_file(path, CARGO_TOML_TEMPLATE, dashed_language_name.as_str(), &generate_opts), - |path| { - let cargo_toml = - fs::read_to_string(path).with_context(|| "Failed to read Cargo.toml")?; - if !cargo_toml.contains("tree-sitter-language") { - let start_index = cargo_toml - .find("tree-sitter = \"") - .ok_or_else(|| anyhow!("Failed to find the `tree-sitter` dependency in Cargo.toml"))?; - - let version_start_index = start_index + "tree-sitter = \"".len(); - let version_end_index = cargo_toml[version_start_index..] - .find('\"') - .map(|i| i + version_start_index) - .ok_or_else(|| anyhow!("Failed to find the end of the `tree-sitter` version in Cargo.toml"))?; - - let cargo_toml = format!( - "{}{}{}\n{}\n{}", - &cargo_toml[..start_index], - "tree-sitter-language = \"0.1.0\"", - &cargo_toml[version_end_index + 1..], - "[dev-dependencies]", - "tree-sitter = \"0.23\"", - ); - - write_file(path, cargo_toml)?; - eprintln!("Updated Cargo.toml with the `tree-sitter-language` dependency"); - } - Ok(()) - }, - )?; + missing_path(repo_path.join("Cargo.toml"), |path| { + generate_file( + path, + CARGO_TOML_TEMPLATE, + dashed_language_name.as_str(), + &generate_opts, + ) + })?; Ok(()) })?; @@ -665,20 +476,9 @@ pub fn generate_grammar_files( // Generate Node bindings if tree_sitter_config.bindings.node { missing_path(bindings_dir.join("node"), create_dir)?.apply(|path| { - missing_path_else( - path.join("index.js"), - allow_update, - |path| generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts), - |path| { - let index_js = - fs::read_to_string(path).with_context(|| "Failed to read index.js")?; - if index_js.contains("../../build/Release") { - eprintln!("Replacing index.js with new binding API"); - generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts)?; - } - Ok(()) - }, - )?; + missing_path(path.join("index.js"), |path| { + generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts) + })?; missing_path(path.join("index.d.ts"), |path| { generate_file(path, INDEX_D_TS_TEMPLATE, language_name, &generate_opts) @@ -693,36 +493,13 @@ pub fn generate_grammar_files( ) })?; - missing_path_else( - path.join("binding.cc"), - allow_update, - |path| generate_file(path, JS_BINDING_CC_TEMPLATE, language_name, &generate_opts), - |path| { - let binding_cc = - fs::read_to_string(path).with_context(|| "Failed to read binding.cc")?; - if binding_cc.contains("NAN_METHOD(New) {}") { - eprintln!("Replacing binding.cc with new binding API"); - generate_file(path, JS_BINDING_CC_TEMPLATE, language_name, &generate_opts)?; - } - Ok(()) - }, - )?; + missing_path(path.join("binding.cc"), |path| { + generate_file(path, JS_BINDING_CC_TEMPLATE, language_name, &generate_opts) + })?; - // Create binding.gyp, or update it with new binding API. - missing_path_else( - repo_path.join("binding.gyp"), - allow_update, - |path| generate_file(path, BINDING_GYP_TEMPLATE, language_name, &generate_opts), - |path| { - let binding_gyp = - fs::read_to_string(path).with_context(|| "Failed to read binding.gyp")?; - if binding_gyp.contains("require('nan')") { - eprintln!("Replacing binding.gyp with new binding API"); - generate_file(path, BINDING_GYP_TEMPLATE, language_name, &generate_opts)?; - } - Ok(()) - }, - )?; + missing_path(repo_path.join("binding.gyp"), |path| { + generate_file(path, BINDING_GYP_TEMPLATE, language_name, &generate_opts) + })?; Ok(()) })?; @@ -767,39 +544,14 @@ pub fn generate_grammar_files( generate_file(path, BINDING_GO_TEMPLATE, language_name, &generate_opts) })?; - missing_path_else( - path.join("binding_test.go"), - allow_update, - |path| { - generate_file( - path, - BINDING_TEST_GO_TEMPLATE, - language_name, - &generate_opts, - ) - }, - |path| { - let binding_test_go = fs::read_to_string(path) - .with_context(|| "Failed to read binding_test.go")?; - if binding_test_go.contains("smacker") { - eprintln!("Replacing binding_test.go with new binding API"); - generate_file( - path, - BINDING_TEST_GO_TEMPLATE, - language_name, - &generate_opts, - )?; - } - Ok(()) - }, - )?; - - // Delete the old go.mod file that lives inside bindings/go, it now lives in the root - // dir - let go_mod_path = path.join("go.mod"); - if allow_update && go_mod_path.exists() { - fs::remove_file(go_mod_path).with_context(|| "Failed to remove old go.mod file")?; - } + missing_path(path.join("binding_test.go"), |path| { + generate_file( + path, + BINDING_TEST_GO_TEMPLATE, + language_name, + &generate_opts, + ) + })?; missing_path(repo_path.join("go.mod"), |path| { generate_file(path, GO_MOD_TEMPLATE, language_name, &generate_opts) @@ -815,20 +567,9 @@ pub fn generate_grammar_files( let lang_path = path.join(format!("tree_sitter_{}", language_name.to_snake_case())); missing_path(&lang_path, create_dir)?; - missing_path_else( - lang_path.join("binding.c"), - allow_update, - |path| generate_file(path, PY_BINDING_C_TEMPLATE, language_name, &generate_opts), - |path| { - let binding_c = fs::read_to_string(path) - .with_context(|| "Failed to read bindings/python/binding.c")?; - if !binding_c.contains("PyCapsule_New") { - eprintln!("Replacing bindings/python/binding.c with new binding API"); - generate_file(path, PY_BINDING_C_TEMPLATE, language_name, &generate_opts)?; - } - Ok(()) - }, - )?; + missing_path(lang_path.join("binding.c"), |path| { + generate_file(path, PY_BINDING_C_TEMPLATE, language_name, &generate_opts) + })?; missing_path(lang_path.join("__init__.py"), |path| { generate_file(path, INIT_PY_TEMPLATE, language_name, &generate_opts) From 0d8f143a9d393205657aa72723434bd7e0d43ce3 Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Fri, 4 Oct 2024 17:35:29 +0300 Subject: [PATCH 04/61] build(bindings): improve cmake file - Use placeholders for version, description, homepage - Add option for `TREE_SITTER_REUSE_ALLOCATOR` - Define `TREE_SITTER_DEBUG` in debug mode (cherry picked from commit a397b454a37016e626d13d948a65443c34658699) --- cli/src/templates/cmakelists.txt | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cli/src/templates/cmakelists.txt b/cli/src/templates/cmakelists.txt index 0a671697..24f2507d 100644 --- a/cli/src/templates/cmakelists.txt +++ b/cli/src/templates/cmakelists.txt @@ -1,12 +1,13 @@ cmake_minimum_required(VERSION 3.13) project(tree-sitter-PARSER_NAME - VERSION "0.0.1" - DESCRIPTION "CAMEL_PARSER_NAME grammar for tree-sitter" - HOMEPAGE_URL "https://github.com/tree-sitter/tree-sitter-PARSER_NAME" + VERSION "PARSER_VERSION" + DESCRIPTION "PARSER_DESCRIPTION" + HOMEPAGE_URL "PARSER_URL" LANGUAGES C) option(BUILD_SHARED_LIBS "Build using shared libraries" ON) +option(TREE_SITTER_REUSE_ALLOCATOR "Reuse the library allocator" OFF) set(TREE_SITTER_ABI_VERSION ABI_VERSION_MAX CACHE STRING "Tree-sitter ABI version") if(NOT ${TREE_SITTER_ABI_VERSION} MATCHES "^[0-9]+$") @@ -29,11 +30,16 @@ if(EXISTS src/scanner.c) endif() target_include_directories(tree-sitter-PARSER_NAME PRIVATE src) +target_compile_definitions(tree-sitter-PARSER_NAME PRIVATE + $<$:TREE_SITTER_REUSE_ALLOCATOR> + $<$:TREE_SITTER_DEBUG>) + set_target_properties(tree-sitter-PARSER_NAME PROPERTIES C_STANDARD 11 POSITION_INDEPENDENT_CODE ON - SOVERSION "${TREE_SITTER_ABI_VERSION}.${PROJECT_VERSION_MAJOR}") + SOVERSION "${TREE_SITTER_ABI_VERSION}.${PROJECT_VERSION_MAJOR}" + DEFINE_SYMBOL "") configure_file(bindings/c/tree-sitter-PARSER_NAME.pc.in "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter-PARSER_NAME.pc" @ONLY) From b3f808cc01b4d18ae19fa91ce907f16bbd7584c5 Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Sat, 5 Oct 2024 10:32:25 +0300 Subject: [PATCH 05/61] chore(templates): update npm packages (cherry picked from commit 95f24e036432061cda8561d183077a245c2b6080) --- cli/src/templates/gitignore | 7 +++++-- cli/src/templates/package.json | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cli/src/templates/gitignore b/cli/src/templates/gitignore index 2fd9daca..308fcab2 100644 --- a/cli/src/templates/gitignore +++ b/cli/src/templates/gitignore @@ -5,11 +5,9 @@ target/ build/ prebuilds/ node_modules/ -*.tgz # Swift artifacts .build/ -Package.resolved # Go artifacts _obj/ @@ -35,3 +33,8 @@ dist/ *.wasm *.obj *.o + +# Archives +*.tar.gz +*.tgz +*.zip diff --git a/cli/src/templates/package.json b/cli/src/templates/package.json index be0ca55f..212ffc3b 100644 --- a/cli/src/templates/package.json +++ b/cli/src/templates/package.json @@ -27,8 +27,8 @@ "*.wasm" ], "dependencies": { - "node-addon-api": "^8.0.0", - "node-gyp-build": "^4.8.1" + "node-addon-api": "^8.1.0", + "node-gyp-build": "^4.8.2" }, "devDependencies": { "prebuildify": "^6.0.1", From b759a5fac5e6c705fcd4f675ff9b53823acb8a1d Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Sat, 5 Oct 2024 10:40:15 +0300 Subject: [PATCH 06/61] fix(templates): properly replace author email (cherry picked from commit 21a34f912446a8b37c51df489d79eba9127243c2) --- cli/src/init.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cli/src/init.rs b/cli/src/init.rs index 5e203cd6..fc661aab 100644 --- a/cli/src/init.rs +++ b/cli/src/init.rs @@ -742,7 +742,12 @@ fn generate_file( } if let Some(email) = generate_opts.author_email { - replacement = replacement.replace(AUTHOR_EMAIL_PLACEHOLDER, email); + replacement = match filename { + "Cargo.toml" | "grammar.js" => { + replacement.replace(AUTHOR_EMAIL_PLACEHOLDER, &format!("<{email}>")) + } + _ => replacement.replace(AUTHOR_EMAIL_PLACEHOLDER, email), + } } else { match filename { "package.json" => { From 76fffb0f2d8192d628439dfbc0cac36689ec8115 Mon Sep 17 00:00:00 2001 From: Carlo Cabrera Date: Sat, 5 Oct 2024 23:27:21 +0800 Subject: [PATCH 07/61] build(make): fix `tree-sitter.pc` generation (#3745) (cherry picked from commit 079c69313fa14b9263739b494a47efacc1c91cdc) --- Makefile | 4 ++-- cli/src/templates/makefile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index aafbf20b..97a29f56 100644 --- a/Makefile +++ b/Makefile @@ -62,8 +62,8 @@ endif tree-sitter.pc: lib/tree-sitter.pc.in sed -e 's|@PROJECT_VERSION@|$(VERSION)|' \ - -e 's|@CMAKE_INSTALL_LIBDIR@|$(LIBDIR)|' \ - -e 's|@CMAKE_INSTALL_INCLUDEDIR@|$(INCLUDEDIR)|' \ + -e 's|@CMAKE_INSTALL_LIBDIR@|$(LIBDIR:$(PREFIX)/%=%)|' \ + -e 's|@CMAKE_INSTALL_INCLUDEDIR@|$(INCLUDEDIR:$(PREFIX)/%=%)|' \ -e 's|@PROJECT_DESCRIPTION@|$(DESCRIPTION)|' \ -e 's|@PROJECT_HOMEPAGE_URL@|$(HOMEPAGE_URL)|' \ -e 's|@CMAKE_INSTALL_PREFIX@|$(PREFIX)|' $< > $@ diff --git a/cli/src/templates/makefile b/cli/src/templates/makefile index c9bb86bc..8691b286 100644 --- a/cli/src/templates/makefile +++ b/cli/src/templates/makefile @@ -59,8 +59,8 @@ endif $(LANGUAGE_NAME).pc: bindings/c/$(LANGUAGE_NAME).pc.in sed -e 's|@PROJECT_VERSION@|$(VERSION)|' \ - -e 's|@CMAKE_INSTALL_LIBDIR@|$(LIBDIR)|' \ - -e 's|@CMAKE_INSTALL_INCLUDEDIR@|$(INCLUDEDIR)|' \ + -e 's|@CMAKE_INSTALL_LIBDIR@|$(LIBDIR:$(PREFIX)/%=%)|' \ + -e 's|@CMAKE_INSTALL_INCLUDEDIR@|$(INCLUDEDIR:$(PREFIX)/%=%)|' \ -e 's|@PROJECT_DESCRIPTION@|$(DESCRIPTION)|' \ -e 's|@PROJECT_HOMEPAGE_URL@|$(HOMEPAGE_URL)|' \ -e 's|@CMAKE_INSTALL_PREFIX@|$(PREFIX)|' $< > $@ From 0b167b0782a74e3702fb77cbef1486c6c01c24c8 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sat, 5 Oct 2024 20:17:34 -0400 Subject: [PATCH 08/61] 0.24.2 --- Cargo.lock | 14 +++++++------- Cargo.toml | 14 +++++++------- Makefile | 2 +- build.zig.zon | 2 +- cli/npm/package.json | 2 +- lib/CMakeLists.txt | 2 +- lib/binding_web/package.json | 2 +- 7 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e65d81dd..9c08efd2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1538,7 +1538,7 @@ dependencies = [ [[package]] name = "tree-sitter" -version = "0.24.1" +version = "0.24.2" dependencies = [ "bindgen", "cc", @@ -1551,7 +1551,7 @@ dependencies = [ [[package]] name = "tree-sitter-cli" -version = "0.24.1" +version = "0.24.2" dependencies = [ "anstyle", "anyhow", @@ -1601,7 +1601,7 @@ dependencies = [ [[package]] name = "tree-sitter-config" -version = "0.24.1" +version = "0.24.2" dependencies = [ "anyhow", "dirs", @@ -1611,7 +1611,7 @@ dependencies = [ [[package]] name = "tree-sitter-generate" -version = "0.24.1" +version = "0.24.2" dependencies = [ "anyhow", "heck 0.5.0", @@ -1632,7 +1632,7 @@ dependencies = [ [[package]] name = "tree-sitter-highlight" -version = "0.24.1" +version = "0.24.2" dependencies = [ "lazy_static", "regex", @@ -1647,7 +1647,7 @@ version = "0.1.2" [[package]] name = "tree-sitter-loader" -version = "0.24.1" +version = "0.24.2" dependencies = [ "anyhow", "cc", @@ -1670,7 +1670,7 @@ dependencies = [ [[package]] name = "tree-sitter-tags" -version = "0.24.1" +version = "0.24.2" dependencies = [ "memchr", "regex", diff --git a/Cargo.toml b/Cargo.toml index 3be44012..17f5be44 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.24.1" +version = "0.24.2" authors = ["Max Brunsfeld "] edition = "2021" rust-version = "1.74.1" @@ -96,9 +96,9 @@ walkdir = "2.5.0" wasmparser = "0.217.0" webbrowser = "1.0.2" -tree-sitter = { version = "0.24.0", path = "./lib" } -tree-sitter-generate = { version = "0.24.0", path = "./cli/generate" } -tree-sitter-loader = { version = "0.24.0", path = "./cli/loader" } -tree-sitter-config = { version = "0.24.0", path = "./cli/config" } -tree-sitter-highlight = { version = "0.24.0", path = "./highlight" } -tree-sitter-tags = { version = "0.24.0", path = "./tags" } +tree-sitter = { version = "0.24.2", path = "./lib" } +tree-sitter-generate = { version = "0.24.2", path = "./cli/generate" } +tree-sitter-loader = { version = "0.24.2", path = "./cli/loader" } +tree-sitter-config = { version = "0.24.2", path = "./cli/config" } +tree-sitter-highlight = { version = "0.24.2", path = "./highlight" } +tree-sitter-tags = { version = "0.24.2", path = "./tags" } diff --git a/Makefile b/Makefile index 97a29f56..22e7dbb6 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ ifeq ($(OS),Windows_NT) $(error Windows is not supported) endif -VERSION := 0.24.1 +VERSION := 0.24.2 DESCRIPTION := An incremental parsing system for programming tools HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/ diff --git a/build.zig.zon b/build.zig.zon index dd188771..8b826321 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,6 +1,6 @@ .{ .name = "tree-sitter", - .version = "0.24.1", + .version = "0.24.2", .paths = .{ "build.zig", "build.zig.zon", diff --git a/cli/npm/package.json b/cli/npm/package.json index 52788514..91611a09 100644 --- a/cli/npm/package.json +++ b/cli/npm/package.json @@ -1,6 +1,6 @@ { "name": "tree-sitter-cli", - "version": "0.24.1", + "version": "0.24.2", "author": "Max Brunsfeld", "license": "MIT", "repository": { diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 5426552e..07664727 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.13) project(tree-sitter - VERSION "0.24.1" + VERSION "0.24.2" DESCRIPTION "An incremental parsing system for programming tools" HOMEPAGE_URL "https://tree-sitter.github.io/tree-sitter/" LANGUAGES C) diff --git a/lib/binding_web/package.json b/lib/binding_web/package.json index 04c00b15..60a013b6 100644 --- a/lib/binding_web/package.json +++ b/lib/binding_web/package.json @@ -1,6 +1,6 @@ { "name": "web-tree-sitter", - "version": "0.24.1", + "version": "0.24.2", "description": "Tree-sitter bindings for the web", "main": "tree-sitter.js", "types": "tree-sitter-web.d.ts", From 46bdc14e20cb7145b5f7ac07409f565701e11bd2 Mon Sep 17 00:00:00 2001 From: dundargoc Date: Sun, 6 Oct 2024 14:21:28 +0200 Subject: [PATCH 09/61] build: fix incorrect variable checks Cmake variables should not use `${}` when checking for existence. (cherry picked from commit 94f7a47abd648c40f54f1f572421472ddd899e61) --- lib/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 07664727..34dfb94b 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -30,7 +30,7 @@ if(TREE_SITTER_FEATURE_WASM) message(CHECK_START "Looking for wasmtime headers") find_path(WASMTIME_INCLUDE_DIR wasmtime.h PATHS ENV DEP_WASMTIME_C_API_INCLUDE) - if(NOT ${WASMTIME_INCLUDE_DIR}) + if(NOT WASMTIME_INCLUDE_DIR) unset(WASMTIME_INCLUDE_DIR CACHE) message(FATAL_ERROR "Could not find wasmtime headers.\nDid you forget to set CMAKE_INCLUDE_PATH?") endif() @@ -46,7 +46,7 @@ if(TREE_SITTER_FEATURE_WASM) else() find_library(WASMTIME_LIBRARY libwasmtime.a) endif() - if(NOT ${WASMTIME_LIBRARY}) + if(NOT WASMTIME_LIBRARY) unset(WASMTIME_LIBRARY CACHE) message(FATAL_ERROR "Could not find wasmtime library.\nDid you forget to set CMAKE_LIBRARY_PATH?") endif() From cd1abd93510a87afe930eddef6862551e10d93c0 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Tue, 8 Oct 2024 17:45:25 -0400 Subject: [PATCH 10/61] fix(lib): correct unexpected side effect in `get_column` when the lexer is at EOF (cherry picked from commit 538a19797614e5934d9319f59a0d567b4f246ee2) --- cli/src/tests/parser_test.rs | 14 ++++++++ lib/src/lexer.c | 12 +++---- .../test_grammars/get_col_eof/corpus.txt | 0 .../test_grammars/get_col_eof/grammar.js | 11 ++++++ .../test_grammars/get_col_eof/scanner.c | 34 +++++++++++++++++++ 5 files changed, 65 insertions(+), 6 deletions(-) create mode 100644 test/fixtures/test_grammars/get_col_eof/corpus.txt create mode 100644 test/fixtures/test_grammars/get_col_eof/grammar.js create mode 100644 test/fixtures/test_grammars/get_col_eof/scanner.c diff --git a/cli/src/tests/parser_test.rs b/cli/src/tests/parser_test.rs index e1319395..6c783aee 100644 --- a/cli/src/tests/parser_test.rs +++ b/cli/src/tests/parser_test.rs @@ -1507,6 +1507,20 @@ fn test_parsing_with_scanner_logging() { assert!(found); } +#[test] +fn test_parsing_get_column_at_eof() { + let dir = fixtures_dir().join("test_grammars").join("get_col_eof"); + let grammar_json = load_grammar_file(&dir.join("grammar.js"), None).unwrap(); + let (grammar_name, parser_code) = generate_parser_for_grammar(&grammar_json).unwrap(); + + let mut parser = Parser::new(); + parser + .set_language(&get_test_language(&grammar_name, &parser_code, Some(&dir))) + .unwrap(); + + parser.parse("a", None).unwrap(); +} + const fn simple_range(start: usize, end: usize) -> Range { Range { start_byte: start, diff --git a/lib/src/lexer.c b/lib/src/lexer.c index e795618d..0d60589a 100644 --- a/lib/src/lexer.c +++ b/lib/src/lexer.c @@ -252,12 +252,12 @@ static uint32_t ts_lexer__get_column(TSLexer *_self) { uint32_t goal_byte = self->current_position.bytes; self->did_get_column = true; - self->current_position.bytes -= self->current_position.extent.column; - self->current_position.extent.column = 0; - - if (self->current_position.bytes < self->chunk_start) { - ts_lexer__get_chunk(self); - } + Length start_of_col = { + self->current_position.bytes - self->current_position.extent.column, + {self->current_position.extent.row, 0}, + }; + ts_lexer_goto(self, start_of_col); + ts_lexer__get_chunk(self); uint32_t result = 0; if (!ts_lexer__eof(_self)) { diff --git a/test/fixtures/test_grammars/get_col_eof/corpus.txt b/test/fixtures/test_grammars/get_col_eof/corpus.txt new file mode 100644 index 00000000..e69de29b diff --git a/test/fixtures/test_grammars/get_col_eof/grammar.js b/test/fixtures/test_grammars/get_col_eof/grammar.js new file mode 100644 index 00000000..3b70db2f --- /dev/null +++ b/test/fixtures/test_grammars/get_col_eof/grammar.js @@ -0,0 +1,11 @@ +module.exports = grammar({ + name: "get_col_eof", + + externals: $ => [ + $.char + ], + + rules: { + source_file: $ => repeat($.char), + } +}); diff --git a/test/fixtures/test_grammars/get_col_eof/scanner.c b/test/fixtures/test_grammars/get_col_eof/scanner.c new file mode 100644 index 00000000..1d262cf9 --- /dev/null +++ b/test/fixtures/test_grammars/get_col_eof/scanner.c @@ -0,0 +1,34 @@ +#include "tree_sitter/parser.h" + +enum TokenType { CHAR }; + +void *tree_sitter_get_col_eof_external_scanner_create(void) { return NULL; } + +void tree_sitter_get_col_eof_external_scanner_destroy(void *scanner) {} + +unsigned tree_sitter_get_col_eof_external_scanner_serialize(void *scanner, + char *buffer) { + return 0; +} + +void tree_sitter_get_col_eof_external_scanner_deserialize(void *scanner, + const char *buffer, + unsigned length) {} + +bool tree_sitter_get_col_eof_external_scanner_scan(void *scanner, + TSLexer *lexer, + const bool *valid_symbols) { + if (lexer->eof(lexer)) { + return false; + } + + if (valid_symbols[CHAR]) { + lexer->advance(lexer, false); + lexer->get_column(lexer); + lexer->result_symbol = CHAR; + lexer->mark_end(lexer); + return true; + } + + return false; +} From 8c45b79808a17d7e0dd7083b15cf9960472a92d2 Mon Sep 17 00:00:00 2001 From: Christian Clason Date: Wed, 9 Oct 2024 19:15:57 +0200 Subject: [PATCH 11/61] build(deps): bump wasmtime to v25.0.2 (cherry picked from commit b7a00527bef4070e48dc945e5755f8ace1038053) --- Cargo.lock | 100 ++++++++++++++++++++++++------------------------- lib/Cargo.toml | 2 +- 2 files changed, 51 insertions(+), 51 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9c08efd2..969d1e21 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -311,18 +311,18 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cranelift-bforest" -version = "0.112.1" +version = "0.112.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6e376bd92bddd03dcfc443b14382611cae5d10012aa0b1628bbf18bb73f12f7" +checksum = "7b765ed4349e66bedd9b88c7691da42e24c7f62067a6be17ddffa949367b6e17" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-bitset" -version = "0.112.1" +version = "0.112.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45ecbe07f25a8100e5077933516200e97808f1d7196b5a073edb85fa08fde32e" +checksum = "9eaa2aece6237198afd32bff57699e08d4dccb8d3902c214fc1e6ba907247ca4" dependencies = [ "serde", "serde_derive", @@ -330,9 +330,9 @@ dependencies = [ [[package]] name = "cranelift-codegen" -version = "0.112.1" +version = "0.112.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc60913f32c1de18538c28bef74b8c87cf16de7841a1b0956fcf01b23237853a" +checksum = "351824439e59d42f0e4fa5aac1d13deded155120043565769e55cd4ad3ca8ed9" dependencies = [ "bumpalo", "cranelift-bforest", @@ -353,33 +353,33 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.112.1" +version = "0.112.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bae009e7822f47aa55e7dcef846ccf3aa4eb102ca6b4bcb8a44b36f3f49aa85c" +checksum = "5a0ce0273d7a493ef8f31f606849a4e931c19187a4923f5f87fc1f2b13109981" dependencies = [ "cranelift-codegen-shared", ] [[package]] name = "cranelift-codegen-shared" -version = "0.112.1" +version = "0.112.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c78f01a852536c68e34444450f845ed6e0782a1f047f85397fe460b8fbce8f1" +checksum = "0f72016ac35579051913f4f07f6b36c509ed69412d852fd44c8e1d7b7fa6d92a" [[package]] name = "cranelift-control" -version = "0.112.1" +version = "0.112.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a061b22e00a9e36b31f2660dfb05a9617b7775bd54b79754d3bb75a990dac06" +checksum = "db28951d21512c4fd0554ef179bfb11e4eb6815062957a9173824eee5de0c46c" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.112.1" +version = "0.112.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95e2b261a3e74ae42f4e606906d5ffa44ee2684e8b1ae23bdf75d21908dc9233" +checksum = "14ebe592a2f81af9237cf9be29dd3854ecb72108cfffa59e85ef12389bf939e3" dependencies = [ "cranelift-bitset", "serde", @@ -388,9 +388,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.112.1" +version = "0.112.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe14abba0e6bab42aca0f9ce757f96880f9187e88bc6cb975ed6acd8a42f7770" +checksum = "4437db9d60c7053ac91ded0802740c2ccf123ee6d6898dd906c34f8c530cd119" dependencies = [ "cranelift-codegen", "log", @@ -400,15 +400,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.112.1" +version = "0.112.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "311d91ae72b37d4262b51217baf8c9e01f1afd5148931468da1fdb7e9d011347" +checksum = "230cb33572b9926e210f2ca28145f2bc87f389e1456560932168e2591feb65c1" [[package]] name = "cranelift-native" -version = "0.112.1" +version = "0.112.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a3f84c75e578189ff7a716c24ad83740b553bf583f2510b323bfe4c1a74bb93" +checksum = "364524ac7aef7070b1141478724abebeec297d4ea1e87ad8b8986465e91146d9" dependencies = [ "cranelift-codegen", "libc", @@ -417,9 +417,9 @@ dependencies = [ [[package]] name = "cranelift-wasm" -version = "0.112.1" +version = "0.112.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f56b7b2476c47b2091eee5a20bc54a80fbb29ca5313ae2bd0dea52621abcfca1" +checksum = "0572cbd9d136a62c0f39837b6bce3b0978b96b8586794042bec0c214668fd6f5" dependencies = [ "cranelift-codegen", "cranelift-entity", @@ -1104,9 +1104,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "b3e4daa0dcf6feba26f985457cdf104d4b4256fc5a09547140f3631bb076b19a" dependencies = [ "unicode-ident", ] @@ -1871,9 +1871,9 @@ dependencies = [ [[package]] name = "wasmtime" -version = "25.0.1" +version = "25.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03601559991d459a228236a49135364eac85ac00dc07b65fb95ae61a957793af" +checksum = "ef01f9cb9636ed42a7ec5a09d785c0643590199dc7372dc22c7e2ba7a31a97d4" dependencies = [ "anyhow", "bitflags", @@ -1911,18 +1911,18 @@ dependencies = [ [[package]] name = "wasmtime-asm-macros" -version = "25.0.1" +version = "25.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e453b3bde07312874c0c6703e2de9281daab46646172c1b71fa59a97226f858e" +checksum = "ba5b20797419d6baf2296db2354f864e8bb3447cacca9d151ce7700ae08b4460" dependencies = [ "cfg-if", ] [[package]] name = "wasmtime-c-api-impl" -version = "25.0.1" +version = "25.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4def1c38f8981c88d92e10acc7efb01da5b5775897fca2ab81caad76e930bd6d" +checksum = "2852f09a087c740683a32a33b8f34268d1d33c1298b4f707d25f4bee158ccd75" dependencies = [ "anyhow", "log", @@ -1934,9 +1934,9 @@ dependencies = [ [[package]] name = "wasmtime-c-api-macros" -version = "25.0.1" +version = "25.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c3feb5a461c52a376e80ef7ce7cee37a3a8395cb1794ac8eb340c0cd0b5d715" +checksum = "fa52cecfad085e7a9725bcbf3c2b15a900e5dc14f5ddcc305c9779c19936618b" dependencies = [ "proc-macro2", "quote", @@ -1944,9 +1944,9 @@ dependencies = [ [[package]] name = "wasmtime-component-macro" -version = "25.0.1" +version = "25.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a6faeabbdbfd27e24e8d5204207ba9c247a13cf84181ea721b5f209f281fe01" +checksum = "26593c4b18c76ca3c3fbdd813d6692256537b639b851d8a6fe827e3d6966fc01" dependencies = [ "anyhow", "proc-macro2", @@ -1959,15 +1959,15 @@ dependencies = [ [[package]] name = "wasmtime-component-util" -version = "25.0.1" +version = "25.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b1b24db4aa3dc7c0d3181d1833b4fe9ec0cd3f08780b746415c84c0a9ec9011" +checksum = "a2ed562fbb0cbed20a56c369c8de146c1de06a48c19e26ed9aa45f073514ee60" [[package]] name = "wasmtime-cranelift" -version = "25.0.1" +version = "25.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c737bef9ea94aab874e29ac6a8688b89ceb43c7b51f047079c43387972c07ee3" +checksum = "f389b789cbcb53a8499131182135dea21d7d97ad77e7fb66830f69479ef0e68c" dependencies = [ "anyhow", "cfg-if", @@ -1990,9 +1990,9 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "25.0.1" +version = "25.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "817bfa9ea878ec37aa24f85fd6912844e8d87d321662824cf920d561b698cdfd" +checksum = "84b72debe8899f19bedf66f7071310f06ef62de943a1369ba9b373613e77dd3d" dependencies = [ "anyhow", "cranelift-bitset", @@ -2013,9 +2013,9 @@ dependencies = [ [[package]] name = "wasmtime-jit-icache-coherence" -version = "25.0.1" +version = "25.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48011232c0da424f89c3752a378d0b7f512fae321ea414a43e1e7a302a6a1f7e" +checksum = "1d930bc1325bc0448be6a11754156d770f56f6c3a61f440e9567f36cd2ea3065" dependencies = [ "anyhow", "cfg-if", @@ -2025,15 +2025,15 @@ dependencies = [ [[package]] name = "wasmtime-slab" -version = "25.0.1" +version = "25.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9858a22e656ae8574631221b474b8bebf63f1367fcac3f179873833eabc2ced" +checksum = "055a181b8d03998511294faea14798df436503f14d7fd20edcf7370ec583e80a" [[package]] name = "wasmtime-types" -version = "25.0.1" +version = "25.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d14b8a9206fe94485a03edb1654cd530dbd2a859a85a43502cb4e99653a568c" +checksum = "c8340d976673ac3fdacac781f2afdc4933920c1adc738c3409e825dab3955399" dependencies = [ "anyhow", "cranelift-entity", @@ -2045,9 +2045,9 @@ dependencies = [ [[package]] name = "wasmtime-versioned-export-macros" -version = "25.0.1" +version = "25.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9bb1f01efb8b542eadfda511e8ea1cc54309451aba97b69969e5b1a59cb7ded" +checksum = "a4b0c1f76891f778db9602ee3fbb4eb7e9a3f511847d1fb1b69eddbcea28303c" dependencies = [ "proc-macro2", "quote", @@ -2056,9 +2056,9 @@ dependencies = [ [[package]] name = "wasmtime-wit-bindgen" -version = "25.0.1" +version = "25.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1596caa67b31ac675fd3da61685c4260f8b10832021db42c85d227b7ba8133" +checksum = "b2fca2cbb5bb390f65d4434c19bf8d9873dfc60f10802918ebcd6f819a38d703" dependencies = [ "anyhow", "heck 0.4.1", diff --git a/lib/Cargo.toml b/lib/Cargo.toml index c261e36c..3460a537 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -37,7 +37,7 @@ tree-sitter-language = { version = "0.1", path = "language" } streaming-iterator = "0.1.9" [dependencies.wasmtime-c-api] -version = "25.0.1" +version = "25.0.2" optional = true package = "wasmtime-c-api-impl" default-features = false From bdfe32402e85673bbc693216f0a6ef72c98bb665 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Wed, 9 Oct 2024 17:22:39 -0400 Subject: [PATCH 12/61] 0.24.3 --- Cargo.lock | 14 +++++++------- Cargo.toml | 14 +++++++------- Makefile | 2 +- build.zig.zon | 2 +- cli/npm/package.json | 2 +- lib/CMakeLists.txt | 2 +- lib/binding_web/package.json | 2 +- 7 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 969d1e21..0b8fd339 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1538,7 +1538,7 @@ dependencies = [ [[package]] name = "tree-sitter" -version = "0.24.2" +version = "0.24.3" dependencies = [ "bindgen", "cc", @@ -1551,7 +1551,7 @@ dependencies = [ [[package]] name = "tree-sitter-cli" -version = "0.24.2" +version = "0.24.3" dependencies = [ "anstyle", "anyhow", @@ -1601,7 +1601,7 @@ dependencies = [ [[package]] name = "tree-sitter-config" -version = "0.24.2" +version = "0.24.3" dependencies = [ "anyhow", "dirs", @@ -1611,7 +1611,7 @@ dependencies = [ [[package]] name = "tree-sitter-generate" -version = "0.24.2" +version = "0.24.3" dependencies = [ "anyhow", "heck 0.5.0", @@ -1632,7 +1632,7 @@ dependencies = [ [[package]] name = "tree-sitter-highlight" -version = "0.24.2" +version = "0.24.3" dependencies = [ "lazy_static", "regex", @@ -1647,7 +1647,7 @@ version = "0.1.2" [[package]] name = "tree-sitter-loader" -version = "0.24.2" +version = "0.24.3" dependencies = [ "anyhow", "cc", @@ -1670,7 +1670,7 @@ dependencies = [ [[package]] name = "tree-sitter-tags" -version = "0.24.2" +version = "0.24.3" dependencies = [ "memchr", "regex", diff --git a/Cargo.toml b/Cargo.toml index 17f5be44..e9140f2a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.24.2" +version = "0.24.3" authors = ["Max Brunsfeld "] edition = "2021" rust-version = "1.74.1" @@ -96,9 +96,9 @@ walkdir = "2.5.0" wasmparser = "0.217.0" webbrowser = "1.0.2" -tree-sitter = { version = "0.24.2", path = "./lib" } -tree-sitter-generate = { version = "0.24.2", path = "./cli/generate" } -tree-sitter-loader = { version = "0.24.2", path = "./cli/loader" } -tree-sitter-config = { version = "0.24.2", path = "./cli/config" } -tree-sitter-highlight = { version = "0.24.2", path = "./highlight" } -tree-sitter-tags = { version = "0.24.2", path = "./tags" } +tree-sitter = { version = "0.24.3", path = "./lib" } +tree-sitter-generate = { version = "0.24.3", path = "./cli/generate" } +tree-sitter-loader = { version = "0.24.3", path = "./cli/loader" } +tree-sitter-config = { version = "0.24.3", path = "./cli/config" } +tree-sitter-highlight = { version = "0.24.3", path = "./highlight" } +tree-sitter-tags = { version = "0.24.3", path = "./tags" } diff --git a/Makefile b/Makefile index 22e7dbb6..92cd8395 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ ifeq ($(OS),Windows_NT) $(error Windows is not supported) endif -VERSION := 0.24.2 +VERSION := 0.24.3 DESCRIPTION := An incremental parsing system for programming tools HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/ diff --git a/build.zig.zon b/build.zig.zon index 8b826321..f9f9798c 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,6 +1,6 @@ .{ .name = "tree-sitter", - .version = "0.24.2", + .version = "0.24.3", .paths = .{ "build.zig", "build.zig.zon", diff --git a/cli/npm/package.json b/cli/npm/package.json index 91611a09..8e7c600b 100644 --- a/cli/npm/package.json +++ b/cli/npm/package.json @@ -1,6 +1,6 @@ { "name": "tree-sitter-cli", - "version": "0.24.2", + "version": "0.24.3", "author": "Max Brunsfeld", "license": "MIT", "repository": { diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 34dfb94b..c27db119 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.13) project(tree-sitter - VERSION "0.24.2" + VERSION "0.24.3" DESCRIPTION "An incremental parsing system for programming tools" HOMEPAGE_URL "https://tree-sitter.github.io/tree-sitter/" LANGUAGES C) diff --git a/lib/binding_web/package.json b/lib/binding_web/package.json index 60a013b6..fa9b5e7e 100644 --- a/lib/binding_web/package.json +++ b/lib/binding_web/package.json @@ -1,6 +1,6 @@ { "name": "web-tree-sitter", - "version": "0.24.2", + "version": "0.24.3", "description": "Tree-sitter bindings for the web", "main": "tree-sitter.js", "types": "tree-sitter-web.d.ts", From 657c7b548e931739042a944caee4fea185d04fe2 Mon Sep 17 00:00:00 2001 From: Ryan Patterson Date: Sun, 13 Oct 2024 10:17:12 +0800 Subject: [PATCH 13/61] Memory errors in wasm_store ``` In file included from tree_sitter/core/lib/src/lib.c:14: tree_sitter/core/lib/src/./wasm_store.c:868:94: warning: incompatible pointer types passing 'uint32_t *' (aka 'unsigned int *') to parameter of type 'uint64_t *' (aka 'unsigned long long *') [-Wincompatible-pointer-types] error = wasmtime_table_grow(context, &function_table, lexer_definitions_len, &initializer, &table_index); ^~~~~~~~~~~~ /Users/rpatterson/Projects/amel/py-tree-sitter/.direnv/python-3.11/include/wasmtime/table.h:105:31: note: passing argument to parameter 'prev_size' here uint64_t *prev_size); ^ In file included from tree_sitter/core/lib/src/lib.c:14: tree_sitter/core/lib/src/./wasm_store.c:969:102: warning: incompatible pointer types passing 'uint32_t *' (aka 'unsigned int *') to parameter of type 'uint64_t *' (aka 'unsigned long long *') [-Wincompatible-pointer-types] error = wasmtime_table_grow(context, &self->function_table, dylink_info->table_size, &initializer, &prev_table_size); ^~~~~~~~~~~~~~~~ /Users/rpatterson/Projects/amel/py-tree-sitter/.direnv/python-3.11/include/wasmtime/table.h:105:31: note: passing argument to parameter 'prev_size' here uint64_t *prev_size); ^ 2 warnings generated. ``` (cherry picked from commit 178c5d93f4ca8c962dc977365148148edbce1f24) --- lib/src/wasm_store.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/src/wasm_store.c b/lib/src/wasm_store.c index 81efbfcc..850d8405 100644 --- a/lib/src/wasm_store.c +++ b/lib/src/wasm_store.c @@ -864,7 +864,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) { // Add all of the lexer callback functions to the function table. Store their function table // indices on the in-memory lexer. - uint32_t table_index; + uint64_t table_index; error = wasmtime_table_grow(context, &function_table, lexer_definitions_len, &initializer, &table_index); if (error) { wasmtime_error_message(error, &message); @@ -965,7 +965,7 @@ static bool ts_wasm_store__instantiate( // Grow the function table to make room for the new functions. wasmtime_val_t initializer = {.kind = WASMTIME_FUNCREF}; - uint32_t prev_table_size; + uint64_t prev_table_size; error = wasmtime_table_grow(context, &self->function_table, dylink_info->table_size, &initializer, &prev_table_size); if (error) { format(error_message, "invalid function table size %u", dylink_info->table_size); From f629fd4aed892685c581199da09387d85806b062 Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Sun, 13 Oct 2024 16:21:10 +0300 Subject: [PATCH 14/61] revert: "Memory errors in wasm_store" This reverts commit 178c5d93f4ca8c962dc977365148148edbce1f24. (cherry picked from commit b7421bf89f993fc48d6e8c6f38e3f1aed96372b8) --- lib/src/wasm_store.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/src/wasm_store.c b/lib/src/wasm_store.c index 850d8405..81efbfcc 100644 --- a/lib/src/wasm_store.c +++ b/lib/src/wasm_store.c @@ -864,7 +864,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) { // Add all of the lexer callback functions to the function table. Store their function table // indices on the in-memory lexer. - uint64_t table_index; + uint32_t table_index; error = wasmtime_table_grow(context, &function_table, lexer_definitions_len, &initializer, &table_index); if (error) { wasmtime_error_message(error, &message); @@ -965,7 +965,7 @@ static bool ts_wasm_store__instantiate( // Grow the function table to make room for the new functions. wasmtime_val_t initializer = {.kind = WASMTIME_FUNCREF}; - uint64_t prev_table_size; + uint32_t prev_table_size; error = wasmtime_table_grow(context, &self->function_table, dylink_info->table_size, &initializer, &prev_table_size); if (error) { format(error_message, "invalid function table size %u", dylink_info->table_size); From a498790a8271d18212d24b77c67b66af767e926c Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sun, 27 Oct 2024 15:25:29 -0400 Subject: [PATCH 15/61] fix(lib): simplify edge cases with zero-width tokens (cherry picked from commit 350fff24bf01508b7b8168b36eb373edfbac0830) --- cli/src/tests/node_test.rs | 25 ++++++++++++++ lib/src/node.c | 71 +++++++++++--------------------------- 2 files changed, 46 insertions(+), 50 deletions(-) diff --git a/cli/src/tests/node_test.rs b/cli/src/tests/node_test.rs index fdabc499..20686a4a 100644 --- a/cli/src/tests/node_test.rs +++ b/cli/src/tests/node_test.rs @@ -1026,6 +1026,31 @@ fn test_node_numeric_symbols_respect_simple_aliases() { assert_eq!(unary_minus_node.kind_id(), binary_minus_node.kind_id()); } +#[test] +fn test_hidden_zero_width_node_with_visible_child() { + let code = r" +class Foo { + std:: +private: + std::string s; +}; +"; + + let mut parser = Parser::new(); + parser.set_language(&get_language("cpp")).unwrap(); + let tree = parser.parse(code, None).unwrap(); + let root = tree.root_node(); + + let class_specifier = root.child(0).unwrap(); + let field_decl_list = class_specifier.child_by_field_name("body").unwrap(); + let field_decl = field_decl_list.named_child(0).unwrap(); + let field_ident = field_decl.child_by_field_name("declarator").unwrap(); + assert_eq!( + field_decl.child_with_descendant(field_ident).unwrap(), + field_ident + ); +} + fn get_all_nodes(tree: &Tree) -> Vec { let mut result = Vec::new(); let mut visited_children = false; diff --git a/lib/src/node.c b/lib/src/node.c index 818735a1..40d6024f 100644 --- a/lib/src/node.c +++ b/lib/src/node.c @@ -103,21 +103,6 @@ static inline bool ts_node_child_iterator_next( return true; } -// This will return true if the next sibling is a zero-width token that is adjacent to the current node and is relevant -static inline bool ts_node_child_iterator_next_sibling_is_empty_adjacent(NodeChildIterator *self, TSNode previous) { - if (!self->parent.ptr || ts_node_child_iterator_done(self)) return false; - if (self->child_index == 0) return false; - const Subtree *child = &ts_subtree_children(self->parent)[self->child_index]; - TSSymbol alias = 0; - if (!ts_subtree_extra(*child)) { - if (self->alias_sequence) { - alias = self->alias_sequence[self->structural_child_index]; - } - } - TSNode next = ts_node_new(self->tree, child, self->position, alias); - return ts_node_end_byte(previous) == ts_node_end_byte(next) && ts_node__is_relevant(next, true); -} - // TSNode - private static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous) { @@ -549,9 +534,9 @@ TSNode ts_node_parent(TSNode self) { if (node.id == self.id) return ts_node__null(); while (true) { - TSNode next_node = ts_node_child_containing_descendant(node, self); - if (ts_node_is_null(next_node)) break; - node = next_node; + TSNode next_node = ts_node_child_containing_descendant(node, self); + if (ts_node_is_null(next_node)) break; + node = next_node; } return node; @@ -560,6 +545,7 @@ TSNode ts_node_parent(TSNode self) { TSNode ts_node_child_containing_descendant(TSNode self, TSNode descendant) { uint32_t start_byte = ts_node_start_byte(descendant); uint32_t end_byte = ts_node_end_byte(descendant); + bool is_empty = start_byte == end_byte; do { NodeChildIterator iter = ts_node_iterate_children(&self); @@ -572,24 +558,16 @@ TSNode ts_node_child_containing_descendant(TSNode self, TSNode descendant) { return ts_node__null(); } - // Here we check the current self node and *all* of its zero-width token siblings that follow. - // If any of these nodes contain the target subnode, we return that node. Otherwise, we restore the node we started at - // for the loop condition, and that will continue with the next *non-zero-width* sibling. - TSNode old = self; - // While the next sibling is a zero-width token - while (ts_node_child_iterator_next_sibling_is_empty_adjacent(&iter, self)) { - TSNode current_node = ts_node_child_containing_descendant(self, descendant); - // If the target child is in self, return it - if (!ts_node_is_null(current_node)) { - return current_node; - } - ts_node_child_iterator_next(&iter, &self); - if (self.id == descendant.id) { - return ts_node__null(); + // If the descendant is empty, and the end byte is within `self`, + // we check whether `self` contains it or not. + if (is_empty && iter.position.bytes >= end_byte && ts_node_child_count(self) > 0) { + TSNode child = ts_node_child_with_descendant(self, descendant); + // If the child is not null, return self if it's relevant, else return the child + if (!ts_node_is_null(child)) { + return ts_node__is_relevant(self, true) ? self : child; } } - self = old; - } while (iter.position.bytes < end_byte || ts_node_child_count(self) == 0); + } while ((is_empty ? iter.position.bytes <= end_byte : iter.position.bytes < end_byte) || ts_node_child_count(self) == 0); } while (!ts_node__is_relevant(self, true)); return self; @@ -598,6 +576,7 @@ TSNode ts_node_child_containing_descendant(TSNode self, TSNode descendant) { TSNode ts_node_child_with_descendant(TSNode self, TSNode descendant) { uint32_t start_byte = ts_node_start_byte(descendant); uint32_t end_byte = ts_node_end_byte(descendant); + bool is_empty = start_byte == end_byte; do { NodeChildIterator iter = ts_node_iterate_children(&self); @@ -612,24 +591,16 @@ TSNode ts_node_child_with_descendant(TSNode self, TSNode descendant) { return self; } - // Here we check the current self node and *all* of its zero-width token siblings that follow. - // If any of these nodes contain the target subnode, we return that node. Otherwise, we restore the node we started at - // for the loop condition, and that will continue with the next *non-zero-width* sibling. - TSNode old = self; - // While the next sibling is a zero-width token - while (ts_node_child_iterator_next_sibling_is_empty_adjacent(&iter, self)) { - TSNode current_node = ts_node_child_with_descendant(self, descendant); - // If the target child is in self, return it - if (!ts_node_is_null(current_node)) { - return current_node; - } - ts_node_child_iterator_next(&iter, &self); - if (self.id == descendant.id) { - return self; + // If the descendant is empty, and the end byte is within `self`, + // we check whether `self` contains it or not. + if (is_empty && iter.position.bytes >= end_byte && ts_node_child_count(self) > 0) { + TSNode child = ts_node_child_with_descendant(self, descendant); + // If the child is not null, return self if it's relevant, else return the child + if (!ts_node_is_null(child)) { + return ts_node__is_relevant(self, true) ? self : child; } } - self = old; - } while (iter.position.bytes < end_byte || ts_node_child_count(self) == 0); + } while ((is_empty ? iter.position.bytes <= end_byte : iter.position.bytes < end_byte) || ts_node_child_count(self) == 0); } while (!ts_node__is_relevant(self, true)); return self; From 12bc174205778a1140bf24102aca35a309ead6e6 Mon Sep 17 00:00:00 2001 From: Sam Estep Date: Wed, 30 Oct 2024 17:09:21 -0400 Subject: [PATCH 16/61] fix(cli): pass all fields to `tree-sitter.json` in `init` (cherry picked from commit f3f7230ee3e6703bda2caafe26c67bbe5946eaf0) --- cli/src/init.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/src/init.rs b/cli/src/init.rs index fc661aab..c6a95d49 100644 --- a/cli/src/init.rs +++ b/cli/src/init.rs @@ -138,7 +138,7 @@ impl JsonConfigOpts { scope: self.scope, path: PathBuf::from("."), external_files: PathsJSON::Empty, - file_types: None, + file_types: Some(self.file_types), highlights: PathsJSON::Empty, injections: PathsJSON::Empty, locals: PathsJSON::Empty, @@ -154,7 +154,7 @@ impl JsonConfigOpts { authors: Some(vec![Author { name: self.author, email: self.email, - url: None, + url: self.url.map(|url| url.to_string()), }]), links: Some(Links { repository: self.repository.unwrap_or_else(|| { From 89bd9e302e09b6eca68cac0da17180f141edd787 Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Thu, 31 Oct 2024 10:11:33 +0200 Subject: [PATCH 17/61] build(bindings): rename cmake test target CTest creates a test target which breaks the build when the parser is included via FetchContent in a CMake project that uses CTest (cherry picked from commit 02ff0af69ca762c5b86b07155ddd2cd8d4a95c97) --- cli/src/templates/cmakelists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/templates/cmakelists.txt b/cli/src/templates/cmakelists.txt index 24f2507d..d3b7024b 100644 --- a/cli/src/templates/cmakelists.txt +++ b/cli/src/templates/cmakelists.txt @@ -53,7 +53,7 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter-PARSER_NAME.pc" install(TARGETS tree-sitter-PARSER_NAME LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}") -add_custom_target(test "${TREE_SITTER_CLI}" test +add_custom_target(ts-test "${TREE_SITTER_CLI}" test WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" COMMENT "tree-sitter test") From b1493f9b35ff7201839f70eb55b145dbfeff6146 Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Sat, 2 Nov 2024 14:01:26 +0200 Subject: [PATCH 18/61] fix(bindings): update CMakeLists.txt file (cherry picked from commit 66a1bc50fa31e3f127c7750e87c70bf2eaba1a9a) --- cli/src/init.rs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/cli/src/init.rs b/cli/src/init.rs index c6a95d49..88e098b1 100644 --- a/cli/src/init.rs +++ b/cli/src/init.rs @@ -361,7 +361,7 @@ pub fn migrate_package_json(repo_path: &Path) -> Result { pub fn generate_grammar_files( repo_path: &Path, language_name: &str, - _allow_update: bool, + allow_update: bool, opts: Option, ) -> Result<()> { let dashed_language_name = language_name.to_kebab_case(); @@ -529,9 +529,20 @@ pub fn generate_grammar_files( generate_file(path, MAKEFILE_TEMPLATE, language_name, &generate_opts) })?; - missing_path(repo_path.join("CMakeLists.txt"), |path| { - generate_file(path, CMAKELISTS_TXT_TEMPLATE, language_name, &generate_opts) - })?; + missing_path_else( + repo_path.join("CMakeLists.txt"), + allow_update, + |path| generate_file(path, CMAKELISTS_TXT_TEMPLATE, language_name, &generate_opts), + |path| { + let contents = fs::read_to_string(path)?; + let old = "add_custom_target(test"; + if contents.contains(old) { + write_file(path, contents.replace(old, "add_custom_target(ts-test")) + } else { + Ok(()) + } + }, + )?; Ok(()) })?; From 5e93499f20d606aa236df193de5663903861137d Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Sun, 20 Oct 2024 13:06:29 +0300 Subject: [PATCH 19/61] chore(bindings): more small changes - Rename cmakelists.txt to cmakelists.cmake - Bump node-addon-api version in package.json - Remove License classifier from pyproject.toml - Move require call to top level in Node.js test (cherry picked from commit d3a127a48fe8ec2c1552f82f6b25621e69d46140) --- cli/src/init.rs | 2 +- cli/src/templates/binding_test.js | 6 +++--- cli/src/templates/{cmakelists.txt => cmakelists.cmake} | 2 -- cli/src/templates/package.json | 2 +- cli/src/templates/pyproject.toml | 1 - 5 files changed, 5 insertions(+), 8 deletions(-) rename cli/src/templates/{cmakelists.txt => cmakelists.cmake} (99%) diff --git a/cli/src/init.rs b/cli/src/init.rs index 88e098b1..6294f2e1 100644 --- a/cli/src/init.rs +++ b/cli/src/init.rs @@ -76,7 +76,7 @@ const BINDING_GYP_TEMPLATE: &str = include_str!("./templates/binding.gyp"); const BINDING_TEST_JS_TEMPLATE: &str = include_str!("./templates/binding_test.js"); const MAKEFILE_TEMPLATE: &str = include_str!("./templates/makefile"); -const CMAKELISTS_TXT_TEMPLATE: &str = include_str!("./templates/cmakelists.txt"); +const CMAKELISTS_TXT_TEMPLATE: &str = include_str!("./templates/cmakelists.cmake"); const PARSER_NAME_H_TEMPLATE: &str = include_str!("./templates/PARSER_NAME.h"); const PARSER_NAME_PC_IN_TEMPLATE: &str = include_str!("./templates/PARSER_NAME.pc.in"); diff --git a/cli/src/templates/binding_test.js b/cli/src/templates/binding_test.js index afede30a..55becacf 100644 --- a/cli/src/templates/binding_test.js +++ b/cli/src/templates/binding_test.js @@ -1,9 +1,9 @@ -/// - const assert = require("node:assert"); const { test } = require("node:test"); +const Parser = require("tree-sitter"); + test("can load grammar", () => { - const parser = new (require("tree-sitter"))(); + const parser = new Parser(); assert.doesNotThrow(() => parser.setLanguage(require("."))); }); diff --git a/cli/src/templates/cmakelists.txt b/cli/src/templates/cmakelists.cmake similarity index 99% rename from cli/src/templates/cmakelists.txt rename to cli/src/templates/cmakelists.cmake index d3b7024b..a09a61fb 100644 --- a/cli/src/templates/cmakelists.txt +++ b/cli/src/templates/cmakelists.cmake @@ -56,5 +56,3 @@ install(TARGETS tree-sitter-PARSER_NAME add_custom_target(ts-test "${TREE_SITTER_CLI}" test WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" COMMENT "tree-sitter test") - -# vim:ft=cmake: diff --git a/cli/src/templates/package.json b/cli/src/templates/package.json index 212ffc3b..22488e45 100644 --- a/cli/src/templates/package.json +++ b/cli/src/templates/package.json @@ -27,7 +27,7 @@ "*.wasm" ], "dependencies": { - "node-addon-api": "^8.1.0", + "node-addon-api": "^8.2.1", "node-gyp-build": "^4.8.2" }, "devDependencies": { diff --git a/cli/src/templates/pyproject.toml b/cli/src/templates/pyproject.toml index 4df75f74..e0db4e21 100644 --- a/cli/src/templates/pyproject.toml +++ b/cli/src/templates/pyproject.toml @@ -9,7 +9,6 @@ version = "PARSER_VERSION" keywords = ["incremental", "parsing", "tree-sitter", "PARSER_NAME"] classifiers = [ "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", "Topic :: Software Development :: Compilers", "Topic :: Text Processing :: Linguistic", "Typing :: Typed", From 61d67adbf7415b616f722ad2dbb40ab7ad6a06a8 Mon Sep 17 00:00:00 2001 From: Bastiaan Marinus van de Weerd Date: Sat, 9 Nov 2024 17:09:50 -0300 Subject: [PATCH 20/61] feat(loader): add `language_for_configuration` (cherry picked from commit 0dc1044d0d2a0d74c194ac8c1a9c200bd2d9da87) --- cli/loader/src/lib.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cli/loader/src/lib.rs b/cli/loader/src/lib.rs index 1b8083c0..f07f5eac 100644 --- a/cli/loader/src/lib.rs +++ b/cli/loader/src/lib.rs @@ -600,6 +600,13 @@ impl Loader { } } + pub fn language_for_configuration( + &self, + configuration: &LanguageConfiguration, + ) -> Result { + self.language_for_id(configuration.language_id) + } + fn language_for_id(&self, id: usize) -> Result { let (path, language, externals) = &self.languages_by_id[id]; language From 4cf96126d486a5ef68eb2a0c99b8070f6c6fb6fd Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sat, 9 Nov 2024 18:27:46 -0500 Subject: [PATCH 21/61] fix(lib): correctly fetch the node name in query errors (cherry picked from commit 49ff53cc83b9ac90684f2675e05a81e6c9771781) --- lib/binding_rust/lib.rs | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/lib/binding_rust/lib.rs b/lib/binding_rust/lib.rs index bba21bea..c2bae2c4 100644 --- a/lib/binding_rust/lib.rs +++ b/lib/binding_rust/lib.rs @@ -8,7 +8,6 @@ extern crate alloc; #[cfg(not(feature = "std"))] use alloc::{boxed::Box, format, string::String, string::ToString, vec::Vec}; use core::{ - char, ffi::{c_char, c_void, CStr}, fmt::{self, Write}, hash, iter, @@ -1854,9 +1853,28 @@ impl Query { // Error types that report names ffi::TSQueryErrorNodeType | ffi::TSQueryErrorField | ffi::TSQueryErrorCapture => { let suffix = source.split_at(offset).1; - let end_offset = suffix - .find(|c| !char::is_alphanumeric(c) && c != '_' && c != '-') - .unwrap_or(suffix.len()); + let in_quotes = source.as_bytes()[offset - 1] == b'"'; + let mut end_offset = suffix.len(); + if let Some(pos) = suffix + .char_indices() + .take_while(|(_, c)| *c != '\n') + .find_map(|(i, c)| match c { + '"' if in_quotes + && i > 0 + && suffix.chars().nth(i - 1) != Some('\\') => + { + Some(i) + } + c if !in_quotes + && (c.is_whitespace() || c == '(' || c == ')' || c == ':') => + { + Some(i) + } + _ => None, + }) + { + end_offset = pos; + } message = suffix.split_at(end_offset).0.to_string(); kind = match error_type { ffi::TSQueryErrorNodeType => QueryErrorKind::NodeType, From aac741dfd1e0825d17749a9aec756d2d6ac60e6a Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sat, 12 Oct 2024 00:57:51 -0400 Subject: [PATCH 22/61] perf(loader): improve language lookup speed (cherry picked from commit 72f114fa126bb43472549f9a1e6f2b4d7b6bfb75) --- Cargo.lock | 1 + cli/loader/Cargo.toml | 1 + cli/loader/src/lib.rs | 89 +++++++++++++++++++++---------------------- cli/src/init.rs | 30 +++++++-------- cli/src/main.rs | 8 ++-- 5 files changed, 63 insertions(+), 66 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0b8fd339..3d52abb9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1654,6 +1654,7 @@ dependencies = [ "dirs", "fs4", "indoc", + "lazy_static", "libloading", "once_cell", "path-slash", diff --git a/cli/loader/Cargo.toml b/cli/loader/Cargo.toml index ec2b35d1..65025127 100644 --- a/cli/loader/Cargo.toml +++ b/cli/loader/Cargo.toml @@ -24,6 +24,7 @@ cc.workspace = true dirs.workspace = true fs4.workspace = true indoc.workspace = true +lazy_static.workspace = true libloading.workspace = true once_cell.workspace = true path-slash.workspace = true diff --git a/cli/loader/src/lib.rs b/cli/loader/src/lib.rs index f07f5eac..1eb094c3 100644 --- a/cli/loader/src/lib.rs +++ b/cli/loader/src/lib.rs @@ -21,6 +21,7 @@ use anyhow::Error; use anyhow::{anyhow, Context, Result}; use fs4::fs_std::FileExt; use indoc::indoc; +use lazy_static::lazy_static; use libloading::{Library, Symbol}; use once_cell::unsync::OnceCell; use path_slash::PathBufExt as _; @@ -38,6 +39,10 @@ use tree_sitter_highlight::HighlightConfiguration; use tree_sitter_tags::{Error as TagsError, TagsConfiguration}; use url::Url; +lazy_static! { + static ref GRAMMAR_NAME_REGEX: Regex = Regex::new(r#""name":\s*"(.*?)""#).unwrap(); +} + pub const EMSCRIPTEN_TAG: &str = concat!("docker.io/emscripten/emsdk:", env!("EMSCRIPTEN_VERSION")); #[derive(Default, Deserialize, Serialize)] @@ -142,11 +147,7 @@ pub struct TreeSitterJSON { impl TreeSitterJSON { pub fn from_file(path: &Path) -> Option { - if let Ok(file) = fs::File::open(path.join("tree-sitter.json")) { - Some(serde_json::from_reader(file).ok()?) - } else { - None - } + serde_json::from_str(&fs::read_to_string(path.join("tree-sitter.json")).ok()?).ok() } pub fn has_multiple_language_configs(&self) -> bool { @@ -192,7 +193,6 @@ pub struct Metadata { pub authors: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub links: Option, - // #[serde(skip_serializing_if = "Option::is_none")] #[serde(skip)] pub namespace: Option, } @@ -635,27 +635,7 @@ impl Loader { pub fn load_language_at_path(&self, mut config: CompileConfig) -> Result { let grammar_path = config.src_path.join("grammar.json"); - - #[derive(Deserialize)] - struct GrammarJSON { - name: String, - } - let mut grammar_file = fs::File::open(&grammar_path).with_context(|| { - format!( - "Failed to read grammar.json file at the following path:\n{:?}", - &grammar_path - ) - })?; - let grammar_json: GrammarJSON = serde_json::from_reader(BufReader::new(&mut grammar_file)) - .with_context(|| { - format!( - "Failed to parse grammar.json file at the following path:\n{:?}", - &grammar_path - ) - })?; - - config.name = grammar_json.name; - + config.name = Self::grammar_json_name(&grammar_path)?; self.load_language_at_path_with_name(config) } @@ -1132,11 +1112,6 @@ impl Loader { parser_path: &Path, set_current_path_config: bool, ) -> Result<&[LanguageConfiguration]> { - #[derive(Deserialize)] - struct GrammarJSON { - name: String, - } - let initial_language_configuration_count = self.language_configurations.len(); if let Some(config) = TreeSitterJSON::from_file(parser_path) { @@ -1147,13 +1122,6 @@ impl Loader { // package.json. let language_path = parser_path.join(grammar.path); - let grammar_path = language_path.join("src").join("grammar.json"); - let mut grammar_file = - fs::File::open(grammar_path).with_context(|| "Failed to read grammar.json")?; - let grammar_json: GrammarJSON = - serde_json::from_reader(BufReader::new(&mut grammar_file)) - .with_context(|| "Failed to parse grammar.json")?; - // Determine if a previous language configuration in this package.json file // already uses the same language. let mut language_id = None; @@ -1191,7 +1159,7 @@ impl Loader { let configuration = LanguageConfiguration { root_path: parser_path.to_path_buf(), - language_name: grammar_json.name, + language_name: grammar.name, scope: Some(grammar.scope), language_id, file_types: grammar.file_types.unwrap_or_default(), @@ -1239,18 +1207,17 @@ impl Loader { } } + // If we didn't find any language configurations in the tree-sitter.json file, + // but there is a grammar.json file, then use the grammar file to form a simple + // language configuration. if self.language_configurations.len() == initial_language_configuration_count && parser_path.join("src").join("grammar.json").exists() { let grammar_path = parser_path.join("src").join("grammar.json"); - let mut grammar_file = - fs::File::open(grammar_path).with_context(|| "Failed to read grammar.json")?; - let grammar_json: GrammarJSON = - serde_json::from_reader(BufReader::new(&mut grammar_file)) - .with_context(|| "Failed to parse grammar.json")?; + let language_name = Self::grammar_json_name(&grammar_path)?; let configuration = LanguageConfiguration { root_path: parser_path.to_owned(), - language_name: grammar_json.name, + language_name, language_id: self.languages_by_id.len(), file_types: Vec::new(), scope: None, @@ -1286,6 +1253,36 @@ impl Loader { pattern.and_then(|r| RegexBuilder::new(r).multi_line(true).build().ok()) } + fn grammar_json_name(grammar_path: &Path) -> Result { + let file = fs::File::open(grammar_path).with_context(|| { + format!("Failed to open grammar.json at {}", grammar_path.display()) + })?; + + let first_three_lines = BufReader::new(file) + .lines() + .take(3) + .collect::, _>>() + .with_context(|| { + format!( + "Failed to read the first three lines of grammar.json at {}", + grammar_path.display() + ) + })? + .join("\n"); + + let name = GRAMMAR_NAME_REGEX + .captures(&first_three_lines) + .and_then(|c| c.get(1)) + .ok_or_else(|| { + anyhow!( + "Failed to parse the language name from grammar.json at {}", + grammar_path.display() + ) + })?; + + Ok(name.as_str().to_string()) + } + pub fn select_language( &mut self, path: &Path, diff --git a/cli/src/init.rs b/cli/src/init.rs index 6294f2e1..724583d3 100644 --- a/cli/src/init.rs +++ b/cli/src/init.rs @@ -1,6 +1,5 @@ use std::{ - fs::{self, File}, - io::BufReader, + fs, path::{Path, PathBuf}, str::{self, FromStr}, }; @@ -211,9 +210,9 @@ pub fn migrate_package_json(repo_path: &Path) -> Result { root_path.join("tree-sitter.json"), ); - let old_config = serde_json::from_reader::<_, PackageJSON>( - File::open(&package_json_path) - .with_context(|| format!("Failed to open package.json in {}", root_path.display()))?, + let old_config = serde_json::from_str::( + &fs::read_to_string(&package_json_path) + .with_context(|| format!("Failed to read package.json in {}", root_path.display()))?, )?; if old_config.tree_sitter.is_none() { @@ -339,9 +338,9 @@ pub fn migrate_package_json(repo_path: &Path) -> Result { )?; // Remove the `tree-sitter` field in-place - let mut package_json = serde_json::from_reader::<_, Map>( - File::open(&package_json_path) - .with_context(|| format!("Failed to open package.json in {}", root_path.display()))?, + let mut package_json = serde_json::from_str::>( + &fs::read_to_string(&package_json_path) + .with_context(|| format!("Failed to read package.json in {}", root_path.display()))?, ) .unwrap(); package_json.remove("tree-sitter"); @@ -388,9 +387,9 @@ pub fn generate_grammar_files( }, )?; - let tree_sitter_config = serde_json::from_reader::<_, TreeSitterJSON>( - File::open(tree_sitter_config.as_path()) - .with_context(|| "Failed to open tree-sitter.json")?, + let tree_sitter_config = serde_json::from_str::( + &fs::read_to_string(tree_sitter_config.as_path()) + .with_context(|| "Failed to read tree-sitter.json")?, )?; let authors = tree_sitter_config.metadata.authors.as_ref(); @@ -671,15 +670,14 @@ pub fn get_root_path(path: &Path) -> Result { let json = pathbuf .exists() .then(|| { - let file = File::open(pathbuf.as_path()) - .with_context(|| format!("Failed to open {filename}"))?; - let reader = BufReader::new(file); + let contents = fs::read_to_string(pathbuf.as_path()) + .with_context(|| format!("Failed to read {filename}"))?; if is_package_json { - serde_json::from_reader::<_, Map>(reader) + serde_json::from_str::>(&contents) .context(format!("Failed to parse {filename}")) .map(|v| v.contains_key("tree-sitter")) } else { - serde_json::from_reader::<_, TreeSitterJSON>(reader) + serde_json::from_str::(&contents) .context(format!("Failed to parse {filename}")) .map(|_| true) } diff --git a/cli/src/main.rs b/cli/src/main.rs index 1758fada..4926f256 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -653,11 +653,11 @@ impl Init { (opts.name.clone(), Some(opts)) } else { - let json = serde_json::from_reader::<_, TreeSitterJSON>( - fs::File::open(current_dir.join("tree-sitter.json")) - .with_context(|| "Failed to open tree-sitter.json")?, + let mut json = serde_json::from_str::( + &fs::read_to_string(current_dir.join("tree-sitter.json")) + .with_context(|| "Failed to read tree-sitter.json")?, )?; - (json.grammars[0].name.clone(), None) + (json.grammars.swap_remove(0).name, None) }; generate_grammar_files(current_dir, &language_name, self.update, json_config_opts)?; From 85a42f48bef9d88184ae19c6e61527489ee273c4 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sat, 5 Oct 2024 00:56:02 -0400 Subject: [PATCH 23/61] chore(cli): minor correction in comments (cherry picked from commit c611e15a310d9af2ed18172ef1faafaf18506176) --- cli/loader/src/lib.rs | 4 ++-- cli/src/tests/detect_language.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/loader/src/lib.rs b/cli/loader/src/lib.rs index 1eb094c3..8e20b9a3 100644 --- a/cli/loader/src/lib.rs +++ b/cli/loader/src/lib.rs @@ -1118,8 +1118,8 @@ impl Loader { let language_count = self.languages_by_id.len(); for grammar in config.grammars { // Determine the path to the parser directory. This can be specified in - // the package.json, but defaults to the directory containing the - // package.json. + // the tree-sitter.json, but defaults to the directory containing the + // tree-sitter.json. let language_path = parser_path.join(grammar.path); // Determine if a previous language configuration in this package.json file diff --git a/cli/src/tests/detect_language.rs b/cli/src/tests/detect_language.rs index 5bad36d3..aed4ae18 100644 --- a/cli/src/tests/detect_language.rs +++ b/cli/src/tests/detect_language.rs @@ -32,7 +32,7 @@ fn detect_language_by_first_line_regex() { .find_language_configurations_at_path(strace_dir.path(), false) .unwrap(); - // this is just to validate that we can read the package.json correctly + // this is just to validate that we can read the tree-sitter.json correctly assert_eq!(config[0].scope.as_ref().unwrap(), "source.strace"); let file_name = strace_dir.path().join("strace.log"); From 9d0313af0f071a1bd493e6fd35a8e6dfe3599379 Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Wed, 16 Oct 2024 17:19:52 +0300 Subject: [PATCH 24/61] fix(init): use current path if unspecified (cherry picked from commit f9a4e8ecdcbe64c9692e9f9dd18058b17d52e9df) --- cli/loader/src/lib.rs | 5 +++-- cli/src/init.rs | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cli/loader/src/lib.rs b/cli/loader/src/lib.rs index 8e20b9a3..5c09a06b 100644 --- a/cli/loader/src/lib.rs +++ b/cli/loader/src/lib.rs @@ -162,7 +162,8 @@ pub struct Grammar { #[serde(skip_serializing_if = "Option::is_none")] pub camelcase: Option, pub scope: String, - pub path: PathBuf, + #[serde(skip_serializing_if = "Option::is_none")] + pub path: Option, #[serde(default, skip_serializing_if = "PathsJSON::is_empty")] pub external_files: PathsJSON, pub file_types: Option>, @@ -1120,7 +1121,7 @@ impl Loader { // Determine the path to the parser directory. This can be specified in // the tree-sitter.json, but defaults to the directory containing the // tree-sitter.json. - let language_path = parser_path.join(grammar.path); + let language_path = parser_path.join(grammar.path.unwrap_or(PathBuf::from("."))); // Determine if a previous language configuration in this package.json file // already uses the same language. diff --git a/cli/src/init.rs b/cli/src/init.rs index 724583d3..6e95cdba 100644 --- a/cli/src/init.rs +++ b/cli/src/init.rs @@ -135,7 +135,7 @@ impl JsonConfigOpts { name: self.name.clone(), camelcase: Some(self.camelcase), scope: self.scope, - path: PathBuf::from("."), + path: None, external_files: PathsJSON::Empty, file_types: Some(self.file_types), highlights: PathsJSON::Empty, @@ -231,7 +231,7 @@ pub fn migrate_package_json(repo_path: &Path) -> Result { name: name.clone(), camelcase: Some(name.to_upper_camel_case()), scope: l.scope.unwrap_or_else(|| format!("source.{name}")), - path: l.path, + path: Some(l.path), external_files: l.external_files, file_types: l.file_types, highlights: l.highlights, From 1aa6567c7b6362153f2c3e70220e786aa9a20b4e Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 14 Oct 2024 18:16:46 -0400 Subject: [PATCH 25/61] fix(init): use camel name from config in missing spots (cherry picked from commit 3b55003fd5c903869a4fb7085fd7ef100409b210) --- cli/src/init.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/cli/src/init.rs b/cli/src/init.rs index 6e95cdba..7a8ee6a3 100644 --- a/cli/src/init.rs +++ b/cli/src/init.rs @@ -198,6 +198,7 @@ struct GenerateOpts<'a> { description: Option<&'a str>, repository: Option<&'a str>, version: &'a Version, + camel_parser_name: &'a str, } // TODO: remove in 0.25 @@ -334,7 +335,7 @@ pub fn migrate_package_json(repo_path: &Path) -> Result { write_file( &tree_sitter_json_path, - serde_json::to_string_pretty(&new_config)?, + serde_json::to_string_pretty(&new_config)? + "\n", )?; // Remove the `tree-sitter` field in-place @@ -346,7 +347,7 @@ pub fn migrate_package_json(repo_path: &Path) -> Result { package_json.remove("tree-sitter"); write_file( &root_path.join("package.json"), - serde_json::to_string_pretty(&package_json)?, + serde_json::to_string_pretty(&package_json)? + "\n", )?; println!("Warning: your package.json's `tree-sitter` field has been automatically migrated to the new `tree-sitter.json` config file"); @@ -393,6 +394,10 @@ pub fn generate_grammar_files( )?; let authors = tree_sitter_config.metadata.authors.as_ref(); + let camel_name = tree_sitter_config.grammars[0] + .camelcase + .clone() + .unwrap_or_else(|| language_name.to_upper_camel_case()); let generate_opts = GenerateOpts { author_name: authors @@ -412,6 +417,7 @@ pub fn generate_grammar_files( .as_ref() .map(|l| l.repository.as_str()), version: &tree_sitter_config.metadata.version, + camel_parser_name: &camel_name, }; // Create package.json @@ -625,7 +631,7 @@ pub fn generate_grammar_files( // Generate Swift bindings if tree_sitter_config.bindings.swift { missing_path(bindings_dir.join("swift"), create_dir)?.apply(|path| { - let lang_path = path.join(format!("TreeSitter{}", language_name.to_upper_camel_case())); + let lang_path = path.join(format!("TreeSitter{camel_name}",)); missing_path(&lang_path, create_dir)?; missing_path(lang_path.join(format!("{language_name}.h")), |path| { @@ -633,18 +639,12 @@ pub fn generate_grammar_files( })?; missing_path( - path.join(format!( - "TreeSitter{}Tests", - language_name.to_upper_camel_case() - )), + path.join(format!("TreeSitter{camel_name}Tests",)), create_dir, )? .apply(|path| { missing_path( - path.join(format!( - "TreeSitter{}Tests.swift", - language_name.to_upper_camel_case() - )), + path.join(format!("TreeSitter{camel_name}Tests.swift")), |path| generate_file(path, TESTS_SWIFT_TEMPLATE, language_name, &generate_opts), )?; @@ -711,7 +711,7 @@ fn generate_file( let mut replacement = template .replace( CAMEL_PARSER_NAME_PLACEHOLDER, - &language_name.to_upper_camel_case(), + generate_opts.camel_parser_name, ) .replace( UPPER_PARSER_NAME_PLACEHOLDER, @@ -854,7 +854,7 @@ fn generate_file( PARSER_DESCRIPTION_PLACEHOLDER, &format!( "{} grammar for tree-sitter", - language_name.to_upper_camel_case() + generate_opts.camel_parser_name, ), ) } From 0358feda056d8483781fb124ca4fd47dc184243c Mon Sep 17 00:00:00 2001 From: Mrmaxmeier <3913977+Mrmaxmeier@users.noreply.github.com> Date: Thu, 17 Oct 2024 21:44:08 +0200 Subject: [PATCH 26/61] fix: return `LanguageRef` in `Parser::language` `ts_parser_language` doesn't do any refcounting, so we can't return the resulting pointer as an owned Language object (cherry picked from commit c03977a87ecdf1927ff72b3a38fc48a3367ba63d) --- lib/binding_rust/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/binding_rust/lib.rs b/lib/binding_rust/lib.rs index c2bae2c4..bb752f35 100644 --- a/lib/binding_rust/lib.rs +++ b/lib/binding_rust/lib.rs @@ -488,9 +488,9 @@ impl Parser { /// Get the parser's current language. #[doc(alias = "ts_parser_language")] #[must_use] - pub fn language(&self) -> Option { + pub fn language(&self) -> Option> { let ptr = unsafe { ffi::ts_parser_language(self.0.as_ptr()) }; - (!ptr.is_null()).then(|| Language(ptr)) + (!ptr.is_null()).then_some(LanguageRef(ptr, PhantomData)) } /// Get the parser's current logger. From 2303b7d2c59f1140b16f038ea95cc90e2b93ac3d Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Sat, 19 Oct 2024 11:00:04 +0300 Subject: [PATCH 27/61] chore(bindings): clean up package.json - Use PARSER_URL in repository - Remove tree-sitter section (cherry picked from commit 5df4706b19623feba384ee76984f6a8341602786) --- cli/src/templates/package.json | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/cli/src/templates/package.json b/cli/src/templates/package.json index 22488e45..aae7cbcb 100644 --- a/cli/src/templates/package.json +++ b/cli/src/templates/package.json @@ -2,7 +2,7 @@ "name": "tree-sitter-PARSER_NAME", "version": "PARSER_VERSION", "description": "PARSER_DESCRIPTION", - "repository": "github:tree-sitter/tree-sitter-PARSER_NAME", + "repository": "PARSER_URL", "license": "PARSER_LICENSE", "author": { "name": "PARSER_AUTHOR_NAME", @@ -47,11 +47,5 @@ "prestart": "tree-sitter build --wasm", "start": "tree-sitter playground", "test": "node --test bindings/node/*_test.js" - }, - "tree-sitter": [ - { - "scope": "source.LOWER_PARSER_NAME", - "injection-regex": "^LOWER_PARSER_NAME$" - } - ] + } } From 1fd07fd619192df595e4dcb2c0fac546cd9c380d Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Sat, 19 Oct 2024 13:28:58 +0300 Subject: [PATCH 28/61] chore(bindings): improve gitattributes file - Separate bindings into language sections - Mark all metadata & lockfiles as generated (cherry picked from commit 4089569934e997844c20ba4c735845ee1e04a4a6) --- cli/src/templates/gitattributes | 34 ++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/cli/src/templates/gitattributes b/cli/src/templates/gitattributes index 9d5c5d49..7e2cae0c 100644 --- a/cli/src/templates/gitattributes +++ b/cli/src/templates/gitattributes @@ -1,13 +1,37 @@ * text=auto eol=lf +# Generated source files src/*.json linguist-generated src/parser.c linguist-generated src/tree_sitter/* linguist-generated -bindings/** linguist-generated -binding.gyp linguist-generated -setup.py linguist-generated -Makefile linguist-generated +# C bindings +bindings/c/* linguist-generated CMakeLists.txt linguist-generated -Package.swift linguist-generated +Makefile linguist-generated + +# Rust bindings +bindings/rust/* linguist-generated +Cargo.toml linguist-generated +Cargo.lock linguist-generated + +# Node.js bindings +bindings/node/* linguist-generated +binding.gyp linguist-generated +package.json linguist-generated +package-lock.json linguist-generated + +# Python bindings +bindings/python/** linguist-generated +setup.py linguist-generated +pyproject.toml linguist-generated + +# Go bindings +bindings/go/* linguist-generated go.mod linguist-generated +go.sum linguist-generated + +# Swift bindings +bindings/swift/** linguist-generated +Package.swift linguist-generated +Package.resolved linguist-generated From 8745e5d2cea915018022f5028563c69ff5b2b093 Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Sat, 19 Oct 2024 15:31:17 +0300 Subject: [PATCH 29/61] chore(bindings): drop pkg-config Requires field (cherry picked from commit c23670264f485ec82346d3b852643772ed461393) --- cli/src/templates/PARSER_NAME.pc.in | 1 - 1 file changed, 1 deletion(-) diff --git a/cli/src/templates/PARSER_NAME.pc.in b/cli/src/templates/PARSER_NAME.pc.in index 8567e9e2..e92fc803 100644 --- a/cli/src/templates/PARSER_NAME.pc.in +++ b/cli/src/templates/PARSER_NAME.pc.in @@ -6,6 +6,5 @@ Name: tree-sitter-PARSER_NAME Description: @PROJECT_DESCRIPTION@ URL: @PROJECT_HOMEPAGE_URL@ Version: @PROJECT_VERSION@ -Requires: @TS_REQUIRES@ Libs: -L${libdir} -ltree-sitter-PARSER_NAME Cflags: -I${includedir} From 54299d3d87e118e45b2b615c4884992b3a2cd23d Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Sat, 19 Oct 2024 17:01:47 +0300 Subject: [PATCH 30/61] chore(bindings): correct editorconfig indent size (cherry picked from commit 8681960fbcb220579cd0d3fea22899741d2cbec2) --- cli/src/templates/.editorconfig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cli/src/templates/.editorconfig b/cli/src/templates/.editorconfig index 7756ee95..65330c40 100644 --- a/cli/src/templates/.editorconfig +++ b/cli/src/templates/.editorconfig @@ -41,3 +41,6 @@ indent_size = 8 [parser.c] indent_size = 2 + +[{alloc,array,parser}.h] +indent_size = 2 From 32c14662244a93e06b1419404b8ab1ffd6103bd3 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sat, 2 Nov 2024 03:53:06 -0400 Subject: [PATCH 31/61] fix(generate): do not set the unit reduction symbol if it's in the extras array (cherry picked from commit 998d2c9d8ce4296c31080955abc9f6cb4c3f40c0) --- .../src/build_tables/minimize_parse_table.rs | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/cli/generate/src/build_tables/minimize_parse_table.rs b/cli/generate/src/build_tables/minimize_parse_table.rs index 74f70869..70950dff 100644 --- a/cli/generate/src/build_tables/minimize_parse_table.rs +++ b/cli/generate/src/build_tables/minimize_parse_table.rs @@ -70,18 +70,17 @@ impl<'a> Minimizer<'a> { production_id: 0, symbol, .. - } => { - if !self.simple_aliases.contains_key(symbol) - && !self.syntax_grammar.supertype_symbols.contains(symbol) - && !aliased_symbols.contains(symbol) - && self.syntax_grammar.variables[symbol.index].kind - != VariableType::Named - && (unit_reduction_symbol.is_none() - || unit_reduction_symbol == Some(symbol)) - { - unit_reduction_symbol = Some(symbol); - continue; - } + } if !self.simple_aliases.contains_key(symbol) + && !self.syntax_grammar.supertype_symbols.contains(symbol) + && !self.syntax_grammar.extra_symbols.contains(symbol) + && !aliased_symbols.contains(symbol) + && self.syntax_grammar.variables[symbol.index].kind + != VariableType::Named + && (unit_reduction_symbol.is_none() + || unit_reduction_symbol == Some(symbol)) => + { + unit_reduction_symbol = Some(symbol); + continue; } _ => {} } From 0258a41e15c9854d2c1f53a4079e91c7e540621a Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sat, 2 Nov 2024 01:31:16 -0400 Subject: [PATCH 32/61] fix(cli): use `contains` over `is` in warning (cherry picked from commit 7baefa86093882dbdd91ac9fca8befb7afb56a55) --- cli/generate/src/prepare_grammar/intern_symbols.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/generate/src/prepare_grammar/intern_symbols.rs b/cli/generate/src/prepare_grammar/intern_symbols.rs index 0941676d..02845f35 100644 --- a/cli/generate/src/prepare_grammar/intern_symbols.rs +++ b/cli/generate/src/prepare_grammar/intern_symbols.rs @@ -149,7 +149,7 @@ impl<'a> Interner<'a> { fn check_single(&self, elements: &[Rule], name: Option<&str>) { if elements.len() == 1 && matches!(elements[0], Rule::String(_) | Rule::Pattern(_, _)) { eprintln!( - "Warning: rule {} is just a `seq` or `choice` rule with a single element. This is unnecessary.", + "Warning: rule {} contains a `seq` or `choice` rule with a single element. This is unnecessary.", name.unwrap_or_default() ); } From 35f119db03b1e74c6fe1cc1f7936450e6ab05199 Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Tue, 5 Nov 2024 10:28:08 +0200 Subject: [PATCH 33/61] chore(bindings): include tree-sitter.json file (cherry picked from commit 2d5a4b6bd68265386c00613c93161dc26913a6d8) --- cli/src/templates/_cargo.toml | 2 +- cli/src/templates/package.json | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cli/src/templates/_cargo.toml b/cli/src/templates/_cargo.toml index 04540d34..4a1b6409 100644 --- a/cli/src/templates/_cargo.toml +++ b/cli/src/templates/_cargo.toml @@ -12,7 +12,7 @@ edition = "2021" autoexamples = false build = "bindings/rust/build.rs" -include = ["bindings/rust/*", "grammar.js", "queries/*", "src/*"] +include = ["bindings/rust/*", "grammar.js", "queries/*", "src/*", "tree-sitter.json"] [lib] path = "bindings/rust/lib.rs" diff --git a/cli/src/templates/package.json b/cli/src/templates/package.json index aae7cbcb..1db11b2d 100644 --- a/cli/src/templates/package.json +++ b/cli/src/templates/package.json @@ -19,6 +19,7 @@ ], "files": [ "grammar.js", + "tree-sitter.json", "binding.gyp", "prebuilds/**", "bindings/node/*", From 70fd2c02f103743d64ff03e3ea5c455d56cd546d Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Tue, 5 Nov 2024 12:02:04 +0200 Subject: [PATCH 34/61] chore(bindings): update go binding version (cherry picked from commit 1d5502d7b26f508bcaeb94b7f8a6844a56c37a1e) --- cli/src/templates/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/templates/go.mod b/cli/src/templates/go.mod index f5887715..93436c82 100644 --- a/cli/src/templates/go.mod +++ b/cli/src/templates/go.mod @@ -2,4 +2,4 @@ module PARSER_URL_STRIPPED go 1.22 -require github.com/tree-sitter/go-tree-sitter v0.23.1 +require github.com/tree-sitter/go-tree-sitter v0.24.0 From 28cbc771f1b944906a0eb981fa3c38c736129ad9 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sat, 9 Nov 2024 21:55:06 -0500 Subject: [PATCH 35/61] fix(cmake): use current source dir for EXISTS check (cherry picked from commit d73b66cbbcd1392884486bcbadadaa827cc5c579) --- cli/src/templates/cmakelists.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/templates/cmakelists.cmake b/cli/src/templates/cmakelists.cmake index a09a61fb..3ce70239 100644 --- a/cli/src/templates/cmakelists.cmake +++ b/cli/src/templates/cmakelists.cmake @@ -25,7 +25,7 @@ add_custom_command(OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/src/parser.c" COMMENT "Generating parser.c") add_library(tree-sitter-PARSER_NAME src/parser.c) -if(EXISTS src/scanner.c) +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/src/scanner.c) target_sources(tree-sitter-PARSER_NAME PRIVATE src/scanner.c) endif() target_include_directories(tree-sitter-PARSER_NAME PRIVATE src) From 340d3eeb41e15bac5d33ae0e45faa0a632655834 Mon Sep 17 00:00:00 2001 From: Will Lillis Date: Sat, 9 Nov 2024 20:46:29 -0500 Subject: [PATCH 36/61] feat(loader): add error message when a `tree-sitter.json` file is invalid (cherry picked from commit 7b90dbf189d7e8e8eabb36bfd5d1910465c1b8b9) --- cli/loader/src/lib.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/cli/loader/src/lib.rs b/cli/loader/src/lib.rs index 5c09a06b..1ee006f0 100644 --- a/cli/loader/src/lib.rs +++ b/cli/loader/src/lib.rs @@ -146,8 +146,10 @@ pub struct TreeSitterJSON { } impl TreeSitterJSON { - pub fn from_file(path: &Path) -> Option { - serde_json::from_str(&fs::read_to_string(path.join("tree-sitter.json")).ok()?).ok() + pub fn from_file(path: &Path) -> Result { + Ok(serde_json::from_str(&fs::read_to_string( + path.join("tree-sitter.json"), + )?)?) } pub fn has_multiple_language_configs(&self) -> bool { @@ -1115,7 +1117,8 @@ impl Loader { ) -> Result<&[LanguageConfiguration]> { let initial_language_configuration_count = self.language_configurations.len(); - if let Some(config) = TreeSitterJSON::from_file(parser_path) { + let ts_json = TreeSitterJSON::from_file(parser_path); + if let Ok(config) = ts_json { let language_count = self.languages_by_id.len(); for grammar in config.grammars { // Determine the path to the parser directory. This can be specified in @@ -1206,6 +1209,11 @@ impl Loader { Some(self.language_configurations.len() - 1); } } + } else if let Err(e) = ts_json { + eprintln!( + "Warning: Failed to read {} -- {e}", + parser_path.join("tree-sitter.json").display() + ); } // If we didn't find any language configurations in the tree-sitter.json file, From 2f6583aae2f85fcc2761b73c3298a29cb71d097f Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sat, 9 Nov 2024 22:34:03 -0500 Subject: [PATCH 37/61] fix(loader): don't print warnings if the file is not found (cherry picked from commit 18e4a2405b52ecd3cb9aefdd05d6554e76823473) --- cli/loader/src/lib.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cli/loader/src/lib.rs b/cli/loader/src/lib.rs index 1ee006f0..13b4fabf 100644 --- a/cli/loader/src/lib.rs +++ b/cli/loader/src/lib.rs @@ -1210,10 +1210,16 @@ impl Loader { } } } else if let Err(e) = ts_json { - eprintln!( - "Warning: Failed to read {} -- {e}", - parser_path.join("tree-sitter.json").display() - ); + match e.downcast_ref::() { + // This is noisy, and not really an issue. + Some(e) if e.kind() == std::io::ErrorKind::NotFound => {} + _ => { + eprintln!( + "Warning: Failed to parse {} -- {e}", + parser_path.join("tree-sitter.json").display() + ); + } + } } // If we didn't find any language configurations in the tree-sitter.json file, From d25a5e488673f1e53293c225251895211f43e8b3 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sat, 9 Nov 2024 21:46:33 -0500 Subject: [PATCH 38/61] feat(node): support single-file executables via bun build --compile (cherry picked from commit 134233f33abdf80d4d55eebf91a1c1e11d02a50f) --- cli/src/templates/index.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cli/src/templates/index.js b/cli/src/templates/index.js index 6657bcf4..a269e6da 100644 --- a/cli/src/templates/index.js +++ b/cli/src/templates/index.js @@ -1,6 +1,10 @@ const root = require("path").join(__dirname, "..", ".."); -module.exports = require("node-gyp-build")(root); +module.exports = + typeof process.versions.bun === "string" + // Support `bun build --compile` by being statically analyzable enough to find the .node file at build-time + ? require(`../../prebuilds/${process.platform}-${process.arch}/tree-sitter-typescript.node`) + : require("node-gyp-build")(root); try { module.exports.nodeTypeInfo = require("../../src/node-types.json"); From 2507aabc04744bc7e213bc142888aeae60258886 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sat, 9 Nov 2024 21:52:03 -0500 Subject: [PATCH 39/61] feat(node): update bindings when necessary (cherry picked from commit ac4595e80864c9112fbc4df3ffa522ef6e3bedc6) --- cli/src/init.rs | 15 ++++++++++++--- cli/src/templates/index.js | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/cli/src/init.rs b/cli/src/init.rs index 7a8ee6a3..d9d0fb3e 100644 --- a/cli/src/init.rs +++ b/cli/src/init.rs @@ -481,9 +481,18 @@ pub fn generate_grammar_files( // Generate Node bindings if tree_sitter_config.bindings.node { missing_path(bindings_dir.join("node"), create_dir)?.apply(|path| { - missing_path(path.join("index.js"), |path| { - generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts) - })?; + missing_path_else( + path.join("index.js"), + allow_update, + |path| generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts), + |path| { + let contents = fs::read_to_string(path)?; + if !contents.contains("bun") { + generate_file(path, INDEX_JS_TEMPLATE, language_name, &generate_opts)?; + } + Ok(()) + }, + )?; missing_path(path.join("index.d.ts"), |path| { generate_file(path, INDEX_D_TS_TEMPLATE, language_name, &generate_opts) diff --git a/cli/src/templates/index.js b/cli/src/templates/index.js index a269e6da..88437495 100644 --- a/cli/src/templates/index.js +++ b/cli/src/templates/index.js @@ -3,7 +3,7 @@ const root = require("path").join(__dirname, "..", ".."); module.exports = typeof process.versions.bun === "string" // Support `bun build --compile` by being statically analyzable enough to find the .node file at build-time - ? require(`../../prebuilds/${process.platform}-${process.arch}/tree-sitter-typescript.node`) + ? require(`../../prebuilds/${process.platform}-${process.arch}/tree-sitter-PARSER_NAME.node`) : require("node-gyp-build")(root); try { From fc8c1863e2e5724a0c40bb6e6cfc8631bfe5908b Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sat, 9 Nov 2024 23:07:55 -0500 Subject: [PATCH 40/61] 0.24.4 --- Cargo.lock | 14 +++++++------- Cargo.toml | 14 +++++++------- Makefile | 2 +- build.zig.zon | 2 +- cli/npm/package.json | 2 +- lib/CMakeLists.txt | 2 +- lib/binding_web/package.json | 2 +- 7 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3d52abb9..9c280e66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1538,7 +1538,7 @@ dependencies = [ [[package]] name = "tree-sitter" -version = "0.24.3" +version = "0.24.4" dependencies = [ "bindgen", "cc", @@ -1551,7 +1551,7 @@ dependencies = [ [[package]] name = "tree-sitter-cli" -version = "0.24.3" +version = "0.24.4" dependencies = [ "anstyle", "anyhow", @@ -1601,7 +1601,7 @@ dependencies = [ [[package]] name = "tree-sitter-config" -version = "0.24.3" +version = "0.24.4" dependencies = [ "anyhow", "dirs", @@ -1611,7 +1611,7 @@ dependencies = [ [[package]] name = "tree-sitter-generate" -version = "0.24.3" +version = "0.24.4" dependencies = [ "anyhow", "heck 0.5.0", @@ -1632,7 +1632,7 @@ dependencies = [ [[package]] name = "tree-sitter-highlight" -version = "0.24.3" +version = "0.24.4" dependencies = [ "lazy_static", "regex", @@ -1647,7 +1647,7 @@ version = "0.1.2" [[package]] name = "tree-sitter-loader" -version = "0.24.3" +version = "0.24.4" dependencies = [ "anyhow", "cc", @@ -1671,7 +1671,7 @@ dependencies = [ [[package]] name = "tree-sitter-tags" -version = "0.24.3" +version = "0.24.4" dependencies = [ "memchr", "regex", diff --git a/Cargo.toml b/Cargo.toml index e9140f2a..82f99e99 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.24.3" +version = "0.24.4" authors = ["Max Brunsfeld "] edition = "2021" rust-version = "1.74.1" @@ -96,9 +96,9 @@ walkdir = "2.5.0" wasmparser = "0.217.0" webbrowser = "1.0.2" -tree-sitter = { version = "0.24.3", path = "./lib" } -tree-sitter-generate = { version = "0.24.3", path = "./cli/generate" } -tree-sitter-loader = { version = "0.24.3", path = "./cli/loader" } -tree-sitter-config = { version = "0.24.3", path = "./cli/config" } -tree-sitter-highlight = { version = "0.24.3", path = "./highlight" } -tree-sitter-tags = { version = "0.24.3", path = "./tags" } +tree-sitter = { version = "0.24.4", path = "./lib" } +tree-sitter-generate = { version = "0.24.4", path = "./cli/generate" } +tree-sitter-loader = { version = "0.24.4", path = "./cli/loader" } +tree-sitter-config = { version = "0.24.4", path = "./cli/config" } +tree-sitter-highlight = { version = "0.24.4", path = "./highlight" } +tree-sitter-tags = { version = "0.24.4", path = "./tags" } diff --git a/Makefile b/Makefile index 92cd8395..0de508d7 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ ifeq ($(OS),Windows_NT) $(error Windows is not supported) endif -VERSION := 0.24.3 +VERSION := 0.24.4 DESCRIPTION := An incremental parsing system for programming tools HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/ diff --git a/build.zig.zon b/build.zig.zon index f9f9798c..f1c2f7f4 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,6 +1,6 @@ .{ .name = "tree-sitter", - .version = "0.24.3", + .version = "0.24.4", .paths = .{ "build.zig", "build.zig.zon", diff --git a/cli/npm/package.json b/cli/npm/package.json index 8e7c600b..a884529f 100644 --- a/cli/npm/package.json +++ b/cli/npm/package.json @@ -1,6 +1,6 @@ { "name": "tree-sitter-cli", - "version": "0.24.3", + "version": "0.24.4", "author": "Max Brunsfeld", "license": "MIT", "repository": { diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index c27db119..d6e758b0 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.13) project(tree-sitter - VERSION "0.24.3" + VERSION "0.24.4" DESCRIPTION "An incremental parsing system for programming tools" HOMEPAGE_URL "https://tree-sitter.github.io/tree-sitter/" LANGUAGES C) diff --git a/lib/binding_web/package.json b/lib/binding_web/package.json index fa9b5e7e..faa4dbe4 100644 --- a/lib/binding_web/package.json +++ b/lib/binding_web/package.json @@ -1,6 +1,6 @@ { "name": "web-tree-sitter", - "version": "0.24.3", + "version": "0.24.4", "description": "Tree-sitter bindings for the web", "main": "tree-sitter.js", "types": "tree-sitter-web.d.ts", From e3c82633389256ccc2c5ab2e509046cbf20453d3 Mon Sep 17 00:00:00 2001 From: Will Lillis Date: Tue, 12 Nov 2024 18:17:45 -0500 Subject: [PATCH 41/61] fix(lib): correct next sibling of zero width node (cherry picked from commit 5d1be545c439eba4810f34a14fef17e5f76df6c0) --- cli/src/tests/node_test.rs | 27 +++++++++++++++++++ lib/src/node.c | 8 +++--- .../next_sibling_from_zwt/corpus.txt | 10 +++++++ .../next_sibling_from_zwt/grammar.js | 22 +++++++++++++++ 4 files changed, 63 insertions(+), 4 deletions(-) create mode 100644 test/fixtures/test_grammars/next_sibling_from_zwt/corpus.txt create mode 100644 test/fixtures/test_grammars/next_sibling_from_zwt/grammar.js diff --git a/cli/src/tests/node_test.rs b/cli/src/tests/node_test.rs index 20686a4a..7217ee27 100644 --- a/cli/src/tests/node_test.rs +++ b/cli/src/tests/node_test.rs @@ -306,6 +306,33 @@ fn test_parent_of_zero_width_node() { assert_eq!(parent, script_element); } +#[test] +fn test_next_sibling_of_zero_width_node() { + let grammar_json = load_grammar_file( + &fixtures_dir() + .join("test_grammars") + .join("next_sibling_from_zwt") + .join("grammar.js"), + None, + ) + .unwrap(); + + let (parser_name, parser_code) = generate_parser_for_grammar(&grammar_json).unwrap(); + + let mut parser = Parser::new(); + let language = get_test_language(&parser_name, &parser_code, None); + parser.set_language(&language).unwrap(); + + let tree = parser.parse("abdef", None).unwrap(); + + let root_node = tree.root_node(); + let missing_c = root_node.child(2).unwrap(); + assert!(missing_c.is_missing()); + assert_eq!(missing_c.kind(), "c"); + let node_d = root_node.child(3).unwrap(); + assert_eq!(missing_c.next_sibling().unwrap(), node_d); +} + #[test] fn test_node_field_name_for_child() { let mut parser = Parser::new(); diff --git a/lib/src/node.c b/lib/src/node.c index 40d6024f..08b99879 100644 --- a/lib/src/node.c +++ b/lib/src/node.c @@ -262,8 +262,8 @@ static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous) TSNode child; NodeChildIterator iterator = ts_node_iterate_children(&node); while (ts_node_child_iterator_next(&iterator, &child)) { - if (iterator.position.bytes < target_end_byte) continue; - if (ts_node_start_byte(child) <= ts_node_start_byte(self)) { + if (iterator.position.bytes <= target_end_byte) continue; + if (ts_node_start_byte(child) < ts_node_start_byte(self)) { if (ts_node__subtree(child).ptr != ts_node__subtree(self).ptr) { child_containing_target = child; } @@ -534,8 +534,8 @@ TSNode ts_node_parent(TSNode self) { if (node.id == self.id) return ts_node__null(); while (true) { - TSNode next_node = ts_node_child_containing_descendant(node, self); - if (ts_node_is_null(next_node)) break; + TSNode next_node = ts_node_child_with_descendant(node, self); + if (next_node.id == self.id || ts_node_is_null(next_node)) break; node = next_node; } diff --git a/test/fixtures/test_grammars/next_sibling_from_zwt/corpus.txt b/test/fixtures/test_grammars/next_sibling_from_zwt/corpus.txt new file mode 100644 index 00000000..61c67140 --- /dev/null +++ b/test/fixtures/test_grammars/next_sibling_from_zwt/corpus.txt @@ -0,0 +1,10 @@ +=========================== +missing c node +=========================== + +abdef + +--- + +(source + (MISSING "c")) diff --git a/test/fixtures/test_grammars/next_sibling_from_zwt/grammar.js b/test/fixtures/test_grammars/next_sibling_from_zwt/grammar.js new file mode 100644 index 00000000..857b94ad --- /dev/null +++ b/test/fixtures/test_grammars/next_sibling_from_zwt/grammar.js @@ -0,0 +1,22 @@ +module.exports = grammar({ + name: "next_sibling_from_zwt", + extras: $ => [ + /\s|\\\r?\n/, + ], + + rules: { + source: $ => seq( + 'a', + $._bc, + 'd', + 'e', + 'f', + ), + + _bc: $ => seq( + 'b', + 'c', + ), + } +}); + From 5e645f11b2c9ffb5ea6b6672732ffdbfb792880b Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Tue, 3 Dec 2024 16:25:56 -0500 Subject: [PATCH 42/61] fix(lib): use `clock_gettime` on macOS again (cherry picked from commit 69d977d73648010d7060001fa518f3198a41a7e5) --- lib/src/clock.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/src/clock.h b/lib/src/clock.h index 5d246ca7..7a13185e 100644 --- a/lib/src/clock.h +++ b/lib/src/clock.h @@ -49,9 +49,9 @@ static inline bool clock_is_gt(TSClock self, TSClock other) { return self > other; } -#elif defined(CLOCK_MONOTONIC) && !defined(__APPLE__) +#elif defined(CLOCK_MONOTONIC) -// POSIX with monotonic clock support (Linux) +// POSIX with monotonic clock support (Linux, macOS) // * Represent a time as a monotonic (seconds, nanoseconds) pair. // * Represent a duration as a number of microseconds. // @@ -102,7 +102,7 @@ static inline bool clock_is_gt(TSClock self, TSClock other) { #else -// macOS or POSIX without monotonic clock support +// POSIX without monotonic clock support // * Represent a time as a process clock value. // * Represent a duration as a number of process clock ticks. // From 6c4c5227244126d0bdeb18539324de37428c8d4f Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sun, 6 Oct 2024 13:41:47 -0400 Subject: [PATCH 43/61] feat: move scripts to xtasks (cherry picked from commit dbe8bbf4808c4cad889595c2054fd9de0d734521) --- .github/workflows/build.yml | 63 ++++---- .github/workflows/release.yml | 4 +- .github/workflows/sanitize.yml | 4 +- Makefile | 10 +- cli/loader/emscripten-version | 2 +- script/benchmark | 62 -------- script/benchmark.cmd | 4 - script/build-fuzzers | 76 ---------- script/build-wasm-stdlib | 28 ---- script/check-mallocs | 12 -- script/fetch-emscripten | 26 ---- script/fetch-fixtures | 37 ----- script/fetch-fixtures.cmd | 32 ----- script/generate-bindings | 44 ------ script/generate-fixtures | 27 ---- script/generate-fixtures-wasm | 33 ----- script/generate-fixtures.cmd | 13 -- script/heap-profile | 36 ----- script/reproduce | 35 ----- script/run-fuzzer | 42 ------ script/serve-docs | 29 ---- script/show-symbol-sizes | 50 ------- script/test | 101 ------------- script/test-wasm | 12 -- script/test.cmd | 10 -- script/util/scan-build.sh | 24 ---- script/util/valgrind.supp | 256 --------------------------------- xtask/Cargo.toml | 9 ++ xtask/src/benchmark.rs | 75 ++++++++++ xtask/src/build_wasm.rs | 228 +++++++++++++++++++++++++++++ xtask/src/bump.rs | 78 +++++----- xtask/src/clippy.rs | 33 +++++ xtask/src/fetch.rs | 119 +++++++++++++++ xtask/src/generate.rs | 118 +++++++++++++++ xtask/src/main.rs | 246 ++++++++++++++++++++++++++++--- xtask/src/test.rs | 122 ++++++++++++++++ 36 files changed, 1013 insertions(+), 1087 deletions(-) delete mode 100755 script/benchmark delete mode 100644 script/benchmark.cmd delete mode 100755 script/build-fuzzers delete mode 100755 script/build-wasm-stdlib delete mode 100755 script/check-mallocs delete mode 100755 script/fetch-emscripten delete mode 100755 script/fetch-fixtures delete mode 100644 script/fetch-fixtures.cmd delete mode 100755 script/generate-bindings delete mode 100755 script/generate-fixtures delete mode 100755 script/generate-fixtures-wasm delete mode 100644 script/generate-fixtures.cmd delete mode 100755 script/heap-profile delete mode 100755 script/reproduce delete mode 100755 script/run-fuzzer delete mode 100755 script/serve-docs delete mode 100755 script/show-symbol-sizes delete mode 100755 script/test delete mode 100755 script/test-wasm delete mode 100644 script/test.cmd delete mode 100755 script/util/scan-build.sh delete mode 100644 script/util/valgrind.supp create mode 100644 xtask/src/benchmark.rs create mode 100644 xtask/src/build_wasm.rs create mode 100644 xtask/src/clippy.rs create mode 100644 xtask/src/fetch.rs create mode 100644 xtask/src/generate.rs create mode 100644 xtask/src/test.rs diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 85624cbf..65979bed 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -36,22 +36,22 @@ jobs: # When adding a new `target`: # 1. Define a new platform alias above # 2. Add a new record to a matrix map in `cli/npm/install.js` - - { platform: linux-arm64 , target: aarch64-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } - - { platform: linux-arm , target: arm-unknown-linux-gnueabi , os: ubuntu-latest , use-cross: true } - - { platform: linux-x64 , target: x86_64-unknown-linux-gnu , os: ubuntu-20.04 , cli_features: wasm } #2272 - - { platform: linux-x86 , target: i686-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } - - { platform: linux-powerpc64 , target: powerpc64-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } - - { platform: windows-arm64 , target: aarch64-pc-windows-msvc , os: windows-latest } - - { platform: windows-x64 , target: x86_64-pc-windows-msvc , os: windows-latest , cli_features: wasm } - - { platform: windows-x86 , target: i686-pc-windows-msvc , os: windows-latest } - - { platform: macos-arm64 , target: aarch64-apple-darwin , os: macos-14 , cli_features: wasm } - - { platform: macos-x64 , target: x86_64-apple-darwin , os: macos-12 , cli_features: wasm } + - { platform: linux-arm64 , target: aarch64-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } + - { platform: linux-arm , target: arm-unknown-linux-gnueabi , os: ubuntu-latest , use-cross: true } + - { platform: linux-x64 , target: x86_64-unknown-linux-gnu , os: ubuntu-20.04 , cli_features: wasm } #2272 + - { platform: linux-x86 , target: i686-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } + - { platform: linux-powerpc64 , target: powerpc64-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } + - { platform: windows-arm64 , target: aarch64-pc-windows-msvc , os: windows-latest } + - { platform: windows-x64 , target: x86_64-pc-windows-msvc , os: windows-latest , cli_features: wasm } + - { platform: windows-x86 , target: i686-pc-windows-msvc , os: windows-latest } + - { platform: macos-arm64 , target: aarch64-apple-darwin , os: macos-14 , cli_features: wasm } + - { platform: macos-x64 , target: x86_64-apple-darwin , os: macos-12 , cli_features: wasm } # Cross compilers for C library - - { platform: linux-arm64 , cc: aarch64-linux-gnu-gcc , ar: aarch64-linux-gnu-ar } - - { platform: linux-arm , cc: arm-linux-gnueabi-gcc , ar: arm-linux-gnueabi-ar } - - { platform: linux-x86 , cc: i686-linux-gnu-gcc , ar: i686-linux-gnu-ar } - - { platform: linux-powerpc64 , cc: powerpc64-linux-gnu-gcc , ar: powerpc64-linux-gnu-ar } + - { platform: linux-arm64 , cc: aarch64-linux-gnu-gcc , ar: aarch64-linux-gnu-ar } + - { platform: linux-arm , cc: arm-linux-gnueabi-gcc , ar: arm-linux-gnueabi-ar } + - { platform: linux-x86 , cc: i686-linux-gnu-gcc , ar: i686-linux-gnu-ar } + - { platform: linux-powerpc64 , cc: powerpc64-linux-gnu-gcc , ar: powerpc64-linux-gnu-ar } # See #2041 tree-sitter issue - { platform: windows-x64 , rust-test-threads: 1 } @@ -86,23 +86,30 @@ jobs: - name: Install cross if: ${{ matrix.use-cross }} - uses: taiki-e/install-action@v2 - with: - tool: cross + run: cargo install cross --git https://github.com/cross-rs/cross - name: Build custom cross image - if: ${{ matrix.use-cross && matrix.os == 'ubuntu-latest' }} + if: ${{ matrix.use-cross }} run: | target="${{ matrix.target }}" image=ghcr.io/cross-rs/$target:custom - echo "CROSS_IMAGE=$image" >> $GITHUB_ENV - echo "[target.$target]" >> Cross.toml - echo "image = \"$image\"" >> Cross.toml - echo "CROSS_CONFIG=$PWD/Cross.toml" >> $GITHUB_ENV + echo "[target.$target]" >> Cross.toml + echo "image = \"$image\"" >> Cross.toml + echo "[build]" >> Cross.toml + echo "pre-build = [" >> Cross.toml + echo " \"dpkg --add-architecture \$CROSS_DEB_ARCH\"," >> Cross.toml + echo " \"apt-get update && apt-get -y install libssl-dev\"" >> Cross.toml + echo "]" >> Cross.toml + + echo "Cross.toml:" + cat Cross.toml + + echo "CROSS_IMAGE=$image" >> $GITHUB_ENV + echo "CROSS_CONFIG=$PWD/Cross.toml" >> $GITHUB_ENV echo "FROM ghcr.io/cross-rs/$target:edge" >> Dockerfile - echo "RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash -" >> Dockerfile + echo "RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash -" >> Dockerfile echo "RUN apt-get update && apt-get -y install nodejs" >> Dockerfile docker build -t $image . @@ -139,22 +146,22 @@ jobs: - name: Build wasm library if: ${{ !matrix.cli-only && !matrix.use-cross }} # No sense to build on the same Github runner hosts many times - run: script/build-wasm + run: $BUILD_CMD run --package xtask -- build-wasm - run: $BUILD_CMD build --release --target=${{ matrix.target }} --features=${{ matrix.cli_features }} - - run: script/fetch-fixtures + - run: $BUILD_CMD run --package xtask -- fetch-fixtures - uses: ./.github/actions/cache id: cache - name: Generate fixtures if: ${{ !matrix.cli-only && inputs.run_test && steps.cache.outputs.cache-hit != 'true' }} # Can't natively run CLI on Github runner's host - run: script/generate-fixtures + run: $BUILD_CMD run --package xtask -- generate-fixtures - name: Generate WASM fixtures if: ${{ !matrix.cli-only && !matrix.use-cross && inputs.run_test && steps.cache.outputs.cache-hit != 'true' }} # See comment for the "Build wasm library" step - run: script/generate-fixtures-wasm + run: $BUILD_CMD run --package xtask -- generate-fixtures --wasm - name: Run main tests if: ${{ !matrix.cli-only && inputs.run_test }} # Can't natively run CLI on Github runner's host @@ -162,7 +169,7 @@ jobs: - name: Run wasm tests if: ${{ !matrix.cli-only && !matrix.use-cross && inputs.run_test }} # See comment for the "Build wasm library" step - run: script/test-wasm + run: $BUILD_CMD run --package xtask -- test-wasm - name: Run benchmarks if: ${{ !matrix.cli-only && !matrix.use-cross && inputs.run_test }} # Cross-compiled benchmarks make no sense diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cd25fd71..3bb201eb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -84,12 +84,12 @@ jobs: - name: Build wasm if: matrix.directory == 'lib/binding_web' - run: ./script/build-wasm + run: cargo xtask build-wasm - name: Setup Node uses: actions/setup-node@v4 with: - node-version: 18 + node-version: 20 registry-url: "https://registry.npmjs.org" - name: Publish lib to npmjs.com diff --git a/.github/workflows/sanitize.yml b/.github/workflows/sanitize.yml index 72bdd079..cfbfbfe2 100644 --- a/.github/workflows/sanitize.yml +++ b/.github/workflows/sanitize.yml @@ -24,13 +24,13 @@ jobs: - run: rustup toolchain install stable --profile minimal - uses: Swatinem/rust-cache@v2 - run: cargo build --release - - run: script/fetch-fixtures + - run: cargo xtask fetch-fixtures - uses: ./.github/actions/cache id: cache - if: ${{ steps.cache.outputs.cache-hit != 'true' }} - run: script/generate-fixtures + run: cargo xtask generate-fixtures - name: Run main tests with undefined behaviour sanitizer (UBSAN) env: diff --git a/Makefile b/Makefile index 0de508d7..fdec43fe 100644 --- a/Makefile +++ b/Makefile @@ -94,13 +94,13 @@ uninstall: ##### Dev targets ##### test: - script/fetch-fixtures - script/generate-fixtures - script/test + cargo xtask fetch-fixtures + cargo xtask generate-fixtures + cargo xtask test test_wasm: - script/generate-fixtures-wasm - script/test-wasm + cargo xtask generate-fixtures-wasm + cargo xtask test-wasm lint: cargo update --workspace --locked --quiet diff --git a/cli/loader/emscripten-version b/cli/loader/emscripten-version index 331ba4bc..96cef7dd 100644 --- a/cli/loader/emscripten-version +++ b/cli/loader/emscripten-version @@ -1 +1 @@ -3.1.64 +3.1.64 \ No newline at end of file diff --git a/script/benchmark b/script/benchmark deleted file mode 100755 index 1fd08f28..00000000 --- a/script/benchmark +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash - -set -e - -function usage { - cat < /dev/null | - jq -rs 'map(select(.target.name == "benchmark" and .executable))[0].executable' - ) - env | grep TREE_SITTER - echo "$test_binary" -else - exec cargo bench benchmark -p tree-sitter-cli -fi diff --git a/script/benchmark.cmd b/script/benchmark.cmd deleted file mode 100644 index fff4c885..00000000 --- a/script/benchmark.cmd +++ /dev/null @@ -1,4 +0,0 @@ -@echo off - -cargo bench benchmark -p tree-sitter-cli -exit /b %errorlevel% diff --git a/script/build-fuzzers b/script/build-fuzzers deleted file mode 100755 index 439221bd..00000000 --- a/script/build-fuzzers +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env bash - -# shellcheck disable=SC2086 - -set -e - -if [[ $(uname -s) != Linux ]]; then - printf 'Fuzzing is only supported on Linux\n' >&2 - exit 1 -fi - -CC=${CC:-clang} -CXX=${CXX:-clang++} - -default_fuzz_flags=-fsanitize=fuzzer,address,undefined - -export CFLAGS="$default_fuzz_flags $CFLAGS" -export CXXFLAGS="$default_fuzz_flags $CXXFLAGS" - -make CC="$CC" CXX="$CXX" libtree-sitter.a - -if [[ -z $* ]]; then - mapfile -t languages < <(ls test/fixtures/grammars) -else - languages=("$@") -fi - -mkdir -p test/fuzz/out - -for lang in "${languages[@]}"; do - # skip typescript & php - if [[ $lang == typescript || $lang == php ]]; then - continue - fi - printf 'Building %s fuzzer...\n' "$lang" - lang_dir="test/fixtures/grammars/$lang" - lang_grammar="${lang_dir}/src/grammar.json" - - # The following assumes each language is implemented as src/parser.c plus an - # optional scanner in src/scanner.c - objects=() - - lang_scanner="${lang_dir}/src/scanner" - if [[ -f "${lang_scanner}.c" ]]; then - $CC $CFLAGS -std=c11 -g -O1 -I "${lang_dir}/src" -c "${lang_scanner}.c" -o "${lang_scanner}.o" - objects+=("${lang_scanner}.o") - fi - - # Compiling with -O0 speeds up the build dramatically - $CC $CFLAGS -g -O0 -I "${lang_dir}/src" "${lang_dir}/src/parser.c" -c -o "${lang_dir}/src/parser.o" - objects+=("${lang_dir}/src/parser.o") - - highlights_filename="${lang_dir}/queries/highlights.scm" - if [[ -f "${highlights_filename}" ]]; then - ts_lang_query_filename="${lang}.scm" - cp "${highlights_filename}" "test/fuzz/out/${ts_lang_query_filename}" - else - ts_lang_query_filename="" - fi - - ts_lang="tree_sitter_$(jq -r .name "$lang_grammar")" - $CXX $CXXFLAGS -std=c++11 -Ilib/include \ - -D TS_LANG="$ts_lang" \ - -D TS_LANG_QUERY_FILENAME="\"${ts_lang_query_filename}\"" \ - test/fuzz/fuzzer.cc \ - "${objects[@]}" \ - libtree-sitter.a \ - -o "test/fuzz/out/${lang}_fuzzer" - - jq ' - [ .. - | if .type? == "STRING" or (.type? == "ALIAS" and .named? == false) then .value else empty end - | select(test("\\S") and length == utf8bytelength) - ] | unique | .[] - ' "$lang_grammar" | sort > "test/fuzz/out/${lang}.dict" -done diff --git a/script/build-wasm-stdlib b/script/build-wasm-stdlib deleted file mode 100755 index 9ef904c2..00000000 --- a/script/build-wasm-stdlib +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -set -e - -declare -a EXPORT_FLAGS -while read -r -d, function; do - EXPORT_FLAGS+=("-Wl,--export=${function:1:-1}") -done < lib/src/wasm/stdlib-symbols.txt - -target/wasi-sdk-21.0/bin/clang-17 \ - -o stdlib.wasm \ - -Os \ - -fPIC \ - -Wl,--no-entry \ - -Wl,--stack-first \ - -Wl,-z -Wl,stack-size=65536 \ - -Wl,--import-undefined \ - -Wl,--import-memory \ - -Wl,--import-table \ - -Wl,--strip-debug \ - -Wl,--export=reset_heap \ - -Wl,--export=__wasm_call_ctors \ - -Wl,--export=__stack_pointer \ - "${EXPORT_FLAGS[@]}" \ - lib/src/wasm/stdlib.c - -xxd -C -i stdlib.wasm > lib/src/wasm/wasm-stdlib.h -mv stdlib.wasm target/ diff --git a/script/check-mallocs b/script/check-mallocs deleted file mode 100755 index cf0aeff5..00000000 --- a/script/check-mallocs +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -src_dir=lib/src -allocation_functions=(malloc calloc realloc free) - -for function in "${allocation_functions[@]}"; do - usages=$(grep -n -E "\b${function}\(" -r $src_dir --exclude alloc.c --exclude stdlib.c) - if [[ -n $usages ]]; then - printf 'The %s function should not be called directly, but is called here:\n%s\n' "$function" "$usages" >&2 - exit 1 - fi -done diff --git a/script/fetch-emscripten b/script/fetch-emscripten deleted file mode 100755 index 772c9ec5..00000000 --- a/script/fetch-emscripten +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -set -e - -EMSDK_DIR=target/emsdk -EMSCRIPTEN_VERSION=$(< cli/loader/emscripten-version) - -{ - if [[ ! -f $EMSDK_DIR/emsdk ]]; then - printf 'Downloading emscripten SDK...\n' - git clone https://github.com/emscripten-core/emsdk.git $EMSDK_DIR - fi - - cd $EMSDK_DIR - - printf 'Updating emscripten SDK...\n' - git reset --hard - git pull - ./emsdk list - - printf 'Installing emscripten...\n' - ./emsdk install "$EMSCRIPTEN_VERSION" - - printf 'Activating emscripten...\n' - ./emsdk activate "$EMSCRIPTEN_VERSION" -} >&2 diff --git a/script/fetch-fixtures b/script/fetch-fixtures deleted file mode 100755 index 14e32759..00000000 --- a/script/fetch-fixtures +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash - -set -e - -GRAMMARS_DIR="$PWD/test/fixtures/grammars" - -fetch_grammar() { - local grammar=$1 - local ref=$2 - local grammar_dir="${GRAMMARS_DIR}/${grammar}" - local grammar_url=https://github.com/tree-sitter/tree-sitter-${grammar} - - printf 'Updating %s grammar...\n' "$grammar" - - if [[ ! -d "$grammar_dir" ]]; then - git clone "$grammar_url" "$grammar_dir" --depth=1 - fi - - git -C "$grammar_dir" fetch origin "$ref" --depth=1 - git -C "$grammar_dir" reset --hard FETCH_HEAD -} - -fetch_grammar bash master -fetch_grammar c master -fetch_grammar cpp master -fetch_grammar embedded-template master -fetch_grammar go master -fetch_grammar html master -fetch_grammar java master -fetch_grammar javascript master -fetch_grammar jsdoc master -fetch_grammar json master -fetch_grammar php master -fetch_grammar python master -fetch_grammar ruby master -fetch_grammar rust master -fetch_grammar typescript master diff --git a/script/fetch-fixtures.cmd b/script/fetch-fixtures.cmd deleted file mode 100644 index 32727b0c..00000000 --- a/script/fetch-fixtures.cmd +++ /dev/null @@ -1,32 +0,0 @@ -@echo off - -call:fetch_grammar bash master -call:fetch_grammar c master -call:fetch_grammar cpp master -call:fetch_grammar embedded-template master -call:fetch_grammar go master -call:fetch_grammar html master -call:fetch_grammar java master -call:fetch_grammar javascript master -call:fetch_grammar jsdoc master -call:fetch_grammar json master -call:fetch_grammar php master -call:fetch_grammar python master -call:fetch_grammar ruby master -call:fetch_grammar rust master -call:fetch_grammar typescript master -exit /B 0 - -:fetch_grammar -setlocal -set grammar_dir=test\fixtures\grammars\%~1 -set grammar_url=https://github.com/tree-sitter/tree-sitter-%~1 -set grammar_branch=%~2 -@if not exist %grammar_dir% ( - git clone %grammar_url% %grammar_dir% --depth=1 -) -pushd %grammar_dir% -git fetch origin %2 --depth=1 -git reset --hard FETCH_HEAD -popd -exit /B 0 diff --git a/script/generate-bindings b/script/generate-bindings deleted file mode 100755 index a0022d8f..00000000 --- a/script/generate-bindings +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -output_path=lib/binding_rust/bindings.rs -header_path=lib/include/tree_sitter/api.h -no_derive_copy=( - TSInput - TSLanguage - TSLogger - TSLookaheadIterator - TSParser - TSTree - TSQuery - TSQueryCursor - TSQueryCapture - TSQueryMatch - TSQueryPredicateStep -) -no_copy=$(IFS='|'; echo "${no_derive_copy[*]}") - -file_version=$(head -n1 "$output_path" | cut -d' ' -f6) -tool_version=$(bindgen --version | cut -d' ' -f2) -higher_version=$(printf '%s\n' "$file_version" "$tool_version" | sort -V | tail -n1) - -if [[ "$higher_version" != "$tool_version" ]]; then - printf 'Latest used bindgen version was %s\n' "$file_version" >&2 - printf 'Currently installed bindgen CLI version is %s\n\n' "$tool_version" >&2 - # shellcheck disable=SC2016 - printf 'You must upgrade bindgen CLI first with `cargo install bindgen-cli`\n' >&2 - exit 1 -fi - -bindgen \ - --no-layout-tests \ - --allowlist-type '^TS.*' \ - --allowlist-function '^ts_.*' \ - --allowlist-var '^TREE_SITTER.*' \ - --blocklist-type '^__.*' \ - --no-prepend-enum-name \ - --no-copy "$no_copy" \ - --use-core \ - "$header_path" \ - -- \ - -D TREE_SITTER_FEATURE_WASM \ - > "$output_path" diff --git a/script/generate-fixtures b/script/generate-fixtures deleted file mode 100755 index ac148649..00000000 --- a/script/generate-fixtures +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash - -set -e - -ROOT_DIR="$PWD" -GRAMMARS_DIR="$ROOT_DIR/test/fixtures/grammars" - -if [[ $CI == true ]]; then - set -x -else - cargo build --release - TREE_SITTER="$ROOT_DIR/target/release/tree-sitter" -fi - -filter_grammar_name="$1" - -while read -r grammar_file; do - grammar_dir="${grammar_file%/*}" - grammar_name="${grammar_dir##*/}" - - if [[ -n $filter_grammar_name && "$filter_grammar_name" != "$grammar_name" ]]; then - continue - fi - - printf 'Regenerating %s parser\n' "$grammar_name" - (cd "$grammar_dir" && "$TREE_SITTER" generate src/grammar.json --abi=latest) -done < <(find "$GRAMMARS_DIR" -name grammar.js -not -path '*/node_modules/*') diff --git a/script/generate-fixtures-wasm b/script/generate-fixtures-wasm deleted file mode 100755 index 353594ec..00000000 --- a/script/generate-fixtures-wasm +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -set -e - -ROOT_DIR="$PWD" -GRAMMARS_DIR="$ROOT_DIR/test/fixtures/grammars" - -if [[ $CI == true ]]; then - set -x -else - cargo build --release - TREE_SITTER="$ROOT_DIR/target/release/tree-sitter" -fi - -build_wasm_args= -if [[ $1 == --docker ]]; then - build_wasm_args=--docker - shift -fi - -filter_grammar_name="$1" - -while read -r grammar_file; do - grammar_dir="${grammar_file%/*}" - grammar_name="${grammar_dir##*/}" - - if [[ -n $filter_grammar_name && "$filter_grammar_name" != "$grammar_name" ]]; then - continue - fi - - printf 'Compiling %s parser to wasm\n' "$grammar_name" - "$TREE_SITTER" build --wasm $build_wasm_args -o "target/release/tree-sitter-${grammar_name}.wasm" "$grammar_dir" -done < <(find "$GRAMMARS_DIR" -name grammar.js -not -path '*/node_modules/*') diff --git a/script/generate-fixtures.cmd b/script/generate-fixtures.cmd deleted file mode 100644 index c137d6f9..00000000 --- a/script/generate-fixtures.cmd +++ /dev/null @@ -1,13 +0,0 @@ -@echo off - -setlocal EnableDelayedExpansion -set tree_sitter="%cd%\target\release\tree-sitter" - -for /f "tokens=*" %%f in ('dir test\fixtures\grammars\grammar.js /b/s') do ( - pushd "%%f\.." - echo Regenerating parser !cd! - %tree_sitter% generate src\grammar.json --abi=latest - popd -) - -exit /B 0 diff --git a/script/heap-profile b/script/heap-profile deleted file mode 100755 index 1bedd9cd..00000000 --- a/script/heap-profile +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash - -# Usage: -# script/heap-profile -# -# Parse an example source file and record memory usage -# -# Dependencies: -# * `pprof` executable: https://github.com/google/pprof -# * `gperftools` package: https://github.com/gperftools/gperftools - -set -e - -GRAMMARS_DIR="$PWD/test/fixtures/grammars" - -# Build the library -make libtree-sitter.a - -# Build the heap-profiling harness -clang++ \ - -Wno-reorder-init-list \ - -Wno-c99-designator \ - -I lib/include \ - -I "$GRAMMARS_DIR" \ - -D GRAMMARS_DIR="\"${GRAMMARS_DIR}/\"" \ - test/profile/heap.cc \ - -l tcmalloc \ - libtree-sitter.a \ - -o target/heap-profile - -# Run the harness with heap profiling enabled. -export HEAPPROFILE="$PWD/profile" -target/heap-profile "$@" - -# Extract statistics using pprof. -pprof -top -cum profile.0001.heap diff --git a/script/reproduce b/script/reproduce deleted file mode 100755 index 7caebfbf..00000000 --- a/script/reproduce +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -if (($# < 3)); then - echo "usage: $0 [libFuzzer args...]" >&2 - exit 1 -fi - -set -eu - -export ASAN_OPTIONS=quarantine_size_mb=10:detect_leaks=1:symbolize=1 -export UBSAN=print_stacktrace=1:halt_on_error=1:symbolize=1 - -# check if CI env var exists -if [[ -z ${CI:-} ]]; then - declare -A mode_config=( - [halt]='-timeout=1 -rss_limit_mb=2048' - [recover]='-timeout=10 -rss_limit_mb=2048' - ) -else - declare -A mode_config=( - [halt]='-max_total_time=120 -timeout=1 -rss_limit_mb=2048' - [recover]='-time=120 -timeout=10 -rss_limit_mb=2048' - ) -fi - -lang="$1" -shift -mode="$1" -shift -testcase="$1" -shift -# Treat remainder of arguments as libFuzzer arguments - -# shellcheck disable=SC2086 -test/fuzz/out/${lang}_fuzzer ${mode_config[$mode]} -runs=1 "$testcase" "$@" diff --git a/script/run-fuzzer b/script/run-fuzzer deleted file mode 100755 index 42cc1bae..00000000 --- a/script/run-fuzzer +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash - -if (($# < 2)); then - echo "usage: $0 [libFuzzer args...]" >&2 - exit 1 -fi - -set -eu - -export ASAN_OPTIONS=quarantine_size_mb=10:detect_leaks=1:symbolize=1 -export UBSAN=print_stacktrace=1:halt_on_error=1:symbolize=1 - -# check if CI env var exists -if [[ -z ${CI:-} ]]; then - declare -A mode_config=( - [halt]='-timeout=1 -rss_limit_mb=2048' - [recover]='-timeout=10 -rss_limit_mb=2048' - ) -else - declare -A mode_config=( - [halt]='-max_total_time=120 -timeout=1 -rss_limit_mb=2048' - [recover]='-time=120 -timeout=10 -rss_limit_mb=2048' - ) -fi - -lang="$1" -shift -mode="$1" -shift -# Treat remainder of arguments as libFuzzer arguments - -# Fuzzing logs and testcases are always written to `pwd`, so `cd` there first -results="$PWD/test/fuzz/out/fuzz-results/${lang}" -mkdir -p "${results}" -cd "${results}" - -# Create a corpus directory, so new discoveries are stored on disk. These will -# then be loaded on subsequent fuzzing runs -mkdir -p corpus - -# shellcheck disable=SC2086 -../../${lang}_fuzzer -dict="../../${lang}.dict" -artifact_prefix=${lang}_ -max_len=2048 ${mode_config[$mode]} corpus "$@" diff --git a/script/serve-docs b/script/serve-docs deleted file mode 100755 index 9639016e..00000000 --- a/script/serve-docs +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -root=$PWD -cd docs - -bundle exec jekyll serve "$@" & - -bundle exec ruby <`); - process.exit(0) -} - -// Get total file size -const totalSize = statSync(libPath).size - -// Dump symbols with addresses -const output = execFileSync( - 'nm', - ['-t', 'd', libPath], - {encoding: 'utf8'} -); - -// Parse addresses -const addressEntries = []; -for (const line of output.split('\n')) { - const [address, _, name] = line.split(/\s+/); - if (address && name) { - addressEntries.push({name, address: parseInt(address)}) - } -} - -// Compute sizes by subtracting addresses -addressEntries.sort((a, b) => a.address - b.address); -const sizeEntries = addressEntries.map(({name, address}, i) => { - const next = addressEntries[i + 1] ? addressEntries[i + 1].address : totalSize; - const size = next - address; - return {name, size} -}) - -function formatSize(sizeInBytes) { - return sizeInBytes > 1024 - ? `${(sizeInBytes / 1024).toFixed(1)} kb` - : `${sizeInBytes} b` -} - -// Display sizes -sizeEntries.sort((a, b) => b.size - a.size); -console.log('total'.padEnd(64, ' '), '\t', formatSize(totalSize)); -for (const entry of sizeEntries) { - console.log(entry.name.padEnd(64, ' '), '\t', formatSize(entry.size)); -} diff --git a/script/test b/script/test deleted file mode 100755 index 3722857e..00000000 --- a/script/test +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env bash - -set -e - -function usage { - cat <&2 - fi - - test_flags+=("--target=$current_target") - ;; - e) - export TREE_SITTER_EXAMPLE=${OPTARG} - ;; - s) - export TREE_SITTER_SEED=${OPTARG} - ;; - i) - export TREE_SITTER_ITERATIONS=${OPTARG} - ;; - d) - export TREE_SITTER_LOG=1 - ;; - D) - export TREE_SITTER_LOG_GRAPHS=1 - ;; - g) - mode=debug - ;; - *) - usage - exit 1 - ;; - esac -done - -shift $((OPTIND - 1)) - -if [[ ${mode} == debug ]]; then - test_binary=$( - cargo test "${test_flags[@]}" --no-run --message-format=json 2> /dev/null | - jq -rs 'map(select(.target.name == "tree-sitter-cli" and .executable))[0].executable' - ) - lldb "${test_binary}" -- "$1" -else - cargo test "${test_flags[@]}" "$1" -- --nocapture -fi diff --git a/script/test-wasm b/script/test-wasm deleted file mode 100755 index 6dca2c56..00000000 --- a/script/test-wasm +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -set -e - -cd lib/binding_web - -if [[ ! -d node_modules/chai ]] || [[ ! -d node_modules/mocha ]]; then - printf 'Installing test dependencies...\n' - npm install -fi - -node_modules/.bin/mocha diff --git a/script/test.cmd b/script/test.cmd deleted file mode 100644 index 4dc97ef3..00000000 --- a/script/test.cmd +++ /dev/null @@ -1,10 +0,0 @@ -@echo off - -setlocal -set RUST_TEST_THREADS=1 -set RUST_BACKTRACE=full -cargo test "%~1" -if %errorlevel% NEQ 0 ( - exit /b %errorlevel% -) -endlocal diff --git a/script/util/scan-build.sh b/script/util/scan-build.sh deleted file mode 100755 index d19c1744..00000000 --- a/script/util/scan-build.sh +++ /dev/null @@ -1,24 +0,0 @@ -function scan_build { - extra_args=() - - # AFAICT, in the trusty travis container the scan-build tool is from the 3.4 - # installation. Therefore, by default it will use clang-3.4 when analysing code - # which doesn't support the '-std=c++14' (it is available via '-std=c++1y'). - # Use the system-wide installed clang instead which is 3.5 and does support - # '-std=c++14'. - extra_args+=("--use-analyzer=$(command -v clang)") - - # scan-build will try to guess which CXX should be used to compile the actual - # code, which is usually g++ but we need g++5 in the CI. Explicitly pass - # $CC/$CXX to scan-build if they are set in the environment. - - if [[ -n $CC ]]; then - extra_args+=("--use-cc=$CC") - fi - - if [[ -n $CXX ]]; then - extra_args+=("--use-c++=$CXX") - fi - - scan-build "${extra_args[@]}" --status-bugs -disable-checker deadcode.DeadStores "$@" -} diff --git a/script/util/valgrind.supp b/script/util/valgrind.supp deleted file mode 100644 index 090e58ae..00000000 --- a/script/util/valgrind.supp +++ /dev/null @@ -1,256 +0,0 @@ -# Errors - -{ - - Memcheck:Cond - fun:_ZN6option6Parser9workhorseEbPKNS_10DescriptorEiPPKcRNS0_6ActionEbbi - fun:_ZN6option6Parser5parseEbPKNS_10DescriptorEiPPKcPNS_6OptionES8_ibi - fun:_ZN6option6ParserC1EPKNS_10DescriptorEiPPcPNS_6OptionES7_ibi - fun:_ZN6bandit6detail7optionsC1EiPPc - fun:_ZN6bandit3runEiPPc - fun:main -} - -{ - - Memcheck:Cond - fun:_ZN6option6Parser9workhorseEbPKNS_10DescriptorEiPPKcRNS0_6ActionEbbi - fun:_ZN6option5Stats3addEbPKNS_10DescriptorEiPPKcib - fun:_ZN6option5StatsC1EPKNS_10DescriptorEiPPcib - fun:_ZN6bandit6detail7optionsC1EiPPc - fun:_ZN6bandit3runEiPPc - fun:main -} - -{ - - Memcheck:Cond - fun:_ZN6option6Parser9workhorseEbPKNS_10DescriptorEiPPKcRNS0_6ActionEbbi - fun:_ZN6bandit6detail7optionsC2EiPPc - fun:_ZN6bandit3runEiPPc - fun:main -} - -{ - - Memcheck:Value8 - fun:_platform_memcmp - fun:_ZNKSt3__112basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7compareEmmPKcm - fun:_ZNKSt3__112basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7compareEPKc - fun:_ZN9snowhouse6Assert4ThatIPKcNS_16EqualsConstraintINSt3__112basic_stringIcNS5_11char_traitsIcEENS5_9allocatorIcEEEEEEEEvRKT_RKT0_S3_i - fun:_ZNSt3__110__function6__funcIZZZZNK3$_0clEvENKUlvE_clEvENKUlNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEE9ts_parserE_clES9_SA_ENKUlvE_clEvEUlvE0_NS7_ISD_EEFvvEEclEv - fun:_ZNSt3__110__function6__funcIZN6bandit2itEPKcNS_8functionIFvvEEERNS2_6detail8listenerERNS_5dequeIPNS8_7contextENS_9allocatorISD_EEEERNS2_8adapters17assertion_adapterERKNS8_10run_policyEEUlvE1_NSE_ISO_EES6_EclEv - fun:_ZN6bandit8adapters17snowhouse_adapter16adapt_exceptionsENSt3__18functionIFvvEEE - fun:_ZN6bandit2itEPKcNSt3__18functionIFvvEEERNS_6detail8listenerERNS2_5dequeIPNS6_7contextENS2_9allocatorISB_EEEERNS_8adapters17assertion_adapterERKNS6_10run_policyE - fun:_ZN6bandit2itEPKcNSt3__18functionIFvvEEE - fun:_ZNSt3__110__function6__funcIZZZNK3$_0clEvENKUlvE_clEvENKUlNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEE9ts_parserE_clES9_SA_EUlvE_NS7_ISC_EEFvvEEclEv - fun:_ZN6bandit8describeEPKcNSt3__18functionIFvvEEERNS_6detail8listenerERNS2_5dequeIPNS6_7contextENS2_9allocatorISB_EEEEb - fun:_ZN6bandit8describeEPKcNSt3__18functionIFvvEEE -} - -{ - - Memcheck:Addr1 - fun:_platform_memcmp - fun:_ZNKSt3__112basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7compareEmmPKcm - fun:_ZNKSt3__112basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7compareEPKc - fun:_ZN9snowhouse6Assert4ThatIPKcNS_16EqualsConstraintINSt3__112basic_stringIcNS5_11char_traitsIcEENS5_9allocatorIcEEEEEEEEvRKT_RKT0_S3_i - fun:_ZNSt3__110__function6__funcIZZZZNK3$_0clEvENKUlvE_clEvENKUlNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEE9ts_parserE_clES9_SA_ENKUlvE_clEvEUlvE0_NS7_ISD_EEFvvEEclEv - fun:_ZNSt3__110__function6__funcIZN6bandit2itEPKcNS_8functionIFvvEEERNS2_6detail8listenerERNS_5dequeIPNS8_7contextENS_9allocatorISD_EEEERNS2_8adapters17assertion_adapterERKNS8_10run_policyEEUlvE1_NSE_ISO_EES6_EclEv - fun:_ZN6bandit8adapters17snowhouse_adapter16adapt_exceptionsENSt3__18functionIFvvEEE - fun:_ZN6bandit2itEPKcNSt3__18functionIFvvEEERNS_6detail8listenerERNS2_5dequeIPNS6_7contextENS2_9allocatorISB_EEEERNS_8adapters17assertion_adapterERKNS6_10run_policyE - fun:_ZN6bandit2itEPKcNSt3__18functionIFvvEEE - fun:_ZNSt3__110__function6__funcIZZZNK3$_0clEvENKUlvE_clEvENKUlNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEE9ts_parserE_clES9_SA_EUlvE_NS7_ISC_EEFvvEEclEv - fun:_ZN6bandit8describeEPKcNSt3__18functionIFvvEEERNS_6detail8listenerERNS2_5dequeIPNS6_7contextENS2_9allocatorISB_EEEEb - fun:_ZN6bandit8describeEPKcNSt3__18functionIFvvEEE -} - -{ - - Memcheck:Cond - fun:_ZNKSt3__112basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7compareEmmPKcm - fun:_ZNKSt3__112basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7compareEPKc - fun:_ZN9snowhouse6Assert4ThatIPKcNS_16EqualsConstraintINSt3__112basic_stringIcNS5_11char_traitsIcEENS5_9allocatorIcEEEEEEEEvRKT_RKT0_S3_i - fun:_ZNSt3__110__function6__funcIZZZZNK3$_0clEvENKUlvE_clEvENKUlNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEE9ts_parserE_clES9_SA_ENKUlvE_clEvEUlvE0_NS7_ISD_EEFvvEEclEv - fun:_ZNSt3__110__function6__funcIZN6bandit2itEPKcNS_8functionIFvvEEERNS2_6detail8listenerERNS_5dequeIPNS8_7contextENS_9allocatorISD_EEEERNS2_8adapters17assertion_adapterERKNS8_10run_policyEEUlvE1_NSE_ISO_EES6_EclEv - fun:_ZN6bandit8adapters17snowhouse_adapter16adapt_exceptionsENSt3__18functionIFvvEEE - fun:_ZN6bandit2itEPKcNSt3__18functionIFvvEEERNS_6detail8listenerERNS2_5dequeIPNS6_7contextENS2_9allocatorISB_EEEERNS_8adapters17assertion_adapterERKNS6_10run_policyE - fun:_ZN6bandit2itEPKcNSt3__18functionIFvvEEE - fun:_ZNSt3__110__function6__funcIZZZNK3$_0clEvENKUlvE_clEvENKUlNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEE9ts_parserE_clES9_SA_EUlvE_NS7_ISC_EEFvvEEclEv - fun:_ZN6bandit8describeEPKcNSt3__18functionIFvvEEERNS_6detail8listenerERNS2_5dequeIPNS6_7contextENS2_9allocatorISB_EEEEb - fun:_ZN6bandit8describeEPKcNSt3__18functionIFvvEEE - fun:_ZNSt3__110__function6__funcIZNK3$_0clEvEUlvE_NS_9allocatorIS3_EEFvvEEclEv -} - -{ - - Memcheck:Cond - fun:_ZN9snowhouse6Assert4ThatINSt3__112basic_stringIcNS2_11char_traitsIcEENS2_9allocatorIcEEEENS_16EqualsConstraintIS8_EEEEvRKT_RKT0_PKci - fun:_ZNSt3__110__function6__funcIZZZNK3$_0clEvENKUlvE_clEvENKUlvE3_clEvEUlvE0_NS_9allocatorIS5_EEFvvEEclEv - fun:_ZNSt3__110__function6__funcIZN6bandit2itEPKcNS_8functionIFvvEEERNS2_6detail8listenerERNS_5dequeIPNS8_7contextENS_9allocatorISD_EEEERNS2_8adapters17assertion_adapterERKNS8_10run_policyEEUlvE1_NSE_ISO_EES6_EclEv - fun:_ZN6bandit8adapters17snowhouse_adapter16adapt_exceptionsENSt3__18functionIFvvEEE - fun:_ZN6bandit2itEPKcNSt3__18functionIFvvEEERNS_6detail8listenerERNS2_5dequeIPNS6_7contextENS2_9allocatorISB_EEEERNS_8adapters17assertion_adapterERKNS6_10run_policyE - fun:_ZN6bandit2itEPKcNSt3__18functionIFvvEEE - fun:_ZNSt3__110__function6__funcIZZNK3$_0clEvENKUlvE_clEvEUlvE3_NS_9allocatorIS4_EEFvvEEclEv - fun:_ZN6bandit8describeEPKcNSt3__18functionIFvvEEERNS_6detail8listenerERNS2_5dequeIPNS6_7contextENS2_9allocatorISB_EEEEb - fun:_ZN6bandit8describeEPKcNSt3__18functionIFvvEEE - fun:_ZNSt3__110__function6__funcIZNK3$_0clEvEUlvE_NS_9allocatorIS3_EEFvvEEclEv - fun:_ZN6bandit8describeEPKcNSt3__18functionIFvvEEERNS_6detail8listenerERNS2_5dequeIPNS6_7contextENS2_9allocatorISB_EEEEb - fun:_ZN6bandit8describeEPKcNSt3__18functionIFvvEEE -} - -{ - - Memcheck:Value8 - fun:_platform_memcmp - fun:_ZNK9snowhouse16EqualsConstraintINSt3__112basic_stringIcNS1_11char_traitsIcEENS1_9allocatorIcEEEEEclIS7_EEbRKT_ - fun:_ZN9snowhouse6Assert4ThatINSt3__112basic_stringIcNS2_11char_traitsIcEENS2_9allocatorIcEEEENS_16EqualsConstraintIS8_EEEEvRKT_RKT0_PKci - fun:_ZNSt3__110__function6__funcIZZZNK3$_0clEvENKUlvE_clEvENKUlvE3_clEvEUlvE0_NS_9allocatorIS5_EEFvvEEclEv - fun:_ZNSt3__110__function6__funcIZN6bandit2itEPKcNS_8functionIFvvEEERNS2_6detail8listenerERNS_5dequeIPNS8_7contextENS_9allocatorISD_EEEERNS2_8adapters17assertion_adapterERKNS8_10run_policyEEUlvE1_NSE_ISO_EES6_EclEv - fun:_ZN6bandit8adapters17snowhouse_adapter16adapt_exceptionsENSt3__18functionIFvvEEE - fun:_ZN6bandit2itEPKcNSt3__18functionIFvvEEERNS_6detail8listenerERNS2_5dequeIPNS6_7contextENS2_9allocatorISB_EEEERNS_8adapters17assertion_adapterERKNS6_10run_policyE - fun:_ZN6bandit2itEPKcNSt3__18functionIFvvEEE - fun:_ZNSt3__110__function6__funcIZZNK3$_0clEvENKUlvE_clEvEUlvE3_NS_9allocatorIS4_EEFvvEEclEv - fun:_ZN6bandit8describeEPKcNSt3__18functionIFvvEEERNS_6detail8listenerERNS2_5dequeIPNS6_7contextENS2_9allocatorISB_EEEEb - fun:_ZN6bandit8describeEPKcNSt3__18functionIFvvEEE - fun:_ZNSt3__110__function6__funcIZNK3$_0clEvEUlvE_NS_9allocatorIS3_EEFvvEEclEv -} - -{ - - Memcheck:Cond - fun:_ZN9snowhouse6Assert4ThatINSt3__112basic_stringIcNS2_11char_traitsIcEENS2_9allocatorIcEEEENS_16EqualsConstraintIS8_EEEEvRKT_RKT0_PKci - fun:_ZNSt3__110__function6__funcIZZNK3$_0clEvENKUlvE_clEvEUlvE1_NS_9allocatorIS4_EEFvvEEclEv - fun:_ZNSt3__110__function6__funcIZN6bandit2itEPKcNS_8functionIFvvEEERNS2_6detail8listenerERNS_5dequeIPNS8_7contextENS_9allocatorISD_EEEERNS2_8adapters17assertion_adapterERKNS8_10run_policyEEUlvE1_NSE_ISO_EES6_EclEv - fun:_ZN6bandit8adapters17snowhouse_adapter16adapt_exceptionsENSt3__18functionIFvvEEE - fun:_ZN6bandit2itEPKcNSt3__18functionIFvvEEERNS_6detail8listenerERNS2_5dequeIPNS6_7contextENS2_9allocatorISB_EEEERNS_8adapters17assertion_adapterERKNS6_10run_policyE - fun:_ZN6bandit2itEPKcNSt3__18functionIFvvEEE - fun:_ZNSt3__110__function6__funcIZNK3$_0clEvEUlvE_NS_9allocatorIS3_EEFvvEEclEv - fun:_ZN6bandit8describeEPKcNSt3__18functionIFvvEEERNS_6detail8listenerERNS2_5dequeIPNS6_7contextENS2_9allocatorISB_EEEEb - fun:_ZN6bandit8describeEPKcNSt3__18functionIFvvEEE - fun:_ZNSt3__110__function6__funcI3$_0NS_9allocatorIS2_EEFvvEEclEv - fun:_ZN6bandit3runERKNS_6detail7optionsERKNSt3__14listINS4_8functionIFvvEEENS4_9allocatorIS8_EEEERNS4_5dequeIPNS0_7contextENS9_ISG_EEEERNS0_8listenerE - fun:_ZN6bandit3runEiPPc -} - -{ - - Memcheck:Value8 - fun:_platform_memcmp - fun:_ZNK9snowhouse16EqualsConstraintINSt3__112basic_stringIcNS1_11char_traitsIcEENS1_9allocatorIcEEEEEclIS7_EEbRKT_ - fun:_ZN9snowhouse6Assert4ThatINSt3__112basic_stringIcNS2_11char_traitsIcEENS2_9allocatorIcEEEENS_16EqualsConstraintIS8_EEEEvRKT_RKT0_PKci - fun:_ZNSt3__110__function6__funcIZZNK3$_0clEvENKUlvE_clEvEUlvE1_NS_9allocatorIS4_EEFvvEEclEv - fun:_ZNSt3__110__function6__funcIZN6bandit2itEPKcNS_8functionIFvvEEERNS2_6detail8listenerERNS_5dequeIPNS8_7contextENS_9allocatorISD_EEEERNS2_8adapters17assertion_adapterERKNS8_10run_policyEEUlvE1_NSE_ISO_EES6_EclEv - fun:_ZN6bandit8adapters17snowhouse_adapter16adapt_exceptionsENSt3__18functionIFvvEEE - fun:_ZN6bandit2itEPKcNSt3__18functionIFvvEEERNS_6detail8listenerERNS2_5dequeIPNS6_7contextENS2_9allocatorISB_EEEERNS_8adapters17assertion_adapterERKNS6_10run_policyE - fun:_ZN6bandit2itEPKcNSt3__18functionIFvvEEE - fun:_ZNSt3__110__function6__funcIZNK3$_0clEvEUlvE_NS_9allocatorIS3_EEFvvEEclEv - fun:_ZN6bandit8describeEPKcNSt3__18functionIFvvEEERNS_6detail8listenerERNS2_5dequeIPNS6_7contextENS2_9allocatorISB_EEEEb - fun:_ZN6bandit8describeEPKcNSt3__18functionIFvvEEE - fun:_ZNSt3__110__function6__funcI3$_0NS_9allocatorIS2_EEFvvEEclEv -} - -{ - - Memcheck:Cond - fun:_ZN9snowhouse6Assert4ThatINSt3__112basic_stringIcNS2_11char_traitsIcEENS2_9allocatorIcEEEENS_16EqualsConstraintIS8_EEEEvRKT_RKT0_PKci - fun:_ZNSt3__110__function6__funcIZZZNK3$_0clEvENKUlvE_clEvENKUlvE4_clEvEUlvE0_NS_9allocatorIS5_EEFvvEEclEv - fun:_ZNSt3__110__function6__funcIZN6bandit2itEPKcNS_8functionIFvvEEERNS2_6detail8listenerERNS_5dequeIPNS8_7contextENS_9allocatorISD_EEEERNS2_8adapters17assertion_adapterERKNS8_10run_policyEEUlvE1_NSE_ISO_EES6_EclEv - fun:_ZN6bandit8adapters17snowhouse_adapter16adapt_exceptionsENSt3__18functionIFvvEEE - fun:_ZN6bandit2itEPKcNSt3__18functionIFvvEEERNS_6detail8listenerERNS2_5dequeIPNS6_7contextENS2_9allocatorISB_EEEERNS_8adapters17assertion_adapterERKNS6_10run_policyE - fun:_ZN6bandit2itEPKcNSt3__18functionIFvvEEE - fun:_ZNSt3__110__function6__funcIZZNK3$_0clEvENKUlvE_clEvEUlvE4_NS_9allocatorIS4_EEFvvEEclEv - fun:_ZN6bandit8describeEPKcNSt3__18functionIFvvEEERNS_6detail8listenerERNS2_5dequeIPNS6_7contextENS2_9allocatorISB_EEEEb - fun:_ZN6bandit8describeEPKcNSt3__18functionIFvvEEE - fun:_ZNSt3__110__function6__funcIZNK3$_0clEvEUlvE_NS_9allocatorIS3_EEFvvEEclEv - fun:_ZN6bandit8describeEPKcNSt3__18functionIFvvEEERNS_6detail8listenerERNS2_5dequeIPNS6_7contextENS2_9allocatorISB_EEEEb - fun:_ZN6bandit8describeEPKcNSt3__18functionIFvvEEE -} - -{ - - Memcheck:Value8 - fun:_platform_memcmp - fun:_ZNK9snowhouse16EqualsConstraintINSt3__112basic_stringIcNS1_11char_traitsIcEENS1_9allocatorIcEEEEEclIS7_EEbRKT_ - fun:_ZN9snowhouse6Assert4ThatINSt3__112basic_stringIcNS2_11char_traitsIcEENS2_9allocatorIcEEEENS_16EqualsConstraintIS8_EEEEvRKT_RKT0_PKci - fun:_ZNSt3__110__function6__funcIZZZNK3$_0clEvENKUlvE_clEvENKUlvE4_clEvEUlvE0_NS_9allocatorIS5_EEFvvEEclEv - fun:_ZNSt3__110__function6__funcIZN6bandit2itEPKcNS_8functionIFvvEEERNS2_6detail8listenerERNS_5dequeIPNS8_7contextENS_9allocatorISD_EEEERNS2_8adapters17assertion_adapterERKNS8_10run_policyEEUlvE1_NSE_ISO_EES6_EclEv - fun:_ZN6bandit8adapters17snowhouse_adapter16adapt_exceptionsENSt3__18functionIFvvEEE - fun:_ZN6bandit2itEPKcNSt3__18functionIFvvEEERNS_6detail8listenerERNS2_5dequeIPNS6_7contextENS2_9allocatorISB_EEEERNS_8adapters17assertion_adapterERKNS6_10run_policyE - fun:_ZN6bandit2itEPKcNSt3__18functionIFvvEEE - fun:_ZNSt3__110__function6__funcIZZNK3$_0clEvENKUlvE_clEvEUlvE4_NS_9allocatorIS4_EEFvvEEclEv - fun:_ZN6bandit8describeEPKcNSt3__18functionIFvvEEERNS_6detail8listenerERNS2_5dequeIPNS6_7contextENS2_9allocatorISB_EEEEb - fun:_ZN6bandit8describeEPKcNSt3__18functionIFvvEEE - fun:_ZNSt3__110__function6__funcIZNK3$_0clEvEUlvE_NS_9allocatorIS3_EEFvvEEclEv -} - -{ - - Memcheck:Cond - fun:_ZNKSt3__112basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7compareEmmPKcm - fun:_ZNKSt3__112basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7compareEPKc - fun:_ZN9snowhouse6Assert4ThatIPKcNS_16EqualsConstraintINSt3__112basic_stringIcNS5_11char_traitsIcEENS5_9allocatorIcEEEEEEEEvRKT_RKT0_S3_i - fun:_ZNSt3__110__function6__funcIZZZZNK3$_0clEvENKUlvE_clEvENKUlNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPF9ts_parservEE_clES9_SC_ENKUlvE_clEvEUlvE0_NS7_ISF_EEFvvEEclEv - fun:_ZNSt3__110__function6__funcIZN6bandit2itEPKcNS_8functionIFvvEEERNS2_6detail8listenerERNS_5dequeIPNS8_7contextENS_9allocatorISD_EEEERNS2_8adapters17assertion_adapterERKNS8_10run_policyEEUlvE1_NSE_ISO_EES6_EclEv - fun:_ZN6bandit8adapters17snowhouse_adapter16adapt_exceptionsENSt3__18functionIFvvEEE - fun:_ZN6bandit2itEPKcNSt3__18functionIFvvEEERNS_6detail8listenerERNS2_5dequeIPNS6_7contextENS2_9allocatorISB_EEEERNS_8adapters17assertion_adapterERKNS6_10run_policyE - fun:_ZN6bandit2itEPKcNSt3__18functionIFvvEEE - fun:_ZNSt3__110__function6__funcIZZZNK3$_0clEvENKUlvE_clEvENKUlNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPF9ts_parservEE_clES9_SC_EUlvE_NS7_ISE_EEFvvEEclEv - fun:_ZN6bandit8describeEPKcNSt3__18functionIFvvEEERNS_6detail8listenerERNS2_5dequeIPNS6_7contextENS2_9allocatorISB_EEEEb - fun:_ZN6bandit8describeEPKcNSt3__18functionIFvvEEE - fun:_ZNSt3__110__function6__funcIZNK3$_0clEvEUlvE_NS_9allocatorIS3_EEFvvEEclEv -} - -{ - - Memcheck:Addr1 - fun:_platform_memcmp - fun:_ZNKSt3__112basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7compareEmmPKcm - fun:_ZNKSt3__112basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7compareEPKc - fun:_ZN9snowhouse6Assert4ThatIPKcNS_16EqualsConstraintINSt3__112basic_stringIcNS5_11char_traitsIcEENS5_9allocatorIcEEEEEEEEvRKT_RKT0_S3_i - fun:_ZNSt3__110__function6__funcIZZZZNK3$_0clEvENKUlvE_clEvENKUlNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPF9ts_parservEE_clES9_SC_ENKUlvE_clEvEUlvE0_NS7_ISF_EEFvvEEclEv - fun:_ZNSt3__110__function6__funcIZN6bandit2itEPKcNS_8functionIFvvEEERNS2_6detail8listenerERNS_5dequeIPNS8_7contextENS_9allocatorISD_EEEERNS2_8adapters17assertion_adapterERKNS8_10run_policyEEUlvE1_NSE_ISO_EES6_EclEv - fun:_ZN6bandit8adapters17snowhouse_adapter16adapt_exceptionsENSt3__18functionIFvvEEE - fun:_ZN6bandit2itEPKcNSt3__18functionIFvvEEERNS_6detail8listenerERNS2_5dequeIPNS6_7contextENS2_9allocatorISB_EEEERNS_8adapters17assertion_adapterERKNS6_10run_policyE - fun:_ZN6bandit2itEPKcNSt3__18functionIFvvEEE - fun:_ZNSt3__110__function6__funcIZZZNK3$_0clEvENKUlvE_clEvENKUlNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPF9ts_parservEE_clES9_SC_EUlvE_NS7_ISE_EEFvvEEclEv - fun:_ZN6bandit8describeEPKcNSt3__18functionIFvvEEERNS_6detail8listenerERNS2_5dequeIPNS6_7contextENS2_9allocatorISB_EEEEb - fun:_ZN6bandit8describeEPKcNSt3__18functionIFvvEEE -} - -{ - - Memcheck:Value8 - fun:_platform_memcmp - fun:_ZNKSt3__112basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7compareEmmPKcm - fun:_ZNKSt3__112basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7compareEPKc - fun:_ZN9snowhouse6Assert4ThatIPKcNS_16EqualsConstraintINSt3__112basic_stringIcNS5_11char_traitsIcEENS5_9allocatorIcEEEEEEEEvRKT_RKT0_S3_i - fun:_ZNSt3__110__function6__funcIZZZZNK3$_0clEvENKUlvE_clEvENKUlNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPF9ts_parservEE_clES9_SC_ENKUlvE_clEvEUlvE0_NS7_ISF_EEFvvEEclEv - fun:_ZNSt3__110__function6__funcIZN6bandit2itEPKcNS_8functionIFvvEEERNS2_6detail8listenerERNS_5dequeIPNS8_7contextENS_9allocatorISD_EEEERNS2_8adapters17assertion_adapterERKNS8_10run_policyEEUlvE1_NSE_ISO_EES6_EclEv - fun:_ZN6bandit8adapters17snowhouse_adapter16adapt_exceptionsENSt3__18functionIFvvEEE - fun:_ZN6bandit2itEPKcNSt3__18functionIFvvEEERNS_6detail8listenerERNS2_5dequeIPNS6_7contextENS2_9allocatorISB_EEEERNS_8adapters17assertion_adapterERKNS6_10run_policyE - fun:_ZN6bandit2itEPKcNSt3__18functionIFvvEEE - fun:_ZNSt3__110__function6__funcIZZZNK3$_0clEvENKUlvE_clEvENKUlNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEPF9ts_parservEE_clES9_SC_EUlvE_NS7_ISE_EEFvvEEclEv - fun:_ZN6bandit8describeEPKcNSt3__18functionIFvvEEERNS_6detail8listenerERNS2_5dequeIPNS6_7contextENS2_9allocatorISB_EEEEb - fun:_ZN6bandit8describeEPKcNSt3__18functionIFvvEEE -} - -# Leaks - -{ - - Memcheck:Leak - match-leak-kinds: possible - fun:malloc_zone_malloc - fun:_objc_copyClassNamesForImage - fun:_ZL9protocolsv - fun:_Z9readClassP10objc_classbb - fun:gc_init - fun:_ZL33objc_initializeClassPair_internalP10objc_classPKcS0_S0_ - fun:layout_string_create - fun:_ZL12realizeClassP10objc_class - fun:_ZL22copySwiftV1MangledNamePKcb - fun:_ZL22copySwiftV1MangledNamePKcb - fun:_ZL22copySwiftV1MangledNamePKcb - fun:_ZL22copySwiftV1MangledNamePKcb -} diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 4f270db2..83bb3008 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -11,10 +11,19 @@ keywords.workspace = true categories.workspace = true publish = false +[lints] +workspace = true + [dependencies] +anstyle.workspace = true +anyhow.workspace = true +bindgen = { version = "0.70.1" } +cc.workspace = true +clap.workspace = true git2.workspace = true indoc.workspace = true toml.workspace = true +regex.workspace = true semver.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/xtask/src/benchmark.rs b/xtask/src/benchmark.rs new file mode 100644 index 00000000..2b15db32 --- /dev/null +++ b/xtask/src/benchmark.rs @@ -0,0 +1,75 @@ +use anyhow::Result; + +use crate::{bail_on_err, Benchmark}; + +pub fn run(args: &Benchmark) -> Result<()> { + if let Some(ref example) = args.example_file_name { + std::env::set_var("TREE_SITTER_BENCHMARK_EXAMPLE_FILTER", example); + } + + if let Some(ref language) = args.language { + std::env::set_var("TREE_SITTER_BENCHMARK_LANGUAGE_FILTER", language); + } + + if args.repetition_count != 5 { + std::env::set_var( + "TREE_SITTER_BENCHMARK_REPETITION_COUNT", + args.repetition_count.to_string(), + ); + } + + if args.debug { + let output = std::process::Command::new("cargo") + .arg("bench") + .arg("benchmark") + .arg("-p") + .arg("tree-sitter-cli") + .arg("--no-run") + .arg("--message-format=json") + .spawn()? + .wait_with_output()?; + + bail_on_err(&output, "Failed to run `cargo bench`")?; + + let json_output = serde_json::from_slice::(&output.stdout)?; + + let test_binary = json_output + .as_array() + .ok_or_else(|| anyhow::anyhow!("Invalid JSON output"))? + .iter() + .find_map(|message| { + if message + .get("target") + .and_then(|target| target.get("name")) + .and_then(|name| name.as_str()) + .is_some_and(|name| name == "benchmark") + && message + .get("executable") + .and_then(|executable| executable.as_str()) + .is_some() + { + message + .get("executable") + .and_then(|executable| executable.as_str()) + } else { + None + } + }) + .ok_or_else(|| anyhow::anyhow!("Failed to find benchmark executable"))?; + + println!("{test_binary}"); + } else { + let status = std::process::Command::new("cargo") + .arg("bench") + .arg("benchmark") + .arg("-p") + .arg("tree-sitter-cli") + .status()?; + + if !status.success() { + anyhow::bail!("Failed to run `cargo bench`"); + } + } + + Ok(()) +} diff --git a/xtask/src/build_wasm.rs b/xtask/src/build_wasm.rs new file mode 100644 index 00000000..5ff63a22 --- /dev/null +++ b/xtask/src/build_wasm.rs @@ -0,0 +1,228 @@ +use std::{ + ffi::{OsStr, OsString}, + fmt::Write, + fs, + process::Command, +}; + +use anyhow::{anyhow, Result}; + +use crate::{bail_on_err, BuildWasm, EMSCRIPTEN_TAG}; + +#[derive(PartialEq, Eq)] +enum EmccSource { + Native, + Docker, + Podman, +} + +pub fn run_wasm(args: &BuildWasm) -> Result<()> { + let mut emscripten_flags = vec!["-O3", "--minify", "0"]; + + if args.debug { + emscripten_flags.extend(["-s", "ASSERTIONS=1", "-s", "SAFE_HEAP=1", "-O0", "-g"]); + } + + if args.verbose { + emscripten_flags.extend(["-s", "VERBOSE=1", "-v"]); + } + + let emcc_name = if cfg!(windows) { "emcc.bat" } else { "emcc" }; + + // Order of preference: emscripten > docker > podman > error + let source = if !args.docker && Command::new(emcc_name).output().is_ok() { + EmccSource::Native + } else if Command::new("docker") + .arg("info") + .output() + .map_or(false, |out| out.status.success()) + { + EmccSource::Docker + } else if Command::new("podman") + .arg("--version") + .output() + .map_or(false, |out| out.status.success()) + { + EmccSource::Podman + } else { + return Err(anyhow!( + "You must have either emcc, docker, or podman on your PATH to run this command" + )); + }; + + let mut command = match source { + EmccSource::Native => Command::new(emcc_name), + EmccSource::Docker | EmccSource::Podman => { + let mut command = match source { + EmccSource::Docker => Command::new("docker"), + EmccSource::Podman => Command::new("podman"), + _ => unreachable!(), + }; + command.args(["run", "--rm"]); + + // Mount the root directory as a volume, which is the repo root + let mut volume_string = OsString::from(std::env::current_dir().unwrap()); + volume_string.push(":/src:Z"); + command.args([OsStr::new("--volume"), &volume_string]); + + // In case `docker` is an alias to `podman`, ensure that podman + // mounts the current directory as writable by the container + // user which has the same uid as the host user. Setting the + // podman-specific variable is more reliable than attempting to + // detect whether `docker` is an alias for `podman`. + // see https://docs.podman.io/en/latest/markdown/podman-run.1.html#userns-mode + command.env("PODMAN_USERNS", "keep-id"); + + // Get the current user id so that files created in the docker container will have + // the same owner. + #[cfg(unix)] + { + #[link(name = "c")] + extern "C" { + fn getuid() -> u32; + } + // don't need to set user for podman since PODMAN_USERNS=keep-id is already set + if source == EmccSource::Docker { + let user_id = unsafe { getuid() }; + command.args(["--user", &user_id.to_string()]); + } + }; + + // Run `emcc` in a container using the `emscripten-slim` image + command.args([EMSCRIPTEN_TAG, "emcc"]); + command + } + }; + + fs::create_dir_all("target/scratch").unwrap(); + + let exported_functions = concat!( + include_str!("../../lib/src/wasm/stdlib-symbols.txt"), + include_str!("../../lib/binding_web/exports.txt") + ) + .replace('"', "") + .lines() + .fold(String::new(), |mut output, line| { + let _ = write!(output, "_{line}"); + output + }) + .trim_end_matches(',') + .to_string(); + + let exported_functions = format!("EXPORTED_FUNCTIONS={exported_functions}"); + let exported_runtime_methods = "EXPORTED_RUNTIME_METHODS=stringToUTF16,AsciiToString"; + + emscripten_flags.extend([ + "-s", + "WASM=1", + "-s", + "INITIAL_MEMORY=33554432", + "-s", + "ALLOW_MEMORY_GROWTH=1", + "-s", + "SUPPORT_BIG_ENDIAN=1", + "-s", + "MAIN_MODULE=2", + "-s", + "FILESYSTEM=0", + "-s", + "NODEJS_CATCH_EXIT=0", + "-s", + "NODEJS_CATCH_REJECTION=0", + "-s", + &exported_functions, + "-s", + exported_runtime_methods, + "-fno-exceptions", + "-std=c11", + "-D", + "fprintf(...)=", + "-D", + "NDEBUG=", + "-D", + "_POSIX_C_SOURCE=200112L", + "-D", + "_DEFAULT_SOURCE=", + "-I", + "lib/src", + "-I", + "lib/include", + "--js-library", + "lib/binding_web/imports.js", + "--pre-js", + "lib/binding_web/prefix.js", + "--post-js", + "lib/binding_web/binding.js", + "--post-js", + "lib/binding_web/suffix.js", + "lib/src/lib.c", + "lib/binding_web/binding.c", + "-o", + "target/scratch/tree-sitter.js", + ]); + + bail_on_err( + &command.args(emscripten_flags).spawn()?.wait_with_output()?, + "Failed to compile the Tree-sitter WASM library", + )?; + + fs::rename( + "target/scratch/tree-sitter.js", + "lib/binding_web/tree-sitter.js", + )?; + + fs::rename( + "target/scratch/tree-sitter.wasm", + "lib/binding_web/tree-sitter.wasm", + )?; + + Ok(()) +} + +pub fn run_wasm_stdlib() -> Result<()> { + let export_flags = include_str!("../../lib/src/wasm/stdlib-symbols.txt") + .lines() + .map(|line| format!("-Wl,--export={}", &line[1..line.len() - 1])) + .collect::>(); + + let mut command = Command::new("target/wasi-sdk-21.0/bin/clang-17"); + + let output = command + .args([ + "-o", + "stdlib.wasm", + "-Os", + "-fPIC", + "-Wl,--no-entry", + "-Wl,--stack-first", + "-Wl,-z", + "-Wl,stack-size=65536", + "-Wl,--import-undefined", + "-Wl,--import-memory", + "-Wl,--import-table", + "-Wl,--strip-debug", + "-Wl,--export=reset_heap", + "-Wl,--export=__wasm_call_ctors", + "-Wl,--export=__stack_pointer", + ]) + .args(export_flags) + .arg("lib/src/wasm/stdlib.c") + .output()?; + + bail_on_err(&output, "Failed to compile the Tree-sitter WASM stdlib")?; + + let xxd = Command::new("xxd") + .args(["-C", "-i", "stdlib.wasm"]) + .output()?; + + bail_on_err( + &xxd, + "Failed to run xxd on the compiled Tree-sitter WASM stdlib", + )?; + + fs::write("lib/src/wasm/wasm-stdlib.h", xxd.stdout)?; + + fs::rename("stdlib.wasm", "target/stdlib.wasm")?; + + Ok(()) +} diff --git a/xtask/src/bump.rs b/xtask/src/bump.rs index b4090ea5..2f7c74f1 100644 --- a/xtask/src/bump.rs +++ b/xtask/src/bump.rs @@ -1,11 +1,14 @@ use std::{cmp::Ordering, path::Path}; +use anyhow::{anyhow, Result}; use git2::{DiffOptions, Repository}; use indoc::indoc; use semver::{BuildMetadata, Prerelease, Version}; use toml::Value; -pub fn get_latest_tag(repo: &Repository) -> Result> { +use crate::BumpVersion; + +pub fn get_latest_tag(repo: &Repository) -> Result { let mut tags = repo .tag_names(None)? .into_iter() @@ -23,10 +26,10 @@ pub fn get_latest_tag(repo: &Repository) -> Result Result<(), Box> { +pub fn run(args: BumpVersion) -> Result<()> { let repo = Repository::open(".")?; let latest_tag = get_latest_tag(&repo)?; let current_version = Version::parse(&latest_tag)?; @@ -104,35 +107,39 @@ pub fn bump_versions() -> Result<(), Box> { } } - let mut version = current_version.clone(); - if should_increment_minor { - version.minor += 1; - version.patch = 0; - version.pre = Prerelease::EMPTY; - version.build = BuildMetadata::EMPTY; - } else if should_increment_patch { - version.patch += 1; - version.pre = Prerelease::EMPTY; - version.build = BuildMetadata::EMPTY; + let next_version = if let Some(version) = args.version { + version } else { - return Err(format!("No source code changed since {current_version}").into()); - } + let mut next_version = current_version.clone(); + if should_increment_minor { + next_version.minor += 1; + next_version.patch = 0; + next_version.pre = Prerelease::EMPTY; + next_version.build = BuildMetadata::EMPTY; + } else if should_increment_patch { + next_version.patch += 1; + next_version.pre = Prerelease::EMPTY; + next_version.build = BuildMetadata::EMPTY; + } else { + return Err(anyhow!(format!( + "No source code changed since {current_version}" + ))); + } + next_version + }; - println!("Bumping from {current_version} to {version}"); - update_crates(¤t_version, &version)?; - update_makefile(&version)?; - update_cmake(&version)?; - update_npm(&version)?; - update_zig(&version)?; - tag_next_version(&repo, &version)?; + println!("Bumping from {current_version} to {next_version}"); + update_crates(¤t_version, &next_version)?; + update_makefile(&next_version)?; + update_cmake(&next_version)?; + update_npm(&next_version)?; + update_zig(&next_version)?; + tag_next_version(&repo, &next_version)?; Ok(()) } -fn tag_next_version( - repo: &Repository, - next_version: &Version, -) -> Result<(), Box> { +fn tag_next_version(repo: &Repository, next_version: &Version) -> Result<()> { // first add the manifests let mut index = repo.index()?; @@ -184,7 +191,7 @@ fn tag_next_version( Ok(()) } -fn update_makefile(next_version: &Version) -> Result<(), Box> { +fn update_makefile(next_version: &Version) -> Result<()> { let makefile = std::fs::read_to_string("Makefile")?; let makefile = makefile .lines() @@ -204,7 +211,7 @@ fn update_makefile(next_version: &Version) -> Result<(), Box Result<(), Box> { +fn update_cmake(next_version: &Version) -> Result<()> { let cmake = std::fs::read_to_string("lib/CMakeLists.txt")?; let cmake = cmake .lines() @@ -230,10 +237,7 @@ fn update_cmake(next_version: &Version) -> Result<(), Box Ok(()) } -fn update_crates( - current_version: &Version, - next_version: &Version, -) -> Result<(), Box> { +fn update_crates(current_version: &Version, next_version: &Version) -> Result<()> { let mut cmd = std::process::Command::new("cargo"); cmd.arg("workspaces").arg("version"); @@ -253,20 +257,20 @@ fn update_crates( let status = cmd.status()?; if !status.success() { - return Err("Failed to update crates".into()); + return Err(anyhow!("Failed to update crates")); } Ok(()) } -fn update_npm(next_version: &Version) -> Result<(), Box> { +fn update_npm(next_version: &Version) -> Result<()> { for path in ["lib/binding_web/package.json", "cli/npm/package.json"] { let package_json = serde_json::from_str::(&std::fs::read_to_string(path)?)?; let mut package_json = package_json .as_object() - .ok_or("Invalid package.json")? + .ok_or_else(|| anyhow!("Invalid package.json"))? .clone(); package_json.insert( "version".to_string(), @@ -281,7 +285,7 @@ fn update_npm(next_version: &Version) -> Result<(), Box> Ok(()) } -fn update_zig(next_version: &Version) -> Result<(), Box> { +fn update_zig(next_version: &Version) -> Result<()> { let zig = std::fs::read_to_string("build.zig.zon")?; let zig = zig @@ -303,7 +307,7 @@ fn update_zig(next_version: &Version) -> Result<(), Box> } /// read Cargo.toml and get the version -fn fetch_workspace_version() -> Result> { +fn fetch_workspace_version() -> Result { let cargo_toml = toml::from_str::(&std::fs::read_to_string("Cargo.toml")?)?; Ok(cargo_toml["workspace"]["package"]["version"] diff --git a/xtask/src/clippy.rs b/xtask/src/clippy.rs new file mode 100644 index 00000000..c8d33348 --- /dev/null +++ b/xtask/src/clippy.rs @@ -0,0 +1,33 @@ +use std::process::Command; + +use anyhow::Result; + +use crate::{bail_on_err, Clippy}; + +pub fn run(args: &Clippy) -> Result<()> { + let mut clippy_command = Command::new("cargo"); + clippy_command.arg("+nightly").arg("clippy"); + + if let Some(package) = args.package.as_ref() { + clippy_command.args(["--package", package]); + } else { + clippy_command.arg("--workspace"); + } + + clippy_command + .arg("--release") + .arg("--all-targets") + .arg("--all-features") + .arg("--") + .arg("-D") + .arg("warnings"); + + if args.fix { + clippy_command.arg("--fix"); + } + + bail_on_err( + &clippy_command.spawn()?.wait_with_output()?, + "Clippy failed", + ) +} diff --git a/xtask/src/fetch.rs b/xtask/src/fetch.rs new file mode 100644 index 00000000..f48373db --- /dev/null +++ b/xtask/src/fetch.rs @@ -0,0 +1,119 @@ +use std::{path::Path, process::Command}; + +use anyhow::Result; + +use crate::{bail_on_err, EMSCRIPTEN_VERSION}; + +pub fn run_fixtures() -> Result<()> { + let grammars_dir = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .join("test") + .join("fixtures") + .join("grammars"); + + [ + ("bash", "master"), + ("c", "master"), + ("cpp", "master"), + ("embedded-template", "master"), + ("go", "master"), + ("html", "master"), + ("java", "master"), + ("javascript", "master"), + ("jsdoc", "master"), + ("json", "master"), + ("php", "master"), + ("python", "master"), + ("ruby", "master"), + ("rust", "master"), + ("typescript", "master"), + ] + .iter() + .try_for_each(|(grammar, r#ref)| { + let grammar_dir = grammars_dir.join(grammar); + let grammar_url = format!("https://github.com/tree-sitter/tree-sitter-{grammar}"); + + println!("Updating the {grammar} grammar..."); + + if !grammar_dir.exists() { + let mut command = Command::new("git"); + command.args([ + "clone", + "--depth", + "1", + &grammar_url, + &grammar_dir.to_string_lossy(), + ]); + bail_on_err( + &command.spawn()?.wait_with_output()?, + "Failed to clone the {grammar} grammar", + )?; + } + + std::env::set_current_dir(&grammar_dir)?; + + let mut command = Command::new("git"); + command.args(["fetch", "origin", r#ref, "--depth", "1"]); + bail_on_err( + &command.spawn()?.wait_with_output()?, + "Failed to fetch the {grammar} grammar", + )?; + + let mut command = Command::new("git"); + command.args(["reset", "--hard", "FETCH_HEAD"]); + bail_on_err( + &command.spawn()?.wait_with_output()?, + "Failed to reset the {grammar} grammar", + )?; + + Ok(()) + }) +} + +pub fn run_emscripten() -> Result<()> { + let emscripten_dir = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .join("target") + .join("emsdk"); + + if emscripten_dir.exists() { + println!("Emscripten SDK already exists"); + return Ok(()); + } + println!("Cloning the Emscripten SDK..."); + + let mut command = Command::new("git"); + command.args([ + "clone", + "https://github.com/emscripten-core/emsdk.git", + &emscripten_dir.to_string_lossy(), + ]); + bail_on_err( + &command.spawn()?.wait_with_output()?, + "Failed to clone the Emscripten SDK", + )?; + + std::env::set_current_dir(&emscripten_dir)?; + + let emsdk = if cfg!(windows) { + "emsdk.bat" + } else { + "./emsdk" + }; + + let mut command = Command::new(emsdk); + command.args(["install", EMSCRIPTEN_VERSION]); + bail_on_err( + &command.spawn()?.wait_with_output()?, + "Failed to install Emscripten", + )?; + + let mut command = Command::new(emsdk); + command.args(["activate", EMSCRIPTEN_VERSION]); + bail_on_err( + &command.spawn()?.wait_with_output()?, + "Failed to activate Emscripten", + ) +} diff --git a/xtask/src/generate.rs b/xtask/src/generate.rs new file mode 100644 index 00000000..e9971b51 --- /dev/null +++ b/xtask/src/generate.rs @@ -0,0 +1,118 @@ +use std::{ffi::OsStr, fs, process::Command}; + +use anyhow::{Context, Result}; + +use crate::{bail_on_err, GenerateFixtures}; + +const HEADER_PATH: &str = "include/tree_sitter/api.h"; + +pub fn run_fixtures(args: &GenerateFixtures) -> Result<()> { + let output = std::process::Command::new("cargo") + .args(["build", "--release"]) + .spawn()? + .wait_with_output()?; + bail_on_err(&output, "Failed to run cargo build")?; + + let tree_sitter_binary = std::env::current_dir()? + .join("target") + .join("release") + .join("tree-sitter"); + + let grammars_dir = std::env::current_dir()? + .join("test") + .join("fixtures") + .join("grammars"); + + for grammar_file in find_grammar_files(grammars_dir.to_str().unwrap()).flatten() { + let grammar_dir = grammar_file.parent().unwrap(); + let grammar_name = grammar_dir.file_name().and_then(OsStr::to_str).unwrap(); + + println!( + "Regenerating {grammar_name} parser{}", + if args.wasm { " to wasm" } else { "" } + ); + + if args.wasm { + let mut cmd = Command::new(&tree_sitter_binary); + let cmd = cmd.args([ + "build", + "--wasm", + "-o", + &format!("target/release/tree-sitter-{grammar_name}.wasm"), + grammar_dir.to_str().unwrap(), + ]); + if args.docker { + cmd.arg("--docker"); + } + bail_on_err( + &cmd.spawn()?.wait_with_output()?, + &format!("Failed to regenerate {grammar_name} parser to wasm"), + )?; + } else { + let output = Command::new(&tree_sitter_binary) + .arg("generate") + .arg("src/grammar.json") + .arg("--abi=latest") + .current_dir(grammar_dir) + .spawn()? + .wait_with_output()?; + bail_on_err( + &output, + &format!("Failed to regenerate {grammar_name} parser"), + )?; + } + } + + Ok(()) +} + +pub fn run_bindings() -> Result<()> { + let no_copy = [ + "TSInput", + "TSLanguage", + "TSLogger", + "TSLookaheadIterator", + "TSParser", + "TSTree", + "TSQuery", + "TSQueryCursor", + "TSQueryCapture", + "TSQueryMatch", + "TSQueryPredicateStep", + ]; + + let bindings = bindgen::Builder::default() + .header(HEADER_PATH) + .layout_tests(false) + .allowlist_type("^TS.*") + .allowlist_function("^ts_.*") + .allowlist_var("^TREE_SITTER.*") + .no_copy(no_copy.join("|")) + .prepend_enum_name(false) + .use_core() + .clang_arg("-D TREE_SITTER_FEATURE_WASM") + .generate() + .expect("Failed to generate bindings"); + + bindings + .write_to_file("lib/binding_rust/bindings.rs") + .with_context(|| "Failed to write bindings") +} + +fn find_grammar_files( + dir: &str, +) -> impl Iterator> { + fs::read_dir(dir) + .expect("Failed to read directory") + .filter_map(Result::ok) + .flat_map(|entry| { + let path = entry.path(); + if path.is_dir() && !path.to_string_lossy().contains("node_modules") { + Box::new(find_grammar_files(path.to_str().unwrap())) as Box> + } else if path.is_file() && path.file_name() == Some(OsStr::new("grammar.js")) { + Box::new(std::iter::once(Ok(path))) as _ + } else { + Box::new(std::iter::empty()) as _ + } + }) +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 80bbbdd5..90ce1d7e 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -1,35 +1,235 @@ +mod benchmark; +mod build_wasm; mod bump; +mod clippy; +mod fetch; +mod generate; +mod test; -use bump::bump_versions; +use anstyle::{AnsiColor, Color, Style}; +use anyhow::Result; +use clap::{crate_authors, Args, Command, FromArgMatches as _, Subcommand}; +use semver::Version; -fn print_help() { - println!( - " -xtask must specify a task to run. - -Usage: `cargo xtask ` - -Tasks: - bump-version -" - ); +#[derive(Subcommand)] +#[command(about="Run various tasks", author=crate_authors!("\n"), styles=get_styles())] +enum Commands { + /// Runs `cargo benchmark` with some optional environment variables set. + Benchmark(Benchmark), + /// Compile the Tree-sitter WASM library. This will create two files in the + /// `lib/binding_web` directory: `tree-sitter.js` and `tree-sitter.wasm`. + BuildWasm(BuildWasm), + /// Compile the Tree-sitter WASM standard library. + BuildWasmStdlib, + /// Bumps the version of the workspace. + BumpVersion(BumpVersion), + /// Runs `cargo clippy`. + Clippy(Clippy), + /// Fetches emscripten. + FetchEmscripten, + /// Fetches the fixtures for testing tree-sitter. + FetchFixtures, + /// Generate the Rust bindings from the C library. + GenerateBindings, + /// Generates the fixtures for testing tree-sitter. + GenerateFixtures(GenerateFixtures), + /// Run the test suite + Test(Test), + /// Run the WASM test suite + TestWasm, } -fn main() -> Result<(), Box> { - let Some(task) = std::env::args().nth(1) else { - print_help(); - std::process::exit(0); - }; +#[derive(Args)] +struct Benchmark { + /// The language to run the benchmarks for. + #[arg(long, short)] + language: Option, + /// The example file to run the benchmarks for. + #[arg(long, short)] + example_file_name: Option, + /// The number of times to parse each sample (default is 5). + #[arg(long, short, default_value = "5")] + repetition_count: u32, + /// Whether to run the benchmarks in debug mode. + #[arg(long, short = 'g')] + debug: bool, +} - match task.as_str() { - "bump-version" => { - bump_versions()?; +#[derive(Args)] +struct BuildWasm { + /// Compile the library more quickly, with fewer optimizations + /// and more runtime assertions. + #[arg(long, short = '0')] + debug: bool, + /// Run emscripten using docker, even if \`emcc\` is installed. + /// By default, \`emcc\` will be run directly when available. + #[arg(long, short)] + docker: bool, + /// Run emscripten with verbose output. + #[arg(long, short)] + verbose: bool, +} + +#[derive(Args)] +struct BumpVersion { + /// The version to bump to. + #[arg(long, short)] + version: Option, +} + +#[derive(Args)] +struct Clippy { + /// Automatically apply lint suggestions (`clippy --fix`). + #[arg(long, short)] + fix: bool, + /// The package to run Clippy against (`cargo -p clippy`). + #[arg(long, short)] + package: Option, +} + +#[derive(Args)] +struct GenerateFixtures { + /// Generates the parser to WASM + #[arg(long, short)] + wasm: bool, + /// Run emscripten via docker even if it is installed locally. + #[arg(long, short, requires = "wasm")] + docker: bool, +} + +#[derive(Args)] +struct Test { + /// Compile C code with the Clang address sanitizer. + #[arg(long, short)] + address_sanitizer: bool, + /// Run only the corpus tests whose name contain the given string. + #[arg(long, short)] + example: Option, + /// Run the given number of iterations of randomized tests (default 10). + #[arg(long, short)] + iterations: Option, + /// Set the seed used to control random behavior. + #[arg(long, short)] + seed: Option, + /// Print parsing log to stderr. + #[arg(long, short)] + debug: bool, + /// Generate an SVG graph of parsing logs. + #[arg(long, short = 'D')] + debug_graph: bool, + /// Run the tests with a debugger. + #[arg(short)] + g: bool, + #[arg(trailing_var_arg = true)] + args: Vec, + /// Don't capture the output + #[arg(long)] + nocapture: bool, +} + +const BUILD_VERSION: &str = env!("CARGO_PKG_VERSION"); +const BUILD_SHA: Option<&str> = option_env!("BUILD_SHA"); +const EMSCRIPTEN_VERSION: &str = include_str!("../../cli/loader/emscripten-version"); +const EMSCRIPTEN_TAG: &str = concat!( + "docker.io/emscripten/emsdk:", + include_str!("../../cli/loader/emscripten-version") +); + +fn main() { + let result = run(); + if let Err(err) = &result { + // Ignore BrokenPipe errors + if let Some(error) = err.downcast_ref::() { + if error.kind() == std::io::ErrorKind::BrokenPipe { + return; + } } - _ => { - println!("invalid task: {task}"); - std::process::exit(1); + if !err.to_string().is_empty() { + eprintln!("{err:?}"); } + std::process::exit(1); + } +} + +fn run() -> Result<()> { + let version = BUILD_SHA.map_or_else( + || BUILD_VERSION.to_string(), + |build_sha| format!("{BUILD_VERSION} ({build_sha})"), + ); + let version: &'static str = Box::leak(version.into_boxed_str()); + + let cli = Command::new("xtask") + .help_template( + "\ +{before-help}{name} {version} +{author-with-newline}{about-with-newline} +{usage-heading} {usage} + +{all-args}{after-help} +", + ) + .version(version) + .subcommand_required(true) + .arg_required_else_help(true) + .disable_help_subcommand(true) + .disable_colored_help(false); + let command = Commands::from_arg_matches(&Commands::augment_subcommands(cli).get_matches())?; + + match command { + Commands::Benchmark(benchmark_options) => benchmark::run(&benchmark_options)?, + Commands::BuildWasm(build_wasm_options) => build_wasm::run_wasm(&build_wasm_options)?, + Commands::BuildWasmStdlib => build_wasm::run_wasm_stdlib()?, + Commands::BumpVersion(bump_options) => bump::run(bump_options)?, + Commands::Clippy(clippy_options) => clippy::run(&clippy_options)?, + Commands::FetchEmscripten => fetch::run_emscripten()?, + Commands::FetchFixtures => fetch::run_fixtures()?, + Commands::GenerateBindings => generate::run_bindings()?, + Commands::GenerateFixtures(generate_fixtures_options) => { + generate::run_fixtures(&generate_fixtures_options)?; + } + Commands::Test(test_options) => test::run(&test_options)?, + Commands::TestWasm => test::run_wasm()?, } Ok(()) } + +fn bail_on_err(output: &std::process::Output, prefix: &str) -> Result<()> { + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + anyhow::bail!("{prefix}:\n{stderr}"); + } + Ok(()) +} + +#[must_use] +const fn get_styles() -> clap::builder::Styles { + clap::builder::Styles::styled() + .usage( + Style::new() + .bold() + .fg_color(Some(Color::Ansi(AnsiColor::Yellow))), + ) + .header( + Style::new() + .bold() + .fg_color(Some(Color::Ansi(AnsiColor::Yellow))), + ) + .literal(Style::new().fg_color(Some(Color::Ansi(AnsiColor::Green)))) + .invalid( + Style::new() + .bold() + .fg_color(Some(Color::Ansi(AnsiColor::Red))), + ) + .error( + Style::new() + .bold() + .fg_color(Some(Color::Ansi(AnsiColor::Red))), + ) + .valid( + Style::new() + .bold() + .fg_color(Some(Color::Ansi(AnsiColor::Green))), + ) + .placeholder(Style::new().fg_color(Some(Color::Ansi(AnsiColor::White)))) +} diff --git a/xtask/src/test.rs b/xtask/src/test.rs new file mode 100644 index 00000000..fd09bd94 --- /dev/null +++ b/xtask/src/test.rs @@ -0,0 +1,122 @@ +use std::{ + env, + path::Path, + process::{Command, Stdio}, +}; + +use anyhow::{anyhow, Result}; +use regex::Regex; + +use crate::{bail_on_err, Test}; + +pub fn run(args: &Test) -> Result<()> { + let test_flags = if args.address_sanitizer { + env::set_var("CFLAGS", "-fsanitize=undefined,address"); + + // When the Tree-sitter C library is compiled with the address sanitizer, the address + // sanitizer runtime library needs to be linked into the final test executable. When + // using Xcode clang, the Rust linker doesn't know where to find that library, so we + // need to specify linker flags directly. + let output = Command::new("cc").arg("-print-runtime-dir").output()?; + bail_on_err(&output, "Failed to get clang runtime dir")?; + let runtime_dir = String::from_utf8(output.stdout)?; + if runtime_dir.contains("/Xcode.app/") { + env::set_var( + "RUSTFLAGS", + format!( + "-C link-arg=-L{runtime_dir} -C link-arg=-lclang_rt.asan_osx_dynamic -C link-arg=-Wl,-rpath,{runtime_dir}" + ), + ); + } + + // Specify a `--target` explicitly. This is required for address sanitizer support. + let output = Command::new("rustup") + .arg("show") + .arg("active-toolchain") + .output()?; + bail_on_err(&output, "Failed to get active Rust toolchain")?; + let toolchain = String::from_utf8(output.stdout)?; + let re = Regex::new(r"(stable|beta|nightly)-([_a-z0-9-]+).*")?; + let captures = re + .captures(&toolchain) + .ok_or_else(|| anyhow!("Failed to parse toolchain '{toolchain}'"))?; + let current_target = captures.get(2).unwrap().as_str(); + format!("--target={current_target}") + } else { + String::new() + }; + if let Some(example) = &args.example { + env::set_var("TREE_SITTER_EXAMPLE", example); + } + if let Some(seed) = args.seed { + env::set_var("TREE_SITTER_SEED", seed.to_string()); + } + if let Some(iterations) = args.iterations { + env::set_var("TREE_SITTER_ITERATIONS", iterations.to_string()); + } + if args.debug { + env::set_var("TREE_SITTER_LOG", "1"); + } + if args.debug_graph { + env::set_var("TREE_SITTER_LOG_GRAPHS", "1"); + } + + if args.g { + let cargo_cmd = Command::new("cargo") + .arg("test") + .arg(test_flags) + .arg("--no-run") + .arg("--message-format=json") + .stdout(Stdio::piped()) + .spawn()?; + + let jq_cmd = Command::new("jq") + .arg("-rs") + .arg(r#"map(select(.target.name == "tree_sitter_cli" and .executable))[0].executable"#) + .stdin(cargo_cmd.stdout.unwrap()) + .output()?; + + let test_binary = String::from_utf8(jq_cmd.stdout)?; + + let mut lldb_cmd = Command::new("lldb"); + lldb_cmd.arg(test_binary.trim()).arg("--").args(&args.args); + bail_on_err( + &lldb_cmd.spawn()?.wait_with_output()?, + &format!("Failed to run {lldb_cmd:?}"), + )?; + } else { + let mut cargo_cmd = Command::new("cargo"); + cargo_cmd.arg("test").arg(test_flags).args(&args.args); + if args.nocapture { + cargo_cmd.arg("--").arg("--nocapture"); + } + bail_on_err( + &cargo_cmd.spawn()?.wait_with_output()?, + &format!("Failed to run {cargo_cmd:?}"), + )?; + } + + Ok(()) +} + +pub fn run_wasm() -> Result<()> { + std::env::set_current_dir("lib/binding_web")?; + + let node_modules_dir = Path::new("node_modules"); + let npm = if cfg!(target_os = "windows") { + "npm.cmd" + } else { + "npm" + }; + + if !node_modules_dir.join("chai").exists() || !node_modules_dir.join("mocha").exists() { + println!("Installing test dependencies..."); + let output = Command::new(npm).arg("install").output()?; + bail_on_err(&output, "Failed to install test dependencies")?; + } + + let output = Command::new(npm).arg("test").output()?; + bail_on_err(&output, &format!("Failed to run {npm} test"))?; + + Ok(()) +} From bb8eb83f50660d2cec42fd6be687368a21467df9 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sun, 6 Oct 2024 13:42:14 -0400 Subject: [PATCH 44/61] build: configure clippy lints at the workspace level (cherry picked from commit 9c08edb066647126bf4bb65176d57a8426ae4b81) --- Cargo.lock | 38 ++++++++++++---------- Cargo.toml | 49 +++++++++++++++++++++++++++++ cli/Cargo.toml | 3 ++ cli/config/Cargo.toml | 3 ++ cli/generate/Cargo.toml | 3 ++ cli/loader/Cargo.toml | 3 ++ cli/src/tests/proc_macro/Cargo.toml | 3 ++ highlight/Cargo.toml | 3 ++ lib/Cargo.toml | 3 ++ lib/language/Cargo.toml | 3 ++ tags/Cargo.toml | 3 ++ 11 files changed, 98 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9c280e66..f0f8f34d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -156,9 +156,9 @@ checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" [[package]] name = "cc" -version = "1.1.24" +version = "1.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812acba72f0a070b003d3697490d2b55b837230ae7c6c6497f05cc2ddbb8d938" +checksum = "2e80e3b6a3ab07840e1cae9b0666a63970dc28e8ed5ffbcdacbfc760c281bfc1" dependencies = [ "jobserver", "libc", @@ -565,6 +565,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "foldhash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -651,6 +657,9 @@ name = "hashbrown" version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +dependencies = [ + "foldhash", +] [[package]] name = "heck" @@ -982,24 +991,21 @@ dependencies = [ [[package]] name = "object" -version = "0.36.4" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "crc32fast", - "hashbrown 0.14.5", + "hashbrown 0.15.0", "indexmap", "memchr", ] [[package]] name = "once_cell" -version = "1.20.1" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1" -dependencies = [ - "portable-atomic", -] +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "openssl-probe" @@ -1055,12 +1061,6 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" -[[package]] -name = "portable-atomic" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" - [[package]] name = "postcard" version = "1.0.10" @@ -2349,8 +2349,14 @@ dependencies = [ name = "xtask" version = "0.1.0" dependencies = [ + "anstyle", + "anyhow", + "bindgen", + "cc", + "clap", "git2", "indoc", + "regex", "semver", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 82f99e99..5e7921c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,55 @@ license = "MIT" keywords = ["incremental", "parsing"] categories = ["command-line-utilities", "parsing"] +[workspace.lints.clippy] +dbg_macro = "deny" +todo = "deny" +pedantic = { level = "warn", priority = -1 } +nursery = { level = "warn", priority = -1 } +cargo = { level = "warn", priority = -1 } + +# The lints below are a specific subset of the pedantic+nursery lints +# that we explicitly allow in the tree-sitter codebase because they either: +# +# 1. Contain false positives, +# 2. Are unnecessary, or +# 3. Worsen the code + +branches_sharing_code = "allow" +cast_lossless = "allow" +cast_possible_truncation = "allow" +cast_possible_wrap = "allow" +cast_precision_loss = "allow" +cast_sign_loss = "allow" +checked_conversions = "allow" +cognitive_complexity = "allow" +collection_is_never_read = "allow" +fallible_impl_from = "allow" +fn_params_excessive_bools = "allow" +inline_always = "allow" +if_not_else = "allow" +items_after_statements = "allow" +match_wildcard_for_single_variants = "allow" +missing_errors_doc = "allow" +missing_panics_doc = "allow" +module_name_repetitions = "allow" +multiple_crate_versions = "allow" +option_if_let_else = "allow" +or_fun_call = "allow" +range_plus_one = "allow" +redundant_clone = "allow" +redundant_closure_for_method_calls = "allow" +ref_option = "allow" +similar_names = "allow" +string_lit_as_bytes = "allow" +struct_excessive_bools = "allow" +struct_field_names = "allow" +transmute_undefined_repr = "allow" +too_many_lines = "allow" +unnecessary_wraps = "allow" +unused_self = "allow" +used_underscore_items = "allow" + [profile.optimize] inherits = "release" strip = true # Automatically strip symbols from the binary. diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 37192824..c96dcb78 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -12,6 +12,9 @@ license.workspace = true keywords.workspace = true categories.workspace = true +[lints] +workspace = true + [[bin]] name = "tree-sitter" path = "src/main.rs" diff --git a/cli/config/Cargo.toml b/cli/config/Cargo.toml index 8379a546..5ad7b88f 100644 --- a/cli/config/Cargo.toml +++ b/cli/config/Cargo.toml @@ -12,6 +12,9 @@ license.workspace = true keywords.workspace = true categories.workspace = true +[lints] +workspace = true + [dependencies] anyhow.workspace = true dirs.workspace = true diff --git a/cli/generate/Cargo.toml b/cli/generate/Cargo.toml index b4600343..8f374ae1 100644 --- a/cli/generate/Cargo.toml +++ b/cli/generate/Cargo.toml @@ -12,6 +12,9 @@ license.workspace = true keywords.workspace = true categories.workspace = true +[lints] +workspace = true + [dependencies] anyhow.workspace = true heck.workspace = true diff --git a/cli/loader/Cargo.toml b/cli/loader/Cargo.toml index 65025127..8c5b160b 100644 --- a/cli/loader/Cargo.toml +++ b/cli/loader/Cargo.toml @@ -12,6 +12,9 @@ license.workspace = true keywords.workspace = true categories.workspace = true +[lints] +workspace = true + [features] wasm = ["tree-sitter/wasm"] # TODO: For backward compatibility these must be enabled by default, diff --git a/cli/src/tests/proc_macro/Cargo.toml b/cli/src/tests/proc_macro/Cargo.toml index d25b0cf4..ef991956 100644 --- a/cli/src/tests/proc_macro/Cargo.toml +++ b/cli/src/tests/proc_macro/Cargo.toml @@ -5,6 +5,9 @@ edition.workspace = true rust-version.workspace = true publish = false +[lints] +workspace = true + [lib] proc-macro = true diff --git a/highlight/Cargo.toml b/highlight/Cargo.toml index 1b57fd52..c6f0063c 100644 --- a/highlight/Cargo.toml +++ b/highlight/Cargo.toml @@ -15,6 +15,9 @@ license.workspace = true keywords = ["incremental", "parsing", "syntax", "highlighting"] categories = ["parsing", "text-editors"] +[lints] +workspace = true + [lib] crate-type = ["lib", "staticlib"] diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 3460a537..538b6098 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -25,6 +25,9 @@ include = [ "/include/tree_sitter/api.h", ] +[lints] +workspace = true + [features] default = ["std"] std = ["regex/std", "regex/perf", "regex-syntax/unicode"] diff --git a/lib/language/Cargo.toml b/lib/language/Cargo.toml index 59e5f738..e3d8a6ff 100644 --- a/lib/language/Cargo.toml +++ b/lib/language/Cargo.toml @@ -10,5 +10,8 @@ repository.workspace = true license.workspace = true keywords.workspace = true +[lints] +workspace = true + [lib] path = "language.rs" diff --git a/tags/Cargo.toml b/tags/Cargo.toml index b7d0846c..9db54306 100644 --- a/tags/Cargo.toml +++ b/tags/Cargo.toml @@ -15,6 +15,9 @@ license.workspace = true keywords = ["incremental", "parsing", "syntax", "tagging"] categories = ["parsing", "text-editors"] +[lints] +workspace = true + [lib] crate-type = ["lib", "staticlib"] From 9be7dda48f9794dd9f4624357224cfc99785b1fd Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 7 Oct 2024 23:23:54 -0400 Subject: [PATCH 45/61] chore: remove unnecessary fuzz and profile helpers (cherry picked from commit cad2d03101341296906971464ee68f811debea0d) --- test/fuzz/README.md | 43 ------------------------ test/fuzz/fuzzer.cc | 79 -------------------------------------------- test/profile/heap.cc | 42 ----------------------- 3 files changed, 164 deletions(-) delete mode 100644 test/fuzz/README.md delete mode 100644 test/fuzz/fuzzer.cc delete mode 100644 test/profile/heap.cc diff --git a/test/fuzz/README.md b/test/fuzz/README.md deleted file mode 100644 index 5adc1b04..00000000 --- a/test/fuzz/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# Fuzzing tree-sitter - -The tree-sitter fuzzing support requires 1) the `libFuzzer` runtime library and 2) a recent version of clang - -## libFuzzer - -The main fuzzing logic is implemented by `libFuzzer` which is part of the compiler-rt project but is not shipped by distros. `libFuzzer` will need to be built from source, e.g.: - -``` -cd ~/src -git clone https://github.com/llvm-mirror/compiler-rt -cd compiler-rt/lib/fuzzer -./build.sh -``` - -## clang - -Using libFuzzer requires at least version 7 of `clang` and may _not_ work with your system-installed version. If your system-installed version is too old, the easiest way to get started is to use the version provided by the Chromium team. Instructions are available at [libFuzzer.info](http://libfuzzer.info). - -The fuzzers can then be built with: -``` -export CLANG_DIR=$HOME/src/third_party/llvm-build/Release+Asserts/bin -CC="$CLANG_DIR/clang" CXX="$CLANG_DIR/clang++" LINK="$CLANG_DIR/clang++" \ - LIB_FUZZER_PATH=$HOME/src/compiler-rt/lib/fuzzer/libFuzzer.a \ - ./script/build-fuzzers -``` - -This will generate a separate fuzzer for each grammar defined in `test/fixtures/grammars` and will be instrumented with [AddressSanitizer](https://clang.llvm.org/docs/AddressSanitizer.html) and [UndefinedBehaviorSanitizer](https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html). Individual fuzzers can be built with, for example, `./script/build-fuzzers python ruby`. - -The `run-fuzzer` script handles running an individual fuzzer with a sensible default set of arguments: -``` -./script/run-fuzzer (halt|recover) -``` - -which will log information to stdout. Failing testcases and a fuzz corpus will be saved to `fuzz-results/`. The most important extra `libFuzzer` options are `-jobs` and `-workers` which allow parallel fuzzing. This is can done with, e.g.: -``` -./script/run-fuzzer halt -jobs=32 -workers=32 -``` - -The testcase can be used to reproduce the crash by running: -``` -./script/reproduce (halt|recover) -``` diff --git a/test/fuzz/fuzzer.cc b/test/fuzz/fuzzer.cc deleted file mode 100644 index 3b933746..00000000 --- a/test/fuzz/fuzzer.cc +++ /dev/null @@ -1,79 +0,0 @@ -#include -#include -#include "tree_sitter/api.h" - -extern "C" const TSLanguage *TS_LANG(); - -static TSQuery *lang_query; - -extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv) { - if(TS_LANG_QUERY_FILENAME[0]) { - // The query filename is relative to the fuzzing binary. Convert it - // to an absolute path first - auto binary_filename = std::string((*argv)[0]); - auto binary_directory = binary_filename.substr(0, binary_filename.find_last_of("\\/")); - auto lang_query_filename = binary_directory + "/" + TS_LANG_QUERY_FILENAME; - - auto f = std::ifstream(lang_query_filename); - assert(f.good()); - std::string lang_query_source((std::istreambuf_iterator(f)), std::istreambuf_iterator()); - - uint32_t error_offset = 0; - TSQueryError error_type = TSQueryErrorNone; - - lang_query = ts_query_new( - TS_LANG(), - lang_query_source.c_str(), - lang_query_source.size(), - &error_offset, - &error_type - ); - - assert(lang_query); - } - - return 0; -} - -extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { - const char *str = reinterpret_cast(data); - - TSParser *parser = ts_parser_new(); - - // This can fail if the language version doesn't match the runtime version - bool language_ok = ts_parser_set_language(parser, TS_LANG()); - assert(language_ok); - - TSTree *tree = ts_parser_parse_string(parser, NULL, str, size); - TSNode root_node = ts_tree_root_node(tree); - - if (lang_query != nullptr) { - { - TSQueryCursor *cursor = ts_query_cursor_new(); - - ts_query_cursor_exec(cursor, lang_query, root_node); - TSQueryMatch match; - while (ts_query_cursor_next_match(cursor, &match)) { - } - - ts_query_cursor_delete(cursor); - } - - { - TSQueryCursor *cursor = ts_query_cursor_new(); - - ts_query_cursor_exec(cursor, lang_query, root_node); - TSQueryMatch match; - uint32_t capture_index; - while (ts_query_cursor_next_capture(cursor, &match, &capture_index)) { - } - - ts_query_cursor_delete(cursor); - } - } - - ts_tree_delete(tree); - ts_parser_delete(parser); - - return 0; -} diff --git a/test/profile/heap.cc b/test/profile/heap.cc deleted file mode 100644 index 6c0027e8..00000000 --- a/test/profile/heap.cc +++ /dev/null @@ -1,42 +0,0 @@ -#include -#include -#include -#include - -extern "C" { -#include "javascript/src/parser.c" -#include "javascript/src/scanner.c" -} - -#define LANGUAGE tree_sitter_javascript -#define SOURCE_PATH "javascript/examples/jquery.js" - -int main() { - TSParser *parser = ts_parser_new(); - if (!ts_parser_set_language(parser, LANGUAGE())) { - fprintf(stderr, "Invalid language\n"); - exit(1); - } - - const char *source_path = GRAMMARS_DIR SOURCE_PATH; - - printf("Parsing %s\n", source_path); - - std::ifstream source_file(source_path); - if (!source_file.good()) { - fprintf(stderr, "Invalid source path %s\n", source_path); - exit(1); - } - - std::string source_code( - (std::istreambuf_iterator(source_file)), - std::istreambuf_iterator() - ); - - TSTree *tree = ts_parser_parse_string( - parser, - NULL, - source_code.c_str(), - source_code.size() - ); -} From 8d466ead300782fbdf704b86b7a2d9a7d2e36591 Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Fri, 11 Oct 2024 21:08:51 +0300 Subject: [PATCH 46/61] build: tune compiler warnings (cherry picked from commit 7715001692ba2955d762028e919c6a9ba59765f8) --- lib/CMakeLists.txt | 6 ++++-- lib/src/parser.c | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index d6e758b0..a505ef48 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -21,8 +21,10 @@ add_library(tree-sitter ${TS_SOURCE_FILES}) target_include_directories(tree-sitter PRIVATE src src/wasm include) -if(NOT MSVC) - target_compile_options(tree-sitter PRIVATE -Wall -Wextra -Wshadow -Wno-unused-parameter -pedantic) +if(MSVC) + target_compile_options(tree-sitter PRIVATE /W4 /wd4018 /wd4701 /wd4702 /wd4100 /wd4232 /wd4244) +else() + target_compile_options(tree-sitter PRIVATE -Wall -Wextra -Wshadow -pedantic) endif() if(TREE_SITTER_FEATURE_WASM) diff --git a/lib/src/parser.c b/lib/src/parser.c index d38ace38..5fccc2ec 100644 --- a/lib/src/parser.c +++ b/lib/src/parser.c @@ -350,7 +350,7 @@ static bool ts_parser__call_main_lex_fn(TSParser *self, TSLexMode lex_mode) { } } -static bool ts_parser__call_keyword_lex_fn(TSParser *self, TSLexMode lex_mode) { +static bool ts_parser__call_keyword_lex_fn(TSParser *self) { if (ts_language_is_wasm(self->language)) { return ts_wasm_store_call_lex_keyword(self->wasm_store, 0); } else { @@ -651,7 +651,7 @@ static Subtree ts_parser__lex( ts_lexer_reset(&self->lexer, self->lexer.token_start_position); ts_lexer_start(&self->lexer); - is_keyword = ts_parser__call_keyword_lex_fn(self, lex_mode); + is_keyword = ts_parser__call_keyword_lex_fn(self); if ( is_keyword && From 4e870b92078688016c012eb957b29b36509526cd Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Fri, 11 Oct 2024 20:10:24 +0300 Subject: [PATCH 47/61] ci: major overhaul - Simplify some workflow steps and auxiliary scripts - Build library using cmake when not cross-compiling - Try to fetch fixtures from cache first - Use `actions-rust-lang/setup-rust-toolchain` (cherry picked from commit e8e56255bde4d07c63562a22318dd82a77bfb0b4) --- .github/actions/cache/action.yml | 23 ++- .github/scripts/cross.sh | 18 +-- .github/scripts/make.sh | 20 +-- .github/scripts/tree-sitter.sh | 27 +--- .github/workflows/backport.yml | 19 ++- .github/workflows/build.yml | 213 ++++++++++++++----------- .github/workflows/ci.yml | 26 ++- .github/workflows/release.yml | 66 ++++---- .github/workflows/response.yml | 37 +++-- .github/workflows/reviewers_remove.yml | 18 ++- .github/workflows/sanitize.yml | 55 ++++--- 11 files changed, 268 insertions(+), 254 deletions(-) diff --git a/.github/actions/cache/action.yml b/.github/actions/cache/action.yml index cc816682..a32cc294 100644 --- a/.github/actions/cache/action.yml +++ b/.github/actions/cache/action.yml @@ -1,24 +1,23 @@ -name: 'Cache' -description: "This action caches fixtures" +name: Cache + +description: This action caches fixtures + outputs: cache-hit: - description: 'Cache hit' - value: ${{ steps.cache_output.outputs.cache-hit }} + description: Cache hit + value: ${{ steps.cache.outputs.cache-hit }} + runs: - using: "composite" + using: composite steps: - uses: actions/cache@v4 - id: cache_fixtures + id: cache with: path: | test/fixtures/grammars target/release/tree-sitter-*.wasm key: fixtures-${{ join(matrix.*, '_') }}-${{ hashFiles( - 'cli/src/generate/**', - 'script/generate-fixtures*', + 'cli/generate/src/**', + 'xtask/src/*', 'test/fixtures/grammars/*/**/src/*.c', '.github/actions/cache/action.yml') }} - - - run: echo "cache-hit=${{ steps.cache_fixtures.outputs.cache-hit }}" >> $GITHUB_OUTPUT - shell: bash - id: cache_output diff --git a/.github/scripts/cross.sh b/.github/scripts/cross.sh index a52f0873..de1d4e94 100755 --- a/.github/scripts/cross.sh +++ b/.github/scripts/cross.sh @@ -1,17 +1,3 @@ -#!/bin/bash +#!/bin/bash -eu -# set -x -set -e - -if [ "$BUILD_CMD" != "cross" ]; then - echo "cross.sh - is a helper to assist only in cross compiling environments" >&2 - echo "To use this tool set the BUILD_CMD env var to the \"cross\" value" >&2 - exit 111 -fi - -if [ -z "$CROSS_IMAGE" ]; then - echo "The CROSS_IMAGE env var should be provided" >&2 - exit 111 -fi - -docker run --rm -v /home/runner:/home/runner -w "$PWD" "$CROSS_IMAGE" "$@" +exec docker run --rm -v /home/runner:/home/runner -w "$PWD" "$CROSS_IMAGE" "$@" diff --git a/.github/scripts/make.sh b/.github/scripts/make.sh index 79192541..175074f9 100755 --- a/.github/scripts/make.sh +++ b/.github/scripts/make.sh @@ -1,19 +1,9 @@ -#!/bin/bash +#!/bin/bash -eu -# set -x -set -e +tree_sitter="$ROOT"/target/"$TARGET"/release/tree-sitter -if [ "$BUILD_CMD" == "cross" ]; then - if [ -z "$CC" ]; then - echo "make.sh: CC is not set" >&2 - exit 111 - fi - if [ -z "$AR" ]; then - echo "make.sh: AR is not set" >&2 - exit 111 - fi - - cross.sh make CC=$CC AR=$AR "$@" +if [[ $BUILD_CMD == cross ]]; then + cross.sh make CC="$CC" AR="$AR" "$@" else - make "$@" + exec make "$@" fi diff --git a/.github/scripts/tree-sitter.sh b/.github/scripts/tree-sitter.sh index 0cac9153..125a2d92 100755 --- a/.github/scripts/tree-sitter.sh +++ b/.github/scripts/tree-sitter.sh @@ -1,28 +1,9 @@ -#!/bin/bash - -# set -x -set -e - -if [ -z "$ROOT" ]; then - echo "The ROOT env var should be set to absolute path of a repo root folder" >&2 - exit 111 -fi - -if [ -z "$TARGET" ]; then - echo "The TARGET env var should be equal to a \`cargo build --target \` command value" >&2 - exit 111 -fi +#!/bin/bash -eu tree_sitter="$ROOT"/target/"$TARGET"/release/tree-sitter -if [ "$BUILD_CMD" == "cross" ]; then - if [ -z "$CROSS_RUNNER" ]; then - echo "The CROSS_RUNNER env var should be set to a CARGO_TARGET_*_RUNNER env var value" >&2 - echo "that is available in a docker image used by the cross tool under the hood" >&2 - exit 111 - fi - - cross.sh $CROSS_RUNNER "$tree_sitter" "$@" +if [[ $BUILD_CMD == cross ]]; then + cross.sh "$CROSS_RUNNER" "$tree_sitter" "$@" else - "$tree_sitter" "$@" + exec "$tree_sitter" "$@" fi diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 0c3ba6be..a0c15e01 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -1,26 +1,29 @@ -name: backport +name: Backport Pull Request + on: pull_request_target: types: [closed, labeled] + +permissions: + contents: write + pull-requests: write + jobs: backport: - permissions: - contents: write - pull-requests: write - name: Backport Pull Request if: github.event.pull_request.merged runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v4 - - uses: actions/create-github-app-token@v1 + - name: Create app token + uses: actions/create-github-app-token@v1 id: app-token with: app-id: ${{ vars.BACKPORT_APP }} private-key: ${{ secrets.BACKPORT_KEY }} - name: Create backport PR - id: backport uses: korthout/backport-action@v3 with: pull_title: "${pull_title}" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 65979bed..6d227eec 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,7 +8,7 @@ env: on: workflow_call: inputs: - run_test: + run-test: default: true type: boolean @@ -21,99 +21,93 @@ jobs: fail-fast: false matrix: platform: - - linux-arm64 # - - linux-arm # - - linux-x64 # - - linux-x86 # - - linux-powerpc64 # - - windows-arm64 # - - windows-x64 # <-- No C library build - requires an additional adapted Makefile for `cl.exe` compiler - - windows-x86 # -- // -- - - macos-arm64 # - - macos-x64 # + - linux-arm64 + - linux-arm + - linux-x64 + - linux-x86 + - linux-powerpc64 + - windows-arm64 + - windows-x64 + - windows-x86 + - macos-arm64 + - macos-x64 include: - # When adding a new `target`: - # 1. Define a new platform alias above - # 2. Add a new record to a matrix map in `cli/npm/install.js` - - { platform: linux-arm64 , target: aarch64-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } - - { platform: linux-arm , target: arm-unknown-linux-gnueabi , os: ubuntu-latest , use-cross: true } - - { platform: linux-x64 , target: x86_64-unknown-linux-gnu , os: ubuntu-20.04 , cli_features: wasm } #2272 - - { platform: linux-x86 , target: i686-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } - - { platform: linux-powerpc64 , target: powerpc64-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } - - { platform: windows-arm64 , target: aarch64-pc-windows-msvc , os: windows-latest } - - { platform: windows-x64 , target: x86_64-pc-windows-msvc , os: windows-latest , cli_features: wasm } - - { platform: windows-x86 , target: i686-pc-windows-msvc , os: windows-latest } - - { platform: macos-arm64 , target: aarch64-apple-darwin , os: macos-14 , cli_features: wasm } - - { platform: macos-x64 , target: x86_64-apple-darwin , os: macos-12 , cli_features: wasm } + # When adding a new `target`: + # 1. Define a new platform alias above + # 2. Add a new record to the matrix map in `cli/npm/install.js` + - { platform: linux-arm64 , target: aarch64-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } + - { platform: linux-arm , target: arm-unknown-linux-gnueabi , os: ubuntu-latest , use-cross: true } + - { platform: linux-x64 , target: x86_64-unknown-linux-gnu , os: ubuntu-20.04 , features: wasm } # See #2272 + - { platform: linux-x86 , target: i686-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } + - { platform: linux-powerpc64 , target: powerpc64-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } + - { platform: windows-arm64 , target: aarch64-pc-windows-msvc , os: windows-latest } + - { platform: windows-x64 , target: x86_64-pc-windows-msvc , os: windows-latest , features: wasm } + - { platform: windows-x86 , target: i686-pc-windows-msvc , os: windows-latest } + - { platform: macos-arm64 , target: aarch64-apple-darwin , os: macos-14 , features: wasm } + - { platform: macos-x64 , target: x86_64-apple-darwin , os: macos-12 , features: wasm } - # Cross compilers for C library - - { platform: linux-arm64 , cc: aarch64-linux-gnu-gcc , ar: aarch64-linux-gnu-ar } - - { platform: linux-arm , cc: arm-linux-gnueabi-gcc , ar: arm-linux-gnueabi-ar } - - { platform: linux-x86 , cc: i686-linux-gnu-gcc , ar: i686-linux-gnu-ar } - - { platform: linux-powerpc64 , cc: powerpc64-linux-gnu-gcc , ar: powerpc64-linux-gnu-ar } + # Cross compilers for C library + - { platform: linux-arm64 , cc: aarch64-linux-gnu-gcc , ar: aarch64-linux-gnu-ar } + - { platform: linux-arm , cc: arm-linux-gnueabi-gcc , ar: arm-linux-gnueabi-ar } + - { platform: linux-x86 , cc: i686-linux-gnu-gcc , ar: i686-linux-gnu-ar } + - { platform: linux-powerpc64 , cc: powerpc64-linux-gnu-gcc , ar: powerpc64-linux-gnu-ar } - # See #2041 tree-sitter issue - - { platform: windows-x64 , rust-test-threads: 1 } - - { platform: windows-x86 , rust-test-threads: 1 } + # Prevent race condition (see #2041) + - { platform: windows-x64 , rust-test-threads: 1 } + - { platform: windows-x86 , rust-test-threads: 1 } - # CLI only build - - { platform: windows-arm64 , cli-only: true } + # Can't natively run CLI on Github runner's host + - { platform: windows-arm64 , no-run: true } env: BUILD_CMD: cargo - EXE: ${{ contains(matrix.target, 'windows') && '.exe' || '' }} + SUFFIX: ${{ contains(matrix.target, 'windows') && '.exe' || '' }} defaults: run: shell: bash steps: - - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v4 - name: Read Emscripten version - run: echo "EMSCRIPTEN_VERSION=$(cat cli/loader/emscripten-version)" >> $GITHUB_ENV + run: printf 'EMSCRIPTEN_VERSION=%s\n' "$(> $GITHUB_ENV - name: Install Emscripten - if: ${{ !matrix.cli-only && !matrix.use-cross }} + if: ${{ !matrix.no-run && !matrix.use-cross }} uses: mymindstorm/setup-emsdk@v14 with: version: ${{ env.EMSCRIPTEN_VERSION }} - - run: rustup toolchain install stable --profile minimal - - run: rustup target add ${{ matrix.target }} - - uses: Swatinem/rust-cache@v2 + - name: Set up Rust + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + target: ${{ matrix.target }} - name: Install cross if: ${{ matrix.use-cross }} run: cargo install cross --git https://github.com/cross-rs/cross - - name: Build custom cross image + - name: Configure cross if: ${{ matrix.use-cross }} run: | - target="${{ matrix.target }}" - image=ghcr.io/cross-rs/$target:custom + printf '%s\n' > Cross.toml \ + '[target.${{ matrix.target }}]' \ + 'image = "ghcr.io/cross-rs/${{ matrix.target }}:edge"' \ + '[build]' \ + 'pre-build = [' \ + ' "dpkg --add-architecture $CROSS_DEB_ARCH",' \ + ' "curl -fsSL https://deb.nodesource.com/setup_22.x | bash -",' \ + ' "apt-get update && apt-get -y install libssl-dev nodejs"' \ + ']' + cat - Cross.toml <<< 'Cross.toml:' + printf '%s\n' >> $GITHUB_ENV \ + "CROSS_CONFIG=$PWD/Cross.toml" \ + "CROSS_IMAGE=ghcr.io/cross-rs/${{ matrix.target }}:edge" - echo "[target.$target]" >> Cross.toml - echo "image = \"$image\"" >> Cross.toml - echo "[build]" >> Cross.toml - echo "pre-build = [" >> Cross.toml - echo " \"dpkg --add-architecture \$CROSS_DEB_ARCH\"," >> Cross.toml - echo " \"apt-get update && apt-get -y install libssl-dev\"" >> Cross.toml - echo "]" >> Cross.toml - - echo "Cross.toml:" - cat Cross.toml - - echo "CROSS_IMAGE=$image" >> $GITHUB_ENV - echo "CROSS_CONFIG=$PWD/Cross.toml" >> $GITHUB_ENV - - echo "FROM ghcr.io/cross-rs/$target:edge" >> Dockerfile - echo "RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash -" >> Dockerfile - echo "RUN apt-get update && apt-get -y install nodejs" >> Dockerfile - docker build -t $image . - - - name: Setup env extras + - name: Set up environment env: RUST_TEST_THREADS: ${{ matrix.rust-test-threads }} USE_CROSS: ${{ matrix.use-cross }} @@ -122,68 +116,93 @@ jobs: AR: ${{ matrix.ar }} run: | PATH="$PWD/.github/scripts:$PATH" - echo "$PWD/.github/scripts" >> $GITHUB_PATH + printf '%s/.github/scripts\n' "$PWD" >> $GITHUB_PATH - echo "TREE_SITTER=tree-sitter.sh" >> $GITHUB_ENV - echo "TARGET=$TARGET" >> $GITHUB_ENV - echo "ROOT=$PWD" >> $GITHUB_ENV + printf '%s\n' >> $GITHUB_ENV \ + 'TREE_SITTER=tree-sitter.sh' \ + "TARGET=$TARGET" \ + "ROOT=$PWD" - [ -n "$RUST_TEST_THREADS" ] && \ - echo "RUST_TEST_THREADS=$RUST_TEST_THREADS" >> $GITHUB_ENV + [[ -n $RUST_TEST_THREADS ]] && \ + printf 'RUST_TEST_THREADS=%s\n' "$RUST_TEST_THREADS" >> $GITHUB_ENV - [ -n "$CC" ] && echo "CC=$CC" >> $GITHUB_ENV - [ -n "$AR" ] && echo "AR=$AR" >> $GITHUB_ENV + [[ -n $CC ]] && printf 'CC=%s\n' "$CC" >> $GITHUB_ENV + [[ -n $AR ]] && printf 'AR=%s\n' "$AR" >> $GITHUB_ENV - if [ "$USE_CROSS" == "true" ]; then - echo "BUILD_CMD=cross" >> $GITHUB_ENV - runner=$(BUILD_CMD=cross cross.sh bash -c "env | sed -nr '/^CARGO_TARGET_.*_RUNNER=/s///p'") - [ -n "$runner" ] && echo "CROSS_RUNNER=$runner" >> $GITHUB_ENV + if [[ $USE_CROSS == true ]]; then + printf 'BUILD_CMD=cross\n' >> $GITHUB_ENV + runner=$(cross.sh bash -c "env | sed -n 's/^CARGO_TARGET_.*_RUNNER=//p'") + [[ -n $runner ]] && printf 'CROSS_RUNNER=%s\n' "$runner" >> $GITHUB_ENV fi - - name: Build C library - if: ${{ !contains(matrix.os, 'windows') }} # Requires an additional adapted Makefile for `cl.exe` compiler - run: make.sh -j CFLAGS="-Werror" + - name: Build C library (make) + if: ${{ runner.os != 'Windows' }} + run: make.sh -j CFLAGS="$CFLAGS" + env: + CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wno-unused-parameter -pedantic + + - name: Build C library (CMake) + if: ${{ !matrix.use-cross }} + run: |- + cmake -S lib -B build \ + -DBUILD_SHARED_LIBS=OFF \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_COMPILE_WARNING_AS_ERROR=ON + cmake --build build --verbose + + cmake -S lib -B build \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_COMPILE_WARNING_AS_ERROR=ON + cmake --build build --verbose - name: Build wasm library - if: ${{ !matrix.cli-only && !matrix.use-cross }} # No sense to build on the same Github runner hosts many times - run: $BUILD_CMD run --package xtask -- build-wasm + # No reason to build on the same Github runner hosts many times + if: ${{ !matrix.no-run && !matrix.use-cross }} + run: $BUILD_CMD run -p xtask -- build-wasm - - run: $BUILD_CMD build --release --target=${{ matrix.target }} --features=${{ matrix.cli_features }} + - name: Build target + run: $BUILD_CMD build --release --target=${{ matrix.target }} --features=${{ matrix.features }} - - run: $BUILD_CMD run --package xtask -- fetch-fixtures - - - uses: ./.github/actions/cache + - name: Cache fixtures id: cache + if: ${{ !matrix.no-run && inputs.run-test }} + uses: ./.github/actions/cache + + - name: Fetch fixtures + if: ${{ !matrix.no-run && inputs.run-test }} + run: $BUILD_CMD run -p xtask -- fetch-fixtures - name: Generate fixtures - if: ${{ !matrix.cli-only && inputs.run_test && steps.cache.outputs.cache-hit != 'true' }} # Can't natively run CLI on Github runner's host - run: $BUILD_CMD run --package xtask -- generate-fixtures + if: ${{ !matrix.no-run && inputs.run-test && steps.cache.outputs.cache-hit != 'true' }} + run: $BUILD_CMD run -p xtask -- generate-fixtures - - name: Generate WASM fixtures - if: ${{ !matrix.cli-only && !matrix.use-cross && inputs.run_test && steps.cache.outputs.cache-hit != 'true' }} # See comment for the "Build wasm library" step - run: $BUILD_CMD run --package xtask -- generate-fixtures --wasm + - name: Generate Wasm fixtures + if: ${{ !matrix.no-run && !matrix.use-cross && inputs.run-test && steps.cache.outputs.cache-hit != 'true' }} + run: $BUILD_CMD run -p xtask -- generate-fixtures --wasm - name: Run main tests - if: ${{ !matrix.cli-only && inputs.run_test }} # Can't natively run CLI on Github runner's host - run: $BUILD_CMD test --target=${{ matrix.target }} --features=${{ matrix.cli_features }} + if: ${{ !matrix.no-run && inputs.run-test }} + run: $BUILD_CMD test --target=${{ matrix.target }} --features=${{ matrix.features }} - name: Run wasm tests - if: ${{ !matrix.cli-only && !matrix.use-cross && inputs.run_test }} # See comment for the "Build wasm library" step - run: $BUILD_CMD run --package xtask -- test-wasm + if: ${{ !matrix.no-run && !matrix.use-cross && inputs.run-test }} + run: $BUILD_CMD run -p xtask -- test-wasm - name: Run benchmarks - if: ${{ !matrix.cli-only && !matrix.use-cross && inputs.run_test }} # Cross-compiled benchmarks make no sense + # Cross-compiled benchmarks are pointless + if: ${{ !matrix.no-run && !matrix.use-cross && inputs.run-test }} run: $BUILD_CMD bench benchmark -p tree-sitter-cli --target=${{ matrix.target }} - name: Upload CLI artifact uses: actions/upload-artifact@v4 with: name: tree-sitter.${{ matrix.platform }} - path: target/${{ matrix.target }}/release/tree-sitter${{ env.EXE }} + path: target/${{ matrix.target }}/release/tree-sitter${{ env.SUFFIX }} if-no-files-found: error retention-days: 7 - - name: Upload WASM artifacts + - name: Upload Wasm artifacts if: ${{ matrix.platform == 'linux-x64' }} uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de51faf5..dac4cdbf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,9 +1,9 @@ name: CI + on: pull_request: push: - branches: - - 'master' + branches: [master] concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -13,12 +13,22 @@ jobs: checks: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - run: rustup toolchain install stable --profile minimal - - run: rustup toolchain install nightly --profile minimal - - run: rustup component add --toolchain nightly rustfmt - - uses: Swatinem/rust-cache@v2 - - run: make lint + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up stable Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: stable + + - name: Set up nightly Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: nightly + components: clippy, rustfmt + + - name: Lint files + run: make lint sanitize: uses: ./.github/workflows/sanitize.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3bb201eb..7c0152ff 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,5 @@ name: Release + on: workflow_dispatch: push: @@ -9,16 +10,17 @@ jobs: build: uses: ./.github/workflows/build.yml with: - run_test: false + run-test: false release: - name: Release + name: Release on GitHub runs-on: ubuntu-latest needs: build permissions: contents: write steps: - - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v4 - name: Download build artifacts uses: actions/download-artifact@v4 @@ -42,29 +44,24 @@ jobs: ls -l target/ - name: Create release - uses: softprops/action-gh-release@v2 - with: - name: ${{ github.ref_name }} - tag_name: ${{ github.ref_name }} - fail_on_unmatched_files: true - files: | - target/tree-sitter-*.gz - target/tree-sitter.wasm + run: |- + gh release create \ + target/tree-sitter-*.gz \ + target/tree-sitter.wasm \ target/tree-sitter.js + env: + GH_TOKEN: ${{ github.token }} crates_io: - name: Publish CLI to Crates.io + name: Publish packages to Crates.io runs-on: ubuntu-latest needs: release steps: - - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v4 - - name: Setup Rust - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true + - name: Set up Rust + uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Publish crates to Crates.io uses: katyo/publish-crates@v2 @@ -72,29 +69,32 @@ jobs: registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }} npm: - name: Publish lib to npmjs.com + name: Publish packages to npmjs.com runs-on: ubuntu-latest needs: release strategy: fail-fast: false matrix: - directory: ["cli/npm", "lib/binding_web"] + directory: [cli/npm, lib/binding_web] steps: - - uses: actions/checkout@v4 + - name: CHeckout repository + uses: actions/checkout@v4 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: 20 + registry-url: https://registry.npmjs.org + + - name: Set up Rust + uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Build wasm if: matrix.directory == 'lib/binding_web' run: cargo xtask build-wasm - - name: Setup Node - uses: actions/setup-node@v4 - with: - node-version: 20 - registry-url: "https://registry.npmjs.org" - - - name: Publish lib to npmjs.com + - name: Publish to npmjs.com + working-directory: ${{ matrix.directory }} + run: npm publish env: - NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} - run: | - cd ${{ matrix.directory }} - npm publish + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.github/workflows/response.yml b/.github/workflows/response.yml index 663ae6ad..576b9474 100644 --- a/.github/workflows/response.yml +++ b/.github/workflows/response.yml @@ -1,34 +1,47 @@ -name: no_response +name: No response + on: schedule: - - cron: '30 1 * * *' # Run every day at 01:30 + - cron: "30 1 * * *" # Run every day at 01:30 workflow_dispatch: issue_comment: +permissions: + issues: write + pull-requests: write + jobs: close: + name: Close issues with no response if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest - permissions: - issues: write - pull-requests: write steps: - - uses: actions/checkout@v4 - - uses: actions/github-script@v7 + - name: Checkout script + uses: actions/checkout@v4 + with: + sparse-checkout: .github/scripts/close_unresponsive.js + sparse-checkout-cone-mode: false + + - name: Run script + uses: actions/github-script@v7 with: script: | const script = require('./.github/scripts/close_unresponsive.js') await script({github, context}) remove_label: + name: Remove response label if: github.event_name == 'issue_comment' runs-on: ubuntu-latest - permissions: - issues: write - pull-requests: write steps: - - uses: actions/checkout@v4 - - uses: actions/github-script@v7 + - name: Checkout script + uses: actions/checkout@v4 + with: + sparse-checkout: .github/scripts/remove_response_label.js + sparse-checkout-cone-mode: false + + - name: Run script + uses: actions/github-script@v7 with: script: | const script = require('./.github/scripts/remove_response_label.js') diff --git a/.github/workflows/reviewers_remove.yml b/.github/workflows/reviewers_remove.yml index b10d8c3d..b99f0caa 100644 --- a/.github/workflows/reviewers_remove.yml +++ b/.github/workflows/reviewers_remove.yml @@ -1,15 +1,23 @@ -name: "reviewers: remove" +name: Remove Reviewers + on: pull_request_target: types: [converted_to_draft, closed] + +permissions: + pull-requests: write + jobs: remove-reviewers: runs-on: ubuntu-latest - permissions: - pull-requests: write steps: - - uses: actions/checkout@v4 - - name: 'Remove reviewers' + - name: Checkout script + uses: actions/checkout@v4 + with: + sparse-checkout: .github/scripts/reviewers_remove.js + sparse-checkout-cone-mode: false + + - name: Run script uses: actions/github-script@v7 with: script: | diff --git a/.github/workflows/sanitize.yml b/.github/workflows/sanitize.yml index cfbfbfe2..875b8278 100644 --- a/.github/workflows/sanitize.yml +++ b/.github/workflows/sanitize.yml @@ -8,39 +8,44 @@ on: workflow_call: jobs: - check_undefined_behaviour: - name: Sanitizer checks + check-undefined-behaviour: runs-on: ubuntu-latest timeout-minutes: 20 env: TREE_SITTER: ${{ github.workspace }}/target/release/tree-sitter steps: - - name: Checkout source code - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v4 - - name: Install UBSAN library - run: sudo apt-get update -y && sudo apt-get install -y libubsan1 + - name: Install UBSAN library + run: sudo apt-get update -y && sudo apt-get install -y libubsan1 - - run: rustup toolchain install stable --profile minimal - - uses: Swatinem/rust-cache@v2 - - run: cargo build --release - - run: cargo xtask fetch-fixtures + - name: Set up Rust + uses: actions-rust-lang/setup-rust-toolchain@v1 - - uses: ./.github/actions/cache - id: cache + - name: Build project + run: cargo build --release - - if: ${{ steps.cache.outputs.cache-hit != 'true' }} - run: cargo xtask generate-fixtures + - name: Cache fixtures + uses: ./.github/actions/cache + id: cache - - name: Run main tests with undefined behaviour sanitizer (UBSAN) - env: - CFLAGS: -fsanitize=undefined - RUSTFLAGS: ${{ env.RUSTFLAGS }} -lubsan - run: cargo test -- --test-threads 1 + - name: Fetch fixtures + run: cargo xtask fetch-fixtures - - name: Run main tests with address sanitizer (ASAN) - env: - ASAN_OPTIONS: verify_asan_link_order=0 - CFLAGS: -fsanitize=address - RUSTFLAGS: ${{ env.RUSTFLAGS }} -lasan --cfg sanitizing - run: cargo test -- --test-threads 1 + - name: Generate fixtures + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + run: cargo xtask generate-fixtures + + - name: Run main tests with undefined behaviour sanitizer (UBSAN) + run: cargo test -- --test-threads 1 + env: + CFLAGS: -fsanitize=undefined + RUSTFLAGS: ${{ env.RUSTFLAGS }} -lubsan + + - name: Run main tests with address sanitizer (ASAN) + run: cargo test -- --test-threads 1 + env: + ASAN_OPTIONS: verify_asan_link_order=0 + CFLAGS: -fsanitize=address + RUSTFLAGS: ${{ env.RUSTFLAGS }} -lasan --cfg sanitizing From bc4186776aa55c42c61767997607e0892298744f Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Tue, 12 Nov 2024 18:16:29 -0500 Subject: [PATCH 48/61] fix: compiler warning (cherry picked from commit 015547c526f8e3ae9ed95c19dc7a13e2df2fd30d) --- cli/loader/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/loader/src/lib.rs b/cli/loader/src/lib.rs index 13b4fabf..31ef448c 100644 --- a/cli/loader/src/lib.rs +++ b/cli/loader/src/lib.rs @@ -846,7 +846,7 @@ impl Loader { format!("Failed to execute the C compiler with the following command:\n{command:?}") })?; - lock_file.unlock()?; + FileExt::unlock(lock_file)?; fs::remove_file(lock_path)?; if output.status.success() { From 3473ca9a6b793465e27a6d7929109eb6ec091ce8 Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Sun, 13 Oct 2024 11:13:57 +0300 Subject: [PATCH 49/61] build: treat incompatible pointer warning as error (cherry picked from commit 70c0cba15b30324ab2e2826ad93cb702c6737118) --- Makefile | 2 +- lib/CMakeLists.txt | 21 ++++++++++++--------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index fdec43fe..9b4d30ef 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ OBJ := $(SRC:.c=.o) # define default flags, and override to append mandatory flags ARFLAGS := rcs -CFLAGS ?= -O3 -Wall -Wextra -Wshadow -pedantic +CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types override CFLAGS += -std=c11 -fPIC -fvisibility=hidden override CFLAGS += -Ilib/src -Ilib/src/wasm -Ilib/include diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index a505ef48..32a81011 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -22,9 +22,18 @@ add_library(tree-sitter ${TS_SOURCE_FILES}) target_include_directories(tree-sitter PRIVATE src src/wasm include) if(MSVC) - target_compile_options(tree-sitter PRIVATE /W4 /wd4018 /wd4701 /wd4702 /wd4100 /wd4232 /wd4244) + target_compile_options(tree-sitter PRIVATE + /wd4018 # disable 'signed/unsigned mismatch' + /wd4232 # disable 'nonstandard extension used' + /wd4244 # disable 'possible loss of data' + /wd4267 # disable 'possible loss of data (size_t)' + /wd4701 # disable 'potentially uninitialized local variable' + /we4022 # treat 'incompatible types' as an error + /W4) else() - target_compile_options(tree-sitter PRIVATE -Wall -Wextra -Wshadow -pedantic) + target_compile_options(tree-sitter PRIVATE + -Wall -Wextra -Wshadow -Wpedantic + -Werror=incompatible-pointer-types) endif() if(TREE_SITTER_FEATURE_WASM) @@ -41,13 +50,7 @@ if(TREE_SITTER_FEATURE_WASM) if(NOT DEFINED CACHE{WASMTIME_LIBRARY}) message(CHECK_START "Looking for wasmtime library") - if(BUILD_SHARED_LIBS) - find_library(WASMTIME_LIBRARY wasmtime) - elseif(MSVC) - find_library(WASMTIME_LIBRARY wasmtime.lib) - else() - find_library(WASMTIME_LIBRARY libwasmtime.a) - endif() + find_library(WASMTIME_LIBRARY wasmtime) if(NOT WASMTIME_LIBRARY) unset(WASMTIME_LIBRARY CACHE) message(FATAL_ERROR "Could not find wasmtime library.\nDid you forget to set CMAKE_LIBRARY_PATH?") From 6919d8406de89ae362a52f449ea322816a2aa7d6 Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Sun, 13 Oct 2024 12:54:57 +0300 Subject: [PATCH 50/61] fix(lib): handle compiler warnings (cherry picked from commit 881c54e462891c4c27e4f04d13528238cee2bed2) --- cli/generate/src/templates/array.h | 3 ++- lib/src/array.h | 3 ++- lib/src/tree_cursor.c | 3 --- lib/src/wasm_store.c | 26 +++++++++++++++++++------- 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/cli/generate/src/templates/array.h b/cli/generate/src/templates/array.h index 15a3b233..a17a574f 100644 --- a/cli/generate/src/templates/array.h +++ b/cli/generate/src/templates/array.h @@ -14,6 +14,7 @@ extern "C" { #include #ifdef _MSC_VER +#pragma warning(push) #pragma warning(disable : 4101) #elif defined(__GNUC__) || defined(__clang__) #pragma GCC diagnostic push @@ -278,7 +279,7 @@ static inline void _array__splice(Array *self, size_t element_size, #define _compare_int(a, b) ((int)*(a) - (int)(b)) #ifdef _MSC_VER -#pragma warning(default : 4101) +#pragma warning(pop) #elif defined(__GNUC__) || defined(__clang__) #pragma GCC diagnostic pop #endif diff --git a/lib/src/array.h b/lib/src/array.h index bbf6c756..d965c617 100644 --- a/lib/src/array.h +++ b/lib/src/array.h @@ -14,6 +14,7 @@ extern "C" { #include #ifdef _MSC_VER +#pragma warning(push) #pragma warning(disable : 4101) #elif defined(__GNUC__) || defined(__clang__) #pragma GCC diagnostic push @@ -278,7 +279,7 @@ static inline void _array__splice(Array *self, size_t element_size, #define _compare_int(a, b) ((int)*(a) - (int)(b)) #ifdef _MSC_VER -#pragma warning(default : 4101) +#pragma warning(pop) #elif defined(__GNUC__) || defined(__clang__) #pragma GCC diagnostic pop #endif diff --git a/lib/src/tree_cursor.c b/lib/src/tree_cursor.c index 250b42da..888e7781 100644 --- a/lib/src/tree_cursor.c +++ b/lib/src/tree_cursor.c @@ -1,5 +1,4 @@ #include "tree_sitter/api.h" -#include "./alloc.h" #include "./tree_cursor.h" #include "./language.h" #include "./tree.h" @@ -212,7 +211,6 @@ bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) { return false; } } - return false; } TreeCursorStep ts_tree_cursor_goto_last_child_internal(TSTreeCursor *_self) { @@ -253,7 +251,6 @@ bool ts_tree_cursor_goto_last_child(TSTreeCursor *self) { return false; } } - return false; } static inline int64_t ts_tree_cursor_goto_first_child_for_byte_and_point( diff --git a/lib/src/wasm_store.c b/lib/src/wasm_store.c index 81efbfcc..51c376a3 100644 --- a/lib/src/wasm_store.c +++ b/lib/src/wasm_store.c @@ -16,6 +16,14 @@ #include #include +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4100) +#elif defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + #define array_len(a) (sizeof(a) / sizeof(a[0])) // The following symbols from the C and C++ standard libraries are available @@ -159,8 +167,6 @@ typedef struct { int32_t eof; } LexerInWasmMemory; -static volatile uint32_t NEXT_LANGUAGE_ID; - // Linear memory layout: // [ <-- stack | stdlib statics | lexer | language statics --> | serialization_buffer | heap --> ] #define MAX_MEMORY_SIZE (128 * 1024 * 1024 / MEMORY_PAGE_SIZE) @@ -169,7 +175,7 @@ static volatile uint32_t NEXT_LANGUAGE_ID; * WasmDylinkMemoryInfo ***********************/ -static uint8_t read_u8(const uint8_t **p, const uint8_t *end) { +static uint8_t read_u8(const uint8_t **p) { return *(*p)++; } @@ -204,7 +210,7 @@ static bool wasm_dylink_info__parse( p += 4; while (p < end) { - uint8_t section_id = read_u8(&p, end); + uint8_t section_id = read_u8(&p); uint32_t section_length = read_uleb128(&p, end); const uint8_t *section_end = p + section_length; if (section_end > end) return false; @@ -217,7 +223,7 @@ static bool wasm_dylink_info__parse( if (name_length == 8 && memcmp(p, "dylink.0", 8) == 0) { p = name_end; while (p < section_end) { - uint8_t subsection_type = read_u8(&p, section_end); + uint8_t subsection_type = read_u8(&p); uint32_t subsection_size = read_uleb128(&p, section_end); const uint8_t *subsection_end = p + subsection_size; if (subsection_end > section_end) return false; @@ -545,6 +551,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) { wasm_trap_t *trap = NULL; wasm_message_t message = WASM_EMPTY_VEC; wasm_exporttype_vec_t export_types = WASM_EMPTY_VEC; + wasm_importtype_vec_t import_types = WASM_EMPTY_VEC; wasmtime_extern_t *imports = NULL; wasmtime_module_t *stdlib_module = NULL; wasm_memorytype_t *memory_type = NULL; @@ -660,11 +667,10 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) { } // Retrieve the stdlib module's imports. - wasm_importtype_vec_t import_types = WASM_EMPTY_VEC; wasmtime_module_imports(stdlib_module, &import_types); // Find the initial number of memory pages needed by the stdlib. - const wasm_memorytype_t *stdlib_memory_type; + const wasm_memorytype_t *stdlib_memory_type = NULL; for (unsigned i = 0; i < import_types.size; i++) { wasm_importtype_t *import_type = import_types.data[i]; const wasm_name_t *import_name = wasm_importtype_name(import_type); @@ -1743,6 +1749,12 @@ void ts_wasm_language_release(const TSLanguage *self) { } } +#ifdef _MSC_VER +#pragma warning(pop) +#elif defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic pop +#endif + #else // If the WASM feature is not enabled, define dummy versions of all of the From 29263440c76d3a664539047772b36a0867f64d78 Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Sun, 13 Oct 2024 11:47:31 +0300 Subject: [PATCH 51/61] ci: build lib with wasmtime (cherry picked from commit 60c5057617c0b2ca3804a362594c484728e3e5bf) --- .github/workflows/build.yml | 40 ++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6d227eec..6504f32c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,7 +45,7 @@ jobs: - { platform: windows-x64 , target: x86_64-pc-windows-msvc , os: windows-latest , features: wasm } - { platform: windows-x86 , target: i686-pc-windows-msvc , os: windows-latest } - { platform: macos-arm64 , target: aarch64-apple-darwin , os: macos-14 , features: wasm } - - { platform: macos-x64 , target: x86_64-apple-darwin , os: macos-12 , features: wasm } + - { platform: macos-x64 , target: x86_64-apple-darwin , os: macos-13 , features: wasm } # Cross compilers for C library - { platform: linux-arm64 , cc: aarch64-linux-gnu-gcc , ar: aarch64-linux-gnu-ar } @@ -135,26 +135,48 @@ jobs: [[ -n $runner ]] && printf 'CROSS_RUNNER=%s\n' "$runner" >> $GITHUB_ENV fi + - name: Build wasmtime library + if: ${{ !matrix.use-cross && contains(matrix.features, 'wasm') }} + run: | + WASMTIME_VERSION=$(cargo metadata --format-version=1 --locked --features wasm | \ + jq -r '.packages[] | select(.name == "wasmtime-c-api-impl") | .version') + curl -LSs "$WASMTIME_REPO/archive/refs/tags/v${WASMTIME_VERSION}.tar.gz" | tar xzf - -C target + cd target/wasmtime-${WASMTIME_VERSION} + cmake -S crates/c-api -B target/c-api \ + -DCMAKE_INSTALL_PREFIX="$PWD/artifacts" \ + -DWASMTIME_DISABLE_ALL_FEATURES=ON \ + -DWASMTIME_FEATURE_CRANELIFT=ON \ + -DWASMTIME_TARGET='${{ matrix.target }}' + cmake --build target/c-api && cmake --install target/c-api + printf 'CMAKE_PREFIX_PATH=%s\n' "$PWD/artifacts" >> $GITHUB_ENV + env: + WASMTIME_REPO: https://github.com/bytecodealliance/wasmtime + - name: Build C library (make) if: ${{ runner.os != 'Windows' }} run: make.sh -j CFLAGS="$CFLAGS" env: - CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wno-unused-parameter -pedantic + CFLAGS: -g -Werror -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types - name: Build C library (CMake) if: ${{ !matrix.use-cross }} - run: |- - cmake -S lib -B build \ + run: | + cmake -S lib -B build/static \ -DBUILD_SHARED_LIBS=OFF \ -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_COMPILE_WARNING_AS_ERROR=ON - cmake --build build --verbose + -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \ + -DTREE_SITTER_FEATURE_WASM=$WASM + cmake --build build/static --verbose - cmake -S lib -B build \ + cmake -S lib -B build/shared \ -DBUILD_SHARED_LIBS=ON \ -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_COMPILE_WARNING_AS_ERROR=ON - cmake --build build --verbose + -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \ + -DTREE_SITTER_FEATURE_WASM=$WASM + cmake --build build/shared --verbose + env: + CC: ${{ contains(matrix.target, 'linux') && 'clang' || '' }} + WASM: ${{ contains(matrix.features, 'wasm') && 'ON' || 'OFF' }} - name: Build wasm library # No reason to build on the same Github runner hosts many times From 62ce991e843d0ae765737ff9466c47f090b2061b Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Wed, 2 Oct 2024 22:27:56 +0300 Subject: [PATCH 52/61] build(cmake): link wasmtime dependencies (cherry picked from commit 6f08a684aaefe19dc195e4f1dd5f479e70d05058) --- lib/CMakeLists.txt | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 32a81011..da3a402b 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -21,19 +21,16 @@ add_library(tree-sitter ${TS_SOURCE_FILES}) target_include_directories(tree-sitter PRIVATE src src/wasm include) -if(MSVC) - target_compile_options(tree-sitter PRIVATE - /wd4018 # disable 'signed/unsigned mismatch' - /wd4232 # disable 'nonstandard extension used' - /wd4244 # disable 'possible loss of data' - /wd4267 # disable 'possible loss of data (size_t)' - /wd4701 # disable 'potentially uninitialized local variable' - /we4022 # treat 'incompatible types' as an error - /W4) -else() - target_compile_options(tree-sitter PRIVATE - -Wall -Wextra -Wshadow -Wpedantic - -Werror=incompatible-pointer-types) +if(NOT MSVC) + target_compile_options(tree-sitter PRIVATE -Wall -Wextra -Wshadow -Wno-unused-parameter -pedantic) +endif() + +if(NOT BUILD_SHARED_LIBS) + if(WIN32) + set(CMAKE_FIND_LIBRARY_SUFFIXES .lib .a) + else() + set(CMAKE_FIND_LIBRARY_SUFFIXES .a) + endif() endif() if(TREE_SITTER_FEATURE_WASM) @@ -66,9 +63,9 @@ if(TREE_SITTER_FEATURE_WASM) if(NOT BUILD_SHARED_LIBS) if(WIN32) target_compile_definitions(tree-sitter PRIVATE WASM_API_EXTERN= WASI_API_EXTERN=) - target_link_libraries(tree-sitter INTERFACE ws2_32 advapi32 userenv ntdll shell32 ole32 bcrypt) + target_link_libraries(tree-sitter PRIVATE ws2_32 advapi32 userenv ntdll shell32 ole32 bcrypt) elseif(NOT APPLE) - target_link_libraries(tree-sitter INTERFACE pthread dl m) + target_link_libraries(tree-sitter PRIVATE pthread dl m) endif() endif() endif() From b79f31da80514c67ac7e544e7c134fa95086af3b Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Wed, 4 Dec 2024 11:39:37 -0500 Subject: [PATCH 53/61] fix: sync cmake with master --- lib/CMakeLists.txt | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index da3a402b..42e324ad 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -21,16 +21,19 @@ add_library(tree-sitter ${TS_SOURCE_FILES}) target_include_directories(tree-sitter PRIVATE src src/wasm include) -if(NOT MSVC) - target_compile_options(tree-sitter PRIVATE -Wall -Wextra -Wshadow -Wno-unused-parameter -pedantic) -endif() - -if(NOT BUILD_SHARED_LIBS) - if(WIN32) - set(CMAKE_FIND_LIBRARY_SUFFIXES .lib .a) - else() - set(CMAKE_FIND_LIBRARY_SUFFIXES .a) - endif() +if(MSVC) + target_compile_options(tree-sitter PRIVATE + /wd4018 # disable 'signed/unsigned mismatch' + /wd4232 # disable 'nonstandard extension used' + /wd4244 # disable 'possible loss of data' + /wd4267 # disable 'possible loss of data (size_t)' + /wd4701 # disable 'potentially uninitialized local variable' + /we4022 # treat 'incompatible types' as an error + /W4) +else() + target_compile_options(tree-sitter PRIVATE + -Wall -Wextra -Wshadow -Wpedantic + -Werror=incompatible-pointer-types) endif() if(TREE_SITTER_FEATURE_WASM) @@ -63,9 +66,9 @@ if(TREE_SITTER_FEATURE_WASM) if(NOT BUILD_SHARED_LIBS) if(WIN32) target_compile_definitions(tree-sitter PRIVATE WASM_API_EXTERN= WASI_API_EXTERN=) - target_link_libraries(tree-sitter PRIVATE ws2_32 advapi32 userenv ntdll shell32 ole32 bcrypt) + target_link_libraries(tree-sitter INTERFACE ws2_32 advapi32 userenv ntdll shell32 ole32 bcrypt) elseif(NOT APPLE) - target_link_libraries(tree-sitter PRIVATE pthread dl m) + target_link_libraries(tree-sitter INTERFACE pthread dl m) endif() endif() endif() @@ -78,6 +81,8 @@ set_target_properties(tree-sitter SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}" DEFINE_SYMBOL "") +target_compile_definitions(tree-sitter PRIVATE _POSIX_C_SOURCE=200112L _DEFAULT_SOURCE) + configure_file(tree-sitter.pc.in "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter.pc" @ONLY) include(GNUInstallDirs) From ede1960eef4569cd8c8f53a3b3473e20ec816376 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Tue, 10 Dec 2024 20:03:42 -0500 Subject: [PATCH 54/61] ci: fix minor issues --- .github/workflows/build.yml | 4 +++- Cargo.lock | 4 ++-- Cargo.toml | 2 +- lib/language/Cargo.toml | 2 ++ lib/language/README.md | 4 ++++ 5 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 lib/language/README.md diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6504f32c..74abe69d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -88,7 +88,7 @@ jobs: - name: Install cross if: ${{ matrix.use-cross }} - run: cargo install cross --git https://github.com/cross-rs/cross + run: RUSTFLAGS="" cargo install cross --git https://github.com/cross-rs/cross - name: Configure cross if: ${{ matrix.use-cross }} @@ -138,6 +138,7 @@ jobs: - name: Build wasmtime library if: ${{ !matrix.use-cross && contains(matrix.features, 'wasm') }} run: | + mkdir -p target WASMTIME_VERSION=$(cargo metadata --format-version=1 --locked --features wasm | \ jq -r '.packages[] | select(.name == "wasmtime-c-api-impl") | .version') curl -LSs "$WASMTIME_REPO/archive/refs/tags/v${WASMTIME_VERSION}.tar.gz" | tar xzf - -C target @@ -151,6 +152,7 @@ jobs: printf 'CMAKE_PREFIX_PATH=%s\n' "$PWD/artifacts" >> $GITHUB_ENV env: WASMTIME_REPO: https://github.com/bytecodealliance/wasmtime + RUSTFLAGS: "" - name: Build C library (make) if: ${{ runner.os != 'Windows' }} diff --git a/Cargo.lock b/Cargo.lock index f0f8f34d..656c5f77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -442,9 +442,9 @@ dependencies = [ [[package]] name = "ctor" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f" +checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501" dependencies = [ "quote", "syn", diff --git a/Cargo.toml b/Cargo.toml index 5e7921c9..0a2a378b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -105,7 +105,7 @@ clap = { version = "4.5.18", features = [ "unstable-styles", ] } clap_complete = "4.5.29" -ctor = "0.2.8" +ctor = "0.2.9" ctrlc = { version = "3.4.5", features = ["termination"] } dialoguer = { version = "0.11.0", features = ["fuzzy-select"] } dirs = "5.0.1" diff --git a/lib/language/Cargo.toml b/lib/language/Cargo.toml index e3d8a6ff..7de3dca5 100644 --- a/lib/language/Cargo.toml +++ b/lib/language/Cargo.toml @@ -5,10 +5,12 @@ version = "0.1.2" authors.workspace = true edition.workspace = true rust-version.workspace = true +readme = "README.md" homepage.workspace = true repository.workspace = true license.workspace = true keywords.workspace = true +categories = ["api-bindings", "development-tools::ffi", "parsing"] [lints] workspace = true diff --git a/lib/language/README.md b/lib/language/README.md new file mode 100644 index 00000000..ddeabb92 --- /dev/null +++ b/lib/language/README.md @@ -0,0 +1,4 @@ +# Tree-sitter Language + +This crate provides a `LanguageFn` type for grammars to create `Language` instances from a parser, +without having to worry about the `tree-sitter` crate version not matching. From 1a983b7e2c45d9b9f45b2133a4e3a829f7e961be Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Wed, 11 Dec 2024 01:00:54 -0500 Subject: [PATCH 55/61] 0.24.5 --- Cargo.lock | 16 ++++++++-------- Cargo.toml | 14 +++++++------- Makefile | 2 +- build.zig.zon | 2 +- cli/npm/package.json | 2 +- lib/CMakeLists.txt | 2 +- lib/binding_web/package.json | 2 +- lib/language/Cargo.toml | 2 +- 8 files changed, 21 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 656c5f77..0f868cc5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1538,7 +1538,7 @@ dependencies = [ [[package]] name = "tree-sitter" -version = "0.24.4" +version = "0.24.5" dependencies = [ "bindgen", "cc", @@ -1551,7 +1551,7 @@ dependencies = [ [[package]] name = "tree-sitter-cli" -version = "0.24.4" +version = "0.24.5" dependencies = [ "anstyle", "anyhow", @@ -1601,7 +1601,7 @@ dependencies = [ [[package]] name = "tree-sitter-config" -version = "0.24.4" +version = "0.24.5" dependencies = [ "anyhow", "dirs", @@ -1611,7 +1611,7 @@ dependencies = [ [[package]] name = "tree-sitter-generate" -version = "0.24.4" +version = "0.24.5" dependencies = [ "anyhow", "heck 0.5.0", @@ -1632,7 +1632,7 @@ dependencies = [ [[package]] name = "tree-sitter-highlight" -version = "0.24.4" +version = "0.24.5" dependencies = [ "lazy_static", "regex", @@ -1643,11 +1643,11 @@ dependencies = [ [[package]] name = "tree-sitter-language" -version = "0.1.2" +version = "0.1.3" [[package]] name = "tree-sitter-loader" -version = "0.24.4" +version = "0.24.5" dependencies = [ "anyhow", "cc", @@ -1671,7 +1671,7 @@ dependencies = [ [[package]] name = "tree-sitter-tags" -version = "0.24.4" +version = "0.24.5" dependencies = [ "memchr", "regex", diff --git a/Cargo.toml b/Cargo.toml index 0a2a378b..1cb3e603 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.24.4" +version = "0.24.5" authors = ["Max Brunsfeld "] edition = "2021" rust-version = "1.74.1" @@ -145,9 +145,9 @@ walkdir = "2.5.0" wasmparser = "0.217.0" webbrowser = "1.0.2" -tree-sitter = { version = "0.24.4", path = "./lib" } -tree-sitter-generate = { version = "0.24.4", path = "./cli/generate" } -tree-sitter-loader = { version = "0.24.4", path = "./cli/loader" } -tree-sitter-config = { version = "0.24.4", path = "./cli/config" } -tree-sitter-highlight = { version = "0.24.4", path = "./highlight" } -tree-sitter-tags = { version = "0.24.4", path = "./tags" } +tree-sitter = { version = "0.24.5", path = "./lib" } +tree-sitter-generate = { version = "0.24.5", path = "./cli/generate" } +tree-sitter-loader = { version = "0.24.5", path = "./cli/loader" } +tree-sitter-config = { version = "0.24.5", path = "./cli/config" } +tree-sitter-highlight = { version = "0.24.5", path = "./highlight" } +tree-sitter-tags = { version = "0.24.5", path = "./tags" } diff --git a/Makefile b/Makefile index 9b4d30ef..16b31ea1 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ ifeq ($(OS),Windows_NT) $(error Windows is not supported) endif -VERSION := 0.24.4 +VERSION := 0.24.5 DESCRIPTION := An incremental parsing system for programming tools HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/ diff --git a/build.zig.zon b/build.zig.zon index f1c2f7f4..42bbe1d0 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,6 +1,6 @@ .{ .name = "tree-sitter", - .version = "0.24.4", + .version = "0.24.5", .paths = .{ "build.zig", "build.zig.zon", diff --git a/cli/npm/package.json b/cli/npm/package.json index a884529f..ca6cee88 100644 --- a/cli/npm/package.json +++ b/cli/npm/package.json @@ -1,6 +1,6 @@ { "name": "tree-sitter-cli", - "version": "0.24.4", + "version": "0.24.5", "author": "Max Brunsfeld", "license": "MIT", "repository": { diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 42e324ad..ee2c1eae 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.13) project(tree-sitter - VERSION "0.24.4" + VERSION "0.24.5" DESCRIPTION "An incremental parsing system for programming tools" HOMEPAGE_URL "https://tree-sitter.github.io/tree-sitter/" LANGUAGES C) diff --git a/lib/binding_web/package.json b/lib/binding_web/package.json index faa4dbe4..1a191a8c 100644 --- a/lib/binding_web/package.json +++ b/lib/binding_web/package.json @@ -1,6 +1,6 @@ { "name": "web-tree-sitter", - "version": "0.24.4", + "version": "0.24.5", "description": "Tree-sitter bindings for the web", "main": "tree-sitter.js", "types": "tree-sitter-web.d.ts", diff --git a/lib/language/Cargo.toml b/lib/language/Cargo.toml index 7de3dca5..5e4b975d 100644 --- a/lib/language/Cargo.toml +++ b/lib/language/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "tree-sitter-language" description = "The tree-sitter Language type, used by the library and by language implementations" -version = "0.1.2" +version = "0.1.3" authors.workspace = true edition.workspace = true rust-version.workspace = true From 62feed07155e75d1670ee5196e11a1c7394062fa Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Thu, 26 Dec 2024 22:25:19 -0500 Subject: [PATCH 56/61] fix(playground): backport new playground --- cli/src/playground.html | 11 + docs/assets/js/playground.js | 393 ++++++++++++++++++++++++----------- 2 files changed, 281 insertions(+), 123 deletions(-) diff --git a/cli/src/playground.html b/cli/src/playground.html index 420cd28d..10a52488 100644 --- a/cli/src/playground.html +++ b/cli/src/playground.html @@ -19,6 +19,11 @@ +
+ + +
+
@@ -67,6 +72,12 @@ + +