From 0b4403294981ffb6a51d153a5509a389b91fed86 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 21:10:03 +0000 Subject: [PATCH 01/22] build(deps): bump the cargo group with 4 updates Bumps the cargo group with 4 updates: [git2](https://github.com/rust-lang/git2-rs), [indexmap](https://github.com/indexmap-rs/indexmap), [indoc](https://github.com/dtolnay/indoc) and [regex](https://github.com/rust-lang/regex). Updates `git2` from 0.18.2 to 0.18.3 - [Changelog](https://github.com/rust-lang/git2-rs/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/git2-rs/compare/git2-0.18.2...git2-0.18.3) Updates `indexmap` from 2.2.5 to 2.2.6 - [Changelog](https://github.com/indexmap-rs/indexmap/blob/master/RELEASES.md) - [Commits](https://github.com/indexmap-rs/indexmap/compare/2.2.5...2.2.6) Updates `indoc` from 2.0.4 to 2.0.5 - [Release notes](https://github.com/dtolnay/indoc/releases) - [Commits](https://github.com/dtolnay/indoc/compare/2.0.4...2.0.5) Updates `regex` from 1.10.3 to 1.10.4 - [Release notes](https://github.com/rust-lang/regex/releases) - [Changelog](https://github.com/rust-lang/regex/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/regex/compare/1.10.3...1.10.4) --- updated-dependencies: - dependency-name: git2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: cargo - dependency-name: indexmap dependency-type: direct:production update-type: version-update:semver-patch dependency-group: cargo - dependency-name: indoc dependency-type: direct:production update-type: version-update:semver-patch dependency-group: cargo - dependency-name: regex dependency-type: direct:production update-type: version-update:semver-patch dependency-group: cargo ... Signed-off-by: dependabot[bot] --- Cargo.lock | 16 ++++++++-------- Cargo.toml | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 532ca329..67f05dea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -546,9 +546,9 @@ dependencies = [ [[package]] name = "git2" -version = "0.18.2" +version = "0.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b3ba52851e73b46a4c3df1d89343741112003f0f6f13beb0dfac9e457c3fdcd" +checksum = "232e6a7bfe35766bf715e55a88b39a700596c0ccfd88cd3680b4cdb40d66ef70" dependencies = [ "bitflags 2.4.2", "libc", @@ -625,9 +625,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.5" +version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", "hashbrown 0.14.3", @@ -636,9 +636,9 @@ dependencies = [ [[package]] name = "indoc" -version = "2.0.4" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" +checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" [[package]] name = "itertools" @@ -1067,9 +1067,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.3" +version = "1.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" dependencies = [ "aho-corasick", "memchr", diff --git a/Cargo.toml b/Cargo.toml index 17c3f4b7..f8355b2d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,12 +55,12 @@ difference = "2.0.0" dirs = "5.0.1" filetime = "0.2.23" fs4 = "0.8.1" -git2 = "0.18.2" +git2 = "0.18.3" glob = "0.3.1" heck = "0.5.0" html-escape = "0.2.13" -indexmap = "2.2.5" -indoc = "2.0.4" +indexmap = "2.2.6" +indoc = "2.0.5" lazy_static = "1.4.0" libloading = "0.8.3" log = { version = "0.4.21", features = ["std"] } @@ -68,7 +68,7 @@ memchr = "2.7.1" once_cell = "1.19.0" pretty_assertions = "1.4.0" rand = "0.8.5" -regex = "1.10.3" +regex = "1.10.4" regex-syntax = "0.8.2" rustc-hash = "1.1.0" semver = "1.0.22" From 80dabf96c7879854fe072c93171640538b70470c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 21:39:15 +0000 Subject: [PATCH 02/22] build(deps): bump the cargo group with 4 updates Bumps the cargo group with 4 updates: [clap](https://github.com/clap-rs/clap), [memchr](https://github.com/BurntSushi/memchr), [regex-syntax](https://github.com/rust-lang/regex) and [serde_json](https://github.com/serde-rs/json). Updates `clap` from 4.5.3 to 4.5.4 - [Release notes](https://github.com/clap-rs/clap/releases) - [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md) - [Commits](https://github.com/clap-rs/clap/compare/v4.5.3...v4.5.4) Updates `memchr` from 2.7.1 to 2.7.2 - [Commits](https://github.com/BurntSushi/memchr/compare/2.7.1...2.7.2) Updates `regex-syntax` from 0.8.2 to 0.8.3 - [Release notes](https://github.com/rust-lang/regex/releases) - [Changelog](https://github.com/rust-lang/regex/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/regex/compare/regex-syntax-0.8.2...regex-syntax-0.8.3) Updates `serde_json` from 1.0.114 to 1.0.115 - [Release notes](https://github.com/serde-rs/json/releases) - [Commits](https://github.com/serde-rs/json/compare/v1.0.114...v1.0.115) --- updated-dependencies: - dependency-name: clap dependency-type: direct:production update-type: version-update:semver-patch dependency-group: cargo - dependency-name: memchr dependency-type: direct:production update-type: version-update:semver-patch dependency-group: cargo - dependency-name: regex-syntax dependency-type: direct:production update-type: version-update:semver-patch dependency-group: cargo - dependency-name: serde_json dependency-type: direct:production update-type: version-update:semver-patch dependency-group: cargo ... Signed-off-by: dependabot[bot] --- Cargo.lock | 20 ++++++++++---------- Cargo.toml | 8 ++++---- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 67f05dea..6e9c56b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -216,9 +216,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.3" +version = "4.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "949626d00e063efc93b6dca932419ceb5432f99769911c0b995f7e884c778813" +checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" dependencies = [ "clap_builder", "clap_derive", @@ -238,9 +238,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.3" +version = "4.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90239a040c80f5e14809ca132ddc4176ab33d5e17e49691793296e3fcb34d72f" +checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" dependencies = [ "heck", "proc-macro2", @@ -812,9 +812,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.1" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" +checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" [[package]] name = "memfd" @@ -1090,9 +1090,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" [[package]] name = "rustc-hash" @@ -1156,9 +1156,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.114" +version = "1.0.115" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" +checksum = "12dc5c46daa8e9fdf4f5e71b6cf9a53f2487da0e86e55808e2d35539666497dd" dependencies = [ "indexmap", "itoa", diff --git a/Cargo.toml b/Cargo.toml index f8355b2d..b0622f40 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,7 +42,7 @@ ansi_term = "0.12.1" anstyle = "1.0.6" anyhow = "1.0.80" cc = "1.0.90" -clap = { version = "4.5.2", features = [ +clap = { version = "4.5.4", features = [ "cargo", "derive", "env", @@ -64,17 +64,17 @@ indoc = "2.0.5" lazy_static = "1.4.0" libloading = "0.8.3" log = { version = "0.4.21", features = ["std"] } -memchr = "2.7.1" +memchr = "2.7.2" once_cell = "1.19.0" pretty_assertions = "1.4.0" rand = "0.8.5" regex = "1.10.4" -regex-syntax = "0.8.2" +regex-syntax = "0.8.3" rustc-hash = "1.1.0" semver = "1.0.22" serde = { version = "1.0.197", features = ["derive"] } serde_derive = "1.0.197" -serde_json = { version = "1.0.114", features = ["preserve_order"] } +serde_json = { version = "1.0.115", features = ["preserve_order"] } smallbitvec = "2.5.3" tempfile = "3.10.1" thiserror = "1.0.57" From 78b6067a5ddf64459530d2dd335a36eb588cd231 Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Thu, 28 Mar 2024 14:27:44 +0200 Subject: [PATCH 03/22] fix(parser): fix variadic macro --- cli/src/generate/render.rs | 6 +++++- lib/src/parser.h | 4 +--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cli/src/generate/render.rs b/cli/src/generate/render.rs index 1de8a069..489364e6 100644 --- a/cli/src/generate/render.rs +++ b/cli/src/generate/render.rs @@ -1296,7 +1296,11 @@ impl Generator { production_id, .. } => { - add!(self, "REDUCE({}, {child_count}", self.symbol_ids[&symbol]); + add!( + self, + "REDUCE(.symbol = {}, .child_count = {child_count}", + self.symbol_ids[&symbol] + ); if dynamic_precedence != 0 { add!(self, ", .dynamic_precedence = {dynamic_precedence}"); } diff --git a/lib/src/parser.h b/lib/src/parser.h index 17b4fde9..70d22982 100644 --- a/lib/src/parser.h +++ b/lib/src/parser.h @@ -203,12 +203,10 @@ struct TSLanguage { } \ }} -#define REDUCE(symbol_val, child_count_val, ...) \ +#define REDUCE(...) \ {{ \ .reduce = { \ .type = TSParseActionTypeReduce, \ - .symbol = symbol_val, \ - .child_count = child_count_val, \ __VA_ARGS__ \ }, \ }} From a7078a1561c630c775465e7c9029f1aac2999213 Mon Sep 17 00:00:00 2001 From: WillLillis Date: Wed, 3 Apr 2024 22:06:19 -0400 Subject: [PATCH 04/22] fix: Add lifetime to mactches function --- lib/binding_rust/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/binding_rust/lib.rs b/lib/binding_rust/lib.rs index c3ee61ae..420f82e4 100644 --- a/lib/binding_rust/lib.rs +++ b/lib/binding_rust/lib.rs @@ -2336,8 +2336,8 @@ impl QueryCursor { /// Because multiple patterns can match the same set of nodes, one match may contain /// captures that appear *before* some of the captures from a previous match. #[doc(alias = "ts_query_cursor_exec")] - pub fn matches<'query, 'tree, T: TextProvider, I: AsRef<[u8]>>( - &mut self, + pub fn matches<'query, 'cursor: 'query, 'tree, T: TextProvider, I: AsRef<[u8]>>( + &'cursor mut self, query: &'query Query, node: Node<'tree>, text_provider: T, From cdd00defdba348a2ae2a6e7f631dd34459ee591d Mon Sep 17 00:00:00 2001 From: Torsten Schmits Date: Thu, 4 Apr 2024 20:51:15 +0200 Subject: [PATCH 05/22] fix(lib): use correct format specifier in log message --- lib/src/parser.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/src/parser.c b/lib/src/parser.c index 9d74c1ae..f3eaf193 100644 --- a/lib/src/parser.c +++ b/lib/src/parser.c @@ -835,14 +835,14 @@ static bool ts_parser__select_tree(TSParser *self, Subtree left, Subtree right) } if (ts_subtree_dynamic_precedence(right) > ts_subtree_dynamic_precedence(left)) { - LOG("select_higher_precedence symbol:%s, prec:%" PRId32 ", over_symbol:%s, other_prec:%u", + LOG("select_higher_precedence symbol:%s, prec:%" PRId32 ", over_symbol:%s, other_prec:%" PRId32, TREE_NAME(right), ts_subtree_dynamic_precedence(right), TREE_NAME(left), ts_subtree_dynamic_precedence(left)); return true; } if (ts_subtree_dynamic_precedence(left) > ts_subtree_dynamic_precedence(right)) { - LOG("select_higher_precedence symbol:%s, prec:%" PRId32 ", over_symbol:%s, other_prec:%u", + LOG("select_higher_precedence symbol:%s, prec:%" PRId32 ", over_symbol:%s, other_prec:%" PRId32, TREE_NAME(left), ts_subtree_dynamic_precedence(left), TREE_NAME(right), ts_subtree_dynamic_precedence(right)); return false; From c76e5fa96052337fe69047557c2b08a5acc08360 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Apr 2024 21:20:24 +0000 Subject: [PATCH 06/22] build(deps): bump wasmtime from 19.0.0 to 19.0.1 Bumps [wasmtime](https://github.com/bytecodealliance/wasmtime) from 19.0.0 to 19.0.1. - [Release notes](https://github.com/bytecodealliance/wasmtime/releases) - [Changelog](https://github.com/bytecodealliance/wasmtime/blob/main/docs/WASI-some-possible-changes.md) - [Commits](https://github.com/bytecodealliance/wasmtime/compare/v19.0.0...v19.0.1) --- updated-dependencies: - dependency-name: wasmtime dependency-type: indirect ... Signed-off-by: dependabot[bot] --- Cargo.lock | 86 +++++++++++++++++++++++++++--------------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6e9c56b4..86227c97 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -288,18 +288,18 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cranelift-bforest" -version = "0.106.0" +version = "0.106.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a535eb1cf5a6003197dc569320c40c1cb2d2f97ef5d5348eebf067f20957381" +checksum = "5b3775cc6cc00c90d29eebea55feedb2b0168e23f5415bab7859c4004d7323d1" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-codegen" -version = "0.106.0" +version = "0.106.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11b5066db32cec1492573827183af2142d2d88fe85a83cfc9e73f0f63d3788d4" +checksum = "637f3184ba5bfa48d425bad1d2e4faf5fcf619f5e0ca107edc6dc02f589d4d74" dependencies = [ "bumpalo", "cranelift-bforest", @@ -318,33 +318,33 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.106.0" +version = "0.106.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64942e5774308e835fbad4dd25f253105412c90324631910e1ec27963147bddb" +checksum = "e4b35b8240462341d94d31aab807cad704683988708261aecae3d57db48b7212" dependencies = [ "cranelift-codegen-shared", ] [[package]] name = "cranelift-codegen-shared" -version = "0.106.0" +version = "0.106.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39c33db9a86dd6d8d04166a10c53deb477aeea3500eaaefca682e4eda9bb986" +checksum = "8f3cd1555aa9df1d6d8375732de41b4cb0d787006948d55b6d004d521e9efeb0" [[package]] name = "cranelift-control" -version = "0.106.0" +version = "0.106.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b7fc4937613aea3156a0538800a17bf56f345a5da2e79ae3df58488c93d867f" +checksum = "14b31a562a10e98ab148fa146801e20665c5f9eda4fce9b2c5a3836575887d74" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.106.0" +version = "0.106.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f85575e79a153ce1ddbfb7fe1813519b4bfe1eb200cc9c8353b45ad123ae4d36" +checksum = "af1e0467700a3f4fccf5feddbaebdf8b0eb82535b06a9600c4bc5df40872e75d" dependencies = [ "serde", "serde_derive", @@ -352,9 +352,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.106.0" +version = "0.106.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbc31d6c0ab2249fe0c21e988256b42f5f401ab2673b4fc40076c82a698bdfb9" +checksum = "6cb918ee2c23939262efd1b99d76a21212ac7bd35129582133e21a22a6ff0467" dependencies = [ "cranelift-codegen", "log", @@ -364,15 +364,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.106.0" +version = "0.106.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc14f37e3314c0e4c53779c2f46753bf242efff76ee9473757a1fff3b495ad37" +checksum = "966e4cfb23cf6d7f1d285d53a912baaffc5f06bcd9c9b0a2d8c66a184fae534b" [[package]] name = "cranelift-native" -version = "0.106.0" +version = "0.106.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ea5375f76ab31f9800a23fb2b440810286a6f669a3eb467cdd7ff255ea64268" +checksum = "bea803aadfc4aabdfae7c3870f1b1f6dd4332f4091859e9758ef5fca6bf8cc87" dependencies = [ "cranelift-codegen", "libc", @@ -381,9 +381,9 @@ dependencies = [ [[package]] name = "cranelift-wasm" -version = "0.106.0" +version = "0.106.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79851dba01b1fa83fad95134aa27beca88dc4b027121d92ab19788582389dc5f" +checksum = "11d18a3572cd897555bba3621e568029417d8f5cc26aeede2d7cb0bad6afd916" dependencies = [ "cranelift-codegen", "cranelift-entity", @@ -740,7 +740,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" dependencies = [ "cfg-if", - "windows-targets 0.52.4", + "windows-targets 0.48.5", ] [[package]] @@ -1626,9 +1626,9 @@ dependencies = [ [[package]] name = "wasmtime" -version = "19.0.0" +version = "19.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a08af88fa3d324cc5cf6d388d90ef396a787b3fb4bbd51ba185f8645dc0f02c" +checksum = "516be5b58a8f75d39b01378516dcb0ff7b9bc39c7f1f10eec5b338d4916cf988" dependencies = [ "anyhow", "bincode", @@ -1657,9 +1657,9 @@ dependencies = [ [[package]] name = "wasmtime-asm-macros" -version = "19.0.0" +version = "19.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16cdbfcf28542bcda0b5fd68d44603e53e5ad126cbe7b9f25c130e1249fd8211" +checksum = "e8d22d88a92d69385f18143c946884bf6aaa9ec206ce54c85a2d320c1362b009" dependencies = [ "cfg-if", ] @@ -1690,9 +1690,9 @@ dependencies = [ [[package]] name = "wasmtime-cranelift" -version = "19.0.0" +version = "19.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67ea025c969a09117818732fa6f96848e858a7953d4659dab8081a6eea3c0523" +checksum = "961ab5ee4b17e627001b18069ee89ef906edbbd3f84955515f6aad5ab6d82299" dependencies = [ "anyhow", "cfg-if", @@ -1715,9 +1715,9 @@ dependencies = [ [[package]] name = "wasmtime-cranelift-shared" -version = "19.0.0" +version = "19.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcd6dd2f8d8d4860b384f61f89b597633a5b5f0943c546210e5084c5d321fe20" +checksum = "bc4db94596be14cd1f85844ce85470bf68acf235143098b9d9bf72b49e47b917" dependencies = [ "anyhow", "cranelift-codegen", @@ -1731,9 +1731,9 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "19.0.0" +version = "19.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f60f3f717658dd77745de03b750d5852126e9be6dad465848c77f90387c44c9" +checksum = "420b13858ef27dfd116f1fdb0513e9593a307a632ade2ea58334b639a3d8d24e" dependencies = [ "anyhow", "bincode", @@ -1752,9 +1752,9 @@ dependencies = [ [[package]] name = "wasmtime-jit-icache-coherence" -version = "19.0.0" +version = "19.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2796e4b4989db62899d2117e1e0258b839d088c044591b14e3a0396e7b3ae53a" +checksum = "59c48eb4223d6556ffbf3decb146d0da124f1fd043f41c98b705252cb6a5c186" dependencies = [ "cfg-if", "libc", @@ -1763,9 +1763,9 @@ dependencies = [ [[package]] name = "wasmtime-runtime" -version = "19.0.0" +version = "19.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bf2b7745df452a4f41b9aab21d3f7ba1347b12da2fdc5241e59306127884a68" +checksum = "7fefac2cb5f5a6f365234a3584bf40bd2e45e7f6cd90a689d9b2afbb9881978f" dependencies = [ "anyhow", "cc", @@ -1790,15 +1790,15 @@ dependencies = [ [[package]] name = "wasmtime-slab" -version = "19.0.0" +version = "19.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83448ef600ad95977019ebaea84a5516fdbc9561d0a8e26b1e099351f993b527" +checksum = "52d7b97b92df126fdbe994a53d2215828ec5ed5087535e6d4703b1fbd299f0e3" [[package]] name = "wasmtime-types" -version = "19.0.0" +version = "19.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf6fe7ed3fd18ed4b1e4465fe5c8674acc9f03523fca5b1b9f975b2560cd741b" +checksum = "509c88abb830819b259c49e2d4e4f22b555db066ba08ded0b76b071a2aa53ddf" dependencies = [ "cranelift-entity", "serde", @@ -1809,9 +1809,9 @@ dependencies = [ [[package]] name = "wasmtime-versioned-export-macros" -version = "19.0.0" +version = "19.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d6d967f01032da7d4c6303da32f6a00d5efe1bac124b156e7342d8ace6ffdfc" +checksum = "f1d81c092a61ca1667013e2eb08fed7c6c53e496dbbaa32d5685dc5152b0a772" dependencies = [ "proc-macro2", "quote", @@ -1820,9 +1820,9 @@ dependencies = [ [[package]] name = "wasmtime-wmemcheck" -version = "19.0.0" +version = "19.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36bd91a4dc55af0bf55e9e2ab0ea13724cfb5c5a1acdf8873039769208f59490" +checksum = "b77212b6874bbc86d220bb1d28632d0c11c6afe996c3e1ddcf746b1a6b4919b9" [[package]] name = "web-sys" From abed43a1697688d5ee8d0dab0c4cfa6b98e8adef Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Tue, 2 Apr 2024 03:34:36 -0400 Subject: [PATCH 07/22] chore: clippy fix --- cli/src/generate/prepare_grammar/expand_tokens.rs | 4 ++-- cli/src/generate/render.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/src/generate/prepare_grammar/expand_tokens.rs b/cli/src/generate/prepare_grammar/expand_tokens.rs index d38719e7..70a92cae 100644 --- a/cli/src/generate/prepare_grammar/expand_tokens.rs +++ b/cli/src/generate/prepare_grammar/expand_tokens.rs @@ -472,7 +472,7 @@ impl NfaBuilder { ) })?; for c in code_points { - if let Some(c) = std::char::from_u32(*c) { + if let Some(c) = char::from_u32(*c) { chars = chars.add_char(c); } } @@ -490,7 +490,7 @@ impl NfaBuilder { for (category, code_points) in UNICODE_CATEGORIES.iter() { if category.starts_with(&category_letter) { for c in code_points { - if let Some(c) = std::char::from_u32(*c) { + if let Some(c) = char::from_u32(*c) { chars = chars.add_char(c); } } diff --git a/cli/src/generate/render.rs b/cli/src/generate/render.rs index 489364e6..659d2cb0 100644 --- a/cli/src/generate/render.rs +++ b/cli/src/generate/render.rs @@ -686,7 +686,7 @@ impl Generator { .advance_actions .iter() .map(|(chars, action)| { - let is_included = !chars.contains(std::char::MAX); + let is_included = !chars.contains(char::MAX); let mut ranges; if is_included { ranges = chars.simplify_ignoring(&ruled_out_chars); From a9172e0caa80898cbb7c18f5ef1cec379aa71d09 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sun, 7 Apr 2024 19:08:07 -0400 Subject: [PATCH 08/22] fix: add a semicolon after SKIP macros --- cli/src/generate/render.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/generate/render.rs b/cli/src/generate/render.rs index 659d2cb0..884ea566 100644 --- a/cli/src/generate/render.rs +++ b/cli/src/generate/render.rs @@ -995,7 +995,7 @@ impl Generator { if action.in_main_token { add!(self, "ADVANCE({});", action.state); } else { - add!(self, "SKIP({})", action.state); + add!(self, "SKIP({});", action.state); } } From 0d2dea9e0e9a4a01dd3bb15853eb7d567e72c45d Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sun, 7 Apr 2024 19:10:28 -0400 Subject: [PATCH 09/22] fix(test): allow colons in test names --- cli/src/test.rs | 52 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/cli/src/test.rs b/cli/src/test.rs index 56617699..142da4f2 100644 --- a/cli/src/test.rs +++ b/cli/src/test.rs @@ -21,8 +21,7 @@ lazy_static! { (?P(?:=+){3,}) (?P[^=\r\n][^\r\n]*)? \r?\n - (?P(?:[^=\r\n:][^\r\n]*\r?\n)+(?:(?:[ \t]*\r?\n)+)?) - (?P((?::(?:skip|error|fail-fast|(language|platform)\([^\r\n)]+\))\r?\n)*)) + (?P(?:[^=\r\n][^\r\n]*\r?\n)+) ===+ (?P[^=\r\n][^\r\n]*)?\r?\n" ) @@ -511,29 +510,46 @@ fn parse_test_content(name: String, content: &str, file_path: Option) - let (mut skip, mut platform, mut fail_fast, mut error, mut languages) = (false, None, false, false, vec![]); - let markers = c.name("markers").map_or("".as_bytes(), |m| m.as_bytes()); + let test_name_and_markers = c + .name("test_name_and_markers") + .map_or("".as_bytes(), |m| m.as_bytes()); - for marker in markers.split(|&c| c == b'\n').filter(|s| !s.is_empty()) { - let marker = str::from_utf8(marker).unwrap(); - let (marker, right) = marker.split_at(marker.find('(').unwrap_or(marker.len())); - match marker { - ":skip" => skip = true, + let mut test_name = String::new(); + let mut seen_marker = false; + + for line in test_name_and_markers + .split(|&c| c == b'\n') + .filter(|s| !s.is_empty()) + { + let line = str::from_utf8(line).unwrap(); + match line.split('(').next().unwrap() { + ":skip" => (seen_marker, skip) = (true, true), ":platform" => { - if let Some(platforms) = - right.strip_prefix('(').and_then(|s| s.strip_suffix(')')) - { + if let Some(platforms) = line.strip_prefix(':').and_then(|s| { + s.strip_prefix("platform(") + .and_then(|s| s.strip_suffix(')')) + }) { + seen_marker = true; platform = Some( platform.unwrap_or(false) || platforms.trim() == std::env::consts::OS, ); } } - ":fail-fast" => fail_fast = true, - ":error" => error = true, + ":fail-fast" => (seen_marker, fail_fast) = (true, true), + ":error" => (seen_marker, error) = (true, true), ":language" => { - if let Some(lang) = right.strip_prefix('(').and_then(|s| s.strip_suffix(')')) { + if let Some(lang) = line.strip_prefix(':').and_then(|s| { + s.strip_prefix("language(") + .and_then(|s| s.strip_suffix(')')) + }) { + seen_marker = true; languages.push(lang.into()); } } + _ if !seen_marker => { + test_name.push_str(line); + test_name.push('\n'); + } _ => {} } } @@ -550,9 +566,11 @@ fn parse_test_content(name: String, content: &str, file_path: Option) - if suffix1 == first_suffix && suffix2 == first_suffix { let header_range = c.get(0).unwrap().range(); - let test_name = c - .name("test_name") - .map(|c| String::from_utf8_lossy(c.as_bytes()).trim_end().to_string()); + let test_name = if test_name.is_empty() { + None + } else { + Some(test_name.trim_end().to_string()) + }; Some(( header_delim_len, header_range, From 21c06101aa3b9088deb843fab08eec3c5447bd5a Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 8 Apr 2024 11:33:47 -0400 Subject: [PATCH 10/22] fix(bindings): remove required platforms for swift --- cli/src/generate/templates/Package.swift | 1 - 1 file changed, 1 deletion(-) diff --git a/cli/src/generate/templates/Package.swift b/cli/src/generate/templates/Package.swift index f0cc1cd1..dfa0c968 100644 --- a/cli/src/generate/templates/Package.swift +++ b/cli/src/generate/templates/Package.swift @@ -3,7 +3,6 @@ import PackageDescription let package = Package( name: "TreeSitterCAMEL_PARSER_NAME", - platforms: [.macOS(.v10_13), .iOS(.v11)], products: [ .library(name: "TreeSitterCAMEL_PARSER_NAME", targets: ["TreeSitterCAMEL_PARSER_NAME"]), ], From c1a63019a473818413dc75e01d8d287331130a51 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 8 Apr 2024 12:29:16 -0400 Subject: [PATCH 11/22] chore: update tests --- lib/binding_web/test/parser-test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/binding_web/test/parser-test.js b/lib/binding_web/test/parser-test.js index cf087599..4fa42c2b 100644 --- a/lib/binding_web/test/parser-test.js +++ b/lib/binding_web/test/parser-test.js @@ -324,7 +324,7 @@ describe('Parser', () => { '(source_file (const_item ' + 'name: (identifier) ' + 'type: (reference_type (lifetime (identifier)) type: (primitive_type)) ' + - 'value: (raw_string_literal)))', + 'value: (raw_string_literal (string_content))))', ); }).timeout(5000); From 4d8b031bcc218977187575e22181106d48f82b86 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 21:58:45 +0000 Subject: [PATCH 12/22] build(deps): bump the cargo group with 3 updates Bumps the cargo group with 3 updates: [cc](https://github.com/rust-lang/cc-rs), [fs4](https://github.com/al8n/fs4-rs) and [webbrowser](https://github.com/amodm/webbrowser-rs). Updates `cc` from 1.0.90 to 1.0.92 - [Release notes](https://github.com/rust-lang/cc-rs/releases) - [Commits](https://github.com/rust-lang/cc-rs/compare/1.0.90...1.0.92) Updates `fs4` from 0.8.1 to 0.8.2 - [Release notes](https://github.com/al8n/fs4-rs/releases) - [Commits](https://github.com/al8n/fs4-rs/commits) Updates `webbrowser` from 0.8.13 to 0.8.14 - [Release notes](https://github.com/amodm/webbrowser-rs/releases) - [Changelog](https://github.com/amodm/webbrowser-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/amodm/webbrowser-rs/compare/v0.8.13...v0.8.14) --- updated-dependencies: - dependency-name: cc dependency-type: direct:production update-type: version-update:semver-patch dependency-group: cargo - dependency-name: fs4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: cargo - dependency-name: webbrowser dependency-type: direct:production update-type: version-update:semver-patch dependency-group: cargo ... Signed-off-by: dependabot[bot] --- Cargo.lock | 14 +++++++------- Cargo.toml | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 86227c97..345907e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -162,9 +162,9 @@ checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] name = "cc" -version = "1.0.90" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" +checksum = "2678b2e3449475e95b0aa6f9b506a28e61b3dc8996592b983695e8ebb58a8b41" dependencies = [ "jobserver", "libc", @@ -514,9 +514,9 @@ dependencies = [ [[package]] name = "fs4" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57b1e34e369d7f0151309821497440bd0266b86c77ccd69717c3b67e5eaeffe4" +checksum = "21dabded2e32cd57ded879041205c60a4a4c4bab47bd0fd2fa8b01f30849f02b" dependencies = [ "rustix", "windows-sys 0.52.0", @@ -740,7 +740,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" dependencies = [ "cfg-if", - "windows-targets 0.48.5", + "windows-targets 0.52.4", ] [[package]] @@ -1836,9 +1836,9 @@ dependencies = [ [[package]] name = "webbrowser" -version = "0.8.13" +version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1b04c569c83a9bb971dd47ec6fd48753315f4bf989b9b04a2e7ca4d7f0dc950" +checksum = "dd595fb70f33583ac61644820ebc144a26c96028b625b96cafcd861f4743fbc8" dependencies = [ "core-foundation", "home", diff --git a/Cargo.toml b/Cargo.toml index b0622f40..e9197ccf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,7 +41,7 @@ strip = false ansi_term = "0.12.1" anstyle = "1.0.6" anyhow = "1.0.80" -cc = "1.0.90" +cc = "1.0.92" clap = { version = "4.5.4", features = [ "cargo", "derive", @@ -54,7 +54,7 @@ ctrlc = { version = "3.4.4", features = ["termination"] } difference = "2.0.0" dirs = "5.0.1" filetime = "0.2.23" -fs4 = "0.8.1" +fs4 = "0.8.2" git2 = "0.18.3" glob = "0.3.1" heck = "0.5.0" @@ -83,7 +83,7 @@ toml = "0.8.12" unindent = "0.2.3" walkdir = "2.5.0" wasmparser = "0.201.0" -webbrowser = "0.8.13" +webbrowser = "0.8.14" tree-sitter = { version = "0.22.2", path = "./lib" } tree-sitter-loader = { version = "0.22.2", path = "./cli/loader" } From cbcb51b8575ca58f8bfde4f5e78cd08d562faa81 Mon Sep 17 00:00:00 2001 From: Will Lillis Date: Tue, 9 Apr 2024 12:19:06 -0400 Subject: [PATCH 13/22] fix: tie the lifetime of the cursor to the query in `QueryCursor::captures()` --- lib/binding_rust/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/binding_rust/lib.rs b/lib/binding_rust/lib.rs index 420f82e4..45626c14 100644 --- a/lib/binding_rust/lib.rs +++ b/lib/binding_rust/lib.rs @@ -2359,8 +2359,8 @@ impl QueryCursor { /// This is useful if you don't care about which pattern matched, and just want a single, /// ordered sequence of captures. #[doc(alias = "ts_query_cursor_exec")] - pub fn captures<'query, 'tree, T: TextProvider, I: AsRef<[u8]>>( - &mut self, + pub fn captures<'query, 'cursor: 'query, 'tree, T: TextProvider, I: AsRef<[u8]>>( + &'cursor mut self, query: &'query Query, node: Node<'tree>, text_provider: T, From 5dc62cc82802a73c6376ff949e736af698bd75c6 Mon Sep 17 00:00:00 2001 From: Sebastian Lackner Date: Thu, 11 Apr 2024 15:01:56 +0200 Subject: [PATCH 14/22] fix(cli): fix mismatched parenthesis when accounting for `&&` --- cli/src/generate/render.rs | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/cli/src/generate/render.rs b/cli/src/generate/render.rs index 884ea566..edc32621 100644 --- a/cli/src/generate/render.rs +++ b/cli/src/generate/render.rs @@ -870,34 +870,32 @@ impl Generator { line_break.push_str(" "); } - // parenthesis needed if we add the `!eof` condition to explicitly avoid confusion with - // precedence of `&&` and `||` - let (mut need_open_paren, mut need_close_paren) = (false, false); for (i, range) in ranges.iter().enumerate() { if is_included { if i > 0 { add!(self, " ||{line_break}"); } + // parenthesis needed if we add the `!eof` condition to explicitly avoid confusion with + // precedence of `&&` and `||` + let mut close_paren = false; if range.start == '\0' { - add!(self, "!eof && "); - (need_open_paren, need_close_paren) = (true, true); + add!(self, "(!eof && "); + close_paren = true; } if range.end == range.start { - if need_open_paren { - add!(self, "("); - need_open_paren = false; - } add!(self, "lookahead == "); self.add_character(range.start); - if need_close_paren && i == ranges.len() - 1 { - add!(self, ")"); - need_close_paren = false; - } } else if range.end as u32 == range.start as u32 + 1 { + if close_paren { + add!(self, "("); + } add!(self, "lookahead == "); self.add_character(range.start); add!(self, " ||{line_break}lookahead == "); self.add_character(range.end); + if close_paren { + add!(self, ")"); + } } else { add!(self, "("); self.add_character(range.start); @@ -905,6 +903,9 @@ impl Generator { self.add_character(range.end); add!(self, ")"); } + if close_paren { + add!(self, ")"); + } } else { if i > 0 { add!(self, " &&{line_break}"); From 895c7680e73c2beae05337b631b80d5776b354b0 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Tue, 26 Mar 2024 23:08:58 -0400 Subject: [PATCH 15/22] fix(windows): add `/utf-8` flag for parsers using unicode symbols --- cli/loader/src/lib.rs | 1 + cli/src/generate/grammar_files.rs | 31 ++++++++++++++++++++++++++--- cli/src/generate/templates/build.rs | 3 +++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/cli/loader/src/lib.rs b/cli/loader/src/lib.rs index 9c1d8dfc..b584abd4 100644 --- a/cli/loader/src/lib.rs +++ b/cli/loader/src/lib.rs @@ -591,6 +591,7 @@ impl Loader { command.arg(scanner_path); } command + .arg("/utf-8") .arg("/link") .arg(format!("/out:{}", output_path.to_str().unwrap())); } else { diff --git a/cli/src/generate/grammar_files.rs b/cli/src/generate/grammar_files.rs index 549bb279..106f9693 100644 --- a/cli/src/generate/grammar_files.rs +++ b/cli/src/generate/grammar_files.rs @@ -1,6 +1,7 @@ use super::write_file; use anyhow::{anyhow, Context, Result}; use heck::{ToKebabCase, ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase}; +use indoc::indoc; use serde::Deserialize; use serde_json::{json, Map, Value}; use std::fs::File; @@ -261,9 +262,33 @@ pub fn generate_grammar_files( generate_file(path, LIB_RS_TEMPLATE, language_name) })?; - missing_path(path.join("build.rs"), |path| { - generate_file(path, BUILD_RS_TEMPLATE, language_name) - })?; + missing_path_else( + path.join("build.rs"), + |path| generate_file(path, BUILD_RS_TEMPLATE, language_name), + |path| { + let build_rs = + fs::read_to_string(path).with_context(|| "Failed to read build.rs")?; + if !build_rs.contains("/utf-8") { + let index = build_rs + .find(" let parser_path = src_dir.join(\"parser.c\")") + .ok_or_else(|| anyhow!(indoc!{ + "Failed to auto-update build.rs with the `/utf-8` flag for windows. + To fix this, remove `bindings/rust/build.rs` and re-run `tree-sitter generate`"}))?; + + let build_rs = format!( + "{}{}{}\n{}", + &build_rs[..index], + " #[cfg(target_env = \"msvc\")]\n", + " c_config.flag(\"-utf-8\");\n", + &build_rs[index..] + ); + + write_file(path, build_rs)?; + eprintln!("Updated build.rs with the /utf-8 flag for Windows compilation"); + } + Ok(()) + }, + )?; missing_path(repo_path.join("Cargo.toml"), |path| { generate_file(path, CARGO_TOML_TEMPLATE, dashed_language_name.as_str()) diff --git a/cli/src/generate/templates/build.rs b/cli/src/generate/templates/build.rs index 3b5e02da..99985b40 100644 --- a/cli/src/generate/templates/build.rs +++ b/cli/src/generate/templates/build.rs @@ -4,6 +4,9 @@ fn main() { let mut c_config = cc::Build::new(); c_config.std("c11").include(src_dir); + #[cfg(target_env = "msvc")] + c_config.flag("-utf-8"); + let parser_path = src_dir.join("parser.c"); c_config.file(&parser_path); println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap()); From 33de5ef849afeba7101c766c276ce43ed8b13c04 Mon Sep 17 00:00:00 2001 From: Ron Panduwana Date: Sat, 23 Mar 2024 09:35:36 +0700 Subject: [PATCH 16/22] fix: default output directory for `build --wasm` should use current_dir --- cli/src/main.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 0337ead0..19df0a7a 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -444,15 +444,11 @@ fn run() -> Result<()> { if build_options.wasm { let grammar_path = current_dir.join(build_options.path.as_deref().unwrap_or_default()); - let (output_dir, output_path) = if let Some(ref path) = build_options.output { - (current_dir.clone(), Some(current_dir.join(path))) - } else { - (loader.parser_lib_path.clone(), None) - }; + let output_path = build_options.output.map(|path| current_dir.join(path)); wasm::compile_language_to_wasm( &loader, &grammar_path, - &output_dir, + ¤t_dir, output_path, build_options.docker, )?; From 8bfe4e1f6b15e2584e3f8a61c58adf1395e1eadf Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Tue, 2 Apr 2024 04:38:56 -0400 Subject: [PATCH 17/22] fix: add back `build-wasm` temporarily --- cli/src/main.rs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/cli/src/main.rs b/cli/src/main.rs index 19df0a7a..9078eece 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -28,6 +28,7 @@ enum Commands { InitConfig(InitConfig), Generate(Generate), Build(Build), + BuildWasm(BuildWasm), Parse(Parse), Test(Test), Query(Query), @@ -115,6 +116,19 @@ struct Build { pub internal_build: bool, } +#[derive(Args)] +#[command(about = "Compile a parser to WASM", alias = "bw")] +struct BuildWasm { + #[arg( + short, + long, + help = "Run emscripten via docker even if it is installed locally" + )] + pub docker: bool, + #[arg(index = 1, num_args = 1, help = "The path to output the wasm file")] + pub path: Option, +} + #[derive(Args)] #[command(about = "Parse files", alias = "p")] struct Parse { @@ -494,6 +508,18 @@ fn run() -> Result<()> { } } + Commands::BuildWasm(wasm_options) => { + eprintln!("`build-wasm` is deprecated and will be removed in v0.24.0. You should use `build --wasm` instead"); + let grammar_path = current_dir.join(wasm_options.path.unwrap_or_default()); + wasm::compile_language_to_wasm( + &loader, + &grammar_path, + ¤t_dir, + None, + wasm_options.docker, + )?; + } + Commands::Parse(parse_options) => { let config = Config::load(parse_options.config_path)?; let output = if parse_options.output_dot { From 818cd8c29199ff3302e383e4ecdb05f7a8f9c0b0 Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Thu, 11 Apr 2024 20:35:09 +0300 Subject: [PATCH 18/22] refactor(parser): make REDUCE macro non-variadic --- cli/src/generate/render.rs | 9 +-------- lib/src/parser.h | 15 +++++++++------ 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/cli/src/generate/render.rs b/cli/src/generate/render.rs index edc32621..bca439fa 100644 --- a/cli/src/generate/render.rs +++ b/cli/src/generate/render.rs @@ -1299,16 +1299,9 @@ impl Generator { } => { add!( self, - "REDUCE(.symbol = {}, .child_count = {child_count}", + "REDUCE({}, {child_count}, {dynamic_precedence}, {production_id})", self.symbol_ids[&symbol] ); - if dynamic_precedence != 0 { - add!(self, ", .dynamic_precedence = {dynamic_precedence}"); - } - if production_id != 0 { - add!(self, ", .production_id = {production_id}"); - } - add!(self, ")"); } } add!(self, ","); diff --git a/lib/src/parser.h b/lib/src/parser.h index 70d22982..a6081cc3 100644 --- a/lib/src/parser.h +++ b/lib/src/parser.h @@ -203,12 +203,15 @@ struct TSLanguage { } \ }} -#define REDUCE(...) \ - {{ \ - .reduce = { \ - .type = TSParseActionTypeReduce, \ - __VA_ARGS__ \ - }, \ +#define REDUCE(symbol_name, children, precedence, prod_id) \ + {{ \ + .reduce = { \ + .type = TSParseActionTypeReduce, \ + .symbol = symbol_name, \ + .child_count = children, \ + .dynamic_precedence = precedence, \ + .production_id = prod_id \ + }, \ }} #define RECOVER() \ From 7830877f63b91a15728e75c1346c8ba37773346a Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Thu, 11 Apr 2024 18:55:19 +0300 Subject: [PATCH 19/22] fix(bindings): add utf-8 flag to python & node --- cli/src/generate/templates/binding.gyp | 13 +++++++++++-- cli/src/generate/templates/setup.py | 9 ++++++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/cli/src/generate/templates/binding.gyp b/cli/src/generate/templates/binding.gyp index 087e6555..9b7feebb 100644 --- a/cli/src/generate/templates/binding.gyp +++ b/cli/src/generate/templates/binding.gyp @@ -13,8 +13,17 @@ "src/parser.c", # NOTE: if your language has an external scanner, add it here. ], - "cflags_c": [ - "-std=c11", + "conditions": [ + ["OS!='win'", { + "cflags_c": [ + "-std=c11", + ], + }, { # OS == "win" + "cflags_c": [ + "/std:c11", + "/utf-8", + ], + }], ], } ] diff --git a/cli/src/generate/templates/setup.py b/cli/src/generate/templates/setup.py index e06337b6..85547e7d 100644 --- a/cli/src/generate/templates/setup.py +++ b/cli/src/generate/templates/setup.py @@ -38,9 +38,12 @@ setup( "src/parser.c", # NOTE: if your language uses an external scanner, add it here. ], - extra_compile_args=( - ["-std=c11"] if system() != 'Windows' else [] - ), + extra_compile_args=[ + "-std=c11", + ] if system() != "Windows" else [ + "/std:c11", + "/utf-8", + ], define_macros=[ ("Py_LIMITED_API", "0x03080000"), ("PY_SSIZE_T_CLEAN", None) From 50dfd409f69c072a6b2b4e6fa085ec0a0fa903ed Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Sun, 17 Mar 2024 23:40:06 +0200 Subject: [PATCH 20/22] fix(bindings): generate parser.c if missing --- cli/src/generate/templates/makefile | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cli/src/generate/templates/makefile b/cli/src/generate/templates/makefile index 98f6ffdf..0492ec02 100644 --- a/cli/src/generate/templates/makefile +++ b/cli/src/generate/templates/makefile @@ -27,11 +27,13 @@ INCLUDEDIR ?= $(PREFIX)/include LIBDIR ?= $(PREFIX)/lib PCLIBDIR ?= $(LIBDIR)/pkgconfig -# object files -OBJS := $(patsubst %.c,%.o,$(wildcard $(SRC_DIR)/*.c)) +# source/object files +PARSER := $(SRC_DIR)/parser.c +EXTRAS := $(filter-out $(PARSER),$(wildcard $(SRC_DIR)/*.c)) +OBJS := $(patsubst %.c,%.o,$(PARSER) $(EXTRAS)) # flags -ARFLAGS := rcs +ARFLAGS ?= rcs override CFLAGS += -I$(SRC_DIR) -std=c11 -fPIC # OS-specific bits @@ -81,8 +83,8 @@ $(LANGUAGE_NAME).pc: bindings/c/$(LANGUAGE_NAME).pc.in -e 's|=$(PREFIX)|=$${prefix}|' \ -e 's|@PREFIX@|$(PREFIX)|' $< > $@ -$(SRC_DIR)/parser.c: grammar.js - $(TS) generate --no-bindings +$(PARSER): $(SRC_DIR)/grammar.json + $(TS) generate --no-bindings $^ install: all install -d '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter '$(DESTDIR)$(PCLIBDIR)' '$(DESTDIR)$(LIBDIR)' From 63babea3014598a2c50d0a810e1e2ac54840d10e Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Mon, 18 Mar 2024 21:11:39 +0200 Subject: [PATCH 21/22] fix: proper function prototypes --- cli/src/generate/render.rs | 2 +- docs/section-3-creating-parsers.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/src/generate/render.rs b/cli/src/generate/render.rs index edc32621..95e4f2e7 100644 --- a/cli/src/generate/render.rs +++ b/cli/src/generate/render.rs @@ -1360,7 +1360,7 @@ impl Generator { add_line!( self, - "TS_PUBLIC const TSLanguage *{language_function_name}() {{", + "TS_PUBLIC const TSLanguage *{language_function_name}(void) {{", ); indent!(self); add_line!(self, "static const TSLanguage language = {{"); diff --git a/docs/section-3-creating-parsers.md b/docs/section-3-creating-parsers.md index 471f2067..d3876b39 100644 --- a/docs/section-3-creating-parsers.md +++ b/docs/section-3-creating-parsers.md @@ -785,7 +785,7 @@ Finally, you must define five functions with specific names, based on your langu #### Create ```c -void * tree_sitter_my_language_external_scanner_create() { +void *tree_sitter_my_language_external_scanner_create(void) { // ... } ``` @@ -891,7 +891,7 @@ For example, assuming you wanted to allocate 100 bytes for your scanner, you'd d // ... -void* tree_sitter_my_language_external_scanner_create() { +void *tree_sitter_my_language_external_scanner_create(void) { return ts_calloc(100, 1); // or ts_malloc(100) } @@ -921,7 +921,7 @@ enum TokenType { // Create the array in your create function -void* tree_sitter_my_language_external_scanner_create() { +void *tree_sitter_my_language_external_scanner_create(void) { return ts_calloc(1, sizeof(Array(int))); // or if you want to zero out the memory yourself From c051a90ac5f48db307d63187b3f032a762c1f3f2 Mon Sep 17 00:00:00 2001 From: Ron Panduwana Date: Wed, 27 Mar 2024 16:08:31 +0700 Subject: [PATCH 22/22] fix: insert "tree-sitter" section in current directory's package.json if it exists Co-authored-by: ObserverOfTime --- cli/src/generate/grammar_files.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/cli/src/generate/grammar_files.rs b/cli/src/generate/grammar_files.rs index 106f9693..1c1b7771 100644 --- a/cli/src/generate/grammar_files.rs +++ b/cli/src/generate/grammar_files.rs @@ -214,6 +214,19 @@ pub fn generate_grammar_files( updated = true; } + // insert `tree-sitter` at the end + if !package_json.contains_key("tree-sitter") { + eprintln!("Adding a `tree-sitter` section to package.json"); + package_json.insert( + "tree-sitter".to_string(), + json!([{ + "scope": format!("source.{language_name}"), + "injection-regex": format!("^{language_name}$"), + }]), + ); + updated = true; + } + if updated { let mut package_json_str = serde_json::to_string_pretty(&package_json)?; package_json_str.push('\n'); @@ -450,8 +463,7 @@ fn lookup_package_json_for_path(path: &Path) -> Result<(PathBuf, PackageJSON)> { .then(|| -> Result { let file = File::open(pathbuf.as_path()).with_context(|| "Failed to open package.json")?; - let package_json: PackageJSON = serde_json::from_reader(BufReader::new(file))?; - Ok(package_json) + Ok(serde_json::from_reader(BufReader::new(file))?) }) .transpose()?; if let Some(package_json) = package_json {