Merge branch 'master' into simpler-large-char-set-code
This commit is contained in:
Max Brunsfeld
commit
be6e6d3708
18 changed files with 248 additions and 160 deletions
132
Cargo.lock
generated
132
Cargo.lock
generated
|
|
@ -162,9 +162,9 @@ checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
|
|||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.0.90"
|
||||
version = "1.0.92"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5"
|
||||
checksum = "2678b2e3449475e95b0aa6f9b506a28e61b3dc8996592b983695e8ebb58a8b41"
|
||||
dependencies = [
|
||||
"jobserver",
|
||||
"libc",
|
||||
|
|
@ -216,9 +216,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.5.3"
|
||||
version = "4.5.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "949626d00e063efc93b6dca932419ceb5432f99769911c0b995f7e884c778813"
|
||||
checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0"
|
||||
dependencies = [
|
||||
"clap_builder",
|
||||
"clap_derive",
|
||||
|
|
@ -238,9 +238,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "clap_derive"
|
||||
version = "4.5.3"
|
||||
version = "4.5.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "90239a040c80f5e14809ca132ddc4176ab33d5e17e49691793296e3fcb34d72f"
|
||||
checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
|
|
@ -288,18 +288,18 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f"
|
|||
|
||||
[[package]]
|
||||
name = "cranelift-bforest"
|
||||
version = "0.106.0"
|
||||
version = "0.106.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a535eb1cf5a6003197dc569320c40c1cb2d2f97ef5d5348eebf067f20957381"
|
||||
checksum = "5b3775cc6cc00c90d29eebea55feedb2b0168e23f5415bab7859c4004d7323d1"
|
||||
dependencies = [
|
||||
"cranelift-entity",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cranelift-codegen"
|
||||
version = "0.106.0"
|
||||
version = "0.106.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "11b5066db32cec1492573827183af2142d2d88fe85a83cfc9e73f0f63d3788d4"
|
||||
checksum = "637f3184ba5bfa48d425bad1d2e4faf5fcf619f5e0ca107edc6dc02f589d4d74"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"cranelift-bforest",
|
||||
|
|
@ -318,33 +318,33 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "cranelift-codegen-meta"
|
||||
version = "0.106.0"
|
||||
version = "0.106.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "64942e5774308e835fbad4dd25f253105412c90324631910e1ec27963147bddb"
|
||||
checksum = "e4b35b8240462341d94d31aab807cad704683988708261aecae3d57db48b7212"
|
||||
dependencies = [
|
||||
"cranelift-codegen-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cranelift-codegen-shared"
|
||||
version = "0.106.0"
|
||||
version = "0.106.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c39c33db9a86dd6d8d04166a10c53deb477aeea3500eaaefca682e4eda9bb986"
|
||||
checksum = "8f3cd1555aa9df1d6d8375732de41b4cb0d787006948d55b6d004d521e9efeb0"
|
||||
|
||||
[[package]]
|
||||
name = "cranelift-control"
|
||||
version = "0.106.0"
|
||||
version = "0.106.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4b7fc4937613aea3156a0538800a17bf56f345a5da2e79ae3df58488c93d867f"
|
||||
checksum = "14b31a562a10e98ab148fa146801e20665c5f9eda4fce9b2c5a3836575887d74"
|
||||
dependencies = [
|
||||
"arbitrary",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cranelift-entity"
|
||||
version = "0.106.0"
|
||||
version = "0.106.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f85575e79a153ce1ddbfb7fe1813519b4bfe1eb200cc9c8353b45ad123ae4d36"
|
||||
checksum = "af1e0467700a3f4fccf5feddbaebdf8b0eb82535b06a9600c4bc5df40872e75d"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_derive",
|
||||
|
|
@ -352,9 +352,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "cranelift-frontend"
|
||||
version = "0.106.0"
|
||||
version = "0.106.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bbc31d6c0ab2249fe0c21e988256b42f5f401ab2673b4fc40076c82a698bdfb9"
|
||||
checksum = "6cb918ee2c23939262efd1b99d76a21212ac7bd35129582133e21a22a6ff0467"
|
||||
dependencies = [
|
||||
"cranelift-codegen",
|
||||
"log",
|
||||
|
|
@ -364,15 +364,15 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "cranelift-isle"
|
||||
version = "0.106.0"
|
||||
version = "0.106.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc14f37e3314c0e4c53779c2f46753bf242efff76ee9473757a1fff3b495ad37"
|
||||
checksum = "966e4cfb23cf6d7f1d285d53a912baaffc5f06bcd9c9b0a2d8c66a184fae534b"
|
||||
|
||||
[[package]]
|
||||
name = "cranelift-native"
|
||||
version = "0.106.0"
|
||||
version = "0.106.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2ea5375f76ab31f9800a23fb2b440810286a6f669a3eb467cdd7ff255ea64268"
|
||||
checksum = "bea803aadfc4aabdfae7c3870f1b1f6dd4332f4091859e9758ef5fca6bf8cc87"
|
||||
dependencies = [
|
||||
"cranelift-codegen",
|
||||
"libc",
|
||||
|
|
@ -381,9 +381,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "cranelift-wasm"
|
||||
version = "0.106.0"
|
||||
version = "0.106.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "79851dba01b1fa83fad95134aa27beca88dc4b027121d92ab19788582389dc5f"
|
||||
checksum = "11d18a3572cd897555bba3621e568029417d8f5cc26aeede2d7cb0bad6afd916"
|
||||
dependencies = [
|
||||
"cranelift-codegen",
|
||||
"cranelift-entity",
|
||||
|
|
@ -514,9 +514,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fs4"
|
||||
version = "0.8.1"
|
||||
version = "0.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57b1e34e369d7f0151309821497440bd0266b86c77ccd69717c3b67e5eaeffe4"
|
||||
checksum = "21dabded2e32cd57ded879041205c60a4a4c4bab47bd0fd2fa8b01f30849f02b"
|
||||
dependencies = [
|
||||
"rustix",
|
||||
"windows-sys 0.52.0",
|
||||
|
|
@ -546,9 +546,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "git2"
|
||||
version = "0.18.2"
|
||||
version = "0.18.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b3ba52851e73b46a4c3df1d89343741112003f0f6f13beb0dfac9e457c3fdcd"
|
||||
checksum = "232e6a7bfe35766bf715e55a88b39a700596c0ccfd88cd3680b4cdb40d66ef70"
|
||||
dependencies = [
|
||||
"bitflags 2.4.2",
|
||||
"libc",
|
||||
|
|
@ -625,9 +625,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "2.2.5"
|
||||
version = "2.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4"
|
||||
checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26"
|
||||
dependencies = [
|
||||
"equivalent",
|
||||
"hashbrown 0.14.3",
|
||||
|
|
@ -636,9 +636,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "indoc"
|
||||
version = "2.0.4"
|
||||
version = "2.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8"
|
||||
checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5"
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
|
|
@ -812,9 +812,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.7.1"
|
||||
version = "2.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"
|
||||
checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d"
|
||||
|
||||
[[package]]
|
||||
name = "memfd"
|
||||
|
|
@ -1067,9 +1067,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.10.3"
|
||||
version = "1.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15"
|
||||
checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
|
|
@ -1090,9 +1090,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.8.2"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
|
||||
checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56"
|
||||
|
||||
[[package]]
|
||||
name = "rustc-hash"
|
||||
|
|
@ -1156,9 +1156,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.114"
|
||||
version = "1.0.115"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0"
|
||||
checksum = "12dc5c46daa8e9fdf4f5e71b6cf9a53f2487da0e86e55808e2d35539666497dd"
|
||||
dependencies = [
|
||||
"indexmap",
|
||||
"itoa",
|
||||
|
|
@ -1626,9 +1626,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wasmtime"
|
||||
version = "19.0.0"
|
||||
version = "19.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a08af88fa3d324cc5cf6d388d90ef396a787b3fb4bbd51ba185f8645dc0f02c"
|
||||
checksum = "516be5b58a8f75d39b01378516dcb0ff7b9bc39c7f1f10eec5b338d4916cf988"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bincode",
|
||||
|
|
@ -1657,9 +1657,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wasmtime-asm-macros"
|
||||
version = "19.0.0"
|
||||
version = "19.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "16cdbfcf28542bcda0b5fd68d44603e53e5ad126cbe7b9f25c130e1249fd8211"
|
||||
checksum = "e8d22d88a92d69385f18143c946884bf6aaa9ec206ce54c85a2d320c1362b009"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
|
@ -1690,9 +1690,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wasmtime-cranelift"
|
||||
version = "19.0.0"
|
||||
version = "19.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67ea025c969a09117818732fa6f96848e858a7953d4659dab8081a6eea3c0523"
|
||||
checksum = "961ab5ee4b17e627001b18069ee89ef906edbbd3f84955515f6aad5ab6d82299"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"cfg-if",
|
||||
|
|
@ -1715,9 +1715,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wasmtime-cranelift-shared"
|
||||
version = "19.0.0"
|
||||
version = "19.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dcd6dd2f8d8d4860b384f61f89b597633a5b5f0943c546210e5084c5d321fe20"
|
||||
checksum = "bc4db94596be14cd1f85844ce85470bf68acf235143098b9d9bf72b49e47b917"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"cranelift-codegen",
|
||||
|
|
@ -1731,9 +1731,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wasmtime-environ"
|
||||
version = "19.0.0"
|
||||
version = "19.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f60f3f717658dd77745de03b750d5852126e9be6dad465848c77f90387c44c9"
|
||||
checksum = "420b13858ef27dfd116f1fdb0513e9593a307a632ade2ea58334b639a3d8d24e"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bincode",
|
||||
|
|
@ -1752,9 +1752,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wasmtime-jit-icache-coherence"
|
||||
version = "19.0.0"
|
||||
version = "19.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2796e4b4989db62899d2117e1e0258b839d088c044591b14e3a0396e7b3ae53a"
|
||||
checksum = "59c48eb4223d6556ffbf3decb146d0da124f1fd043f41c98b705252cb6a5c186"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
|
|
@ -1763,9 +1763,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wasmtime-runtime"
|
||||
version = "19.0.0"
|
||||
version = "19.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4bf2b7745df452a4f41b9aab21d3f7ba1347b12da2fdc5241e59306127884a68"
|
||||
checksum = "7fefac2cb5f5a6f365234a3584bf40bd2e45e7f6cd90a689d9b2afbb9881978f"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"cc",
|
||||
|
|
@ -1790,15 +1790,15 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wasmtime-slab"
|
||||
version = "19.0.0"
|
||||
version = "19.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "83448ef600ad95977019ebaea84a5516fdbc9561d0a8e26b1e099351f993b527"
|
||||
checksum = "52d7b97b92df126fdbe994a53d2215828ec5ed5087535e6d4703b1fbd299f0e3"
|
||||
|
||||
[[package]]
|
||||
name = "wasmtime-types"
|
||||
version = "19.0.0"
|
||||
version = "19.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf6fe7ed3fd18ed4b1e4465fe5c8674acc9f03523fca5b1b9f975b2560cd741b"
|
||||
checksum = "509c88abb830819b259c49e2d4e4f22b555db066ba08ded0b76b071a2aa53ddf"
|
||||
dependencies = [
|
||||
"cranelift-entity",
|
||||
"serde",
|
||||
|
|
@ -1809,9 +1809,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wasmtime-versioned-export-macros"
|
||||
version = "19.0.0"
|
||||
version = "19.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6d6d967f01032da7d4c6303da32f6a00d5efe1bac124b156e7342d8ace6ffdfc"
|
||||
checksum = "f1d81c092a61ca1667013e2eb08fed7c6c53e496dbbaa32d5685dc5152b0a772"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -1820,9 +1820,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wasmtime-wmemcheck"
|
||||
version = "19.0.0"
|
||||
version = "19.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "36bd91a4dc55af0bf55e9e2ab0ea13724cfb5c5a1acdf8873039769208f59490"
|
||||
checksum = "b77212b6874bbc86d220bb1d28632d0c11c6afe996c3e1ddcf746b1a6b4919b9"
|
||||
|
||||
[[package]]
|
||||
name = "web-sys"
|
||||
|
|
@ -1836,9 +1836,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "webbrowser"
|
||||
version = "0.8.13"
|
||||
version = "0.8.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d1b04c569c83a9bb971dd47ec6fd48753315f4bf989b9b04a2e7ca4d7f0dc950"
|
||||
checksum = "dd595fb70f33583ac61644820ebc144a26c96028b625b96cafcd861f4743fbc8"
|
||||
dependencies = [
|
||||
"core-foundation",
|
||||
"home",
|
||||
|
|
|
|||
22
Cargo.toml
22
Cargo.toml
|
|
@ -41,8 +41,8 @@ strip = false
|
|||
ansi_term = "0.12.1"
|
||||
anstyle = "1.0.6"
|
||||
anyhow = "1.0.80"
|
||||
cc = "1.0.90"
|
||||
clap = { version = "4.5.2", features = [
|
||||
cc = "1.0.92"
|
||||
clap = { version = "4.5.4", features = [
|
||||
"cargo",
|
||||
"derive",
|
||||
"env",
|
||||
|
|
@ -54,27 +54,27 @@ ctrlc = { version = "3.4.4", features = ["termination"] }
|
|||
difference = "2.0.0"
|
||||
dirs = "5.0.1"
|
||||
filetime = "0.2.23"
|
||||
fs4 = "0.8.1"
|
||||
git2 = "0.18.2"
|
||||
fs4 = "0.8.2"
|
||||
git2 = "0.18.3"
|
||||
glob = "0.3.1"
|
||||
heck = "0.5.0"
|
||||
html-escape = "0.2.13"
|
||||
indexmap = "2.2.5"
|
||||
indoc = "2.0.4"
|
||||
indexmap = "2.2.6"
|
||||
indoc = "2.0.5"
|
||||
lazy_static = "1.4.0"
|
||||
libloading = "0.8.3"
|
||||
log = { version = "0.4.21", features = ["std"] }
|
||||
memchr = "2.7.1"
|
||||
memchr = "2.7.2"
|
||||
once_cell = "1.19.0"
|
||||
pretty_assertions = "1.4.0"
|
||||
rand = "0.8.5"
|
||||
regex = "1.10.3"
|
||||
regex-syntax = "0.8.2"
|
||||
regex = "1.10.4"
|
||||
regex-syntax = "0.8.3"
|
||||
rustc-hash = "1.1.0"
|
||||
semver = "1.0.22"
|
||||
serde = { version = "1.0.197", features = ["derive"] }
|
||||
serde_derive = "1.0.197"
|
||||
serde_json = { version = "1.0.114", features = ["preserve_order"] }
|
||||
serde_json = { version = "1.0.115", features = ["preserve_order"] }
|
||||
smallbitvec = "2.5.3"
|
||||
tempfile = "3.10.1"
|
||||
thiserror = "1.0.57"
|
||||
|
|
@ -83,7 +83,7 @@ toml = "0.8.12"
|
|||
unindent = "0.2.3"
|
||||
walkdir = "2.5.0"
|
||||
wasmparser = "0.201.0"
|
||||
webbrowser = "0.8.13"
|
||||
webbrowser = "0.8.14"
|
||||
|
||||
tree-sitter = { version = "0.22.2", path = "./lib" }
|
||||
tree-sitter-loader = { version = "0.22.2", path = "./cli/loader" }
|
||||
|
|
|
|||
|
|
@ -591,6 +591,7 @@ impl Loader {
|
|||
command.arg(scanner_path);
|
||||
}
|
||||
command
|
||||
.arg("/utf-8")
|
||||
.arg("/link")
|
||||
.arg(format!("/out:{}", output_path.to_str().unwrap()));
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
use super::write_file;
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use heck::{ToKebabCase, ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase};
|
||||
use indoc::indoc;
|
||||
use serde::Deserialize;
|
||||
use serde_json::{json, Map, Value};
|
||||
use std::fs::File;
|
||||
|
|
@ -213,6 +214,19 @@ pub fn generate_grammar_files(
|
|||
updated = true;
|
||||
}
|
||||
|
||||
// insert `tree-sitter` at the end
|
||||
if !package_json.contains_key("tree-sitter") {
|
||||
eprintln!("Adding a `tree-sitter` section to package.json");
|
||||
package_json.insert(
|
||||
"tree-sitter".to_string(),
|
||||
json!([{
|
||||
"scope": format!("source.{language_name}"),
|
||||
"injection-regex": format!("^{language_name}$"),
|
||||
}]),
|
||||
);
|
||||
updated = true;
|
||||
}
|
||||
|
||||
if updated {
|
||||
let mut package_json_str = serde_json::to_string_pretty(&package_json)?;
|
||||
package_json_str.push('\n');
|
||||
|
|
@ -261,9 +275,33 @@ pub fn generate_grammar_files(
|
|||
generate_file(path, LIB_RS_TEMPLATE, language_name)
|
||||
})?;
|
||||
|
||||
missing_path(path.join("build.rs"), |path| {
|
||||
generate_file(path, BUILD_RS_TEMPLATE, language_name)
|
||||
})?;
|
||||
missing_path_else(
|
||||
path.join("build.rs"),
|
||||
|path| generate_file(path, BUILD_RS_TEMPLATE, language_name),
|
||||
|path| {
|
||||
let build_rs =
|
||||
fs::read_to_string(path).with_context(|| "Failed to read build.rs")?;
|
||||
if !build_rs.contains("/utf-8") {
|
||||
let index = build_rs
|
||||
.find(" let parser_path = src_dir.join(\"parser.c\")")
|
||||
.ok_or_else(|| anyhow!(indoc!{
|
||||
"Failed to auto-update build.rs with the `/utf-8` flag for windows.
|
||||
To fix this, remove `bindings/rust/build.rs` and re-run `tree-sitter generate`"}))?;
|
||||
|
||||
let build_rs = format!(
|
||||
"{}{}{}\n{}",
|
||||
&build_rs[..index],
|
||||
" #[cfg(target_env = \"msvc\")]\n",
|
||||
" c_config.flag(\"-utf-8\");\n",
|
||||
&build_rs[index..]
|
||||
);
|
||||
|
||||
write_file(path, build_rs)?;
|
||||
eprintln!("Updated build.rs with the /utf-8 flag for Windows compilation");
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
)?;
|
||||
|
||||
missing_path(repo_path.join("Cargo.toml"), |path| {
|
||||
generate_file(path, CARGO_TOML_TEMPLATE, dashed_language_name.as_str())
|
||||
|
|
@ -425,8 +463,7 @@ fn lookup_package_json_for_path(path: &Path) -> Result<(PathBuf, PackageJSON)> {
|
|||
.then(|| -> Result<PackageJSON> {
|
||||
let file =
|
||||
File::open(pathbuf.as_path()).with_context(|| "Failed to open package.json")?;
|
||||
let package_json: PackageJSON = serde_json::from_reader(BufReader::new(file))?;
|
||||
Ok(package_json)
|
||||
Ok(serde_json::from_reader(BufReader::new(file))?)
|
||||
})
|
||||
.transpose()?;
|
||||
if let Some(package_json) = package_json {
|
||||
|
|
|
|||
|
|
@ -472,7 +472,7 @@ impl NfaBuilder {
|
|||
)
|
||||
})?;
|
||||
for c in code_points {
|
||||
if let Some(c) = std::char::from_u32(*c) {
|
||||
if let Some(c) = char::from_u32(*c) {
|
||||
chars = chars.add_char(c);
|
||||
}
|
||||
}
|
||||
|
|
@ -490,7 +490,7 @@ impl NfaBuilder {
|
|||
for (category, code_points) in UNICODE_CATEGORIES.iter() {
|
||||
if category.starts_with(&category_letter) {
|
||||
for c in code_points {
|
||||
if let Some(c) = std::char::from_u32(*c) {
|
||||
if let Some(c) = char::from_u32(*c) {
|
||||
chars = chars.add_char(c);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -802,9 +802,6 @@ impl Generator {
|
|||
is_included: bool,
|
||||
line_break: &str,
|
||||
) {
|
||||
// parenthesis needed if we add the `!eof` condition to explicitly avoid confusion with
|
||||
// precedence of `&&` and `||`
|
||||
let (mut need_open_paren, mut need_close_paren) = (false, false);
|
||||
for (i, range) in characters.ranges().enumerate() {
|
||||
let start = *range.start();
|
||||
let end = *range.end();
|
||||
|
|
@ -812,21 +809,20 @@ impl Generator {
|
|||
if i > 0 {
|
||||
add!(self, " ||{line_break}");
|
||||
}
|
||||
|
||||
if start == '\0' {
|
||||
add!(self, "!eof && ");
|
||||
(need_open_paren, need_close_paren) = (true, true);
|
||||
}
|
||||
if end == start {
|
||||
if need_open_paren {
|
||||
add!(self, "(");
|
||||
need_open_paren = false;
|
||||
add!(self, "(!eof && ");
|
||||
if end == '\0' {
|
||||
add!(self, "lookahead == 0");
|
||||
} else {
|
||||
add!(self, "lookahead <= ");
|
||||
}
|
||||
self.add_character(end);
|
||||
add!(self, ")");
|
||||
continue;
|
||||
} else if end == start {
|
||||
add!(self, "lookahead == ");
|
||||
self.add_character(start);
|
||||
if need_close_paren && i == characters.range_count() - 1 {
|
||||
add!(self, ")");
|
||||
need_close_paren = false;
|
||||
}
|
||||
} else if end as u32 == start as u32 + 1 {
|
||||
add!(self, "lookahead == ");
|
||||
self.add_character(start);
|
||||
|
|
@ -908,7 +904,7 @@ impl Generator {
|
|||
if action.in_main_token {
|
||||
add!(self, "ADVANCE({});", action.state);
|
||||
} else {
|
||||
add!(self, "SKIP({})", action.state);
|
||||
add!(self, "SKIP({});", action.state);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1209,14 +1205,11 @@ impl Generator {
|
|||
production_id,
|
||||
..
|
||||
} => {
|
||||
add!(self, "REDUCE({}, {child_count}", self.symbol_ids[&symbol]);
|
||||
if dynamic_precedence != 0 {
|
||||
add!(self, ", .dynamic_precedence = {dynamic_precedence}");
|
||||
}
|
||||
if production_id != 0 {
|
||||
add!(self, ", .production_id = {production_id}");
|
||||
}
|
||||
add!(self, ")");
|
||||
add!(
|
||||
self,
|
||||
"REDUCE({}, {child_count}, {dynamic_precedence}, {production_id})",
|
||||
self.symbol_ids[&symbol]
|
||||
);
|
||||
}
|
||||
}
|
||||
add!(self, ",");
|
||||
|
|
@ -1268,7 +1261,7 @@ impl Generator {
|
|||
|
||||
add_line!(
|
||||
self,
|
||||
"TS_PUBLIC const TSLanguage *{language_function_name}() {{",
|
||||
"TS_PUBLIC const TSLanguage *{language_function_name}(void) {{",
|
||||
);
|
||||
indent!(self);
|
||||
add_line!(self, "static const TSLanguage language = {{");
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ import PackageDescription
|
|||
|
||||
let package = Package(
|
||||
name: "TreeSitterCAMEL_PARSER_NAME",
|
||||
platforms: [.macOS(.v10_13), .iOS(.v11)],
|
||||
products: [
|
||||
.library(name: "TreeSitterCAMEL_PARSER_NAME", targets: ["TreeSitterCAMEL_PARSER_NAME"]),
|
||||
],
|
||||
|
|
|
|||
|
|
@ -13,8 +13,17 @@
|
|||
"src/parser.c",
|
||||
# NOTE: if your language has an external scanner, add it here.
|
||||
],
|
||||
"cflags_c": [
|
||||
"-std=c11",
|
||||
"conditions": [
|
||||
["OS!='win'", {
|
||||
"cflags_c": [
|
||||
"-std=c11",
|
||||
],
|
||||
}, { # OS == "win"
|
||||
"cflags_c": [
|
||||
"/std:c11",
|
||||
"/utf-8",
|
||||
],
|
||||
}],
|
||||
],
|
||||
}
|
||||
]
|
||||
|
|
|
|||
|
|
@ -4,6 +4,9 @@ fn main() {
|
|||
let mut c_config = cc::Build::new();
|
||||
c_config.std("c11").include(src_dir);
|
||||
|
||||
#[cfg(target_env = "msvc")]
|
||||
c_config.flag("-utf-8");
|
||||
|
||||
let parser_path = src_dir.join("parser.c");
|
||||
c_config.file(&parser_path);
|
||||
println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap());
|
||||
|
|
|
|||
|
|
@ -27,11 +27,13 @@ INCLUDEDIR ?= $(PREFIX)/include
|
|||
LIBDIR ?= $(PREFIX)/lib
|
||||
PCLIBDIR ?= $(LIBDIR)/pkgconfig
|
||||
|
||||
# object files
|
||||
OBJS := $(patsubst %.c,%.o,$(wildcard $(SRC_DIR)/*.c))
|
||||
# source/object files
|
||||
PARSER := $(SRC_DIR)/parser.c
|
||||
EXTRAS := $(filter-out $(PARSER),$(wildcard $(SRC_DIR)/*.c))
|
||||
OBJS := $(patsubst %.c,%.o,$(PARSER) $(EXTRAS))
|
||||
|
||||
# flags
|
||||
ARFLAGS := rcs
|
||||
ARFLAGS ?= rcs
|
||||
override CFLAGS += -I$(SRC_DIR) -std=c11 -fPIC
|
||||
|
||||
# OS-specific bits
|
||||
|
|
@ -81,8 +83,8 @@ $(LANGUAGE_NAME).pc: bindings/c/$(LANGUAGE_NAME).pc.in
|
|||
-e 's|=$(PREFIX)|=$${prefix}|' \
|
||||
-e 's|@PREFIX@|$(PREFIX)|' $< > $@
|
||||
|
||||
$(SRC_DIR)/parser.c: grammar.js
|
||||
$(TS) generate --no-bindings
|
||||
$(PARSER): $(SRC_DIR)/grammar.json
|
||||
$(TS) generate --no-bindings $^
|
||||
|
||||
install: all
|
||||
install -d '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter '$(DESTDIR)$(PCLIBDIR)' '$(DESTDIR)$(LIBDIR)'
|
||||
|
|
|
|||
|
|
@ -38,9 +38,12 @@ setup(
|
|||
"src/parser.c",
|
||||
# NOTE: if your language uses an external scanner, add it here.
|
||||
],
|
||||
extra_compile_args=(
|
||||
["-std=c11"] if system() != 'Windows' else []
|
||||
),
|
||||
extra_compile_args=[
|
||||
"-std=c11",
|
||||
] if system() != "Windows" else [
|
||||
"/std:c11",
|
||||
"/utf-8",
|
||||
],
|
||||
define_macros=[
|
||||
("Py_LIMITED_API", "0x03080000"),
|
||||
("PY_SSIZE_T_CLEAN", None)
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ enum Commands {
|
|||
InitConfig(InitConfig),
|
||||
Generate(Generate),
|
||||
Build(Build),
|
||||
BuildWasm(BuildWasm),
|
||||
Parse(Parse),
|
||||
Test(Test),
|
||||
Query(Query),
|
||||
|
|
@ -115,6 +116,19 @@ struct Build {
|
|||
pub internal_build: bool,
|
||||
}
|
||||
|
||||
#[derive(Args)]
|
||||
#[command(about = "Compile a parser to WASM", alias = "bw")]
|
||||
struct BuildWasm {
|
||||
#[arg(
|
||||
short,
|
||||
long,
|
||||
help = "Run emscripten via docker even if it is installed locally"
|
||||
)]
|
||||
pub docker: bool,
|
||||
#[arg(index = 1, num_args = 1, help = "The path to output the wasm file")]
|
||||
pub path: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Args)]
|
||||
#[command(about = "Parse files", alias = "p")]
|
||||
struct Parse {
|
||||
|
|
@ -444,15 +458,11 @@ fn run() -> Result<()> {
|
|||
if build_options.wasm {
|
||||
let grammar_path =
|
||||
current_dir.join(build_options.path.as_deref().unwrap_or_default());
|
||||
let (output_dir, output_path) = if let Some(ref path) = build_options.output {
|
||||
(current_dir.clone(), Some(current_dir.join(path)))
|
||||
} else {
|
||||
(loader.parser_lib_path.clone(), None)
|
||||
};
|
||||
let output_path = build_options.output.map(|path| current_dir.join(path));
|
||||
wasm::compile_language_to_wasm(
|
||||
&loader,
|
||||
&grammar_path,
|
||||
&output_dir,
|
||||
¤t_dir,
|
||||
output_path,
|
||||
build_options.docker,
|
||||
)?;
|
||||
|
|
@ -498,6 +508,18 @@ fn run() -> Result<()> {
|
|||
}
|
||||
}
|
||||
|
||||
Commands::BuildWasm(wasm_options) => {
|
||||
eprintln!("`build-wasm` is deprecated and will be removed in v0.24.0. You should use `build --wasm` instead");
|
||||
let grammar_path = current_dir.join(wasm_options.path.unwrap_or_default());
|
||||
wasm::compile_language_to_wasm(
|
||||
&loader,
|
||||
&grammar_path,
|
||||
¤t_dir,
|
||||
None,
|
||||
wasm_options.docker,
|
||||
)?;
|
||||
}
|
||||
|
||||
Commands::Parse(parse_options) => {
|
||||
let config = Config::load(parse_options.config_path)?;
|
||||
let output = if parse_options.output_dot {
|
||||
|
|
|
|||
|
|
@ -21,8 +21,7 @@ lazy_static! {
|
|||
(?P<equals>(?:=+){3,})
|
||||
(?P<suffix1>[^=\r\n][^\r\n]*)?
|
||||
\r?\n
|
||||
(?P<test_name>(?:[^=\r\n:][^\r\n]*\r?\n)+(?:(?:[ \t]*\r?\n)+)?)
|
||||
(?P<markers>((?::(?:skip|error|fail-fast|(language|platform)\([^\r\n)]+\))\r?\n)*))
|
||||
(?P<test_name_and_markers>(?:[^=\r\n][^\r\n]*\r?\n)+)
|
||||
===+
|
||||
(?P<suffix2>[^=\r\n][^\r\n]*)?\r?\n"
|
||||
)
|
||||
|
|
@ -511,29 +510,46 @@ fn parse_test_content(name: String, content: &str, file_path: Option<PathBuf>) -
|
|||
let (mut skip, mut platform, mut fail_fast, mut error, mut languages) =
|
||||
(false, None, false, false, vec![]);
|
||||
|
||||
let markers = c.name("markers").map_or("".as_bytes(), |m| m.as_bytes());
|
||||
let test_name_and_markers = c
|
||||
.name("test_name_and_markers")
|
||||
.map_or("".as_bytes(), |m| m.as_bytes());
|
||||
|
||||
for marker in markers.split(|&c| c == b'\n').filter(|s| !s.is_empty()) {
|
||||
let marker = str::from_utf8(marker).unwrap();
|
||||
let (marker, right) = marker.split_at(marker.find('(').unwrap_or(marker.len()));
|
||||
match marker {
|
||||
":skip" => skip = true,
|
||||
let mut test_name = String::new();
|
||||
let mut seen_marker = false;
|
||||
|
||||
for line in test_name_and_markers
|
||||
.split(|&c| c == b'\n')
|
||||
.filter(|s| !s.is_empty())
|
||||
{
|
||||
let line = str::from_utf8(line).unwrap();
|
||||
match line.split('(').next().unwrap() {
|
||||
":skip" => (seen_marker, skip) = (true, true),
|
||||
":platform" => {
|
||||
if let Some(platforms) =
|
||||
right.strip_prefix('(').and_then(|s| s.strip_suffix(')'))
|
||||
{
|
||||
if let Some(platforms) = line.strip_prefix(':').and_then(|s| {
|
||||
s.strip_prefix("platform(")
|
||||
.and_then(|s| s.strip_suffix(')'))
|
||||
}) {
|
||||
seen_marker = true;
|
||||
platform = Some(
|
||||
platform.unwrap_or(false) || platforms.trim() == std::env::consts::OS,
|
||||
);
|
||||
}
|
||||
}
|
||||
":fail-fast" => fail_fast = true,
|
||||
":error" => error = true,
|
||||
":fail-fast" => (seen_marker, fail_fast) = (true, true),
|
||||
":error" => (seen_marker, error) = (true, true),
|
||||
":language" => {
|
||||
if let Some(lang) = right.strip_prefix('(').and_then(|s| s.strip_suffix(')')) {
|
||||
if let Some(lang) = line.strip_prefix(':').and_then(|s| {
|
||||
s.strip_prefix("language(")
|
||||
.and_then(|s| s.strip_suffix(')'))
|
||||
}) {
|
||||
seen_marker = true;
|
||||
languages.push(lang.into());
|
||||
}
|
||||
}
|
||||
_ if !seen_marker => {
|
||||
test_name.push_str(line);
|
||||
test_name.push('\n');
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
|
@ -550,9 +566,11 @@ fn parse_test_content(name: String, content: &str, file_path: Option<PathBuf>) -
|
|||
|
||||
if suffix1 == first_suffix && suffix2 == first_suffix {
|
||||
let header_range = c.get(0).unwrap().range();
|
||||
let test_name = c
|
||||
.name("test_name")
|
||||
.map(|c| String::from_utf8_lossy(c.as_bytes()).trim_end().to_string());
|
||||
let test_name = if test_name.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(test_name.trim_end().to_string())
|
||||
};
|
||||
Some((
|
||||
header_delim_len,
|
||||
header_range,
|
||||
|
|
|
|||
|
|
@ -785,7 +785,7 @@ Finally, you must define five functions with specific names, based on your langu
|
|||
#### Create
|
||||
|
||||
```c
|
||||
void * tree_sitter_my_language_external_scanner_create() {
|
||||
void *tree_sitter_my_language_external_scanner_create(void) {
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
|
@ -891,7 +891,7 @@ For example, assuming you wanted to allocate 100 bytes for your scanner, you'd d
|
|||
|
||||
// ...
|
||||
|
||||
void* tree_sitter_my_language_external_scanner_create() {
|
||||
void *tree_sitter_my_language_external_scanner_create(void) {
|
||||
return ts_calloc(100, 1); // or ts_malloc(100)
|
||||
}
|
||||
|
||||
|
|
@ -921,7 +921,7 @@ enum TokenType {
|
|||
|
||||
// Create the array in your create function
|
||||
|
||||
void* tree_sitter_my_language_external_scanner_create() {
|
||||
void *tree_sitter_my_language_external_scanner_create(void) {
|
||||
return ts_calloc(1, sizeof(Array(int)));
|
||||
|
||||
// or if you want to zero out the memory yourself
|
||||
|
|
|
|||
|
|
@ -2336,8 +2336,8 @@ impl QueryCursor {
|
|||
/// Because multiple patterns can match the same set of nodes, one match may contain
|
||||
/// captures that appear *before* some of the captures from a previous match.
|
||||
#[doc(alias = "ts_query_cursor_exec")]
|
||||
pub fn matches<'query, 'tree, T: TextProvider<I>, I: AsRef<[u8]>>(
|
||||
&mut self,
|
||||
pub fn matches<'query, 'cursor: 'query, 'tree, T: TextProvider<I>, I: AsRef<[u8]>>(
|
||||
&'cursor mut self,
|
||||
query: &'query Query,
|
||||
node: Node<'tree>,
|
||||
text_provider: T,
|
||||
|
|
@ -2359,8 +2359,8 @@ impl QueryCursor {
|
|||
/// This is useful if you don't care about which pattern matched, and just want a single,
|
||||
/// ordered sequence of captures.
|
||||
#[doc(alias = "ts_query_cursor_exec")]
|
||||
pub fn captures<'query, 'tree, T: TextProvider<I>, I: AsRef<[u8]>>(
|
||||
&mut self,
|
||||
pub fn captures<'query, 'cursor: 'query, 'tree, T: TextProvider<I>, I: AsRef<[u8]>>(
|
||||
&'cursor mut self,
|
||||
query: &'query Query,
|
||||
node: Node<'tree>,
|
||||
text_provider: T,
|
||||
|
|
|
|||
|
|
@ -324,7 +324,7 @@ describe('Parser', () => {
|
|||
'(source_file (const_item ' +
|
||||
'name: (identifier) ' +
|
||||
'type: (reference_type (lifetime (identifier)) type: (primitive_type)) ' +
|
||||
'value: (raw_string_literal)))',
|
||||
'value: (raw_string_literal (string_content))))',
|
||||
);
|
||||
}).timeout(5000);
|
||||
|
||||
|
|
|
|||
|
|
@ -835,14 +835,14 @@ static bool ts_parser__select_tree(TSParser *self, Subtree left, Subtree right)
|
|||
}
|
||||
|
||||
if (ts_subtree_dynamic_precedence(right) > ts_subtree_dynamic_precedence(left)) {
|
||||
LOG("select_higher_precedence symbol:%s, prec:%" PRId32 ", over_symbol:%s, other_prec:%u",
|
||||
LOG("select_higher_precedence symbol:%s, prec:%" PRId32 ", over_symbol:%s, other_prec:%" PRId32,
|
||||
TREE_NAME(right), ts_subtree_dynamic_precedence(right), TREE_NAME(left),
|
||||
ts_subtree_dynamic_precedence(left));
|
||||
return true;
|
||||
}
|
||||
|
||||
if (ts_subtree_dynamic_precedence(left) > ts_subtree_dynamic_precedence(right)) {
|
||||
LOG("select_higher_precedence symbol:%s, prec:%" PRId32 ", over_symbol:%s, other_prec:%u",
|
||||
LOG("select_higher_precedence symbol:%s, prec:%" PRId32 ", over_symbol:%s, other_prec:%" PRId32,
|
||||
TREE_NAME(left), ts_subtree_dynamic_precedence(left), TREE_NAME(right),
|
||||
ts_subtree_dynamic_precedence(right));
|
||||
return false;
|
||||
|
|
|
|||
|
|
@ -226,14 +226,15 @@ static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t
|
|||
} \
|
||||
}}
|
||||
|
||||
#define REDUCE(symbol_val, child_count_val, ...) \
|
||||
{{ \
|
||||
.reduce = { \
|
||||
.type = TSParseActionTypeReduce, \
|
||||
.symbol = symbol_val, \
|
||||
.child_count = child_count_val, \
|
||||
__VA_ARGS__ \
|
||||
}, \
|
||||
#define REDUCE(symbol_name, children, precedence, prod_id) \
|
||||
{{ \
|
||||
.reduce = { \
|
||||
.type = TSParseActionTypeReduce, \
|
||||
.symbol = symbol_name, \
|
||||
.child_count = children, \
|
||||
.dynamic_precedence = precedence, \
|
||||
.production_id = prod_id \
|
||||
}, \
|
||||
}}
|
||||
|
||||
#define RECOVER() \
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue