diff --git a/Cargo.lock b/Cargo.lock index 96982282..eb488156 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -74,9 +74,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.86" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" [[package]] name = "arbitrary" @@ -92,16 +92,14 @@ checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" [[package]] name = "bindgen" -version = "0.69.4" +version = "0.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f" dependencies = [ "bitflags", "cexpr", "clang-sys", - "itertools", - "lazy_static", - "lazycell", + "itertools 0.13.0", "log", "prettyplease", "proc-macro2", @@ -110,7 +108,6 @@ dependencies = [ "rustc-hash", "shlex", "syn", - "which", ] [[package]] @@ -128,6 +125,17 @@ dependencies = [ "objc2", ] +[[package]] +name = "bstr" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -142,15 +150,15 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" +checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" [[package]] name = "cc" -version = "1.1.14" +version = "1.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d2eb3cd3d1bf4529e31c215ee6f93ec5a3d536d9f578f93d9d33ee19562932" +checksum = "812acba72f0a070b003d3697490d2b55b837230ae7c6c6497f05cc2ddbb8d938" dependencies = [ "jobserver", "libc", @@ -203,9 +211,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.16" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed6719fffa43d0d87e5fd8caeab59be1554fb028cd30edc88fc4369b17971019" +checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3" dependencies = [ "clap_builder", "clap_derive", @@ -213,9 +221,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.15" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216aec2b177652e3846684cbfe25c9964d18ec45234f0f5da5157b207ed1aab6" +checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b" dependencies = [ "anstream", "anstyle", @@ -225,9 +233,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.13" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -265,9 +273,9 @@ dependencies = [ [[package]] name = "core-foundation" -version = "0.9.4" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63" dependencies = [ "core-foundation-sys", "libc", @@ -394,7 +402,7 @@ dependencies = [ "cranelift-codegen", "cranelift-entity", "cranelift-frontend", - "itertools", + "itertools 0.12.1", "log", "smallvec", "wasmparser", @@ -505,9 +513,9 @@ checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" [[package]] name = "filetime" -version = "0.2.24" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf401df4a4e3872c4fe8151134cf483738e74b67fc934d6532c882b3d24a4550" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" dependencies = [ "cfg-if", "libc", @@ -650,9 +658,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93ead53efc7ea8ed3cfb0c79fc8023fbb782a5432b52830b6518941cebe6505c" +checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" dependencies = [ "equivalent", "hashbrown 0.14.5", @@ -680,6 +688,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -732,12 +749,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "leb128" version = "0.2.5" @@ -746,9 +757,9 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "libc" -version = "0.2.158" +version = "0.2.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" +checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" [[package]] name = "libgit2-sys" @@ -923,9 +934,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.3" +version = "0.36.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b64972346851a39438c60b341ebc01bba47464ae329e55cf343eb93964efd9" +checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" dependencies = [ "crc32fast", "hashbrown 0.14.5", @@ -935,9 +946,12 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1" +dependencies = [ + "portable-atomic", +] [[package]] name = "openssl-probe" @@ -969,6 +983,12 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "path-slash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e91099d4268b0e11973f036e885d652fb0b21fedcf69738c627f94db6a44f42" + [[package]] name = "percent-encoding" version = "2.3.1" @@ -983,9 +1003,15 @@ checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" [[package]] name = "pkg-config" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" + +[[package]] +name = "portable-atomic" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" [[package]] name = "postcard" @@ -1010,9 +1036,9 @@ dependencies = [ [[package]] name = "pretty_assertions" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" dependencies = [ "diff", "yansi", @@ -1039,9 +1065,9 @@ dependencies = [ [[package]] name = "psm" -version = "0.1.21" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874" +checksum = "aa37f80ca58604976033fae9515a8a2989fc13797d953f7c04fb8fa36a11f205" dependencies = [ "cc", ] @@ -1087,9 +1113,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.3" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" dependencies = [ "bitflags", ] @@ -1120,9 +1146,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.6" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" dependencies = [ "aho-corasick", "memchr", @@ -1132,9 +1158,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick", "memchr", @@ -1143,9 +1169,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "rustc-hash" @@ -1155,9 +1181,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" dependencies = [ "bitflags", "errno", @@ -1189,18 +1215,18 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.209" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.209" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", @@ -1209,9 +1235,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.127" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad" +checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ "indexmap", "itoa", @@ -1222,9 +1248,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" dependencies = [ "serde", ] @@ -1282,9 +1308,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" -version = "2.0.76" +version = "2.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578e081a14e0cefc3279b0472138c513f37b41a08d5a3cca9b6e4e8ceb6cd525" +checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" dependencies = [ "proc-macro2", "quote", @@ -1299,9 +1325,9 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "tempfile" -version = "3.12.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" +checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" dependencies = [ "cfg-if", "fastrand", @@ -1321,18 +1347,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" +checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" dependencies = [ "proc-macro2", "quote", @@ -1389,9 +1415,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.20" +version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ "indexmap", "serde", @@ -1433,7 +1459,7 @@ dependencies = [ [[package]] name = "tree-sitter" -version = "0.23.0" +version = "0.23.2" dependencies = [ "bindgen", "cc", @@ -1445,10 +1471,11 @@ dependencies = [ [[package]] name = "tree-sitter-cli" -version = "0.23.0" +version = "0.23.2" dependencies = [ "anstyle", "anyhow", + "bstr", "clap", "ctor", "ctrlc", @@ -1490,7 +1517,7 @@ dependencies = [ [[package]] name = "tree-sitter-config" -version = "0.23.0" +version = "0.23.2" dependencies = [ "anyhow", "dirs", @@ -1500,7 +1527,7 @@ dependencies = [ [[package]] name = "tree-sitter-highlight" -version = "0.23.0" +version = "0.23.2" dependencies = [ "lazy_static", "regex", @@ -1510,11 +1537,11 @@ dependencies = [ [[package]] name = "tree-sitter-language" -version = "0.1.0" +version = "0.1.1" [[package]] name = "tree-sitter-loader" -version = "0.23.0" +version = "0.23.2" dependencies = [ "anyhow", "cc", @@ -1523,6 +1550,7 @@ dependencies = [ "indoc", "libloading", "once_cell", + "path-slash", "regex", "serde", "serde_json", @@ -1534,7 +1562,7 @@ dependencies = [ [[package]] name = "tree-sitter-tags" -version = "0.23.0" +version = "0.23.2" dependencies = [ "memchr", "regex", @@ -1560,24 +1588,24 @@ checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-normalization" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" dependencies = [ "tinyvec", ] [[package]] name = "unicode-xid" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "229730647fbc343e3a80e463c1db7f78f3855d3f3739bee0dda773c9a037c90a" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "unindent" @@ -1933,9 +1961,9 @@ dependencies = [ [[package]] name = "webbrowser" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "425ba64c1e13b1c6e8c5d2541c8fac10022ca584f33da781db01b5756aef1f4e" +checksum = "2e5f07fb9bc8de2ddfe6b24a71a75430673fd679e568c48b52716cef1cfae923" dependencies = [ "block2", "core-foundation", @@ -1949,18 +1977,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix", -] - [[package]] name = "winapi-util" version = "0.1.9" @@ -2186,9 +2202,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.18" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" dependencies = [ "memchr", ] @@ -2225,9 +2241,9 @@ dependencies = [ [[package]] name = "yansi" -version = "0.5.1" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "zerocopy" diff --git a/Cargo.toml b/Cargo.toml index 6f1f984a..ddaf5b56 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.23.0" +version = "0.23.2" authors = ["Max Brunsfeld "] edition = "2021" rust-version = "1.74.1" @@ -40,9 +40,10 @@ strip = false [workspace.dependencies] anstyle = "1.0.8" -anyhow = "1.0.86" -cc = "1.1.13" -clap = { version = "4.5.16", features = [ +anyhow = "1.0.89" +bstr = "1.10.0" +cc = "1.1.19" +clap = { version = "4.5.17", features = [ "cargo", "derive", "env", @@ -52,28 +53,29 @@ clap = { version = "4.5.16", features = [ ctor = "0.2.8" ctrlc = { version = "3.4.5", features = ["termination"] } dirs = "5.0.1" -filetime = "0.2.24" +filetime = "0.2.25" fs4 = "0.8.4" git2 = "0.18.3" glob = "0.3.1" heck = "0.5.0" html-escape = "0.2.13" -indexmap = "2.4.0" +indexmap = "2.5.0" indoc = "2.0.5" lazy_static = "1.5.0" libloading = "0.8.5" log = { version = "0.4.22", features = ["std"] } memchr = "2.7.4" once_cell = "1.19.0" -pretty_assertions = "1.4.0" +path-slash = "0.2.1" +pretty_assertions = "1.4.1" rand = "0.8.5" regex = "1.10.6" regex-syntax = "0.8.4" rustc-hash = "1.1.0" semver = "1.0.23" -serde = { version = "1.0.208", features = ["derive"] } +serde = { version = "1.0.210", features = ["derive"] } serde_derive = "1.0.197" -serde_json = { version = "1.0.125", features = ["preserve_order"] } +serde_json = { version = "1.0.128", features = ["preserve_order"] } similar = "2.6.0" smallbitvec = "2.5.3" tempfile = "3.12.0" @@ -83,10 +85,10 @@ toml = "0.8.19" unindent = "0.2.3" walkdir = "2.5.0" wasmparser = "0.215.0" -webbrowser = "1.0.1" +webbrowser = "1.0.2" -tree-sitter = { version = "0.23.0", path = "./lib" } -tree-sitter-loader = { version = "0.23.0", path = "./cli/loader" } -tree-sitter-config = { version = "0.23.0", path = "./cli/config" } -tree-sitter-highlight = { version = "0.23.0", path = "./highlight" } -tree-sitter-tags = { version = "0.23.0", path = "./tags" } +tree-sitter = { version = "0.23.2", path = "./lib" } +tree-sitter-loader = { version = "0.23.2", path = "./cli/loader" } +tree-sitter-config = { version = "0.23.2", path = "./cli/config" } +tree-sitter-highlight = { version = "0.23.2", path = "./highlight" } +tree-sitter-tags = { version = "0.23.2", path = "./tags" } diff --git a/Makefile b/Makefile index 465f940d..57ac8ae0 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ ifeq ($(OS),Windows_NT) $(error Windows is not supported) endif -VERSION := 0.23.0 +VERSION := 0.23.2 # install directory layout PREFIX ?= /usr/local @@ -32,7 +32,7 @@ SONAME_MAJOR := $(word 1,$(subst ., ,$(VERSION))) SONAME_MINOR := $(word 2,$(subst ., ,$(VERSION))) # OS-specific bits -ifeq ($(shell uname),Darwin) +ifneq ($(findstring darwin,$(shell $(CC) -dumpmachine)),) SOEXT = dylib SOEXTVER_MAJOR = $(SONAME_MAJOR).$(SOEXT) SOEXTVER = $(SONAME_MAJOR).$(SONAME_MINOR).$(SOEXT) diff --git a/Package.swift b/Package.swift index 6135da0c..3a4b9744 100644 --- a/Package.swift +++ b/Package.swift @@ -14,27 +14,6 @@ let package = Package( targets: [ .target(name: "TreeSitter", path: "lib", - exclude: [ - "binding_rust", - "binding_web", - "node_modules", - "Cargo.toml", - "README.md", - "src/unicode/README.md", - "src/unicode/LICENSE", - "src/unicode/ICU_SHA", - "src/get_changed_ranges.c", - "src/tree_cursor.c", - "src/stack.c", - "src/node.c", - "src/lexer.c", - "src/parser.c", - "src/language.c", - "src/alloc.c", - "src/subtree.c", - "src/tree.c", - "src/query.c" - ], sources: ["src/lib.c"], cSettings: [.headerSearchPath("src")]), ], diff --git a/build.zig.zon b/build.zig.zon index 97b4cdb9..dbebf231 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,6 +1,6 @@ .{ .name = "tree-sitter", - .version = "0.23.0", + .version = "0.23.2", .paths = .{ "build.zig", "build.zig.zon", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 493503ce..34fa31db 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -27,6 +27,7 @@ wasm = ["tree-sitter/wasm", "tree-sitter-loader/wasm"] [dependencies] anstyle.workspace = true anyhow.workspace = true +bstr.workspace = true clap.workspace = true ctor.workspace = true ctrlc.workspace = true diff --git a/cli/loader/Cargo.toml b/cli/loader/Cargo.toml index 5179b36d..0c4df0af 100644 --- a/cli/loader/Cargo.toml +++ b/cli/loader/Cargo.toml @@ -26,6 +26,7 @@ fs4.workspace = true indoc.workspace = true libloading.workspace = true once_cell.workspace = true +path-slash.workspace = true regex.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/cli/loader/src/lib.rs b/cli/loader/src/lib.rs index 610d0602..9ecd716c 100644 --- a/cli/loader/src/lib.rs +++ b/cli/loader/src/lib.rs @@ -23,6 +23,7 @@ use fs4::FileExt; use indoc::indoc; use libloading::{Library, Symbol}; use once_cell::unsync::OnceCell; +use path_slash::PathBufExt as _; use regex::{Regex, RegexBuilder}; use serde::{Deserialize, Deserializer, Serialize}; use tree_sitter::Language; @@ -823,7 +824,7 @@ impl Loader { path.push(src_path.strip_prefix(root_path).unwrap()); path }; - command.args(["--workdir", &workdir.to_string_lossy()]); + command.args(["--workdir", &workdir.to_slash_lossy()]); // Mount the root directory as a volume, which is the repo root let mut volume_string = OsString::from(&root_path); diff --git a/cli/npm/package.json b/cli/npm/package.json index bdab77fc..33c1194f 100644 --- a/cli/npm/package.json +++ b/cli/npm/package.json @@ -1,6 +1,6 @@ { "name": "tree-sitter-cli", - "version": "0.23.0", + "version": "0.23.2", "author": "Max Brunsfeld", "license": "MIT", "repository": { diff --git a/cli/src/fuzz/mod.rs b/cli/src/fuzz/mod.rs index 14f50c72..e7926df0 100644 --- a/cli/src/fuzz/mod.rs +++ b/cli/src/fuzz/mod.rs @@ -108,11 +108,16 @@ pub fn fuzz_language_corpus( } let tests = flatten_tests(main_tests, options.filter.as_ref()); - let mut skipped = options.skipped.as_ref().map(|x| { - x.iter() - .map(|x| (x.as_str(), 0)) - .collect::>() - }); + let get_test_name = |test: &FlattenedTest| format!("{language_name} - {}", test.name); + + let mut skipped = options + .skipped + .take() + .unwrap_or_default() + .into_iter() + .chain(tests.iter().filter(|x| x.skip).map(get_test_name)) + .map(|x| (x, 0)) + .collect::>(); let mut failure_count = 0; @@ -125,13 +130,11 @@ pub fn fuzz_language_corpus( println!(); for (test_index, test) in tests.iter().enumerate() { - let test_name = format!("{language_name} - {}", test.name); - if let Some(skipped) = skipped.as_mut() { - if let Some(counter) = skipped.get_mut(test_name.as_str()) { - println!(" {test_index}. {test_name} - SKIPPED"); - *counter += 1; - continue; - } + let test_name = get_test_name(test); + if let Some(counter) = skipped.get_mut(test_name.as_str()) { + println!(" {test_index}. {test_name} - SKIPPED"); + *counter += 1; + continue; } println!(" {test_index}. {test_name}"); @@ -143,6 +146,11 @@ pub fn fuzz_language_corpus( set_included_ranges(&mut parser, &test.input, test.template_delimiters); let tree = parser.parse(&test.input, None).unwrap(); + + if test.error { + return true; + } + let mut actual_output = tree.root_node().to_sexp(); if !test.has_fields { actual_output = strip_sexp_fields(&actual_output); @@ -240,7 +248,7 @@ pub fn fuzz_language_corpus( actual_output = strip_sexp_fields(&actual_output); } - if actual_output != test.output { + if actual_output != test.output && !test.error { println!("Incorrect parse for {test_name} - seed {seed}"); print_diff_key(); print_diff(&actual_output, &test.output, true); @@ -272,16 +280,14 @@ pub fn fuzz_language_corpus( eprintln!("{failure_count} {language_name} corpus tests failed fuzzing"); } - if let Some(skipped) = skipped.as_mut() { - skipped.retain(|_, v| *v == 0); + skipped.retain(|_, v| *v == 0); - if !skipped.is_empty() { - println!("Non matchable skip definitions:"); - for k in skipped.keys() { - println!(" {k}"); - } - panic!("Non matchable skip definitions needs to be removed"); + if !skipped.is_empty() { + println!("Non matchable skip definitions:"); + for k in skipped.keys() { + println!(" {k}"); } + panic!("Non matchable skip definitions needs to be removed"); } } @@ -290,6 +296,8 @@ pub struct FlattenedTest { pub input: Vec, pub output: String, pub languages: Vec>, + pub error: bool, + pub skip: bool, pub has_fields: bool, pub template_delimiters: Option<(&'static str, &'static str)>, } @@ -327,6 +335,8 @@ pub fn flatten_tests(test: TestEntry, filter: Option<&Regex>) -> Vec, extracted_usage_counts: Vec, } @@ -181,19 +183,25 @@ struct SymbolReplacer { } impl TokenExtractor { - fn extract_tokens_in_variable(&mut self, variable: &mut Variable) { + fn extract_tokens_in_variable( + &mut self, + is_first: bool, + variable: &mut Variable, + ) -> Result<()> { self.current_variable_name.clear(); self.current_variable_name.push_str(&variable.name); self.current_variable_token_count = 0; + self.is_first_rule = is_first; let mut rule = Rule::Blank; mem::swap(&mut rule, &mut variable.rule); - variable.rule = self.extract_tokens_in_rule(&rule); + variable.rule = self.extract_tokens_in_rule(&rule)?; + Ok(()) } - fn extract_tokens_in_rule(&mut self, input: &Rule) -> Rule { + fn extract_tokens_in_rule(&mut self, input: &Rule) -> Result { match input { - Rule::String(name) => self.extract_token(input, Some(name)).into(), - Rule::Pattern(..) => self.extract_token(input, None).into(), + Rule::String(name) => Ok(self.extract_token(input, Some(name))?.into()), + Rule::Pattern(..) => Ok(self.extract_token(input, None)?.into()), Rule::Metadata { params, rule } => { if params.is_token { let mut params = params.clone(); @@ -210,41 +218,53 @@ impl TokenExtractor { input }; - self.extract_token(rule_to_extract, string_value).into() + Ok(self.extract_token(rule_to_extract, string_value)?.into()) } else { - Rule::Metadata { + Ok(Rule::Metadata { params: params.clone(), - rule: Box::new(self.extract_tokens_in_rule(rule)), - } + rule: Box::new(self.extract_tokens_in_rule(rule)?), + }) } } - Rule::Repeat(content) => Rule::Repeat(Box::new(self.extract_tokens_in_rule(content))), - Rule::Seq(elements) => Rule::Seq( + Rule::Repeat(content) => Ok(Rule::Repeat(Box::new( + self.extract_tokens_in_rule(content)?, + ))), + Rule::Seq(elements) => Ok(Rule::Seq( elements .iter() .map(|e| self.extract_tokens_in_rule(e)) - .collect(), - ), - Rule::Choice(elements) => Rule::Choice( + .collect::>>()?, + )), + Rule::Choice(elements) => Ok(Rule::Choice( elements .iter() .map(|e| self.extract_tokens_in_rule(e)) - .collect(), - ), - _ => input.clone(), + .collect::>>()?, + )), + _ => Ok(input.clone()), } } - fn extract_token(&mut self, rule: &Rule, string_value: Option<&String>) -> Symbol { + fn extract_token(&mut self, rule: &Rule, string_value: Option<&String>) -> Result { for (i, variable) in self.extracted_variables.iter_mut().enumerate() { if variable.rule == *rule { self.extracted_usage_counts[i] += 1; - return Symbol::terminal(i); + return Ok(Symbol::terminal(i)); } } let index = self.extracted_variables.len(); let variable = if let Some(string_value) = string_value { + if string_value.is_empty() && !self.is_first_rule { + return Err(anyhow!( + "The rule `{}` contains an empty string. + +Tree-sitter does not support syntactic rules that contain an empty string +unless they are used only as the grammar's start rule. +", + self.current_variable_name + )); + } Variable { name: string_value.clone(), kind: VariableType::Anonymous, @@ -264,7 +284,7 @@ impl TokenExtractor { self.extracted_variables.push(variable); self.extracted_usage_counts.push(1); - Symbol::terminal(index) + Ok(Symbol::terminal(index)) } } @@ -520,6 +540,15 @@ mod test { ); } + #[test] + fn test_extraction_with_empty_string() { + assert!(extract_tokens(build_grammar(vec![ + Variable::named("rule_0", Rule::non_terminal(1)), + Variable::hidden("_rule_1", Rule::string("")), + ])) + .is_err()); + } + fn build_grammar(variables: Vec) -> InternedGrammar { InternedGrammar { variables, diff --git a/cli/src/generate/prepare_grammar/flatten_grammar.rs b/cli/src/generate/prepare_grammar/flatten_grammar.rs index 4b707bee..04cfe265 100644 --- a/cli/src/generate/prepare_grammar/flatten_grammar.rs +++ b/cli/src/generate/prepare_grammar/flatten_grammar.rs @@ -173,7 +173,7 @@ fn flatten_variable(variable: Variable) -> SyntaxVariable { } } -fn symbol_is_used(variables: &[SyntaxVariable], symbol: Symbol) -> bool { +pub fn symbol_is_used(variables: &[SyntaxVariable], symbol: Symbol) -> bool { for variable in variables { for production in &variable.productions { for step in &production.steps { @@ -192,8 +192,10 @@ pub(super) fn flatten_grammar(grammar: ExtractedSyntaxGrammar) -> Result *const (); } -/// The tree-sitter [`LanguageFn`] for this grammar. +/// The tree-sitter [`LanguageFn`][LanguageFn] for this grammar. +/// +/// [LanguageFn]: https://docs.rs/tree-sitter-language/*/tree_sitter_language/struct.LanguageFn.html pub const LANGUAGE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_PARSER_NAME) }; /// The content of the [`node-types.json`][] file for this grammar. diff --git a/cli/src/generate/templates/package.swift b/cli/src/generate/templates/package.swift index c1be93db..d1053b46 100644 --- a/cli/src/generate/templates/package.swift +++ b/cli/src/generate/templates/package.swift @@ -14,29 +14,6 @@ let package = Package( name: "TreeSitterCAMEL_PARSER_NAME", dependencies: [], path: ".", - exclude: [ - "Cargo.toml", - "Makefile", - "binding.gyp", - "bindings/c", - "bindings/go", - "bindings/node", - "bindings/python", - "bindings/rust", - "prebuilds", - "grammar.js", - "package.json", - "package-lock.json", - "pyproject.toml", - "setup.py", - "test", - "examples", - ".editorconfig", - ".github", - ".gitignore", - ".gitattributes", - ".gitmodules", - ], sources: [ "src/parser.c", // NOTE: if your language has an external scanner, add it here. diff --git a/cli/src/main.rs b/cli/src/main.rs index ce4aae95..0f0d911c 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -263,13 +263,13 @@ struct Fuzz { pub skip: Option>, #[arg(long, help = "Subdirectory to the language")] pub subdir: Option, - #[arg(long, short, help = "Maximum number of edits to perform per fuzz test")] + #[arg(long, help = "Maximum number of edits to perform per fuzz test")] pub edits: Option, - #[arg(long, short, help = "Number of fuzzing iterations to run per test")] + #[arg(long, help = "Number of fuzzing iterations to run per test")] pub iterations: Option, #[arg(long, short, help = "Regex pattern to filter tests")] pub filter: Option, - #[arg(long, short, help = "Enable logging of graphs and input")] + #[arg(long, help = "Enable logging of graphs and input")] pub log_graphs: bool, #[arg(long, short, help = "Enable parser logging")] pub log: bool, diff --git a/cli/src/query.rs b/cli/src/query.rs index bffa0588..f32c5450 100644 --- a/cli/src/query.rs +++ b/cli/src/query.rs @@ -9,7 +9,7 @@ use std::{ use anyhow::{Context, Result}; use tree_sitter::{Language, Parser, Point, Query, QueryCursor}; -use crate::query_testing; +use crate::query_testing::{self, to_utf8_point}; #[allow(clippy::too_many_arguments)] pub fn query_files_at_paths( @@ -70,8 +70,8 @@ pub fn query_files_at_paths( } results.push(query_testing::CaptureInfo { name: (*capture_name).to_string(), - start: capture.node.start_position(), - end: capture.node.end_position(), + start: to_utf8_point(capture.node.start_position(), source_code.as_slice()), + end: to_utf8_point(capture.node.end_position(), source_code.as_slice()), }); } } else { @@ -100,8 +100,8 @@ pub fn query_files_at_paths( } results.push(query_testing::CaptureInfo { name: (*capture_name).to_string(), - start: capture.node.start_position(), - end: capture.node.end_position(), + start: to_utf8_point(capture.node.start_position(), source_code.as_slice()), + end: to_utf8_point(capture.node.end_position(), source_code.as_slice()), }); } } diff --git a/cli/src/query_testing.rs b/cli/src/query_testing.rs index cdf2e988..258770d1 100644 --- a/cli/src/query_testing.rs +++ b/cli/src/query_testing.rs @@ -1,6 +1,7 @@ use std::fs; use anyhow::{anyhow, Result}; +use bstr::{BStr, ByteSlice}; use lazy_static::lazy_static; use regex::Regex; use tree_sitter::{Language, Parser, Point}; @@ -9,16 +10,56 @@ lazy_static! { static ref CAPTURE_NAME_REGEX: Regex = Regex::new("[\\w_\\-.]+").unwrap(); } +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct Utf8Point { + pub row: usize, + pub column: usize, +} + +impl std::fmt::Display for Utf8Point { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "({}, {})", self.row, self.column) + } +} + +impl Utf8Point { + pub const fn new(row: usize, column: usize) -> Self { + Self { row, column } + } +} + +pub fn to_utf8_point(point: Point, source: &[u8]) -> Utf8Point { + if point.column == 0 { + return Utf8Point::new(point.row, 0); + } + + let bstr = BStr::new(source); + let line = bstr.lines_with_terminator().nth(point.row).unwrap(); + let mut utf8_column = 0; + + for (_, grapheme_end, _) in line.grapheme_indices() { + utf8_column += 1; + if grapheme_end >= point.column { + break; + } + } + + Utf8Point { + row: point.row, + column: utf8_column, + } +} + #[derive(Debug, Eq, PartialEq)] pub struct CaptureInfo { pub name: String, - pub start: Point, - pub end: Point, + pub start: Utf8Point, + pub end: Utf8Point, } #[derive(Debug, PartialEq, Eq)] pub struct Assertion { - pub position: Point, + pub position: Utf8Point, pub negative: bool, pub expected_capture_name: String, } @@ -27,7 +68,7 @@ impl Assertion { #[must_use] pub fn new(row: usize, col: usize, negative: bool, expected_capture_name: String) -> Self { Self { - position: Point::new(row, col), + position: Utf8Point::new(row, col), negative, expected_capture_name, } @@ -62,7 +103,7 @@ pub fn parse_position_comments( if let Ok(text) = node.utf8_text(source) { let mut position = node.start_position(); if position.row > 0 { - // Find the arrow character ("^" or '<-") in the comment. A left arrow + // Find the arrow character ("^" or "<-") in the comment. A left arrow // refers to the column where the comment node starts. An up arrow refers // to its own column. let mut has_left_caret = false; @@ -103,7 +144,7 @@ pub fn parse_position_comments( { assertion_ranges.push((node.start_position(), node.end_position())); result.push(Assertion { - position, + position: to_utf8_point(position, source), negative, expected_capture_name: mat.as_str().to_string(), }); diff --git a/cli/src/test.rs b/cli/src/test.rs index 250c8efe..3ce12a48 100644 --- a/cli/src/test.rs +++ b/cli/src/test.rs @@ -1,5 +1,5 @@ use std::{ - collections::{BTreeMap, HashSet}, + collections::BTreeMap, ffi::OsStr, fs, io::{self, Write}, @@ -27,7 +27,7 @@ lazy_static! { (?P(?:=+){3,}) (?P[^=\r\n][^\r\n]*)? \r?\n - (?P(?:[^=][^\r\n]*\r?\n)+) + (?P(?:([^=\r\n]|\s+:)[^\r\n]*\r?\n)+) ===+ (?P[^=\r\n][^\r\n]*)?\r?\n" ) @@ -59,6 +59,7 @@ pub enum TestEntry { header_delim_len: usize, divider_delim_len: usize, has_fields: bool, + attributes_str: String, attributes: TestAttributes, }, } @@ -171,10 +172,22 @@ pub fn run_tests_at_path(parser: &mut Parser, opts: &mut TestOptions) -> Result< print_diff_key(); } for (i, (name, actual, expected)) in failures.iter().enumerate() { - println!("\n {}. {name}:", i + 1); - let actual = format_sexp(actual, 2); - let expected = format_sexp(expected, 2); - print_diff(&actual, &expected, opts.color); + if expected == "NO ERROR" { + println!("\n {}. {name}:\n", i + 1); + println!(" Expected an ERROR node, but got:"); + println!( + " {}", + paint( + opts.color.then_some(AnsiColor::Red), + &format_sexp(actual, 2) + ) + ); + } else { + println!("\n {}. {name}:", i + 1); + let actual = format_sexp(actual, 2); + let expected = format_sexp(expected, 2); + print_diff(&actual, &expected, opts.color); + } } if has_parse_errors { @@ -326,7 +339,7 @@ fn run_tests( opts: &mut TestOptions, mut indent_level: i32, failures: &mut Vec<(String, String, String)>, - corrected_entries: &mut Vec<(String, String, String, usize, usize)>, + corrected_entries: &mut Vec<(String, String, String, String, usize, usize)>, has_parse_errors: &mut bool, ) -> Result { match test_entry { @@ -337,6 +350,7 @@ fn run_tests( header_delim_len, divider_delim_len, has_fields, + attributes_str, attributes, } => { print!("{}", " ".repeat(indent_level as usize)); @@ -376,12 +390,42 @@ fn run_tests( opts.test_num, paint(opts.color.then_some(AnsiColor::Green), &name) ); + if opts.update { + let input = String::from_utf8(input.clone()).unwrap(); + let output = format_sexp(&output, 0); + corrected_entries.push(( + name.clone(), + input, + output, + attributes_str.clone(), + header_delim_len, + divider_delim_len, + )); + } } else { + if opts.update { + let input = String::from_utf8(input.clone()).unwrap(); + // Keep the original `expected` output if the actual output has no error + let output = format_sexp(&output, 0); + corrected_entries.push(( + name.clone(), + input, + output, + attributes_str.clone(), + header_delim_len, + divider_delim_len, + )); + } println!( "{:>3}.  {}", opts.test_num, paint(opts.color.then_some(AnsiColor::Red), &name) ); + failures.push(( + name.clone(), + tree.root_node().to_sexp(), + "NO ERROR".to_string(), + )); } if attributes.fail_fast { @@ -406,6 +450,7 @@ fn run_tests( name.clone(), input, output, + attributes_str.clone(), header_delim_len, divider_delim_len, )); @@ -429,6 +474,7 @@ fn run_tests( name.clone(), input, expected_output, + attributes_str.clone(), header_delim_len, divider_delim_len, )); @@ -437,6 +483,7 @@ fn run_tests( name.clone(), input, actual_output, + attributes_str.clone(), header_delim_len, divider_delim_len, )); @@ -470,64 +517,76 @@ fn run_tests( } TestEntry::Group { name, - mut children, + children, file_path, } => { - // track which tests are being skipped to maintain consistent numbering while using - // filters - let mut skipped_tests = HashSet::new(); - let mut advance_counter = opts.test_num; - children.retain(|child| match child { - TestEntry::Example { name, .. } => { - if let Some(filter) = opts.filter { - if !name.contains(filter) { - skipped_tests.insert(advance_counter); - advance_counter += 1; - return false; - } - } - if let Some(include) = &opts.include { - if !include.is_match(name) { - skipped_tests.insert(advance_counter); - advance_counter += 1; - return false; - } - } - if let Some(exclude) = &opts.exclude { - if exclude.is_match(name) { - skipped_tests.insert(advance_counter); - advance_counter += 1; - return false; - } - } - advance_counter += 1; - true - } - TestEntry::Group { .. } => { - advance_counter += count_subtests(child); - true - } - }); - if children.is_empty() { - opts.test_num = advance_counter; return Ok(true); } - if indent_level > 0 { - print!("{}", " ".repeat(indent_level as usize)); - println!("{name}:"); - } - - let failure_count = failures.len(); - indent_level += 1; + let mut advance_counter = opts.test_num; + let failure_count = failures.len(); + let mut has_printed = false; + let mut skipped_tests = 0; + + let matches_filter = |name: &str, opts: &TestOptions| { + if let Some(filter) = opts.filter { + name.contains(filter) + } else if let Some(include) = &opts.include { + include.is_match(name) + } else if let Some(exclude) = &opts.exclude { + !exclude.is_match(name) + } else { + true + } + }; + + let mut should_skip = |entry: &TestEntry, opts: &TestOptions| match entry { + TestEntry::Example { name, .. } => { + advance_counter += 1; + !matches_filter(name, opts) + } + TestEntry::Group { .. } => { + advance_counter += count_subtests(entry); + false + } + }; + for child in children { - if let TestEntry::Example { .. } = child { - while skipped_tests.remove(&opts.test_num) { + if let TestEntry::Example { + ref name, + ref input, + ref output, + ref attributes_str, + header_delim_len, + divider_delim_len, + .. + } = child + { + if should_skip(&child, opts) { + let input = String::from_utf8(input.clone()).unwrap(); + let output = format_sexp(output, 0); + corrected_entries.push(( + name.clone(), + input, + output, + attributes_str.clone(), + header_delim_len, + divider_delim_len, + )); + opts.test_num += 1; + skipped_tests += 1; + + continue; } } + if !has_printed && indent_level > 1 { + has_printed = true; + print!("{}", " ".repeat((indent_level - 1) as usize)); + println!("{name}:"); + } if !run_tests( parser, child, @@ -542,7 +601,7 @@ fn run_tests( } } - opts.test_num += skipped_tests.len(); + opts.test_num += skipped_tests; if let Some(file_path) = file_path { if opts.update && failures.len() - failure_count > 0 { @@ -566,7 +625,7 @@ fn count_subtests(test_entry: &TestEntry) -> usize { fn write_tests( file_path: &Path, - corrected_entries: &[(String, String, String, usize, usize)], + corrected_entries: &[(String, String, String, String, usize, usize)], ) -> Result<()> { let mut buffer = fs::File::create(file_path)?; write_tests_to_buffer(&mut buffer, corrected_entries) @@ -574,9 +633,9 @@ fn write_tests( fn write_tests_to_buffer( buffer: &mut impl Write, - corrected_entries: &[(String, String, String, usize, usize)], + corrected_entries: &[(String, String, String, String, usize, usize)], ) -> Result<()> { - for (i, (name, input, output, header_delim_len, divider_delim_len)) in + for (i, (name, input, output, attributes_str, header_delim_len, divider_delim_len)) in corrected_entries.iter().enumerate() { if i > 0 { @@ -584,8 +643,13 @@ fn write_tests_to_buffer( } writeln!( buffer, - "{}\n{name}\n{}\n{input}\n{}\n\n{}", + "{}\n{name}\n{}{}\n{input}\n{}\n\n{}", "=".repeat(*header_delim_len), + if attributes_str.is_empty() { + attributes_str.clone() + } else { + format!("{}\n", attributes_str) + }, "=".repeat(*header_delim_len), "-".repeat(*divider_delim_len), output.trim() @@ -643,6 +707,7 @@ fn parse_test_content(name: String, content: &str, file_path: Option) - let mut children = Vec::new(); let bytes = content.as_bytes(); let mut prev_name = String::new(); + let mut prev_attributes_str = String::new(); let mut prev_header_end = 0; // Find the first test header in the file, and determine if it has a @@ -673,17 +738,20 @@ fn parse_test_content(name: String, content: &str, file_path: Option) - .map_or("".as_bytes(), |m| m.as_bytes()); let mut test_name = String::new(); + let mut attributes_str = String::new(); + let mut seen_marker = false; - for line in str::from_utf8(test_name_and_markers) - .unwrap() - .lines() + let test_name_and_markers = str::from_utf8(test_name_and_markers).unwrap(); + for line in test_name_and_markers + .split_inclusive('\n') .filter(|s| !s.is_empty()) { - match line.split('(').next().unwrap() { + let trimmed_line = line.trim(); + match trimmed_line.split('(').next().unwrap() { ":skip" => (seen_marker, skip) = (true, true), ":platform" => { - if let Some(platforms) = line.strip_prefix(':').and_then(|s| { + if let Some(platforms) = trimmed_line.strip_prefix(':').and_then(|s| { s.strip_prefix("platform(") .and_then(|s| s.strip_suffix(')')) }) { @@ -696,7 +764,7 @@ fn parse_test_content(name: String, content: &str, file_path: Option) - ":fail-fast" => (seen_marker, fail_fast) = (true, true), ":error" => (seen_marker, error) = (true, true), ":language" => { - if let Some(lang) = line.strip_prefix(':').and_then(|s| { + if let Some(lang) = trimmed_line.strip_prefix(':').and_then(|s| { s.strip_prefix("language(") .and_then(|s| s.strip_suffix(')')) }) { @@ -706,11 +774,11 @@ fn parse_test_content(name: String, content: &str, file_path: Option) - } _ if !seen_marker => { test_name.push_str(line); - test_name.push('\n'); } _ => {} } } + attributes_str.push_str(test_name_and_markers.strip_prefix(&test_name).unwrap()); // prefer skip over error, both shouldn't be set if skip { @@ -729,10 +797,16 @@ fn parse_test_content(name: String, content: &str, file_path: Option) - } else { Some(test_name.trim_end().to_string()) }; + let attributes_str = if attributes_str.is_empty() { + None + } else { + Some(attributes_str.trim_end().to_string()) + }; Some(( header_delim_len, header_range, test_name, + attributes_str, TestAttributes { skip, platform: platform.unwrap_or(true), @@ -747,12 +821,15 @@ fn parse_test_content(name: String, content: &str, file_path: Option) - }); let (mut prev_header_len, mut prev_attributes) = (80, TestAttributes::default()); - for (header_delim_len, header_range, test_name, attributes) in header_matches.chain(Some(( - 80, - bytes.len()..bytes.len(), - None, - TestAttributes::default(), - ))) { + for (header_delim_len, header_range, test_name, attributes_str, attributes) in header_matches + .chain(Some(( + 80, + bytes.len()..bytes.len(), + None, + None, + TestAttributes::default(), + ))) + { // Find the longest line of dashes following each test description. That line // separates the input from the expected output. Ignore any matches whose suffix // does not match the first suffix in the file. @@ -804,6 +881,7 @@ fn parse_test_content(name: String, content: &str, file_path: Option) - header_delim_len: prev_header_len, divider_delim_len, has_fields, + attributes_str: prev_attributes_str, attributes: prev_attributes, }; @@ -813,6 +891,7 @@ fn parse_test_content(name: String, content: &str, file_path: Option) - } prev_attributes = attributes; prev_name = test_name.unwrap_or_default(); + prev_attributes_str = attributes_str.unwrap_or_default(); prev_header_len = header_delim_len; prev_header_end = header_range.end; } @@ -866,6 +945,7 @@ d header_delim_len: 15, divider_delim_len: 3, has_fields: false, + attributes_str: String::new(), attributes: TestAttributes::default(), }, TestEntry::Example { @@ -875,6 +955,7 @@ d header_delim_len: 16, divider_delim_len: 3, has_fields: false, + attributes_str: String::new(), attributes: TestAttributes::default(), }, ], @@ -925,6 +1006,7 @@ abc header_delim_len: 18, divider_delim_len: 7, has_fields: false, + attributes_str: String::new(), attributes: TestAttributes::default(), }, TestEntry::Example { @@ -934,6 +1016,7 @@ abc header_delim_len: 25, divider_delim_len: 19, has_fields: false, + attributes_str: String::new(), attributes: TestAttributes::default(), }, ], @@ -999,6 +1082,7 @@ abc "title 1".to_string(), "input 1".to_string(), "output 1".to_string(), + String::new(), 80, 80, ), @@ -1006,6 +1090,7 @@ abc "title 2".to_string(), "input 2".to_string(), "output 2".to_string(), + String::new(), 80, 80, ), @@ -1086,6 +1171,7 @@ code header_delim_len: 18, divider_delim_len: 3, has_fields: false, + attributes_str: String::new(), attributes: TestAttributes::default(), }, TestEntry::Example { @@ -1095,6 +1181,7 @@ code header_delim_len: 18, divider_delim_len: 3, has_fields: false, + attributes_str: String::new(), attributes: TestAttributes::default(), }, TestEntry::Example { @@ -1104,6 +1191,7 @@ code header_delim_len: 25, divider_delim_len: 3, has_fields: false, + attributes_str: String::new(), attributes: TestAttributes::default(), } ], @@ -1155,7 +1243,27 @@ NOT A TEST HEADER ---asdf\()[]|{}*+?^$.- (a) - " + +==============================asdf\()[]|{}*+?^$.- +Test containing equals +==============================asdf\()[]|{}*+?^$.- + +=== + +------------------------------asdf\()[]|{}*+?^$.- + +(a) + +==============================asdf\()[]|{}*+?^$.- +Subsequent test containing equals +==============================asdf\()[]|{}*+?^$.- + +=== + +------------------------------asdf\()[]|{}*+?^$.- + +(a) +" .trim(), None, ); @@ -1165,7 +1273,7 @@ NOT A TEST HEADER =========================\n\ -------------------------\n" .to_vec(); - assert_eq!( + pretty_assertions::assert_eq!( entry, TestEntry::Group { name: "the-filename".to_string(), @@ -1177,6 +1285,7 @@ NOT A TEST HEADER header_delim_len: 18, divider_delim_len: 3, has_fields: false, + attributes_str: String::new(), attributes: TestAttributes::default(), }, TestEntry::Example { @@ -1186,6 +1295,7 @@ NOT A TEST HEADER header_delim_len: 18, divider_delim_len: 3, has_fields: false, + attributes_str: String::new(), attributes: TestAttributes::default(), }, TestEntry::Example { @@ -1195,6 +1305,27 @@ NOT A TEST HEADER header_delim_len: 25, divider_delim_len: 3, has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + }, + TestEntry::Example { + name: "Test containing equals".to_string(), + input: "\n===\n".into(), + output: "(a)".into(), + header_delim_len: 30, + divider_delim_len: 30, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + }, + TestEntry::Example { + name: "Subsequent test containing equals".to_string(), + input: "\n===\n".into(), + output: "(a)".into(), + header_delim_len: 30, + divider_delim_len: 30, + has_fields: false, + attributes_str: String::new(), attributes: TestAttributes::default(), } ], @@ -1240,6 +1371,7 @@ code with ---- header_delim_len: 15, divider_delim_len: 3, has_fields: false, + attributes_str: String::new(), attributes: TestAttributes::default(), }, TestEntry::Example { @@ -1249,6 +1381,7 @@ code with ---- header_delim_len: 20, divider_delim_len: 3, has_fields: false, + attributes_str: String::new(), attributes: TestAttributes::default(), } ] @@ -1286,6 +1419,7 @@ a header_delim_len: 21, divider_delim_len: 3, has_fields: false, + attributes_str: ":skip".to_string(), attributes: TestAttributes { skip: true, platform: true, @@ -1313,6 +1447,7 @@ a ============================= Test with bad platform marker :platform({}) + :language(foo) ============================= a @@ -1342,6 +1477,7 @@ a header_delim_len: 25, divider_delim_len: 3, has_fields: false, + attributes_str: format!(":platform({})\n:fail-fast", std::env::consts::OS), attributes: TestAttributes { skip: false, platform: true, @@ -1357,6 +1493,11 @@ a header_delim_len: 29, divider_delim_len: 3, has_fields: false, + attributes_str: if std::env::consts::OS == "linux" { + ":platform(macos)\n\n:language(foo)".to_string() + } else { + ":platform(linux)\n\n:language(foo)".to_string() + }, attributes: TestAttributes { skip: false, platform: false, diff --git a/cli/src/test_highlight.rs b/cli/src/test_highlight.rs index 541d98fd..34be438f 100644 --- a/cli/src/test_highlight.rs +++ b/cli/src/test_highlight.rs @@ -7,7 +7,7 @@ use tree_sitter_highlight::{Highlight, HighlightConfiguration, HighlightEvent, H use tree_sitter_loader::{Config, Loader}; use super::{ - query_testing::{parse_position_comments, Assertion}, + query_testing::{parse_position_comments, to_utf8_point, Assertion, Utf8Point}, test::paint, util, }; @@ -141,7 +141,7 @@ fn test_highlights_indented( } pub fn iterate_assertions( assertions: &[Assertion], - highlights: &[(Point, Point, Highlight)], + highlights: &[(Utf8Point, Utf8Point, Highlight)], highlight_names: &[String], ) -> Result { // Iterate through all of the highlighting assertions, checking each one against the @@ -224,7 +224,7 @@ pub fn get_highlight_positions( highlighter: &mut Highlighter, highlight_config: &HighlightConfiguration, source: &[u8], -) -> Result> { +) -> Result> { let mut row = 0; let mut column = 0; let mut byte_offset = 0; @@ -261,7 +261,10 @@ pub fn get_highlight_positions( } } if let Some(highlight) = highlight_stack.last() { - result.push((start_position, Point::new(row, column), *highlight)); + let utf8_start_position = to_utf8_point(start_position, source.as_bytes()); + let utf8_end_position = + to_utf8_point(Point::new(row, column), source.as_bytes()); + result.push((utf8_start_position, utf8_end_position, *highlight)); } } } diff --git a/cli/src/test_tags.rs b/cli/src/test_tags.rs index c5a1dc02..5b290bda 100644 --- a/cli/src/test_tags.rs +++ b/cli/src/test_tags.rs @@ -2,12 +2,11 @@ use std::{fs, path::Path}; use anstyle::AnsiColor; use anyhow::{anyhow, Result}; -use tree_sitter::Point; use tree_sitter_loader::{Config, Loader}; use tree_sitter_tags::{TagsConfiguration, TagsContext}; use super::{ - query_testing::{parse_position_comments, Assertion}, + query_testing::{parse_position_comments, to_utf8_point, Assertion, Utf8Point}, test::paint, util, }; @@ -168,7 +167,7 @@ pub fn get_tag_positions( tags_context: &mut TagsContext, tags_config: &TagsConfiguration, source: &[u8], -) -> Result> { +) -> Result> { let (tags_iter, _has_error) = tags_context.generate_tags(tags_config, source, None)?; let tag_positions = tags_iter .filter_map(std::result::Result::ok) @@ -179,7 +178,11 @@ pub fn get_tag_positions( } else { format!("reference.{tag_postfix}") }; - (tag.span.start, tag.span.end, tag_name) + ( + to_utf8_point(tag.span.start, source), + to_utf8_point(tag.span.end, source), + tag_name, + ) }) .collect(); Ok(tag_positions) diff --git a/cli/src/tests/node_test.rs b/cli/src/tests/node_test.rs index e05ed932..559f6f59 100644 --- a/cli/src/tests/node_test.rs +++ b/cli/src/tests/node_test.rs @@ -290,6 +290,16 @@ fn test_parent_of_zero_width_node() { function_definition ); assert_eq!(function_definition.child_containing_descendant(block), None); + + let code = ""; + parser.set_language(&get_language("html")).unwrap(); + + let tree = parser.parse(code, None).unwrap(); + let root = tree.root_node(); + let script_element = root.child(0).unwrap(); + let raw_text = script_element.child(1).unwrap(); + let parent = raw_text.parent().unwrap(); + assert_eq!(parent, script_element); } #[test] @@ -603,6 +613,26 @@ fn test_node_descendant_for_range() { assert_eq!(pair_node.end_byte(), string_index + 9); assert_eq!(pair_node.start_position(), Point::new(6, 4)); assert_eq!(pair_node.end_position(), Point::new(6, 13)); + + // Zero-width token + { + let code = ""; + let mut parser = Parser::new(); + parser.set_language(&get_language("html")).unwrap(); + + let tree = parser.parse(code, None).unwrap(); + let root = tree.root_node(); + + let child = root + .named_descendant_for_point_range(Point::new(0, 8), Point::new(0, 8)) + .unwrap(); + assert_eq!(child.kind(), "raw_text"); + + let child2 = root.named_descendant_for_byte_range(8, 8).unwrap(); + assert_eq!(child2.kind(), "raw_text"); + + assert_eq!(child, child2); + } } #[test] diff --git a/cli/src/tests/parser_test.rs b/cli/src/tests/parser_test.rs index bd19717b..7b534be1 100644 --- a/cli/src/tests/parser_test.rs +++ b/cli/src/tests/parser_test.rs @@ -501,6 +501,67 @@ h + i ); } +#[test] +fn test_parsing_after_editing_tree_that_depends_on_column_position() { + let dir = fixtures_dir() + .join("test_grammars") + .join("depends_on_column"); + + let grammar_json = load_grammar_file(&dir.join("grammar.js"), None).unwrap(); + let (grammar_name, parser_code) = generate_parser_for_grammar(grammar_json.as_str()).unwrap(); + + let mut parser = Parser::new(); + parser + .set_language(&get_test_language(&grammar_name, &parser_code, Some(&dir))) + .unwrap(); + + let mut code = b"\n x".to_vec(); + let mut tree = parser.parse(&code, None).unwrap(); + assert_eq!(tree.root_node().to_sexp(), "(x_is_at (odd_column))"); + + perform_edit( + &mut tree, + &mut code, + &Edit { + position: 1, + deleted_length: 0, + inserted_text: b" ".to_vec(), + }, + ) + .unwrap(); + + assert_eq!(code, b"\n x"); + + let mut recorder = ReadRecorder::new(&code); + let mut tree = parser + .parse_with(&mut |i, _| recorder.read(i), Some(&tree)) + .unwrap(); + + assert_eq!(tree.root_node().to_sexp(), "(x_is_at (even_column))",); + assert_eq!(recorder.strings_read(), vec!["\n x"]); + + perform_edit( + &mut tree, + &mut code, + &Edit { + position: 1, + deleted_length: 0, + inserted_text: b"\n".to_vec(), + }, + ) + .unwrap(); + + assert_eq!(code, b"\n\n x"); + + let mut recorder = ReadRecorder::new(&code); + let tree = parser + .parse_with(&mut |i, _| recorder.read(i), Some(&tree)) + .unwrap(); + + assert_eq!(tree.root_node().to_sexp(), "(x_is_at (even_column))",); + assert_eq!(recorder.strings_read(), vec!["\n\n x"]); +} + #[test] fn test_parsing_after_detecting_error_in_the_middle_of_a_string_token() { let mut parser = Parser::new(); diff --git a/cli/src/tests/test_highlight_test.rs b/cli/src/tests/test_highlight_test.rs index 8699c2a6..054e33f8 100644 --- a/cli/src/tests/test_highlight_test.rs +++ b/cli/src/tests/test_highlight_test.rs @@ -1,9 +1,9 @@ -use tree_sitter::{Parser, Point}; +use tree_sitter::Parser; use tree_sitter_highlight::{Highlight, Highlighter}; use super::helpers::fixtures::{get_highlight_config, get_language, test_loader}; use crate::{ - query_testing::{parse_position_comments, Assertion}, + query_testing::{parse_position_comments, Assertion, Utf8Point}, test_highlight::get_highlight_positions, }; @@ -28,6 +28,9 @@ fn test_highlight_test_with_basic_test() { " // ^ variable", " // ^ !variable", "};", + "var y̆y̆y̆y̆ = function() {}", + " // ^ function", + " // ^ keyword", ] .join("\n"); @@ -40,6 +43,8 @@ fn test_highlight_test_with_basic_test() { Assertion::new(1, 11, false, String::from("keyword")), Assertion::new(4, 9, false, String::from("variable")), Assertion::new(4, 11, true, String::from("variable")), + Assertion::new(8, 5, false, String::from("function")), + Assertion::new(8, 11, false, String::from("keyword")), ] ); @@ -50,13 +55,16 @@ fn test_highlight_test_with_basic_test() { assert_eq!( highlight_positions, &[ - (Point::new(1, 0), Point::new(1, 3), Highlight(2)), // "var" - (Point::new(1, 4), Point::new(1, 7), Highlight(0)), // "abc" - (Point::new(1, 10), Point::new(1, 18), Highlight(2)), // "function" - (Point::new(1, 19), Point::new(1, 20), Highlight(1)), // "d" - (Point::new(4, 2), Point::new(4, 8), Highlight(2)), // "return" - (Point::new(4, 9), Point::new(4, 10), Highlight(1)), // "d" - (Point::new(4, 13), Point::new(4, 14), Highlight(1)), // "e" + (Utf8Point::new(1, 0), Utf8Point::new(1, 3), Highlight(2)), // "var" + (Utf8Point::new(1, 4), Utf8Point::new(1, 7), Highlight(0)), // "abc" + (Utf8Point::new(1, 10), Utf8Point::new(1, 18), Highlight(2)), // "function" + (Utf8Point::new(1, 19), Utf8Point::new(1, 20), Highlight(1)), // "d" + (Utf8Point::new(4, 2), Utf8Point::new(4, 8), Highlight(2)), // "return" + (Utf8Point::new(4, 9), Utf8Point::new(4, 10), Highlight(1)), // "d" + (Utf8Point::new(4, 13), Utf8Point::new(4, 14), Highlight(1)), // "e" + (Utf8Point::new(8, 0), Utf8Point::new(8, 3), Highlight(2)), // "var" + (Utf8Point::new(8, 4), Utf8Point::new(8, 8), Highlight(0)), // "y̆y̆y̆y̆" + (Utf8Point::new(8, 11), Utf8Point::new(8, 19), Highlight(2)), // "function" ] ); } diff --git a/cli/src/tests/test_tags_test.rs b/cli/src/tests/test_tags_test.rs index 5e7bf9c9..5f7b88fc 100644 --- a/cli/src/tests/test_tags_test.rs +++ b/cli/src/tests/test_tags_test.rs @@ -1,9 +1,9 @@ -use tree_sitter::{Parser, Point}; +use tree_sitter::Parser; use tree_sitter_tags::TagsContext; use super::helpers::fixtures::{get_language, get_tags_config}; use crate::{ - query_testing::{parse_position_comments, Assertion}, + query_testing::{parse_position_comments, Assertion, Utf8Point}, test_tags::get_tag_positions, }; @@ -43,18 +43,18 @@ fn test_tags_test_with_basic_test() { tag_positions, &[ ( - Point::new(1, 4), - Point::new(1, 7), + Utf8Point::new(1, 4), + Utf8Point::new(1, 7), "definition.function".to_string() ), ( - Point::new(3, 8), - Point::new(3, 11), + Utf8Point::new(3, 8), + Utf8Point::new(3, 11), "reference.call".to_string() ), ( - Point::new(5, 11), - Point::new(5, 12), + Utf8Point::new(5, 11), + Utf8Point::new(5, 12), "reference.call".to_string() ), ] diff --git a/cli/src/tests/tree_test.rs b/cli/src/tests/tree_test.rs index 3e4b2775..62cc23cd 100644 --- a/cli/src/tests/tree_test.rs +++ b/cli/src/tests/tree_test.rs @@ -702,6 +702,33 @@ fn test_consistency_with_mid_codepoint_edit() { assert_eq!(tree3.root_node().to_sexp(), tree.root_node().to_sexp()); } +#[test] +fn test_tree_cursor_on_aliased_root_with_extra_child() { + let source = r#" +fn main() { + C/* hi */::::E; +} +"#; + + let mut parser = Parser::new(); + parser.set_language(&get_language("rust")).unwrap(); + + let tree = parser.parse(source, None).unwrap(); + + let function = tree.root_node().child(0).unwrap(); + let block = function.child(3).unwrap(); + let expression_statement = block.child(1).unwrap(); + let scoped_identifier = expression_statement.child(0).unwrap(); + let generic_type = scoped_identifier.child(0).unwrap(); + assert_eq!(generic_type.kind(), "generic_type"); + + let mut cursor = generic_type.walk(); + assert!(cursor.goto_first_child()); + assert_eq!(cursor.node().kind(), "type_identifier"); + assert!(cursor.goto_next_sibling()); + assert_eq!(cursor.node().kind(), "block_comment"); +} + fn index_of(text: &[u8], substring: &str) -> usize { str::from_utf8(text).unwrap().find(substring).unwrap() } diff --git a/highlight/README.md b/highlight/README.md index 982e510a..4ca76d6c 100644 --- a/highlight/README.md +++ b/highlight/README.md @@ -12,8 +12,8 @@ to parse, to your `Cargo.toml`: ```toml [dependencies] -tree-sitter-highlight = "^0.21.0" -tree-sitter-javascript = "0.20.3" +tree-sitter-highlight = "0.22.0" +tree-sitter-javascript = "0.21.3" ``` Define the list of highlight names that you will recognize: @@ -61,9 +61,8 @@ let mut javascript_config = HighlightConfiguration::new( javascript_language, "javascript", tree_sitter_javascript::HIGHLIGHT_QUERY, - tree_sitter_javascript::INJECTION_QUERY, + tree_sitter_javascript::INJECTIONS_QUERY, tree_sitter_javascript::LOCALS_QUERY, - false, ).unwrap(); ``` diff --git a/lib/Cargo.toml b/lib/Cargo.toml index c5dc919d..a0f04259 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -33,7 +33,7 @@ wasm = ["wasmtime-c-api"] [dependencies] regex = { version = "1.10.6", default-features = false, features = ["unicode"] } regex-syntax = { version = "0.8.4", default-features = false } -tree-sitter-language = { version = "0.1", path = "language" } +tree-sitter-language = { version = "0.1.1", path = "language" } [dependencies.wasmtime-c-api] version = "24.0.0" @@ -43,7 +43,7 @@ default-features = false features = ["cranelift"] [build-dependencies] -bindgen = { version = "0.69.4", optional = true } +bindgen = { version = "0.70.1", optional = true } cc.workspace = true [lib] diff --git a/lib/binding_rust/ffi.rs b/lib/binding_rust/ffi.rs index ba3bcab6..755d2d52 100644 --- a/lib/binding_rust/ffi.rs +++ b/lib/binding_rust/ffi.rs @@ -8,7 +8,7 @@ include!(concat!(env!("OUT_DIR"), "/bindings.rs")); #[cfg(not(feature = "bindgen"))] include!("./bindings.rs"); -#[cfg(any(unix, target_os = "wasi"))] +#[cfg(unix)] #[cfg(feature = "std")] extern "C" { pub(crate) fn _ts_dup(fd: std::os::raw::c_int) -> std::os::raw::c_int; diff --git a/lib/binding_rust/lib.rs b/lib/binding_rust/lib.rs index c97fd5ca..c97b4eaa 100644 --- a/lib/binding_rust/lib.rs +++ b/lib/binding_rust/lib.rs @@ -548,13 +548,14 @@ impl Parser { /// want to pipe these graphs directly to a `dot(1)` process in order to /// generate SVG output. #[doc(alias = "ts_parser_print_dot_graphs")] + #[cfg(not(target_os = "wasi"))] #[cfg(feature = "std")] pub fn print_dot_graphs( &mut self, - #[cfg(any(unix, target_os = "wasi"))] file: &impl AsRawFd, + #[cfg(unix)] file: &impl AsRawFd, #[cfg(windows)] file: &impl AsRawHandle, ) { - #[cfg(any(unix, target_os = "wasi"))] + #[cfg(unix)] { let fd = file.as_raw_fd(); unsafe { @@ -946,13 +947,14 @@ impl Tree { /// graph directly to a `dot(1)` process in order to generate SVG /// output. #[doc(alias = "ts_tree_print_dot_graph")] + #[cfg(not(target_os = "wasi"))] #[cfg(feature = "std")] pub fn print_dot_graph( &self, - #[cfg(any(unix, target_os = "wasi"))] file: &impl AsRawFd, + #[cfg(unix)] file: &impl AsRawFd, #[cfg(windows)] file: &impl AsRawHandle, ) { - #[cfg(any(unix, target_os = "wasi"))] + #[cfg(unix)] { let fd = file.as_raw_fd(); unsafe { ffi::ts_tree_print_dot_graph(self.0.as_ptr(), fd) } @@ -1381,6 +1383,20 @@ impl<'tree> Node<'tree> { Self::new(unsafe { ffi::ts_node_prev_named_sibling(self.0) }) } + /// Get the node's first child that extends beyond the given byte offset. + #[doc(alias = "ts_node_first_child_for_byte")] + #[must_use] + pub fn first_child_for_byte(&self, byte: usize) -> Option { + Self::new(unsafe { ffi::ts_node_first_child_for_byte(self.0, byte as u32) }) + } + + /// Get the node's first named child that extends beyond the given byte offset. + #[doc(alias = "ts_node_first_named_child_for_point")] + #[must_use] + pub fn first_named_child_for_byte(&self, byte: usize) -> Option { + Self::new(unsafe { ffi::ts_node_first_named_child_for_byte(self.0, byte as u32) }) + } + /// Get the node's number of descendants, including one for the node itself. #[doc(alias = "ts_node_descendant_count")] #[must_use] diff --git a/lib/binding_web/package.json b/lib/binding_web/package.json index 06319dc7..755f1be7 100644 --- a/lib/binding_web/package.json +++ b/lib/binding_web/package.json @@ -1,6 +1,6 @@ { "name": "web-tree-sitter", - "version": "0.23.0", + "version": "0.23.2", "description": "Tree-sitter bindings for the web", "main": "tree-sitter.js", "types": "tree-sitter-web.d.ts", diff --git a/lib/binding_web/tree-sitter-web.d.ts b/lib/binding_web/tree-sitter-web.d.ts index 97a48077..fa780336 100644 --- a/lib/binding_web/tree-sitter-web.d.ts +++ b/lib/binding_web/tree-sitter-web.d.ts @@ -150,11 +150,10 @@ declare module 'web-tree-sitter' { rootNodeWithOffset(offsetBytes: number, offsetExtent: Point): SyntaxNode; copy(): Tree; delete(): void; - edit(edit: Edit): Tree; + edit(edit: Edit): void; walk(): TreeCursor; getChangedRanges(other: Tree): Range[]; getIncludedRanges(): Range[]; - getEditedRange(other: Tree): Range; getLanguage(): Language; } diff --git a/lib/language/Cargo.toml b/lib/language/Cargo.toml index 55361fd0..40524b6f 100644 --- a/lib/language/Cargo.toml +++ b/lib/language/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "tree-sitter-language" description = "The tree-sitter Language type, used by the library and by language implementations" -version = "0.1.0" +version = "0.1.1" authors.workspace = true edition.workspace = true rust-version.workspace = true diff --git a/lib/language/language.rs b/lib/language/language.rs index 4c194da7..1997b1f3 100644 --- a/lib/language/language.rs +++ b/lib/language/language.rs @@ -1,6 +1,7 @@ #![no_std] /// LanguageFn wraps a C function that returns a pointer to a tree-sitter grammer. #[repr(transparent)] +#[derive(Clone, Copy)] pub struct LanguageFn(unsafe extern "C" fn() -> *const ()); impl LanguageFn { diff --git a/lib/src/node.c b/lib/src/node.c index 1c0eea73..2768efbb 100644 --- a/lib/src/node.c +++ b/lib/src/node.c @@ -12,6 +12,8 @@ typedef struct { const TSSymbol *alias_sequence; } NodeChildIterator; +static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous); + // TSNode - constructors TSNode ts_node_new( @@ -101,6 +103,21 @@ static inline bool ts_node_child_iterator_next( return true; } +// This will return true if the next sibling is a zero-width token that is adjacent to the current node and is relevant +static inline bool ts_node_child_iterator_next_sibling_is_empty_adjacent(NodeChildIterator *self, TSNode previous) { + if (!self->parent.ptr || ts_node_child_iterator_done(self)) return false; + if (self->child_index == 0) return false; + const Subtree *child = &ts_subtree_children(self->parent)[self->child_index]; + TSSymbol alias = 0; + if (!ts_subtree_extra(*child)) { + if (self->alias_sequence) { + alias = self->alias_sequence[self->structural_child_index]; + } + } + TSNode next = ts_node_new(self->tree, child, self->position, alias); + return ts_node_end_byte(previous) == ts_node_end_byte(next) && ts_node__is_relevant(next, true); +} + // TSNode - private static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous) { @@ -304,22 +321,36 @@ static inline TSNode ts_node__first_child_for_byte( TSNode node = self; bool did_descend = true; + NodeChildIterator last_iterator; + bool has_last_iterator = false; + while (did_descend) { did_descend = false; TSNode child; NodeChildIterator iterator = ts_node_iterate_children(&node); + loop: while (ts_node_child_iterator_next(&iterator, &child)) { if (ts_node_end_byte(child) > goal) { if (ts_node__is_relevant(child, include_anonymous)) { return child; } else if (ts_node_child_count(child) > 0) { + if (iterator.child_index < ts_subtree_child_count(ts_node__subtree(child))) { + last_iterator = iterator; + has_last_iterator = true; + } did_descend = true; node = child; break; } } } + + if (!did_descend && has_last_iterator) { + iterator = last_iterator; + has_last_iterator = false; + goto loop; + } } return ts_node__null(); @@ -344,9 +375,13 @@ static inline TSNode ts_node__descendant_for_byte_range( uint32_t node_end = iterator.position.bytes; // The end of this node must extend far enough forward to touch - // the end of the range and exceed the start of the range. + // the end of the range if (node_end < range_end) continue; - if (node_end <= range_start) continue; + + // ...and exceed the start of the range, unless the node itself is + // empty, in which case it must at least be equal to the start of the range. + bool is_empty = ts_node_start_byte(child) == node_end; + if (is_empty ? node_end < range_start : node_end <= range_start) continue; // The start of this node must extend far enough backward to // touch the start of the range. @@ -383,9 +418,15 @@ static inline TSNode ts_node__descendant_for_point_range( TSPoint node_end = iterator.position.extent; // The end of this node must extend far enough forward to touch - // the end of the range and exceed the start of the range. + // the end of the range if (point_lt(node_end, range_end)) continue; - if (point_lte(node_end, range_start)) continue; + + // ...and exceed the start of the range, unless the node itself is + // empty, in which case it must at least be equal to the start of the range. + bool is_empty = point_eq(ts_node_start_point(child), node_end); + if (is_empty ? point_lt(node_end, range_start) : point_lte(node_end, range_start)) { + continue; + } // The start of this node must extend far enough backward to // touch the start of the range. @@ -530,6 +571,24 @@ TSNode ts_node_child_containing_descendant(TSNode self, TSNode subnode) { ) { return ts_node__null(); } + + // Here we check the current self node and *all* of its zero-width token siblings that follow. + // If any of these nodes contain the target subnode, we return that node. Otherwise, we restore the node we started at + // for the loop condition, and that will continue with the next *non-zero-width* sibling. + TSNode old = self; + // While the next sibling is a zero-width token + while (ts_node_child_iterator_next_sibling_is_empty_adjacent(&iter, self)) { + TSNode current_node = ts_node_child_containing_descendant(self, subnode); + // If the target child is in self, return it + if (!ts_node_is_null(current_node)) { + return current_node; + } + ts_node_child_iterator_next(&iter, &self); + if (self.id == subnode.id) { + return ts_node__null(); + } + } + self = old; } while (iter.position.bytes < end_byte || ts_node_child_count(self) == 0); } while (!ts_node__is_relevant(self, true)); diff --git a/lib/src/subtree.c b/lib/src/subtree.c index 4524e182..2ab8f475 100644 --- a/lib/src/subtree.c +++ b/lib/src/subtree.c @@ -677,7 +677,8 @@ Subtree ts_subtree_edit(Subtree self, const TSInputEdit *input_edit, SubtreePool Edit edit = entry.edit; bool is_noop = edit.old_end.bytes == edit.start.bytes && edit.new_end.bytes == edit.start.bytes; bool is_pure_insertion = edit.old_end.bytes == edit.start.bytes; - bool invalidate_first_row = ts_subtree_depends_on_column(*entry.tree); + bool parent_depends_on_column = ts_subtree_depends_on_column(*entry.tree); + bool column_shifted = edit.new_end.extent.column != edit.old_end.extent.column; Length size = ts_subtree_size(*entry.tree); Length padding = ts_subtree_padding(*entry.tree); @@ -771,8 +772,12 @@ Subtree ts_subtree_edit(Subtree self, const TSInputEdit *input_edit, SubtreePool (child_left.bytes > edit.old_end.bytes) || (child_left.bytes == edit.old_end.bytes && child_size.bytes > 0 && i > 0) ) && ( - !invalidate_first_row || - child_left.extent.row > entry.tree->ptr->padding.extent.row + !parent_depends_on_column || + child_left.extent.row > padding.extent.row + ) && ( + !ts_subtree_depends_on_column(*child) || + !column_shifted || + child_left.extent.row > edit.old_end.extent.row )) { break; } diff --git a/lib/src/tree.c b/lib/src/tree.c index 14936732..55e79a7e 100644 --- a/lib/src/tree.c +++ b/lib/src/tree.c @@ -148,7 +148,7 @@ void ts_tree_print_dot_graph(const TSTree *self, int fd) { fclose(file); } -#else +#elif !defined(__wasi__) // WASI doesn't support dup #include @@ -162,4 +162,11 @@ void ts_tree_print_dot_graph(const TSTree *self, int file_descriptor) { fclose(file); } +#else + +void ts_tree_print_dot_graph(const TSTree *self, int file_descriptor) { + (void)self; + (void)file_descriptor; +} + #endif diff --git a/lib/src/tree_cursor.c b/lib/src/tree_cursor.c index ddd7d66b..24416663 100644 --- a/lib/src/tree_cursor.c +++ b/lib/src/tree_cursor.c @@ -475,8 +475,9 @@ uint32_t ts_tree_cursor_current_descendant_index(const TSTreeCursor *_self) { TSNode ts_tree_cursor_current_node(const TSTreeCursor *_self) { const TreeCursor *self = (const TreeCursor *)_self; TreeCursorEntry *last_entry = array_back(&self->stack); - TSSymbol alias_symbol = self->root_alias_symbol; - if (self->stack.size > 1 && !ts_subtree_extra(*last_entry->subtree)) { + bool is_extra = ts_subtree_extra(*last_entry->subtree); + TSSymbol alias_symbol = is_extra ? 0 : self->root_alias_symbol; + if (self->stack.size > 1 && !is_extra) { TreeCursorEntry *parent_entry = &self->stack.contents[self->stack.size - 2]; alias_symbol = ts_language_alias_at( self->tree->language, diff --git a/test/fixtures/test_grammars/depends_on_column/corpus.txt b/test/fixtures/test_grammars/depends_on_column/corpus.txt new file mode 100644 index 00000000..5c8dbbe6 --- /dev/null +++ b/test/fixtures/test_grammars/depends_on_column/corpus.txt @@ -0,0 +1,21 @@ +================== +X is at odd column +================== + + x + +--- + +(x_is_at + (odd_column)) + +=================== +X is at even column +=================== + + x + +--- + +(x_is_at + (even_column)) diff --git a/test/fixtures/test_grammars/depends_on_column/grammar.js b/test/fixtures/test_grammars/depends_on_column/grammar.js new file mode 100644 index 00000000..6f74810e --- /dev/null +++ b/test/fixtures/test_grammars/depends_on_column/grammar.js @@ -0,0 +1,7 @@ +module.exports = grammar({ + name: "depends_on_column", + rules: { + x_is_at: ($) => seq(/[ \r\n]*/, choice($.odd_column, $.even_column), "x"), + }, + externals: ($) => [$.odd_column, $.even_column], +}); diff --git a/test/fixtures/test_grammars/depends_on_column/scanner.c b/test/fixtures/test_grammars/depends_on_column/scanner.c new file mode 100644 index 00000000..29c2eb1b --- /dev/null +++ b/test/fixtures/test_grammars/depends_on_column/scanner.c @@ -0,0 +1,40 @@ +#include "tree_sitter/parser.h" + +enum TokenType { ODD_COLUMN, EVEN_COLUMN }; + +// The scanner is stateless + +void *tree_sitter_depends_on_column_external_scanner_create() { + return NULL; +} + +void tree_sitter_depends_on_column_external_scanner_destroy( + void *payload +) { + // no-op +} + +unsigned tree_sitter_depends_on_column_external_scanner_serialize( + void *payload, + char *buffer +) { + return 0; +} + +void tree_sitter_depends_on_column_external_scanner_deserialize( + void *payload, + const char *buffer, + unsigned length +) { + // no-op +} + +bool tree_sitter_depends_on_column_external_scanner_scan( + void *payload, + TSLexer *lexer, + const bool *valid_symbols +) { + lexer->result_symbol = + lexer->get_column(lexer) % 2 ? ODD_COLUMN : EVEN_COLUMN; + return true; +}