diff -Nru tree-sitter-0.20.7/.appveyor.yml tree-sitter-0.20.8/.appveyor.yml --- tree-sitter-0.20.7/.appveyor.yml 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/.appveyor.yml 1970-01-01 00:00:00.000000000 +0000 @@ -1,50 +0,0 @@ -build: false -install: - # Terminate early unless building either a tag or a PR. - - if "%APPVEYOR_REPO_TAG%" == "false" if not "%APPVEYOR_REPO_BRANCH%" == "master" appveyor exit - - # Install rust - - appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe - - IF "%PLATFORM%" == "x86" rustup-init -y --default-toolchain stable --default-host i686-pc-windows-msvc - - IF "%PLATFORM%" == "x64" rustup-init -y --default-toolchain stable --default-host x86_64-pc-windows-msvc - - set PATH=%PATH%;C:\Users\appveyor\.cargo\bin - - rustc -vV - - cargo -vV - - # Install dependencies - - git submodule update --init - -platform: - - x64 - - x86 - -test_script: - # Fetch and regenerate the fixture parsers - - script\fetch-fixtures.cmd - - cargo build --release - - script\generate-fixtures.cmd - - # Run tests - - script\test.cmd - - script\benchmark.cmd - -before_deploy: - - move target\release\tree-sitter.exe tree-sitter.exe - - 7z a -tgzip tree-sitter-windows-%PLATFORM%.gz tree-sitter.exe - - appveyor PushArtifact tree-sitter-windows-%PLATFORM%.gz - -deploy: - description: '' - provider: GitHub - auth_token: - secure: VC9ntV5+inKoNteZyLQksKzWMKXF46P+Jx3JHKVSfF+o1rWtZn2iIHAVsQv5LaUi - artifact: /tree-sitter-windows-.*/ - draft: true - force_update: true - on: - APPVEYOR_REPO_TAG: true - -cache: - - target - - test\fixtures\grammars - - C:\Users\appveyor\.cargo diff -Nru tree-sitter-0.20.7/Cargo.lock tree-sitter-0.20.8/Cargo.lock --- tree-sitter-0.20.7/Cargo.lock 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/Cargo.lock 2023-04-04 09:15:07.000000000 +0000 @@ -4,24 +4,15 @@ [[package]] name = "aho-corasick" -version = "0.7.15" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" dependencies = [ "memchr", ] [[package]] name = "ansi_term" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" -dependencies = [ - "winapi", -] - -[[package]] -name = "ansi_term" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" @@ -31,27 +22,15 @@ [[package]] name = "anyhow" -version = "1.0.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b2cd92db5cbd74e8e5028f7e27dd7aa3090e89e4f2a197cc7c8dfb69c7063b" - -[[package]] -name = "arrayref" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" - -[[package]] -name = "arrayvec" -version = "0.5.2" +version = "1.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" +checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" [[package]] name = "ascii" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbf56136a5198c7b01a49e3afcbef6cf84597273d298f54432926024107b0109" +checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" [[package]] name = "atty" @@ -59,51 +38,46 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ - "hermit-abi", + "hermit-abi 0.1.19", "libc", "winapi", ] [[package]] name = "autocfg" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] -name = "base64" -version = "0.13.0" +name = "bitflags" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] -name = "bitflags" -version = "1.2.1" +name = "bumpalo" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] -name = "blake2b_simd" -version = "0.5.11" +name = "bytes" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afa748e348ad3be8263be728124b24a24f268266f6f5d58af9d75f6a40b5c587" -dependencies = [ - "arrayref", - "arrayvec", - "constant_time_eq", -] +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] -name = "bumpalo" -version = "3.6.1" +name = "cc" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" [[package]] -name = "cc" -version = "1.0.67" +name = "cesu8" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" [[package]] name = "cfg-if" @@ -112,31 +86,18 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] -name = "chrono" -version = "0.4.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" -dependencies = [ - "libc", - "num-integer", - "num-traits", - "time", - "winapi", -] - -[[package]] name = "chunked_transfer" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fff857943da45f546682664a79488be82e69e43c1a7a2307679ab9afb3a66d2e" +checksum = "cca491388666e04d7248af3f60f0c40cfb0991c72205595d7c396e3510207d1a" [[package]] name = "clap" -version = "2.33.3" +version = "2.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" dependencies = [ - "ansi_term 0.11.0", + "ansi_term", "atty", "bitflags", "strsim", @@ -146,37 +107,46 @@ ] [[package]] -name = "constant_time_eq" -version = "0.1.5" +name = "combine" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" +dependencies = [ + "bytes", + "memchr", +] [[package]] -name = "crossbeam-utils" -version = "0.8.3" +name = "core-foundation" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7e9d99fa91428effe99c5c6d4634cdeba32b8cf784fc428a2a687f61a952c49" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" dependencies = [ - "autocfg", - "cfg-if", - "lazy_static", + "core-foundation-sys", + "libc", ] [[package]] +name = "core-foundation-sys" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" + +[[package]] name = "ctor" -version = "0.1.20" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e98e2ad1a782e33928b96fc3948e7c355e5af34ba4de7670fe8bac2a3b2006d" +checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" dependencies = [ "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "diff" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e25ea47919b1560c4e3b7fe0aaab9becf5b84a10325ddf7db0f0ba5e1026499" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" [[package]] name = "difference" @@ -186,18 +156,27 @@ [[package]] name = "dirs" -version = "3.0.1" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142995ed02755914747cc6ca76fc7e4583cd18578746716d0508ea6ed558b9ff" +checksum = "30baa043103c9d0c2a57cf537cc2f35623889dc0d405e6c3cccfadbc81c71309" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" dependencies = [ "dirs-sys", ] [[package]] name = "dirs-sys" -version = "0.3.5" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e93d7f5705de3e49895a2b5e0b8855a1c27f080192ae9c32a6432d50741a57a" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" dependencies = [ "libc", "redox_users", @@ -206,104 +185,175 @@ [[package]] name = "either" -version = "1.6.1" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] -name = "form_urlencoded" -version = "1.0.1" +name = "errno" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +checksum = "50d6a0976c999d473fe89ad888d5a284e55366d9dc9038b1ba2aa15128c4afa0" dependencies = [ - "matches", - "percent-encoding", + "errno-dragonfly", + "libc", + "windows-sys", ] [[package]] -name = "getrandom" -version = "0.1.16" +name = "errno-dragonfly" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" dependencies = [ - "cfg-if", + "cc", "libc", - "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + +[[package]] +name = "form_urlencoded" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +dependencies = [ + "percent-encoding", ] [[package]] name = "getrandom" -version = "0.2.2" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ "cfg-if", "libc", - "wasi 0.10.2+wasi-snapshot-preview1", + "wasi", ] [[package]] name = "glob" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "hashbrown" -version = "0.9.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hermit-abi" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" dependencies = [ "libc", ] [[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + +[[package]] name = "html-escape" -version = "0.2.6" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d348900ce941b7474395ba922ed3735a517df4546a2939ddb416ce85eeaa988e" +checksum = "6d1ad449764d627e22bfd7cd5e8868264fc9236e07c752972b4080cd351cb476" dependencies = [ "utf8-width", ] [[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + +[[package]] name = "idna" -version = "0.2.2" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89829a5d69c23d348314a7ac337fe39173b61149a9864deabd260983aed48c21" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" dependencies = [ - "matches", "unicode-bidi", "unicode-normalization", ] [[package]] name = "indexmap" -version = "1.6.1" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown", ] [[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb" +dependencies = [ + "hermit-abi 0.3.1", + "libc", + "windows-sys", +] + +[[package]] name = "itoa" -version = "0.4.7" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" + +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys", + "log", + "thiserror", + "walkdir", + "windows-sys", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" [[package]] name = "js-sys" -version = "0.3.48" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc9f84f9b115ce7843d60706df1422a916680bfdfcbdb0447c5614ff9d7e4d78" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" dependencies = [ "wasm-bindgen", ] @@ -316,86 +366,91 @@ [[package]] name = "libc" -version = "0.2.86" +version = "0.2.141" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c" +checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" [[package]] name = "libloading" -version = "0.7.0" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f84d96438c15fcd6c3f244c8fce01d1e2b9c6b5623e9c711dc9286d8fc92d6a" +checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" dependencies = [ "cfg-if", "winapi", ] [[package]] +name = "linux-raw-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" + +[[package]] name = "log" -version = "0.4.14" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ "cfg-if", ] [[package]] -name = "matches" -version = "0.1.8" +name = "malloc_buf" +version = "0.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +checksum = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb" +dependencies = [ + "libc", +] [[package]] name = "memchr" -version = "2.3.4" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] -name = "num-integer" -version = "0.1.44" +name = "ndk-context" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" -dependencies = [ - "autocfg", - "num-traits", -] +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" [[package]] -name = "num-traits" -version = "0.2.14" +name = "objc" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1" dependencies = [ - "autocfg", + "malloc_buf", ] [[package]] name = "once_cell" -version = "1.7.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10acf907b94fc1b1a152d08ef97e7759650268cf986bf127f387e602b02c7e5a" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "output_vt100" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53cdc5b785b7a58c5aad8216b3dfa114df64b0b06ae6e1501cef91df2fbdf8f9" +checksum = "628223faebab4e3e40667ee0b2336d34a5b960ff60ea743ddfdbcf7770bcfb66" dependencies = [ "winapi", ] [[package]] name = "percent-encoding" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "ppv-lite86" -version = "0.2.10" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "pretty_assertions" @@ -403,7 +458,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cab0e7c02cf376875e9335e0ba1da535775beb5450d21e1dffca068818ed98b" dependencies = [ - "ansi_term 0.12.1", + "ansi_term", "ctor", "diff", "output_vt100", @@ -411,39 +466,48 @@ [[package]] name = "proc-macro2" -version = "1.0.24" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proc_macro" +version = "0.1.0" dependencies = [ - "unicode-xid", + "proc-macro2", + "quote", + "rand", + "syn 1.0.109", ] [[package]] name = "quote" -version = "1.0.9" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" dependencies = [ "proc-macro2", ] [[package]] name = "rand" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha", "rand_core", - "rand_hc", ] [[package]] name = "rand_chacha" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", "rand_core", @@ -451,98 +515,90 @@ [[package]] name = "rand_core" -version = "0.6.2" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.2", + "getrandom", ] [[package]] -name = "rand_hc" -version = "0.3.0" +name = "raw-window-handle" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" -dependencies = [ - "rand_core", -] +checksum = "f2ff9a1f06a88b01621b7ae906ef0211290d1c8a168a15542486a8f61c0833b9" [[package]] name = "redox_syscall" -version = "0.1.57" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] [[package]] name = "redox_syscall" -version = "0.2.5" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ "bitflags", ] [[package]] name = "redox_users" -version = "0.3.5" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ - "getrandom 0.1.16", - "redox_syscall 0.1.57", - "rust-argon2", + "getrandom", + "redox_syscall 0.2.16", + "thiserror", ] [[package]] name = "regex" -version = "1.4.3" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" +checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" dependencies = [ "aho-corasick", "memchr", "regex-syntax", - "thread_local", ] [[package]] name = "regex-syntax" -version = "0.6.22" +version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] -name = "remove_dir_all" -version = "0.5.3" +name = "rustc-hash" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi", -] +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] -name = "rust-argon2" -version = "0.8.3" +name = "rustix" +version = "0.37.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b18820d944b33caa75a71378964ac46f58517c92b6ae5f762636247c09e78fb" +checksum = "2aae838e49b3d63e9274e1c01833cc8139d3fec468c3b84688c628f44b1ae11d" dependencies = [ - "base64", - "blake2b_simd", - "constant_time_eq", - "crossbeam-utils", + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys", ] [[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - -[[package]] name = "ryu" -version = "1.0.5" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" [[package]] name = "same-file" @@ -555,35 +611,35 @@ [[package]] name = "semver" -version = "1.0.5" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0486718e92ec9a68fbed73bb5ef687d71103b142595b406835649bebd33f72c7" +checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" [[package]] name = "serde" -version = "1.0.130" +version = "1.0.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913" +checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.130" +version = "1.0.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7bc1a1ab1961464eae040d96713baa5a724a8152c1222492465b54322ec508b" +checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.13", ] [[package]] name = "serde_json" -version = "1.0.63" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43535db9747a4ba938c0ce0a98cc631a46ebf943c9e1d604e091df6007620bf6" +checksum = "d721eca97ac802aa7777b701877c8004d950fc142651367300d21c1cc0194744" dependencies = [ "indexmap", "itoa", @@ -605,27 +661,37 @@ [[package]] name = "syn" -version = "1.0.67" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6498a9efc342871f91cc2d0d694c674368b4ceb40f62b65a7a08c3792935e702" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2", "quote", - "unicode-xid", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c9da457c5285ac1f936ebd076af6dac17a61cfe7826f2076b4d015cf47bc8ec" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", ] [[package]] name = "tempfile" -version = "3.2.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" dependencies = [ "cfg-if", - "libc", - "rand", - "redox_syscall 0.2.5", - "remove_dir_all", - "winapi", + "fastrand", + "redox_syscall 0.3.5", + "rustix", + "windows-sys", ] [[package]] @@ -639,83 +705,63 @@ [[package]] name = "thiserror" -version = "1.0.25" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa6f76457f59514c7eeb4e59d891395fab0b2fd1d40723ae737d64153392e9c6" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.25" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a36768c0fbf1bb15eca10defa29526bda730a2376c2ab4393ccfa16fb1a318d" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn", -] - -[[package]] -name = "thread_local" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd" -dependencies = [ - "once_cell", -] - -[[package]] -name = "time" -version = "0.1.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" -dependencies = [ - "libc", - "winapi", + "syn 2.0.13", ] [[package]] name = "tiny_http" -version = "0.8.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eded47106b8e52d8ed8119f0ea6e8c0f5881e69783e0297b5a8462958f334bc1" +checksum = "389915df6413a2e74fb181895f933386023c71110878cd0825588928e64cdc82" dependencies = [ "ascii", - "chrono", "chunked_transfer", + "httpdate", "log", - "url", ] [[package]] name = "tinyvec" -version = "1.1.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317cca572a0e89c3ce0ca1f1bdc9369547fe318a683418e42ac8f59d14701023" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" dependencies = [ "tinyvec_macros", ] [[package]] name = "tinyvec_macros" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "toml" -version = "0.5.8" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" dependencies = [ "serde", ] [[package]] name = "tree-sitter" -version = "0.20.9" +version = "0.20.10" dependencies = [ "cc", "lazy_static", @@ -724,21 +770,22 @@ [[package]] name = "tree-sitter-cli" -version = "0.20.7" +version = "0.20.8" dependencies = [ - "ansi_term 0.12.1", + "ansi_term", "anyhow", "atty", "clap", "ctor", "difference", - "dirs", + "dirs 3.0.2", "glob", "html-escape", "indexmap", "lazy_static", "log", "pretty_assertions", + "proc_macro", "rand", "regex", "regex-syntax", @@ -755,6 +802,7 @@ "tree-sitter-highlight", "tree-sitter-loader", "tree-sitter-tags", + "unindent", "walkdir", "webbrowser", "which", @@ -765,7 +813,7 @@ version = "0.19.0" dependencies = [ "anyhow", - "dirs", + "dirs 3.0.2", "serde", "serde_json", ] @@ -785,7 +833,7 @@ dependencies = [ "anyhow", "cc", - "dirs", + "dirs 3.0.2", "libloading", "once_cell", "regex", @@ -808,51 +856,53 @@ [[package]] name = "unicode-bidi" -version = "0.3.4" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" -dependencies = [ - "matches", -] +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" + +[[package]] +name = "unicode-ident" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" [[package]] name = "unicode-normalization" -version = "0.1.17" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" dependencies = [ "tinyvec", ] [[package]] name = "unicode-width" -version = "0.1.8" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" [[package]] -name = "unicode-xid" +name = "unindent" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" +checksum = "5aa30f5ea51ff7edfc797c6d3f9ec8cbd8cfedef5371766b7181d33977f4814f" [[package]] name = "url" -version = "2.2.1" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" dependencies = [ "form_urlencoded", "idna", - "matches", "percent-encoding", ] [[package]] name = "utf8-width" -version = "0.1.4" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9071ac216321a4470a69fb2b28cfc68dcd1a39acd877c8be8e014df6772d8efa" +checksum = "5190c9442dcdaf0ddd50f37420417d219ae5261bbf5db120d0f9bab996c9cba1" [[package]] name = "vec_map" @@ -862,32 +912,25 @@ [[package]] name = "walkdir" -version = "2.3.1" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d" +checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" dependencies = [ "same-file", - "winapi", "winapi-util", ] [[package]] name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" +version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" - -[[package]] -name = "wasi" -version = "0.10.2+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.71" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ee1280240b7c461d6a0071313e08f34a60b0365f14260362e5a2b17d1d31aa7" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -895,24 +938,24 @@ [[package]] name = "wasm-bindgen-backend" -version = "0.2.71" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b7d8b6942b8bb3a9b0e73fc79b98095a27de6fa247615e59d096754a3bc2aa8" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" dependencies = [ "bumpalo", - "lazy_static", "log", + "once_cell", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.71" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ac38da8ef716661f0f36c0d8320b89028efe10c7c0afde65baffb496ce0d3b" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -920,28 +963,28 @@ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.71" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc053ec74d454df287b9374ee8abb36ffd5acb95ba87da3ba5b7d3fe20eb401e" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.71" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d6f8ec44822dd71f5f221a5847fb34acd9060535c1211b70a05844c0f6383b1" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" [[package]] name = "web-sys" -version = "0.3.48" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec600b26223b2948cedfde2a0aa6756dcf1fef616f43d7b3097aaf53a6c4d92b" +checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" dependencies = [ "js-sys", "wasm-bindgen", @@ -949,32 +992,33 @@ [[package]] name = "webbrowser" -version = "0.5.5" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecad156490d6b620308ed411cfee90d280b3cbd13e189ea0d3fada8acc89158a" +checksum = "579cc485bd5ce5bfa0d738e4921dd0b956eca9800be1fd2e5257ebe95bc4617e" dependencies = [ + "core-foundation", + "dirs 4.0.0", + "jni", + "log", + "ndk-context", + "objc", + "raw-window-handle", + "url", "web-sys", - "widestring", - "winapi", ] [[package]] name = "which" -version = "4.1.0" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b55551e42cbdf2ce2bedd2203d0cc08dba002c27510f86dab6d0ce304cba3dfe" +checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" dependencies = [ "either", "libc", + "once_cell", ] [[package]] -name = "widestring" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c168940144dd21fd8046987c16a46a33d5fc84eec29ef9dcddc2ac9e31526b7c" - -[[package]] name = "winapi" version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1004,3 +1048,69 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" diff -Nru tree-sitter-0.20.7/Cargo.toml tree-sitter-0.20.8/Cargo.toml --- tree-sitter-0.20.7/Cargo.toml 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/Cargo.toml 2023-04-04 09:15:07.000000000 +0000 @@ -1,4 +1,10 @@ [workspace] default-members = ["cli"] - members = ["cli", "lib"] +resolver = "2" + +[workspace.package] +rust-version = "1.65" + +[profile.release] +strip = true diff -Nru tree-sitter-0.20.7/cli/build.rs tree-sitter-0.20.8/cli/build.rs --- tree-sitter-0.20.7/cli/build.rs 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/cli/build.rs 2023-04-04 09:15:07.000000000 +0000 @@ -1,3 +1,4 @@ +use std::ffi::OsStr; use std::path::{Path, PathBuf}; use std::{env, fs}; @@ -66,7 +67,39 @@ // If we're on a branch, read the SHA from the ref file. if head_content.starts_with("ref: ") { head_content.replace_range(0.."ref: ".len(), ""); - let ref_filename = git_dir_path.join(&head_content); + let ref_filename = { + // Go to real non-worktree gitdir + let git_dir_path = git_dir_path + .parent() + .map(|p| { + p.file_name() + .map(|n| n == OsStr::new("worktrees")) + .and_then(|x| x.then(|| p.parent())) + }) + .flatten() + .flatten() + .unwrap_or(&git_dir_path); + + let file = git_dir_path.join(&head_content); + if file.is_file() { + file + } else { + let packed_refs = git_dir_path.join("packed-refs"); + if let Ok(packed_refs_content) = fs::read_to_string(&packed_refs) { + for line in packed_refs_content.lines() { + if let Some((hash, r#ref)) = line.split_once(' ') { + if r#ref == head_content { + if let Some(path) = packed_refs.to_str() { + println!("cargo:rerun-if-changed={}", path); + } + return Some(hash.to_string()); + } + } + } + } + return None; + } + }; if let Some(path) = ref_filename.to_str() { println!("cargo:rerun-if-changed={}", path); } diff -Nru tree-sitter-0.20.7/cli/Cargo.toml tree-sitter-0.20.8/cli/Cargo.toml --- tree-sitter-0.20.7/cli/Cargo.toml 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/cli/Cargo.toml 2023-04-04 09:15:07.000000000 +0000 @@ -1,14 +1,15 @@ [package] name = "tree-sitter-cli" description = "CLI tool for developing, testing, and using Tree-sitter parsers" -version = "0.20.7" +version = "0.20.8" authors = ["Max Brunsfeld "] -edition = "2018" +edition = "2021" license = "MIT" readme = "README.md" keywords = ["incremental", "parsing"] categories = ["command-line-utilities", "parsing"] repository = "https://github.com/tree-sitter/tree-sitter" +rust-version.workspace = true [[bin]] name = "tree-sitter" @@ -35,13 +36,13 @@ semver = "1.0" serde = { version = "1.0.130", features = ["derive"] } smallbitvec = "2.5.1" -tiny_http = "0.8" +tiny_http = "0.12.0" walkdir = "2.3" -webbrowser = "0.5.1" +webbrowser = "0.8.3" which = "4.1.0" [dependencies.tree-sitter] -version = "0.20.3" +version = "0.20.10" path = "../lib" [dependencies.tree-sitter-config] @@ -69,10 +70,13 @@ features = ["std"] [dev-dependencies] +proc_macro = { path = "src/tests/proc_macro" } + rand = "0.8" tempfile = "3" pretty_assertions = "0.7.2" ctor = "0.1" +unindent = "0.2" [build-dependencies] toml = "0.5" diff -Nru tree-sitter-0.20.7/cli/config/Cargo.toml tree-sitter-0.20.8/cli/config/Cargo.toml --- tree-sitter-0.20.7/cli/config/Cargo.toml 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/cli/config/Cargo.toml 2023-04-04 09:15:07.000000000 +0000 @@ -9,6 +9,7 @@ keywords = ["incremental", "parsing"] categories = ["command-line-utilities", "parsing"] repository = "https://github.com/tree-sitter/tree-sitter" +rust-version.workspace = true [dependencies] anyhow = "1.0" diff -Nru tree-sitter-0.20.7/cli/config/src/lib.rs tree-sitter-0.20.8/cli/config/src/lib.rs --- tree-sitter-0.20.7/cli/config/src/lib.rs 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/cli/config/src/lib.rs 2023-04-04 09:15:07.000000000 +0000 @@ -25,6 +25,9 @@ if let Ok(path) = env::var("TREE_SITTER_DIR") { let mut path = PathBuf::from(path); path.push("config.json"); + if !path.exists() { + return Ok(None); + } if path.is_file() { return Ok(Some(path)); } @@ -37,7 +40,8 @@ let legacy_path = dirs::home_dir() .ok_or(anyhow!("Cannot determine home directory"))? - .join(".tree-sitter/config.json"); + .join(".tree-sitter") + .join("config.json"); if legacy_path.is_file() { return Ok(Some(legacy_path)); } @@ -48,7 +52,8 @@ fn xdg_config_file() -> Result { let xdg_path = dirs::config_dir() .ok_or(anyhow!("Cannot determine config directory"))? - .join("tree-sitter/config.json"); + .join("tree-sitter") + .join("config.json"); Ok(xdg_path) } @@ -79,7 +84,13 @@ /// /// (Note that this is typically only done by the `tree-sitter init-config` command.) pub fn initial() -> Result { - let location = Self::xdg_config_file()?; + let location = if let Ok(path) = env::var("TREE_SITTER_DIR") { + let mut path = PathBuf::from(path); + path.push("config.json"); + path + } else { + Self::xdg_config_file()? + }; let config = serde_json::json!({}); Ok(Config { location, config }) } diff -Nru tree-sitter-0.20.7/cli/emscripten-version tree-sitter-0.20.8/cli/emscripten-version --- tree-sitter-0.20.7/cli/emscripten-version 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/cli/emscripten-version 2023-04-04 09:15:07.000000000 +0000 @@ -1 +1 @@ -2.0.24 +3.1.29 diff -Nru tree-sitter-0.20.7/cli/loader/Cargo.toml tree-sitter-0.20.8/cli/loader/Cargo.toml --- tree-sitter-0.20.7/cli/loader/Cargo.toml 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/cli/loader/Cargo.toml 2023-04-04 09:15:07.000000000 +0000 @@ -9,6 +9,7 @@ keywords = ["incremental", "parsing"] categories = ["command-line-utilities", "parsing"] repository = "https://github.com/tree-sitter/tree-sitter" +rust-version.workspace = true [dependencies] anyhow = "1.0" diff -Nru tree-sitter-0.20.7/cli/loader/src/lib.rs tree-sitter-0.20.8/cli/loader/src/lib.rs --- tree-sitter-0.20.7/cli/loader/src/lib.rs 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/cli/loader/src/lib.rs 2023-04-04 09:15:07.000000000 +0000 @@ -10,7 +10,7 @@ use std::process::Command; use std::sync::Mutex; use std::time::SystemTime; -use std::{fs, mem}; +use std::{env, fs, mem}; use tree_sitter::{Language, QueryError, QueryErrorKind}; use tree_sitter_highlight::HighlightConfiguration; use tree_sitter_tags::{Error as TagsError, TagsConfiguration}; @@ -108,9 +108,13 @@ impl Loader { pub fn new() -> Result { - let parser_lib_path = dirs::cache_dir() - .ok_or(anyhow!("Cannot determine cache directory"))? - .join("tree-sitter/lib"); + let parser_lib_path = match env::var("TREE_SITTER_LIBDIR") { + Ok(path) => PathBuf::from(path), + _ => dirs::cache_dir() + .ok_or(anyhow!("Cannot determine cache directory"))? + .join("tree-sitter") + .join("lib"), + }; Ok(Self::with_parser_lib_path(parser_lib_path)) } diff -Nru tree-sitter-0.20.7/cli/npm/package.json tree-sitter-0.20.8/cli/npm/package.json --- tree-sitter-0.20.7/cli/npm/package.json 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/cli/npm/package.json 2023-04-04 09:15:07.000000000 +0000 @@ -1,6 +1,6 @@ { "name": "tree-sitter-cli", - "version": "0.20.7", + "version": "0.20.8", "author": "Max Brunsfeld", "license": "MIT", "repository": { diff -Nru tree-sitter-0.20.7/cli/README.md tree-sitter-0.20.8/cli/README.md --- tree-sitter-0.20.7/cli/README.md 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/cli/README.md 2023-04-04 09:15:07.000000000 +0000 @@ -1,8 +1,6 @@ Tree-sitter CLI =============== -[![Build Status](https://travis-ci.org/tree-sitter/tree-sitter.svg?branch=master)](https://travis-ci.org/tree-sitter/tree-sitter) -[![Build status](https://ci.appveyor.com/api/projects/status/vtmbd6i92e97l55w/branch/master?svg=true)](https://ci.appveyor.com/project/maxbrunsfeld/tree-sitter/branch/master) [![Crates.io](https://img.shields.io/crates/v/tree-sitter-cli.svg)](https://crates.io/crates/tree-sitter-cli) The Tree-sitter CLI allows you to develop, test, and use Tree-sitter grammars from the command line. It works on MacOS, Linux, and Windows. diff -Nru tree-sitter-0.20.7/cli/src/main.rs tree-sitter-0.20.8/cli/src/main.rs --- tree-sitter-0.20.7/cli/src/main.rs 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/cli/src/main.rs 2023-04-04 09:15:07.000000000 +0000 @@ -1,8 +1,10 @@ use anyhow::{anyhow, Context, Result}; use clap::{App, AppSettings, Arg, SubCommand}; use glob::glob; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::{env, fs, u64}; +use tree_sitter::Point; +use tree_sitter_cli::parse::ParseOutput; use tree_sitter_cli::{ generate, highlight, logger, parse, playground, query, tags, test, test_highlight, test_tags, util, wasm, @@ -107,12 +109,24 @@ ) .arg(Arg::with_name("no-bindings").long("no-bindings")) .arg( + Arg::with_name("build") + .long("build") + .short("b") + .help("Compile all defined languages in the current dir"), + ) + .arg(&debug_build_arg) + .arg( + Arg::with_name("libdir") + .long("libdir") + .takes_value(true) + .value_name("path"), + ) + .arg( Arg::with_name("report-states-for-rule") .long("report-states-for-rule") .value_name("rule-name") .takes_value(true), - ) - .arg(Arg::with_name("no-minimize").long("no-minimize")), + ), ) .subcommand( SubCommand::with_name("parse") @@ -124,7 +138,8 @@ .arg(&debug_arg) .arg(&debug_build_arg) .arg(&debug_graph_arg) - .arg(Arg::with_name("debug-xml").long("xml").short("x")) + .arg(Arg::with_name("output-dot").long("dot")) + .arg(Arg::with_name("output-xml").long("xml").short("x")) .arg( Arg::with_name("stat") .help("Show parsing statistic") @@ -159,6 +174,8 @@ .index(1) .required(true), ) + .arg(&time_arg) + .arg(&quiet_arg) .arg(&paths_file_arg) .arg(&paths_arg.clone().index(2)) .arg( @@ -167,6 +184,12 @@ .long("byte-range") .takes_value(true), ) + .arg( + Arg::with_name("row-range") + .help("The range of rows in which the query will be executed") + .long("row-range") + .takes_value(true), + ) .arg(&scope_arg) .arg(Arg::with_name("captures").long("captures").short("c")) .arg(Arg::with_name("test").long("test")), @@ -270,6 +293,9 @@ ("generate", Some(matches)) => { let grammar_path = matches.value_of("grammar-path"); + let debug_build = matches.is_present("debug-build"); + let build = matches.is_present("build"); + let libdir = matches.value_of("libdir"); let report_symbol_name = matches.value_of("report-states-for-rule").or_else(|| { if matches.is_present("report-states") { Some("") @@ -298,6 +324,13 @@ generate_bindings, report_symbol_name, )?; + if build { + if let Some(path) = libdir { + loader = loader::Loader::with_parser_lib_path(PathBuf::from(path)); + } + loader.use_debug_build(debug_build); + loader.languages_at_path(¤t_dir)?; + } } ("test", Some(matches)) => { @@ -307,6 +340,11 @@ let update = matches.is_present("update"); let filter = matches.value_of("filter"); + if debug { + // For augmenting debug logging in external scanners + env::set_var("TREE_SITTER_DEBUG", "1"); + } + loader.use_debug_build(debug_build); let languages = loader.languages_at_path(¤t_dir)?; @@ -350,8 +388,17 @@ let debug = matches.is_present("debug"); let debug_graph = matches.is_present("debug-graph"); let debug_build = matches.is_present("debug-build"); - let debug_xml = matches.is_present("debug-xml"); - let quiet = matches.is_present("quiet"); + + let output = if matches.is_present("output-dot") { + ParseOutput::Dot + } else if matches.is_present("output-xml") { + ParseOutput::Xml + } else if matches.is_present("quiet") { + ParseOutput::Quiet + } else { + ParseOutput::Normal + }; + let time = matches.is_present("time"); let edits = matches .values_of("edits") @@ -389,12 +436,11 @@ path, &edits, max_path_length, - quiet, + output, time, timeout, debug, debug_graph, - debug_xml, Some(&cancellation_flag), )?; @@ -419,6 +465,8 @@ ("query", Some(matches)) => { let ordered_captures = matches.values_of("captures").is_some(); + let quiet = matches.values_of("quiet").is_some(); + let time = matches.values_of("time").is_some(); let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?; let loader_config = config.get()?; loader.find_all_languages(&loader_config)?; @@ -428,9 +476,17 @@ matches.value_of("scope"), )?; let query_path = Path::new(matches.value_of("query-path").unwrap()); - let range = matches.value_of("byte-range").map(|br| { - let r: Vec<&str> = br.split(":").collect(); - r[0].parse().unwrap()..r[1].parse().unwrap() + let byte_range = matches.value_of("byte-range").and_then(|arg| { + let mut parts = arg.split(":"); + let start = parts.next()?.parse().ok()?; + let end = parts.next().unwrap().parse().ok()?; + Some(start..end) + }); + let point_range = matches.value_of("row-range").and_then(|arg| { + let mut parts = arg.split(":"); + let start = parts.next()?.parse().ok()?; + let end = parts.next().unwrap().parse().ok()?; + Some(Point::new(start, 0)..Point::new(end, 0)) }); let should_test = matches.is_present("test"); query::query_files_at_paths( @@ -438,8 +494,11 @@ paths, query_path, ordered_captures, - range, + byte_range, + point_range, should_test, + quiet, + time, )?; } diff -Nru tree-sitter-0.20.7/cli/src/parse.rs tree-sitter-0.20.8/cli/src/parse.rs --- tree-sitter-0.20.7/cli/src/parse.rs 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/cli/src/parse.rs 2023-04-04 09:15:07.000000000 +0000 @@ -30,17 +30,24 @@ } } +#[derive(Copy, Clone)] +pub enum ParseOutput { + Normal, + Quiet, + Xml, + Dot, +} + pub fn parse_file_at_path( language: Language, path: &Path, edits: &Vec<&str>, max_path_length: usize, - quiet: bool, + output: ParseOutput, print_time: bool, timeout: u64, debug: bool, debug_graph: bool, - debug_xml: bool, cancellation_flag: Option<&AtomicUsize>, ) -> Result { let mut _log_session = None; @@ -95,7 +102,7 @@ let duration_ms = duration.as_secs() * 1000 + duration.subsec_nanos() as u64 / 1000000; let mut cursor = tree.walk(); - if !quiet { + if matches!(output, ParseOutput::Normal) { let mut needs_newline = false; let mut indent_level = 0; let mut did_visit_children = false; @@ -151,7 +158,7 @@ println!(""); } - if debug_xml { + if matches!(output, ParseOutput::Xml) { let mut needs_newline = false; let mut indent_level = 0; let mut did_visit_children = false; @@ -206,6 +213,10 @@ println!(""); } + if matches!(output, ParseOutput::Dot) { + util::print_tree_graph(&tree, "log.html").unwrap(); + } + let mut first_error = None; loop { let node = cursor.node(); diff -Nru tree-sitter-0.20.7/cli/src/playground.rs tree-sitter-0.20.8/cli/src/playground.rs --- tree-sitter-0.20.7/cli/src/playground.rs 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/cli/src/playground.rs 2023-04-04 09:15:07.000000000 +0000 @@ -45,20 +45,7 @@ } pub fn serve(grammar_path: &Path, open_in_browser: bool) { - let port = env::var("TREE_SITTER_PLAYGROUND_PORT") - .map(|v| v.parse::().expect("Invalid port specification")) - .unwrap_or_else( - |_| get_available_port().expect( - "Couldn't find an available port, try providing a port number via the TREE_SITTER_PLAYGROUND_PORT \ - environment variable" - ) - ); - let addr = format!( - "{}:{}", - env::var("TREE_SITTER_PLAYGROUND_ADDR").unwrap_or("127.0.0.1".to_owned()), - port - ); - let server = Server::http(&addr).expect("Failed to start web server"); + let server = get_server(); let grammar_name = wasm::get_grammar_name(&grammar_path.join("src")) .with_context(|| "Failed to get wasm filename") .unwrap(); @@ -71,7 +58,7 @@ ) }) .unwrap(); - let url = format!("http://{}", addr); + let url = format!("http://{}", server.server_addr()); println!("Started playground on: {}", url); if open_in_browser { if let Err(_) = webbrowser::open(&url) { @@ -135,10 +122,24 @@ .with_header(header.clone()) } -fn get_available_port() -> Option { - (8000..12000).find(port_is_available) +fn get_server() -> Server { + let addr = env::var("TREE_SITTER_PLAYGROUND_ADDR").unwrap_or("127.0.0.1".to_owned()); + let port = env::var("TREE_SITTER_PLAYGROUND_PORT") + .map(|v| v.parse::().expect("Invalid port specification")) + .ok(); + let listener = match port { + Some(port) => bind_to(&*addr, port).expect("Can't bind to the specified port"), + None => { + get_listener_on_available_port(&*addr).expect("Can't find a free port to bind to it") + } + }; + Server::from_listener(listener, None).expect("Failed to start web server") +} + +fn get_listener_on_available_port(addr: &str) -> Option { + (8000..12000).find_map(|port| bind_to(addr, port)) } -fn port_is_available(port: &u16) -> bool { - TcpListener::bind(("127.0.0.1", *port)).is_ok() +fn bind_to(addr: &str, port: u16) -> Option { + TcpListener::bind(format!("{addr}:{port}")).ok() } diff -Nru tree-sitter-0.20.7/cli/src/query.rs tree-sitter-0.20.8/cli/src/query.rs --- tree-sitter-0.20.7/cli/src/query.rs 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/cli/src/query.rs 2023-04-04 09:15:07.000000000 +0000 @@ -5,16 +5,20 @@ io::{self, Write}, ops::Range, path::Path, + time::Instant, }; -use tree_sitter::{Language, Parser, Query, QueryCursor}; +use tree_sitter::{Language, Parser, Point, Query, QueryCursor}; pub fn query_files_at_paths( language: Language, paths: Vec, query_path: &Path, ordered_captures: bool, - range: Option>, + byte_range: Option>, + point_range: Option>, should_test: bool, + quiet: bool, + print_time: bool, ) -> Result<()> { let stdout = io::stdout(); let mut stdout = stdout.lock(); @@ -24,9 +28,12 @@ let query = Query::new(language, &query_source).with_context(|| "Query compilation failed")?; let mut query_cursor = QueryCursor::new(); - if let Some(range) = range { + if let Some(range) = byte_range { query_cursor.set_byte_range(range); } + if let Some(range) = point_range { + query_cursor.set_point_range(range); + } let mut parser = Parser::new(); parser.set_language(language)?; @@ -40,22 +47,25 @@ fs::read(&path).with_context(|| format!("Error reading source file {:?}", path))?; let tree = parser.parse(&source_code, None).unwrap(); + let start = Instant::now(); if ordered_captures { for (mat, capture_index) in query_cursor.captures(&query, tree.root_node(), source_code.as_slice()) { let capture = mat.captures[capture_index]; let capture_name = &query.capture_names()[capture.index as usize]; - writeln!( - &mut stdout, - " pattern: {:>2}, capture: {} - {}, start: {}, end: {}, text: `{}`", - mat.pattern_index, - capture.index, - capture_name, - capture.node.start_position(), - capture.node.end_position(), - capture.node.utf8_text(&source_code).unwrap_or("") - )?; + if !quiet { + writeln!( + &mut stdout, + " pattern: {:>2}, capture: {} - {}, start: {}, end: {}, text: `{}`", + mat.pattern_index, + capture.index, + capture_name, + capture.node.start_position(), + capture.node.end_position(), + capture.node.utf8_text(&source_code).unwrap_or("") + )?; + } results.push(query_testing::CaptureInfo { name: capture_name.to_string(), start: capture.node.start_position(), @@ -64,27 +74,31 @@ } } else { for m in query_cursor.matches(&query, tree.root_node(), source_code.as_slice()) { - writeln!(&mut stdout, " pattern: {}", m.pattern_index)?; + if !quiet { + writeln!(&mut stdout, " pattern: {}", m.pattern_index)?; + } for capture in m.captures { let start = capture.node.start_position(); let end = capture.node.end_position(); let capture_name = &query.capture_names()[capture.index as usize]; - if end.row == start.row { - writeln!( - &mut stdout, - " capture: {} - {}, start: {}, end: {}, text: `{}`", - capture.index, - capture_name, - start, - end, - capture.node.utf8_text(&source_code).unwrap_or("") - )?; - } else { - writeln!( - &mut stdout, - " capture: {}, start: {}, end: {}", - capture_name, start, end, - )?; + if !quiet { + if end.row == start.row { + writeln!( + &mut stdout, + " capture: {} - {}, start: {}, end: {}, text: `{}`", + capture.index, + capture_name, + start, + end, + capture.node.utf8_text(&source_code).unwrap_or("") + )?; + } else { + writeln!( + &mut stdout, + " capture: {}, start: {}, end: {}", + capture_name, start, end, + )?; + } } results.push(query_testing::CaptureInfo { name: capture_name.to_string(), @@ -103,6 +117,9 @@ if should_test { query_testing::assert_expected_captures(results, path, &mut parser, language)? } + if print_time { + writeln!(&mut stdout, "{:?}", start.elapsed())?; + } } Ok(()) diff -Nru tree-sitter-0.20.7/cli/src/test.rs tree-sitter-0.20.8/cli/src/test.rs --- tree-sitter-0.20.7/cli/src/test.rs 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/cli/src/test.rs 2023-04-04 09:15:07.000000000 +0000 @@ -294,15 +294,10 @@ // "(node_name" write!(formatted, "{}", s).unwrap(); - let mut c_iter = s.chars(); - c_iter.next(); - match c_iter.next() { - Some('M') | Some('U') => { - // "(MISSING node_name" or "(UNEXPECTED 'x'" - let s = s_iter.next().unwrap(); - write!(formatted, " {}", s).unwrap(); - } - Some(_) | None => {} + // "(MISSING node_name" or "(UNEXPECTED 'x'" + if s.starts_with("(MISSING") || s.starts_with("(UNEXPECTED") { + let s = s_iter.next().unwrap(); + write!(formatted, " {}", s).unwrap(); } } else if s.ends_with(':') { // "field:" @@ -597,6 +592,14 @@ .to_string() ); assert_eq!(format_sexp(&"()".to_string()), "()".to_string()); + assert_eq!( + format_sexp(&"(A (M (B)))".to_string()), + "(A\n (M\n (B)))" + ); + assert_eq!( + format_sexp(&"(A (U (B)))".to_string()), + "(A\n (U\n (B)))" + ); } #[test] diff -Nru tree-sitter-0.20.7/cli/src/tests/corpus_test.rs tree-sitter-0.20.8/cli/src/tests/corpus_test.rs --- tree-sitter-0.20.7/cli/src/tests/corpus_test.rs 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/cli/src/tests/corpus_test.rs 2023-04-04 09:15:07.000000000 +0000 @@ -2,6 +2,7 @@ allocations, edits::{get_random_edit, invert_edit}, fixtures::{fixtures_dir, get_language, get_test_language}, + new_seed, random::Rand, scope_sequence::ScopeSequence, EDIT_COUNT, EXAMPLE_FILTER, ITERATION_COUNT, LANGUAGE_FILTER, LOG_ENABLED, LOG_GRAPH_ENABLED, @@ -13,106 +14,120 @@ test::{parse_tests, print_diff, print_diff_key, strip_sexp_fields, TestEntry}, util, }; -use std::fs; -use tree_sitter::{LogType, Node, Parser, Tree}; +use proc_macro::test_with_seed; +use std::{env, fs}; +use tree_sitter::{LogType, Node, Parser, Point, Range, Tree}; -#[test] -fn test_bash_corpus() { - test_language_corpus("bash"); +#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] +fn test_corpus_for_bash(seed: usize) { + test_language_corpus(seed, "bash"); } -#[test] -fn test_c_corpus() { - test_language_corpus("c"); +#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] +fn test_corpus_for_c(seed: usize) { + test_language_corpus(seed, "c"); } -#[test] -fn test_cpp_corpus() { - test_language_corpus("cpp"); +#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] +fn test_corpus_for_cpp(seed: usize) { + test_language_corpus(seed, "cpp"); } -#[test] -fn test_embedded_template_corpus() { - test_language_corpus("embedded-template"); +#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] +fn test_corpus_for_embedded_template(seed: usize) { + test_language_corpus(seed, "embedded-template"); } -#[test] -fn test_go_corpus() { - test_language_corpus("go"); +#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] +fn test_corpus_for_go(seed: usize) { + test_language_corpus(seed, "go"); } -#[test] -fn test_html_corpus() { - test_language_corpus("html"); +#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] +fn test_corpus_for_html(seed: usize) { + test_language_corpus(seed, "html"); } -#[test] -fn test_javascript_corpus() { - test_language_corpus("javascript"); +#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] +fn test_corpus_for_javascript(seed: usize) { + test_language_corpus(seed, "javascript"); } -#[test] -fn test_json_corpus() { - test_language_corpus("json"); +#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] +fn test_corpus_for_json(seed: usize) { + test_language_corpus(seed, "json"); } -#[test] -fn test_php_corpus() { - test_language_corpus("php"); +#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] +fn test_corpus_for_php(seed: usize) { + test_language_corpus(seed, "php"); } -#[test] -fn test_python_corpus() { - test_language_corpus("python"); +#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] +fn test_corpus_for_python(seed: usize) { + test_language_corpus(seed, "python"); } -#[test] -fn test_ruby_corpus() { - test_language_corpus("ruby"); +#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] +fn test_corpus_for_ruby(seed: usize) { + test_language_corpus(seed, "ruby"); } -#[test] -fn test_rust_corpus() { - test_language_corpus("rust"); +#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] +fn test_corpus_for_rust(seed: usize) { + test_language_corpus(seed, "rust"); } -fn test_language_corpus(language_name: &str) { +fn test_language_corpus(start_seed: usize, language_name: &str) { let grammars_dir = fixtures_dir().join("grammars"); let error_corpus_dir = fixtures_dir().join("error_corpus"); + let template_corpus_dir = fixtures_dir().join("template_corpus"); let mut corpus_dir = grammars_dir.join(language_name).join("corpus"); if !corpus_dir.is_dir() { corpus_dir = grammars_dir.join(language_name).join("test").join("corpus"); } let error_corpus_file = error_corpus_dir.join(&format!("{}_errors.txt", language_name)); + let template_corpus_file = + template_corpus_dir.join(&format!("{}_templates.txt", language_name)); let main_tests = parse_tests(&corpus_dir).unwrap(); let error_tests = parse_tests(&error_corpus_file).unwrap_or(TestEntry::default()); + let template_tests = parse_tests(&template_corpus_file).unwrap_or(TestEntry::default()); let mut tests = flatten_tests(main_tests); tests.extend(flatten_tests(error_tests)); + tests.extend(flatten_tests(template_tests).into_iter().map(|mut t| { + t.template_delimiters = Some(("<%", "%>")); + t + })); let language = get_language(language_name); let mut failure_count = 0; - for (example_name, input, expected_output, has_fields) in tests { - println!(" {} example - {}", language_name, example_name); + + let log_seed = env::var("TREE_SITTER_LOG_SEED").is_ok(); + + println!(); + for test in tests { + println!(" {} example - {}", language_name, test.name); let passed = allocations::record(|| { let mut log_session = None; let mut parser = get_parser(&mut log_session, "log.html"); parser.set_language(language).unwrap(); + set_included_ranges(&mut parser, &test.input, test.template_delimiters); - let tree = parser.parse(&input, None).unwrap(); + let tree = parser.parse(&test.input, None).unwrap(); let mut actual_output = tree.root_node().to_sexp(); - if !has_fields { + if !test.has_fields { actual_output = strip_sexp_fields(actual_output); } - if actual_output != expected_output { + if actual_output != test.output { println!( "Incorrect initial parse for {} - {}", - language_name, example_name, + language_name, test.name, ); print_diff_key(); - print_diff(&actual_output, &expected_output); + print_diff(&actual_output, &test.output); println!(""); return false; } @@ -127,18 +142,18 @@ let mut parser = Parser::new(); parser.set_language(language).unwrap(); - let tree = parser.parse(&input, None).unwrap(); + let tree = parser.parse(&test.input, None).unwrap(); drop(parser); for trial in 0..*ITERATION_COUNT { - let seed = *START_SEED + trial; + let seed = start_seed + trial; let passed = allocations::record(|| { let mut rand = Rand::new(seed); let mut log_session = None; let mut parser = get_parser(&mut log_session, "log.html"); parser.set_language(language).unwrap(); let mut tree = tree.clone(); - let mut input = input.clone(); + let mut input = test.input.clone(); if *LOG_GRAPH_ENABLED { eprintln!("{}\n", String::from_utf8_lossy(&input)); @@ -152,21 +167,21 @@ perform_edit(&mut tree, &mut input, &edit); } - // println!(" seed: {}", seed); + if log_seed { + println!(" seed: {}", seed); + } if *LOG_GRAPH_ENABLED { eprintln!("{}\n", String::from_utf8_lossy(&input)); } + set_included_ranges(&mut parser, &input, test.template_delimiters); let mut tree2 = parser.parse(&input, Some(&tree)).unwrap(); // Check that the new tree is consistent. check_consistent_sizes(&tree2, &input); if let Err(message) = check_changed_ranges(&tree, &tree2, &input) { - println!( - "\nUnexpected scope change in seed {}\n{}\n\n", - seed, message - ); + println!("\nUnexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n",); return false; } @@ -178,21 +193,22 @@ eprintln!("{}\n", String::from_utf8_lossy(&input)); } + set_included_ranges(&mut parser, &test.input, test.template_delimiters); let tree3 = parser.parse(&input, Some(&tree2)).unwrap(); // Verify that the final tree matches the expectation from the corpus. let mut actual_output = tree3.root_node().to_sexp(); - if !has_fields { + if !test.has_fields { actual_output = strip_sexp_fields(actual_output); } - if actual_output != expected_output { + if actual_output != test.output { println!( "Incorrect parse for {} - {} - seed {}", - language_name, example_name, seed + language_name, test.name, seed ); print_diff_key(); - print_diff(&actual_output, &expected_output); + print_diff(&actual_output, &test.output); println!(""); return false; } @@ -200,7 +216,7 @@ // Check that the edited tree is consistent. check_consistent_sizes(&tree3, &input); if let Err(message) = check_changed_ranges(&tree2, &tree3, &input) { - eprintln!("Unexpected scope change in seed {}\n{}\n\n", seed, message); + println!("Unexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n"); return false; } @@ -293,23 +309,23 @@ eprintln!("test language: {:?}", language_name); } - for (name, input, expected_output, has_fields) in tests { - eprintln!(" example: {:?}", name); + for test in tests { + eprintln!(" example: {:?}", test.name); let passed = allocations::record(|| { let mut log_session = None; let mut parser = get_parser(&mut log_session, "log.html"); parser.set_language(language).unwrap(); - let tree = parser.parse(&input, None).unwrap(); + let tree = parser.parse(&test.input, None).unwrap(); let mut actual_output = tree.root_node().to_sexp(); - if !has_fields { + if !test.has_fields { actual_output = strip_sexp_fields(actual_output); } - if actual_output == expected_output { + if actual_output == test.output { true } else { print_diff_key(); - print_diff(&actual_output, &expected_output); + print_diff(&actual_output, &test.output); println!(""); false } @@ -390,6 +406,7 @@ let old_range = old_tree.root_node().range(); let new_range = new_tree.root_node().range(); + let byte_range = old_range.start_byte.min(new_range.start_byte)..old_range.end_byte.max(new_range.end_byte); let point_range = old_range.start_point.min(new_range.start_point) @@ -407,6 +424,45 @@ old_scope_sequence.check_changes(&new_scope_sequence, &input, &changed_ranges) } +fn set_included_ranges(parser: &mut Parser, input: &[u8], delimiters: Option<(&str, &str)>) { + if let Some((start, end)) = delimiters { + let mut ranges = Vec::new(); + let mut ix = 0; + while ix < input.len() { + let Some(mut start_ix) = input[ix..].windows(2).position(|win| win == start.as_bytes()) else { break }; + start_ix += ix + start.len(); + let end_ix = input[start_ix..] + .windows(2) + .position(|win| win == end.as_bytes()) + .map_or(input.len(), |ix| start_ix + ix); + ix = end_ix; + ranges.push(Range { + start_byte: start_ix, + end_byte: end_ix, + start_point: point_for_offset(input, start_ix), + end_point: point_for_offset(input, end_ix), + }); + } + + parser.set_included_ranges(&ranges).unwrap(); + } else { + parser.set_included_ranges(&[]).unwrap(); + } +} + +fn point_for_offset(text: &[u8], offset: usize) -> Point { + let mut point = Point::default(); + for byte in &text[..offset] { + if *byte == b'\n' { + point.row += 1; + point.column = 0; + } else { + point.column += 1; + } + } + point +} + fn get_parser(session: &mut Option, log_filename: &str) -> Parser { let mut parser = Parser::new(); @@ -425,13 +481,16 @@ parser } -fn flatten_tests(test: TestEntry) -> Vec<(String, Vec, String, bool)> { - fn helper( - test: TestEntry, - is_root: bool, - prefix: &str, - result: &mut Vec<(String, Vec, String, bool)>, - ) { +struct FlattenedTest { + name: String, + input: Vec, + output: String, + has_fields: bool, + template_delimiters: Option<(&'static str, &'static str)>, +} + +fn flatten_tests(test: TestEntry) -> Vec { + fn helper(test: TestEntry, is_root: bool, prefix: &str, result: &mut Vec) { match test { TestEntry::Example { mut name, @@ -448,7 +507,13 @@ return; } } - result.push((name, input, output, has_fields)); + result.push(FlattenedTest { + name, + input, + output, + has_fields, + template_delimiters: None, + }); } TestEntry::Group { mut name, children, .. diff -Nru tree-sitter-0.20.7/cli/src/tests/helpers/mod.rs tree-sitter-0.20.8/cli/src/tests/helpers/mod.rs --- tree-sitter-0.20.7/cli/src/tests/helpers/mod.rs 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/cli/src/tests/helpers/mod.rs 2023-04-04 09:15:07.000000000 +0000 @@ -6,7 +6,8 @@ pub(super) mod scope_sequence; use lazy_static::lazy_static; -use std::{env, time, usize}; +use rand::Rng; +use std::env; lazy_static! { pub static ref LOG_ENABLED: bool = env::var("TREE_SITTER_LOG").is_ok(); @@ -16,11 +17,7 @@ } lazy_static! { - pub static ref START_SEED: usize = - int_env_var("TREE_SITTER_SEED").unwrap_or_else(|| time::SystemTime::now() - .duration_since(time::UNIX_EPOCH) - .unwrap() - .as_secs() as usize,); + pub static ref START_SEED: usize = new_seed(); pub static ref EDIT_COUNT: usize = int_env_var("TREE_SITTER_EDITS").unwrap_or(3); pub static ref ITERATION_COUNT: usize = int_env_var("TREE_SITTER_ITERATIONS").unwrap_or(10); } @@ -28,3 +25,10 @@ fn int_env_var(name: &'static str) -> Option { env::var(name).ok().and_then(|e| e.parse().ok()) } + +pub(crate) fn new_seed() -> usize { + int_env_var("TREE_SITTER_SEED").unwrap_or_else(|| { + let mut rng = rand::thread_rng(); + rng.gen::() + }) +} diff -Nru tree-sitter-0.20.7/cli/src/tests/helpers/random.rs tree-sitter-0.20.8/cli/src/tests/helpers/random.rs --- tree-sitter-0.20.7/cli/src/tests/helpers/random.rs 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/cli/src/tests/helpers/random.rs 2023-04-04 09:15:07.000000000 +0000 @@ -4,7 +4,7 @@ }; const OPERATORS: &[char] = &[ - '+', '-', '<', '>', '(', ')', '*', '/', '&', '|', '!', ',', '.', + '+', '-', '<', '>', '(', ')', '*', '/', '&', '|', '!', ',', '.', '%', ]; pub struct Rand(StdRng); diff -Nru tree-sitter-0.20.7/cli/src/tests/helpers/scope_sequence.rs tree-sitter-0.20.8/cli/src/tests/helpers/scope_sequence.rs --- tree-sitter-0.20.7/cli/src/tests/helpers/scope_sequence.rs 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/cli/src/tests/helpers/scope_sequence.rs 2023-04-04 09:15:07.000000000 +0000 @@ -44,20 +44,10 @@ text: &Vec, known_changed_ranges: &Vec, ) -> Result<(), String> { - if self.0.len() != text.len() { - panic!( - "Inconsistent scope sequence: {:?}", - self.0 - .iter() - .zip(text.iter().map(|c| *c as char)) - .collect::>() - ); - } - - assert_eq!(self.0.len(), other.0.len()); let mut position = Point { row: 0, column: 0 }; - for (i, stack) in self.0.iter().enumerate() { - let other_stack = &other.0[i]; + for i in 0..(self.0.len().max(other.0.len())) { + let stack = &self.0.get(i); + let other_stack = &other.0.get(i); if *stack != *other_stack && ![b'\r', b'\n'].contains(&text[i]) { let containing_range = known_changed_ranges .iter() diff -Nru tree-sitter-0.20.7/cli/src/tests/parser_test.rs tree-sitter-0.20.8/cli/src/tests/parser_test.rs --- tree-sitter-0.20.7/cli/src/tests/parser_test.rs 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/cli/src/tests/parser_test.rs 2023-04-04 09:15:07.000000000 +0000 @@ -8,6 +8,7 @@ generate::generate_parser_for_grammar, parse::{perform_edit, Edit}, }; +use proc_macro::retry; use std::{ sync::atomic::{AtomicUsize, Ordering}, thread, time, @@ -505,7 +506,7 @@ let tree = parser.parse(&source, None).unwrap(); assert_eq!( tree.root_node().to_sexp(), - "(module (expression_statement (assignment left: (identifier) right: (expression_list (identifier) (string)))))" + "(module (expression_statement (assignment left: (identifier) right: (expression_list (identifier) (string string_content: (string_content))))))" ); // Delete a suffix of the source code, starting in the middle of the string @@ -638,6 +639,7 @@ // Timeouts #[test] +#[retry(10)] fn test_parsing_with_a_timeout() { let mut parser = Parser::new(); parser.set_language(get_language("json")).unwrap(); @@ -829,6 +831,7 @@ js_tree.root_node().start_position(), Point::new(0, source_code.find("console").unwrap()) ); + assert_eq!(js_tree.included_ranges(), &[script_content_node.range()]); } #[test] @@ -853,28 +856,27 @@ let close_quote_node = template_string_node.child(3).unwrap(); parser.set_language(get_language("html")).unwrap(); - parser - .set_included_ranges(&[ - Range { - start_byte: open_quote_node.end_byte(), - start_point: open_quote_node.end_position(), - end_byte: interpolation_node1.start_byte(), - end_point: interpolation_node1.start_position(), - }, - Range { - start_byte: interpolation_node1.end_byte(), - start_point: interpolation_node1.end_position(), - end_byte: interpolation_node2.start_byte(), - end_point: interpolation_node2.start_position(), - }, - Range { - start_byte: interpolation_node2.end_byte(), - start_point: interpolation_node2.end_position(), - end_byte: close_quote_node.start_byte(), - end_point: close_quote_node.start_position(), - }, - ]) - .unwrap(); + let html_ranges = &[ + Range { + start_byte: open_quote_node.end_byte(), + start_point: open_quote_node.end_position(), + end_byte: interpolation_node1.start_byte(), + end_point: interpolation_node1.start_position(), + }, + Range { + start_byte: interpolation_node1.end_byte(), + start_point: interpolation_node1.end_position(), + end_byte: interpolation_node2.start_byte(), + end_point: interpolation_node2.start_position(), + }, + Range { + start_byte: interpolation_node2.end_byte(), + start_point: interpolation_node2.end_position(), + end_byte: close_quote_node.start_byte(), + end_point: close_quote_node.start_position(), + }, + ]; + parser.set_included_ranges(html_ranges).unwrap(); let html_tree = parser.parse(source_code, None).unwrap(); assert_eq!( @@ -888,6 +890,7 @@ " (end_tag (tag_name))))", ) ); + assert_eq!(html_tree.included_ranges(), html_ranges); let div_element_node = html_tree.root_node().child(0).unwrap(); let hello_text_node = div_element_node.child(1).unwrap(); @@ -950,7 +953,9 @@ parser.set_included_ranges(&[range_to_parse]).unwrap(); - let html_tree = parser.parse(source_code, None).unwrap(); + let html_tree = parser + .parse_with(&mut chunked_input(source_code, 3), None) + .unwrap(); assert_eq!(html_tree.root_node().range(), range_to_parse); @@ -1077,7 +1082,9 @@ // Parse HTML including the template directive, which will cause an error let mut parser = Parser::new(); parser.set_language(get_language("html")).unwrap(); - let mut first_tree = parser.parse(&source_code, None).unwrap(); + let mut first_tree = parser + .parse_with(&mut chunked_input(&source_code, 3), None) + .unwrap(); // Insert code at the beginning of the document. let prefix = "a very very long line of plain text. "; @@ -1112,7 +1119,9 @@ }, ]) .unwrap(); - let tree = parser.parse(&source_code, Some(&first_tree)).unwrap(); + let tree = parser + .parse_with(&mut chunked_input(&source_code, 3), Some(&first_tree)) + .unwrap(); assert_eq!( tree.root_node().to_sexp(), @@ -1163,7 +1172,9 @@ parser .set_included_ranges(&[simple_range(range1_start, range1_end)]) .unwrap(); - let tree = parser.parse(source_code, None).unwrap(); + let tree = parser + .parse_with(&mut chunked_input(&source_code, 3), None) + .unwrap(); assert_eq!( tree.root_node().to_sexp(), concat!( @@ -1180,7 +1191,9 @@ simple_range(range3_start, range3_end), ]) .unwrap(); - let tree2 = parser.parse(&source_code, Some(&tree)).unwrap(); + let tree2 = parser + .parse_with(&mut chunked_input(&source_code, 3), Some(&tree)) + .unwrap(); assert_eq!( tree2.root_node().to_sexp(), concat!( @@ -1288,3 +1301,7 @@ end_point: Point::new(0, end), } } + +fn chunked_input<'a>(text: &'a str, size: usize) -> impl FnMut(usize, Point) -> &'a [u8] { + move |offset, _| text[offset..text.len().min(offset + size)].as_bytes() +} diff -Nru tree-sitter-0.20.7/cli/src/tests/proc_macro/Cargo.toml tree-sitter-0.20.8/cli/src/tests/proc_macro/Cargo.toml --- tree-sitter-0.20.7/cli/src/tests/proc_macro/Cargo.toml 1970-01-01 00:00:00.000000000 +0000 +++ tree-sitter-0.20.8/cli/src/tests/proc_macro/Cargo.toml 2023-04-04 09:15:07.000000000 +0000 @@ -0,0 +1,15 @@ +[package] +name = "proc_macro" +version = "0.1.0" +edition = "2021" +publish = false +rust-version.workspace = true + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = "1" +quote = "1" +rand = "0.8.5" +syn = { version = "1", features = ["full"] } diff -Nru tree-sitter-0.20.7/cli/src/tests/proc_macro/src/lib.rs tree-sitter-0.20.8/cli/src/tests/proc_macro/src/lib.rs --- tree-sitter-0.20.7/cli/src/tests/proc_macro/src/lib.rs 1970-01-01 00:00:00.000000000 +0000 +++ tree-sitter-0.20.8/cli/src/tests/proc_macro/src/lib.rs 2023-04-04 09:15:07.000000000 +0000 @@ -0,0 +1,137 @@ +use proc_macro::TokenStream; +use proc_macro2::Span; +use quote::quote; +use syn::{ + parse::{Parse, ParseStream}, + parse_macro_input, Error, Expr, Ident, ItemFn, LitInt, Token, +}; + +#[proc_macro_attribute] +pub fn retry(args: TokenStream, input: TokenStream) -> TokenStream { + let count = parse_macro_input!(args as LitInt); + let input = parse_macro_input!(input as ItemFn); + let attrs = input.attrs.clone(); + let name = input.sig.ident.clone(); + + TokenStream::from(quote! { + #(#attrs),* + fn #name() { + #input + + for i in 0..=#count { + let result = std::panic::catch_unwind(|| { + #name(); + }); + + if result.is_ok() { + return; + } + + if i == #count { + std::panic::resume_unwind(result.unwrap_err()); + } + } + } + }) +} + +#[proc_macro_attribute] +pub fn test_with_seed(args: TokenStream, input: TokenStream) -> TokenStream { + struct Args { + retry: LitInt, + seed: Expr, + seed_fn: Option, + } + + impl Parse for Args { + fn parse(input: ParseStream) -> syn::Result { + let mut retry = None; + let mut seed = None; + let mut seed_fn = None; + + while !input.is_empty() { + let name = input.parse::()?; + match name.to_string().as_str() { + "retry" => { + input.parse::()?; + retry.replace(input.parse()?); + } + "seed" => { + input.parse::()?; + seed.replace(input.parse()?); + } + "seed_fn" => { + input.parse::()?; + seed_fn.replace(input.parse()?); + } + x => { + return Err(Error::new( + name.span(), + format!("Unsupported parameter `{x}`"), + )) + } + } + + if !input.is_empty() { + input.parse::()?; + } + } + + if retry.is_none() { + retry.replace(LitInt::new("0", Span::mixed_site())); + } + + Ok(Args { + retry: retry.expect("`retry` parameter is requred"), + seed: seed.expect("`initial_seed` parameter is required"), + seed_fn, + }) + } + } + + let Args { + retry, + seed, + seed_fn, + } = parse_macro_input!(args as Args); + + let seed_fn = seed_fn.iter(); + + let func = parse_macro_input!(input as ItemFn); + let attrs = func.attrs.clone(); + let name = func.sig.ident.clone(); + + // dbg!(quote::ToTokens::into_token_stream(&func)); + + TokenStream::from(quote! { + #[test] + #(#attrs),* + fn #name() { + #func + + let mut seed = #seed; + + for i in 0..=#retry { + let result = std::panic::catch_unwind(|| { + #name(seed); + }); + + if result.is_ok() { + return; + } + + if i == #retry { + std::panic::resume_unwind(result.unwrap_err()); + } + + #( + seed = #seed_fn(); + )* + + if i < #retry { + println!("\nRetry {}/{} with a new seed {}", i + 1, #retry, seed); + } + } + } + }) +} diff -Nru tree-sitter-0.20.7/cli/src/tests/query_test.rs tree-sitter-0.20.8/cli/src/tests/query_test.rs --- tree-sitter-0.20.7/cli/src/tests/query_test.rs 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/cli/src/tests/query_test.rs 2023-04-04 09:15:07.000000000 +0000 @@ -2,6 +2,7 @@ allocations, fixtures::get_language, query_helpers::{Match, Pattern}, + ITERATION_COUNT, }; use lazy_static::lazy_static; use rand::{prelude::StdRng, SeedableRng}; @@ -10,6 +11,7 @@ CaptureQuantifier, Language, Node, Parser, Point, Query, QueryCapture, QueryCursor, QueryError, QueryErrorKind, QueryMatch, QueryPredicate, QueryPredicateArg, QueryProperty, }; +use unindent::Unindent; lazy_static! { static ref EXAMPLE_FILTER: Option = env::var("TREE_SITTER_TEST_EXAMPLE_FILTER").ok(); @@ -1874,7 +1876,6 @@ cursor .set_byte_range(0..8) .matches(&query, tree.root_node(), source.as_bytes()); - assert_eq!( collect_matches(matches, &query, source), &[ @@ -1888,7 +1889,6 @@ cursor .set_byte_range(5..15) .matches(&query, tree.root_node(), source.as_bytes()); - assert_eq!( collect_matches(matches, &query, source), &[ @@ -1902,7 +1902,6 @@ cursor .set_byte_range(12..0) .matches(&query, tree.root_node(), source.as_bytes()); - assert_eq!( collect_matches(matches, &query, source), &[ @@ -1920,20 +1919,28 @@ let language = get_language("javascript"); let query = Query::new(language, "(identifier) @element").unwrap(); - let source = "[a, b,\n c, d,\n e, f,\n g]"; + let source = " + [ + a, b, + c, d, + e, f, + g, h, + i, j, + k, l, + ] + " + .unindent(); let mut parser = Parser::new(); parser.set_language(language).unwrap(); let tree = parser.parse(&source, None).unwrap(); - let mut cursor = QueryCursor::new(); let matches = cursor - .set_point_range(Point::new(0, 0)..Point::new(1, 3)) + .set_point_range(Point::new(1, 0)..Point::new(2, 3)) .matches(&query, tree.root_node(), source.as_bytes()); - assert_eq!( - collect_matches(matches, &query, source), + collect_matches(matches, &query, &source), &[ (0, vec![("element", "a")]), (0, vec![("element", "b")]), @@ -1942,11 +1949,10 @@ ); let matches = cursor - .set_point_range(Point::new(1, 0)..Point::new(2, 3)) + .set_point_range(Point::new(2, 0)..Point::new(3, 3)) .matches(&query, tree.root_node(), source.as_bytes()); - assert_eq!( - collect_matches(matches, &query, source), + collect_matches(matches, &query, &source), &[ (0, vec![("element", "c")]), (0, vec![("element", "d")]), @@ -1954,16 +1960,19 @@ ] ); + // Zero end point is treated like no end point. let matches = cursor - .set_point_range(Point::new(2, 1)..Point::new(0, 0)) + .set_point_range(Point::new(4, 1)..Point::new(0, 0)) .matches(&query, tree.root_node(), source.as_bytes()); - assert_eq!( - collect_matches(matches, &query, source), + collect_matches(matches, &query, &source), &[ - (0, vec![("element", "e")]), - (0, vec![("element", "f")]), (0, vec![("element", "g")]), + (0, vec![("element", "h")]), + (0, vec![("element", "i")]), + (0, vec![("element", "j")]), + (0, vec![("element", "k")]), + (0, vec![("element", "l")]), ] ); }); @@ -2211,6 +2220,57 @@ } #[test] +fn test_query_matches_within_range_of_long_repetition() { + allocations::record(|| { + let language = get_language("rust"); + let query = Query::new( + language, + " + (function_item name: (identifier) @fn-name) + ", + ) + .unwrap(); + + let source = " + fn zero() {} + fn one() {} + fn two() {} + fn three() {} + fn four() {} + fn five() {} + fn six() {} + fn seven() {} + fn eight() {} + fn nine() {} + fn ten() {} + fn eleven() {} + fn twelve() {} + " + .unindent(); + + let mut parser = Parser::new(); + let mut cursor = QueryCursor::new(); + + parser.set_language(language).unwrap(); + let tree = parser.parse(&source, None).unwrap(); + + let matches = cursor + .set_point_range(Point::new(8, 0)..Point::new(20, 0)) + .matches(&query, tree.root_node(), source.as_bytes()); + assert_eq!( + collect_matches(matches, &query, &source), + &[ + (0, vec![("fn-name", "eight")]), + (0, vec![("fn-name", "nine")]), + (0, vec![("fn-name", "ten")]), + (0, vec![("fn-name", "eleven")]), + (0, vec![("fn-name", "twelve")]), + ] + ); + }); +} + +#[test] fn test_query_matches_different_queries_same_cursor() { allocations::record(|| { let language = get_language("javascript"); @@ -3634,17 +3694,22 @@ .parse(include_str!("helpers/query_helpers.rs"), None) .unwrap(); - // let start_seed = *SEED; let start_seed = 0; + let end_seed = start_seed + *ITERATION_COUNT; - for i in 0..100 { - let seed = (start_seed + i) as u64; + for seed in start_seed..(start_seed + end_seed) { + let seed = seed as u64; let mut rand = StdRng::seed_from_u64(seed); let (pattern_ast, _) = Pattern::random_pattern_in_tree(&pattern_tree, &mut rand); let pattern = pattern_ast.to_string(); let expected_matches = pattern_ast.matches_in_tree(&test_tree); - let query = Query::new(language, &pattern).unwrap(); + let query = match Query::new(language, &pattern) { + Ok(query) => query, + Err(e) => { + panic!("failed to build query for pattern {pattern} - {e}. seed: {seed}"); + } + }; let mut actual_matches = cursor .matches( &query, @@ -4062,6 +4127,103 @@ "Description: {}, Pattern: {:?}", row.description, row.pattern + .split_ascii_whitespace() + .collect::>() + .join(" "), + ) + } + }); +} + +#[test] +fn test_query_is_pattern_non_local() { + struct Row { + description: &'static str, + pattern: &'static str, + language: Language, + is_non_local: bool, + } + + let rows = [ + Row { + description: "simple token", + pattern: r#"(identifier)"#, + language: get_language("python"), + is_non_local: false, + }, + Row { + description: "siblings that can occur in an argument list", + pattern: r#"((identifier) (identifier))"#, + language: get_language("python"), + is_non_local: true, + }, + Row { + description: "siblings that can occur in a statement block", + pattern: r#"((return_statement) (return_statement))"#, + language: get_language("python"), + is_non_local: true, + }, + Row { + description: "siblings that can occur in a source file", + pattern: r#"((function_definition) (class_definition))"#, + language: get_language("python"), + is_non_local: true, + }, + Row { + description: "siblings that can't occur in any repetition", + pattern: r#"("{" "}")"#, + language: get_language("python"), + is_non_local: false, + }, + Row { + description: "siblings that can't occur in any repetition, wildcard root", + pattern: r#"(_ "{" "}") @foo"#, + language: get_language("javascript"), + is_non_local: false, + }, + Row { + description: "siblings that can occur in a class body, wildcard root", + pattern: r#"(_ (method_definition) (method_definition)) @foo"#, + language: get_language("javascript"), + is_non_local: true, + }, + Row { + description: "top-level repetitions that can occur in a class body", + pattern: r#"(method_definition)+ @foo"#, + language: get_language("javascript"), + is_non_local: true, + }, + Row { + description: "top-level repetitions that can occur in a statement block", + pattern: r#"(return_statement)+ @foo"#, + language: get_language("javascript"), + is_non_local: true, + }, + Row { + description: "rooted pattern that can occur in a statement block", + pattern: r#"(return_statement) @foo"#, + language: get_language("javascript"), + is_non_local: false, + }, + ]; + + allocations::record(|| { + eprintln!(""); + + for row in &rows { + if let Some(filter) = EXAMPLE_FILTER.as_ref() { + if !row.description.contains(filter.as_str()) { + continue; + } + } + eprintln!(" query example: {:?}", row.description); + let query = Query::new(row.language, row.pattern).unwrap(); + assert_eq!( + query.is_pattern_non_local(0), + row.is_non_local, + "Description: {}, Pattern: {:?}", + row.description, + row.pattern .split_ascii_whitespace() .collect::>() .join(" "), diff -Nru tree-sitter-0.20.7/cli/src/tests/tree_test.rs tree-sitter-0.20.8/cli/src/tests/tree_test.rs --- tree-sitter-0.20.7/cli/src/tests/tree_test.rs 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/cli/src/tests/tree_test.rs 2023-04-04 09:15:07.000000000 +0000 @@ -233,6 +233,71 @@ } #[test] +fn test_tree_edit_with_included_ranges() { + let mut parser = Parser::new(); + parser.set_language(get_language("html")).unwrap(); + + let source = "
<% if a %>a<% else %>b<% end %>
"; + + let ranges = [0..5, 15..29, 39..53, 62..68]; + + parser + .set_included_ranges( + &ranges + .iter() + .map(|range| Range { + start_byte: range.start, + end_byte: range.end, + start_point: Point::new(0, range.start), + end_point: Point::new(0, range.end), + }) + .collect::>(), + ) + .unwrap(); + + let mut tree = parser.parse(source, None).unwrap(); + + tree.edit(&InputEdit { + start_byte: 29, + old_end_byte: 53, + new_end_byte: 29, + start_position: Point::new(0, 29), + old_end_position: Point::new(0, 53), + new_end_position: Point::new(0, 29), + }); + + assert_eq!( + tree.included_ranges(), + &[ + Range { + start_byte: 0, + end_byte: 5, + start_point: Point::new(0, 0), + end_point: Point::new(0, 5), + }, + Range { + start_byte: 15, + end_byte: 29, + start_point: Point::new(0, 15), + end_point: Point::new(0, 29), + }, + Range { + start_byte: 29, + end_byte: 29, + start_point: Point::new(0, 29), + end_point: Point::new(0, 29), + }, + Range { + start_byte: 38, + end_byte: 44, + start_point: Point::new(0, 38), + end_point: Point::new(0, 44), + } + ] + ); +} + +#[test] fn test_tree_cursor() { let mut parser = Parser::new(); parser.set_language(get_language("rust")).unwrap(); diff -Nru tree-sitter-0.20.7/cli/src/util.rs tree-sitter-0.20.8/cli/src/util.rs --- tree-sitter-0.20.7/cli/src/util.rs 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/cli/src/util.rs 2023-04-04 09:15:07.000000000 +0000 @@ -3,7 +3,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::thread; -use tree_sitter::Parser; +use tree_sitter::{Parser, Tree}; #[cfg(unix)] use anyhow::{anyhow, Context}; @@ -13,7 +13,14 @@ use std::process::{Child, ChildStdin, Command, Stdio}; #[cfg(unix)] -const HTML_HEADER: &[u8] = b"\n\n\n"; +const HTML_HEADER: &[u8] = b" + + + + +"; pub fn cancel_on_stdin() -> Arc { let result = Arc::new(AtomicUsize::new(0)); @@ -29,39 +36,66 @@ } result } + #[cfg(windows)] -pub struct LogSession(); +pub struct LogSession; #[cfg(unix)] -pub struct LogSession(PathBuf, Option, Option); +pub struct LogSession { + path: PathBuf, + dot_process: Option, + dot_process_stdin: Option, +} + +#[cfg(windows)] +pub fn print_tree_graph(_tree: &Tree, _path: &str) -> Result<()> { + Ok(()) +} #[cfg(windows)] pub fn log_graphs(_parser: &mut Parser, _path: &str) -> Result { - Ok(LogSession()) + Ok(LogSession) +} + +#[cfg(unix)] +pub fn print_tree_graph(tree: &Tree, path: &str) -> Result<()> { + let session = LogSession::new(path)?; + tree.print_dot_graph(session.dot_process_stdin.as_ref().unwrap()); + Ok(()) } #[cfg(unix)] pub fn log_graphs(parser: &mut Parser, path: &str) -> Result { - use std::io::Write; + let session = LogSession::new(path)?; + parser.print_dot_graphs(session.dot_process_stdin.as_ref().unwrap()); + Ok(session) +} - let mut dot_file = std::fs::File::create(path)?; - dot_file.write(HTML_HEADER)?; - let mut dot_process = Command::new("dot") - .arg("-Tsvg") - .stdin(Stdio::piped()) - .stdout(dot_file) - .spawn() - .with_context(|| "Failed to run the `dot` command. Check that graphviz is installed.")?; - let dot_stdin = dot_process - .stdin - .take() - .ok_or_else(|| anyhow!("Failed to open stdin for `dot` process."))?; - parser.print_dot_graphs(&dot_stdin); - Ok(LogSession( - PathBuf::from(path), - Some(dot_process), - Some(dot_stdin), - )) +#[cfg(unix)] +impl LogSession { + fn new(path: &str) -> Result { + use std::io::Write; + + let mut dot_file = std::fs::File::create(path)?; + dot_file.write(HTML_HEADER)?; + let mut dot_process = Command::new("dot") + .arg("-Tsvg") + .stdin(Stdio::piped()) + .stdout(dot_file) + .spawn() + .with_context(|| { + "Failed to run the `dot` command. Check that graphviz is installed." + })?; + let dot_stdin = dot_process + .stdin + .take() + .ok_or_else(|| anyhow!("Failed to open stdin for `dot` process."))?; + Ok(Self { + path: PathBuf::from(path), + dot_process: Some(dot_process), + dot_process_stdin: Some(dot_stdin), + }) + } } #[cfg(unix)] @@ -69,13 +103,13 @@ fn drop(&mut self) { use std::fs; - drop(self.2.take().unwrap()); - let output = self.1.take().unwrap().wait_with_output().unwrap(); + drop(self.dot_process_stdin.take().unwrap()); + let output = self.dot_process.take().unwrap().wait_with_output().unwrap(); if output.status.success() { if cfg!(target_os = "macos") - && fs::metadata(&self.0).unwrap().len() > HTML_HEADER.len() as u64 + && fs::metadata(&self.path).unwrap().len() > HTML_HEADER.len() as u64 { - Command::new("open").arg(&self.0).output().unwrap(); + Command::new("open").arg(&self.path).output().unwrap(); } } else { eprintln!( diff -Nru tree-sitter-0.20.7/cli/src/wasm.rs tree-sitter-0.20.8/cli/src/wasm.rs --- tree-sitter-0.20.7/cli/src/wasm.rs 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/cli/src/wasm.rs 2023-04-04 09:15:07.000000000 +0000 @@ -65,6 +65,11 @@ // Run `emcc` in a container using the `emscripten-slim` image command.args(&[EMSCRIPTEN_TAG, "emcc"]); } else { + if force_docker { + return Err(anyhow!( + "You must have docker on your PATH to run this command with --docker" + )); + } return Err(anyhow!( "You must have either emcc or docker on your PATH to run this command" )); @@ -83,6 +88,8 @@ "-s", "NODEJS_CATCH_EXIT=0", "-s", + "NODEJS_CATCH_REJECTION=0", + "-s", &format!("EXPORTED_FUNCTIONS=[\"_tree_sitter_{}\"]", grammar_name), "-fno-exceptions", "-I", diff -Nru tree-sitter-0.20.7/debian/changelog tree-sitter-0.20.8/debian/changelog --- tree-sitter-0.20.7/debian/changelog 2022-11-15 12:19:34.000000000 +0000 +++ tree-sitter-0.20.8/debian/changelog 2023-07-04 13:09:44.000000000 +0000 @@ -1,3 +1,18 @@ +tree-sitter (0.20.8-2) unstable; urgency=medium + + * Upload to unstable + + -- James McCoy Tue, 04 Jul 2023 09:09:44 -0400 + +tree-sitter (0.20.8-1) experimental; urgency=medium + + * New upstream release + * Update symbols for 0.20.8 + * Mark all exported symbols not part of the public API as optional + * Declare compliance with Policy 4.6.2, no changes needed + + -- James McCoy Fri, 07 Apr 2023 19:59:30 -0400 + tree-sitter (0.20.7-1) unstable; urgency=medium * New upstream release diff -Nru tree-sitter-0.20.7/debian/control tree-sitter-0.20.8/debian/control --- tree-sitter-0.20.7/debian/control 2022-11-15 12:19:34.000000000 +0000 +++ tree-sitter-0.20.8/debian/control 2023-07-04 13:09:44.000000000 +0000 @@ -5,7 +5,7 @@ James McCoy , Build-Depends: debhelper-compat (= 13), -Standards-Version: 4.6.1 +Standards-Version: 4.6.2 Section: libs Homepage: https://tree-sitter.github.io/tree-sitter/ Rules-Requires-Root: no diff -Nru tree-sitter-0.20.7/debian/copyright tree-sitter-0.20.8/debian/copyright --- tree-sitter-0.20.7/debian/copyright 2022-11-15 12:19:34.000000000 +0000 +++ tree-sitter-0.20.8/debian/copyright 2023-07-04 13:09:44.000000000 +0000 @@ -7,7 +7,7 @@ License: Expat Files: debian/* -Copyright: 2020-2022 James McCoy +Copyright: 2020-2023 James McCoy License: Expat Files: lib/src/unicode/* diff -Nru tree-sitter-0.20.7/debian/libtree-sitter0.symbols tree-sitter-0.20.8/debian/libtree-sitter0.symbols --- tree-sitter-0.20.7/debian/libtree-sitter0.symbols 2022-11-15 12:19:34.000000000 +0000 +++ tree-sitter-0.20.8/debian/libtree-sitter0.symbols 2023-07-04 13:09:44.000000000 +0000 @@ -1,35 +1,13 @@ libtree-sitter.so.0 libtree-sitter0 #MINVER# * Build-Depends-Package: libtree-sitter-dev - ts_current_calloc@Base 0.20.2 - ts_current_free@Base 0.20.2 - ts_current_malloc@Base 0.20.2 - ts_current_realloc@Base 0.20.2 - ts_external_scanner_state_copy@Base 0.19 - ts_external_scanner_state_data@Base 0.19 - ts_external_scanner_state_delete@Base 0.19 - ts_external_scanner_state_eq@Base 0.19 - ts_external_scanner_state_init@Base 0.19 ts_language_field_count@Base 0.19 ts_language_field_id_for_name@Base 0.19 ts_language_field_name_for_id@Base 0.19 - ts_language_public_symbol@Base 0.19 ts_language_symbol_count@Base 0.19 ts_language_symbol_for_name@Base 0.19 - ts_language_symbol_metadata@Base 0.19 ts_language_symbol_name@Base 0.19 ts_language_symbol_type@Base 0.19 - ts_language_table_entry@Base 0.19 ts_language_version@Base 0.19 - ts_lexer_advance_to_end@Base 0.19 - ts_lexer_delete@Base 0.19 - ts_lexer_finish@Base 0.19 - ts_lexer_included_ranges@Base 0.19 - ts_lexer_init@Base 0.19 - ts_lexer_mark_end@Base 0.19 - ts_lexer_reset@Base 0.19 - ts_lexer_set_included_ranges@Base 0.19 - ts_lexer_set_input@Base 0.19 - ts_lexer_start@Base 0.19 ts_node_child@Base 0.19 ts_node_child_by_field_id@Base 0.19 ts_node_child_by_field_name@Base 0.19 @@ -53,10 +31,8 @@ ts_node_named_child_count@Base 0.19 ts_node_named_descendant_for_byte_range@Base 0.19 ts_node_named_descendant_for_point_range@Base 0.19 - ts_node_new@Base 0.19 ts_node_next_named_sibling@Base 0.19 ts_node_next_sibling@Base 0.19 - ts_node_parent@Base 0.19 ts_node_prev_named_sibling@Base 0.19 ts_node_prev_sibling@Base 0.19 ts_node_start_byte@Base 0.19 @@ -74,19 +50,15 @@ ts_parser_parse_string@Base 0.19 ts_parser_parse_string_encoding@Base 0.19 ts_parser_print_dot_graphs@Base 0.19 - ts_parser_reset@Base 0.19 ts_parser_set_cancellation_flag@Base 0.19 ts_parser_set_included_ranges@Base 0.19 ts_parser_set_language@Base 0.19 ts_parser_set_logger@Base 0.19 ts_parser_set_timeout_micros@Base 0.19 ts_parser_timeout_micros@Base 0.19 - ts_query__step_is_fallible@Base 0.20.1 ts_query_capture_count@Base 0.19 ts_query_capture_name_for_id@Base 0.19 ts_query_capture_quantifier_for_id@Base 0.20.3 - ts_query_cursor__compare_captures@Base 0.19 - ts_query_cursor__compare_nodes@Base 0.19 ts_query_cursor_delete@Base 0.19 ts_query_cursor_did_exceed_match_limit@Base 0.19.2 ts_query_cursor_exec@Base 0.19 @@ -102,6 +74,7 @@ ts_query_disable_capture@Base 0.19 ts_query_disable_pattern@Base 0.19 ts_query_is_pattern_guaranteed_at_step@Base 0.20.1 + ts_query_is_pattern_non_local@Base 0.20.8 ts_query_is_pattern_rooted@Base 0.20.7 ts_query_new@Base 0.19 ts_query_pattern_count@Base 0.19 @@ -109,90 +82,121 @@ ts_query_start_byte_for_pattern@Base 0.19 ts_query_string_count@Base 0.19 ts_query_string_value_for_id@Base 0.19 - ts_range_array_get_changed_ranges@Base 0.19 - ts_range_array_intersects@Base 0.19 ts_set_allocator@Base 0.20.2 - ts_stack_can_merge@Base 0.19 - ts_stack_clear@Base 0.19 - ts_stack_copy_version@Base 0.19 - ts_stack_delete@Base 0.19 - ts_stack_dynamic_precedence@Base 0.19 - ts_stack_error_cost@Base 0.19 - ts_stack_get_summary@Base 0.19 - ts_stack_halt@Base 0.19 - ts_stack_has_advanced_since_error@Base 0.19 - ts_stack_is_active@Base 0.19 - ts_stack_is_halted@Base 0.19 - ts_stack_is_paused@Base 0.19 - ts_stack_last_external_token@Base 0.19 - ts_stack_merge@Base 0.19 - ts_stack_new@Base 0.19 - ts_stack_node_count_since_error@Base 0.19 - ts_stack_pause@Base 0.19 - ts_stack_pop_all@Base 0.19 - ts_stack_pop_count@Base 0.19 - ts_stack_pop_error@Base 0.19 - ts_stack_pop_pending@Base 0.19 - ts_stack_position@Base 0.19 - ts_stack_print_dot_graph@Base 0.19 - ts_stack_push@Base 0.19 - ts_stack_record_summary@Base 0.19 - ts_stack_remove_version@Base 0.19 - ts_stack_renumber_version@Base 0.19 - ts_stack_resume@Base 0.19 - ts_stack_set_last_external_token@Base 0.19 - ts_stack_state@Base 0.19 - ts_stack_swap_versions@Base 0.19 - ts_stack_version_count@Base 0.19 - ts_subtree__print_dot_graph@Base 0.19 - ts_subtree_array_clear@Base 0.19 - ts_subtree_array_copy@Base 0.19 - ts_subtree_array_delete@Base 0.19 - ts_subtree_array_remove_trailing_extras@Base 0.19 - ts_subtree_array_reverse@Base 0.19 - ts_subtree_balance@Base 0.19 - ts_subtree_clone@Base 0.19 - ts_subtree_compare@Base 0.19 - ts_subtree_edit@Base 0.19 - ts_subtree_external_scanner_state@Base 0.20.7 - ts_subtree_external_scanner_state_eq@Base 0.19 - ts_subtree_get_changed_ranges@Base 0.19 - ts_subtree_last_external_token@Base 0.19 - ts_subtree_make_mut@Base 0.19 - ts_subtree_new_error@Base 0.19 - ts_subtree_new_error_node@Base 0.19 - ts_subtree_new_leaf@Base 0.19 - ts_subtree_new_missing_leaf@Base 0.19 - ts_subtree_new_node@Base 0.19 - ts_subtree_pool_delete@Base 0.19 - ts_subtree_pool_new@Base 0.19 - ts_subtree_print_dot_graph@Base 0.19 - ts_subtree_release@Base 0.19 - ts_subtree_retain@Base 0.19 - ts_subtree_set_symbol@Base 0.19 - ts_subtree_string@Base 0.19 - ts_subtree_summarize_children@Base 0.19 ts_tree_copy@Base 0.19 ts_tree_cursor_copy@Base 0.19 ts_tree_cursor_current_field_id@Base 0.19 ts_tree_cursor_current_field_name@Base 0.19 ts_tree_cursor_current_node@Base 0.19 - ts_tree_cursor_current_status@Base 0.19 ts_tree_cursor_delete@Base 0.19 ts_tree_cursor_goto_first_child@Base 0.19 ts_tree_cursor_goto_first_child_for_byte@Base 0.19 ts_tree_cursor_goto_first_child_for_point@Base 0.20 ts_tree_cursor_goto_next_sibling@Base 0.19 ts_tree_cursor_goto_parent@Base 0.19 - ts_tree_cursor_init@Base 0.19 ts_tree_cursor_new@Base 0.19 - ts_tree_cursor_parent_node@Base 0.19 ts_tree_cursor_reset@Base 0.19 ts_tree_delete@Base 0.19 ts_tree_edit@Base 0.19 ts_tree_get_changed_ranges@Base 0.19 + ts_tree_included_ranges@Base 0.20.8 ts_tree_language@Base 0.19 - ts_tree_new@Base 0.19 - ts_tree_print_dot_graph@Base 0.19 + ts_tree_print_dot_graph@Base 0.20.8 ts_tree_root_node@Base 0.19 ts_tree_root_node_with_offset@Base 0.20.7 + (optional)ts_current_calloc@Base 0.20.2 + (optional)ts_current_free@Base 0.20.2 + (optional)ts_current_malloc@Base 0.20.2 + (optional)ts_current_realloc@Base 0.20.2 + (optional)ts_external_scanner_state_copy@Base 0.19 + (optional)ts_external_scanner_state_data@Base 0.19 + (optional)ts_external_scanner_state_delete@Base 0.19 + (optional)ts_external_scanner_state_eq@Base 0.19 + (optional)ts_external_scanner_state_init@Base 0.19 + (optional)ts_language_public_symbol@Base 0.19 + (optional)ts_language_symbol_metadata@Base 0.19 + (optional)ts_language_table_entry@Base 0.19 + (optional)ts_lexer_advance_to_end@Base 0.19 + (optional)ts_lexer_delete@Base 0.19 + (optional)ts_lexer_finish@Base 0.19 + (optional)ts_lexer_included_ranges@Base 0.19 + (optional)ts_lexer_init@Base 0.19 + (optional)ts_lexer_mark_end@Base 0.19 + (optional)ts_lexer_reset@Base 0.19 + (optional)ts_lexer_set_included_ranges@Base 0.19 + (optional)ts_lexer_set_input@Base 0.19 + (optional)ts_lexer_start@Base 0.19 + (optional)ts_node_new@Base 0.19 + (optional)ts_node_parent@Base 0.19 + (optional)ts_parser_reset@Base 0.19 + (optional)ts_query__step_is_fallible@Base 0.20.1 + (optional)ts_query_cursor__compare_captures@Base 0.19 + (optional)ts_query_cursor__compare_nodes@Base 0.19 + (optional)ts_range_array_get_changed_ranges@Base 0.19 + (optional)ts_range_array_intersects@Base 0.19 + (optional)ts_stack_can_merge@Base 0.19 + (optional)ts_subtree__print_dot_graph@Base 0.19 + (optional)ts_subtree_array_clear@Base 0.19 + (optional)ts_subtree_array_copy@Base 0.19 + (optional)ts_subtree_array_delete@Base 0.19 + (optional)ts_subtree_array_remove_trailing_extras@Base 0.19 + (optional)ts_subtree_array_reverse@Base 0.19 + (optional)ts_subtree_balance@Base 0.19 + (optional)ts_subtree_clone@Base 0.19 + (optional)ts_subtree_compare@Base 0.19 + (optional)ts_subtree_edit@Base 0.19 + (optional)ts_subtree_external_scanner_state@Base 0.20.7 + (optional)ts_subtree_external_scanner_state_eq@Base 0.19 + (optional)ts_subtree_get_changed_ranges@Base 0.19 + (optional)ts_subtree_last_external_token@Base 0.19 + (optional)ts_subtree_make_mut@Base 0.19 + (optional)ts_subtree_new_error@Base 0.19 + (optional)ts_subtree_new_error_node@Base 0.19 + (optional)ts_subtree_new_leaf@Base 0.19 + (optional)ts_subtree_new_missing_leaf@Base 0.19 + (optional)ts_subtree_new_node@Base 0.19 + (optional)ts_subtree_pool_delete@Base 0.19 + (optional)ts_subtree_pool_new@Base 0.19 + (optional)ts_subtree_print_dot_graph@Base 0.19 + (optional)ts_subtree_release@Base 0.19 + (optional)ts_subtree_retain@Base 0.19 + (optional)ts_subtree_set_symbol@Base 0.19 + (optional)ts_subtree_string@Base 0.19 + (optional)ts_subtree_summarize_children@Base 0.19 + (optional)ts_stack_clear@Base 0.19 + (optional)ts_stack_copy_version@Base 0.19 + (optional)ts_stack_delete@Base 0.19 + (optional)ts_stack_dynamic_precedence@Base 0.19 + (optional)ts_stack_error_cost@Base 0.19 + (optional)ts_stack_get_summary@Base 0.19 + (optional)ts_stack_halt@Base 0.19 + (optional)ts_stack_has_advanced_since_error@Base 0.19 + (optional)ts_stack_is_active@Base 0.19 + (optional)ts_stack_is_halted@Base 0.19 + (optional)ts_stack_is_paused@Base 0.19 + (optional)ts_stack_last_external_token@Base 0.19 + (optional)ts_stack_merge@Base 0.19 + (optional)ts_stack_new@Base 0.19 + (optional)ts_stack_node_count_since_error@Base 0.19 + (optional)ts_stack_pause@Base 0.19 + (optional)ts_stack_pop_all@Base 0.19 + (optional)ts_stack_pop_count@Base 0.19 + (optional)ts_stack_pop_error@Base 0.19 + (optional)ts_stack_pop_pending@Base 0.19 + (optional)ts_stack_position@Base 0.19 + (optional)ts_stack_print_dot_graph@Base 0.19 + (optional)ts_stack_push@Base 0.19 + (optional)ts_stack_record_summary@Base 0.19 + (optional)ts_stack_remove_version@Base 0.19 + (optional)ts_stack_renumber_version@Base 0.19 + (optional)ts_stack_resume@Base 0.19 + (optional)ts_stack_set_last_external_token@Base 0.19 + (optional)ts_stack_state@Base 0.19 + (optional)ts_stack_swap_versions@Base 0.19 + (optional)ts_stack_version_count@Base 0.19 + (optional)ts_tree_cursor_current_status@Base 0.19 + (optional)ts_tree_cursor_goto_first_child_internal@Base 0.20.8 + (optional)ts_tree_cursor_init@Base 0.19 + (optional)ts_tree_cursor_goto_next_sibling_internal@Base 0.20.8 + (optional)ts_tree_cursor_parent_node@Base 0.19 + (optional)ts_tree_new@Base 0.19 diff -Nru tree-sitter-0.20.7/docs/Gemfile tree-sitter-0.20.8/docs/Gemfile --- tree-sitter-0.20.7/docs/Gemfile 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/docs/Gemfile 2023-04-04 09:15:07.000000000 +0000 @@ -1,2 +1,3 @@ source 'https://rubygems.org' -gem 'github-pages', group: :jekyll_plugins \ No newline at end of file +gem 'github-pages', group: :jekyll_plugins +gem "webrick" diff -Nru tree-sitter-0.20.7/docs/Gemfile.lock tree-sitter-0.20.8/docs/Gemfile.lock --- tree-sitter-0.20.7/docs/Gemfile.lock 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/docs/Gemfile.lock 2023-04-04 09:15:07.000000000 +0000 @@ -1,258 +1,260 @@ GEM remote: https://rubygems.org/ specs: - activesupport (4.2.9) - i18n (~> 0.7) - minitest (~> 5.1) - thread_safe (~> 0.3, >= 0.3.4) - tzinfo (~> 1.1) - addressable (2.8.0) - public_suffix (>= 2.0.2, < 5.0) + activesupport (7.0.4.3) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + addressable (2.8.1) + public_suffix (>= 2.0.2, < 6.0) coffee-script (2.4.1) coffee-script-source execjs coffee-script-source (1.11.1) colorator (1.1.0) - commonmarker (0.17.8) - ruby-enum (~> 0.5) - concurrent-ruby (1.0.5) - ethon (0.14.0) + commonmarker (0.23.8) + concurrent-ruby (1.2.2) + dnsruby (1.61.9) + simpleidn (~> 0.1) + em-websocket (0.5.3) + eventmachine (>= 0.12.9) + http_parser.rb (~> 0) + ethon (0.16.0) ffi (>= 1.15.0) - execjs (2.7.0) - faraday (1.5.1) - faraday-em_http (~> 1.0) - faraday-em_synchrony (~> 1.0) - faraday-excon (~> 1.1) - faraday-httpclient (~> 1.0.1) - faraday-net_http (~> 1.0) - faraday-net_http_persistent (~> 1.1) - faraday-patron (~> 1.0) - multipart-post (>= 1.2, < 3) + eventmachine (1.2.7) + execjs (2.8.1) + faraday (2.7.4) + faraday-net_http (>= 2.0, < 3.1) ruby2_keywords (>= 0.0.4) - faraday-em_http (1.0.0) - faraday-em_synchrony (1.0.0) - faraday-excon (1.1.0) - faraday-httpclient (1.0.1) - faraday-net_http (1.0.1) - faraday-net_http_persistent (1.2.0) - faraday-patron (1.0.0) - ffi (1.15.3) + faraday-net_http (3.0.2) + ffi (1.15.5) forwardable-extended (2.6.0) - gemoji (3.0.0) - github-pages (177) - activesupport (= 4.2.9) - github-pages-health-check (= 1.3.5) - jekyll (= 3.6.2) - jekyll-avatar (= 0.5.0) - jekyll-coffeescript (= 1.0.2) - jekyll-commonmark-ghpages (= 0.1.5) + gemoji (3.0.1) + github-pages (228) + github-pages-health-check (= 1.17.9) + jekyll (= 3.9.3) + jekyll-avatar (= 0.7.0) + jekyll-coffeescript (= 1.1.1) + jekyll-commonmark-ghpages (= 0.4.0) jekyll-default-layout (= 0.1.4) - jekyll-feed (= 0.9.2) - jekyll-gist (= 1.4.1) - jekyll-github-metadata (= 2.9.3) - jekyll-mentions (= 1.2.0) - jekyll-optional-front-matter (= 0.3.0) + jekyll-feed (= 0.15.1) + jekyll-gist (= 1.5.0) + jekyll-github-metadata (= 2.13.0) + jekyll-include-cache (= 0.2.1) + jekyll-mentions (= 1.6.0) + jekyll-optional-front-matter (= 0.3.2) jekyll-paginate (= 1.1.0) - jekyll-readme-index (= 0.2.0) - jekyll-redirect-from (= 0.12.1) - jekyll-relative-links (= 0.5.2) - jekyll-remote-theme (= 0.2.3) - jekyll-sass-converter (= 1.5.0) - jekyll-seo-tag (= 2.3.0) - jekyll-sitemap (= 1.1.1) - jekyll-swiss (= 0.4.0) - jekyll-theme-architect (= 0.1.0) - jekyll-theme-cayman (= 0.1.0) - jekyll-theme-dinky (= 0.1.0) - jekyll-theme-hacker (= 0.1.0) - jekyll-theme-leap-day (= 0.1.0) - jekyll-theme-merlot (= 0.1.0) - jekyll-theme-midnight (= 0.1.0) - jekyll-theme-minimal (= 0.1.0) - jekyll-theme-modernist (= 0.1.0) - jekyll-theme-primer (= 0.5.2) - jekyll-theme-slate (= 0.1.0) - jekyll-theme-tactile (= 0.1.0) - jekyll-theme-time-machine (= 0.1.0) - jekyll-titles-from-headings (= 0.5.0) - jemoji (= 0.8.1) - kramdown (= 1.16.2) - liquid (= 4.0.0) - listen (= 3.0.6) + jekyll-readme-index (= 0.3.0) + jekyll-redirect-from (= 0.16.0) + jekyll-relative-links (= 0.6.1) + jekyll-remote-theme (= 0.4.3) + jekyll-sass-converter (= 1.5.2) + jekyll-seo-tag (= 2.8.0) + jekyll-sitemap (= 1.4.0) + jekyll-swiss (= 1.0.0) + jekyll-theme-architect (= 0.2.0) + jekyll-theme-cayman (= 0.2.0) + jekyll-theme-dinky (= 0.2.0) + jekyll-theme-hacker (= 0.2.0) + jekyll-theme-leap-day (= 0.2.0) + jekyll-theme-merlot (= 0.2.0) + jekyll-theme-midnight (= 0.2.0) + jekyll-theme-minimal (= 0.2.0) + jekyll-theme-modernist (= 0.2.0) + jekyll-theme-primer (= 0.6.0) + jekyll-theme-slate (= 0.2.0) + jekyll-theme-tactile (= 0.2.0) + jekyll-theme-time-machine (= 0.2.0) + jekyll-titles-from-headings (= 0.5.3) + jemoji (= 0.12.0) + kramdown (= 2.3.2) + kramdown-parser-gfm (= 1.1.0) + liquid (= 4.0.4) mercenary (~> 0.3) - minima (= 2.1.1) - nokogiri (>= 1.8.1, < 2.0) - rouge (= 2.2.1) + minima (= 2.5.1) + nokogiri (>= 1.13.6, < 2.0) + rouge (= 3.26.0) terminal-table (~> 1.4) - github-pages-health-check (1.3.5) + github-pages-health-check (1.17.9) addressable (~> 2.3) - net-dns (~> 0.8) + dnsruby (~> 1.60) octokit (~> 4.0) - public_suffix (~> 2.0) - typhoeus (~> 0.7) - html-pipeline (2.7.1) + public_suffix (>= 3.0, < 5.0) + typhoeus (~> 1.3) + html-pipeline (2.14.3) activesupport (>= 2) nokogiri (>= 1.4) - i18n (0.9.5) + http_parser.rb (0.8.0) + i18n (1.12.0) concurrent-ruby (~> 1.0) - jekyll (3.6.2) + jekyll (3.9.3) addressable (~> 2.4) colorator (~> 1.0) + em-websocket (~> 0.5) + i18n (>= 0.7, < 2) jekyll-sass-converter (~> 1.0) - jekyll-watch (~> 1.1) - kramdown (~> 1.14) + jekyll-watch (~> 2.0) + kramdown (>= 1.17, < 3) liquid (~> 4.0) mercenary (~> 0.3.3) pathutil (~> 0.9) - rouge (>= 1.7, < 3) + rouge (>= 1.7, < 4) safe_yaml (~> 1.0) - jekyll-avatar (0.5.0) - jekyll (~> 3.0) - jekyll-coffeescript (1.0.2) + jekyll-avatar (0.7.0) + jekyll (>= 3.0, < 5.0) + jekyll-coffeescript (1.1.1) coffee-script (~> 2.2) coffee-script-source (~> 1.11.1) - jekyll-commonmark (1.1.0) - commonmarker (~> 0.14) - jekyll (>= 3.0, < 4.0) - jekyll-commonmark-ghpages (0.1.5) - commonmarker (~> 0.17.6) - jekyll-commonmark (~> 1) - rouge (~> 2) + jekyll-commonmark (1.4.0) + commonmarker (~> 0.22) + jekyll-commonmark-ghpages (0.4.0) + commonmarker (~> 0.23.7) + jekyll (~> 3.9.0) + jekyll-commonmark (~> 1.4.0) + rouge (>= 2.0, < 5.0) jekyll-default-layout (0.1.4) jekyll (~> 3.0) - jekyll-feed (0.9.2) - jekyll (~> 3.3) - jekyll-gist (1.4.1) + jekyll-feed (0.15.1) + jekyll (>= 3.7, < 5.0) + jekyll-gist (1.5.0) octokit (~> 4.2) - jekyll-github-metadata (2.9.3) - jekyll (~> 3.1) + jekyll-github-metadata (2.13.0) + jekyll (>= 3.4, < 5.0) octokit (~> 4.0, != 4.4.0) - jekyll-mentions (1.2.0) - activesupport (~> 4.0) + jekyll-include-cache (0.2.1) + jekyll (>= 3.7, < 5.0) + jekyll-mentions (1.6.0) html-pipeline (~> 2.3) - jekyll (~> 3.0) - jekyll-optional-front-matter (0.3.0) - jekyll (~> 3.0) + jekyll (>= 3.7, < 5.0) + jekyll-optional-front-matter (0.3.2) + jekyll (>= 3.0, < 5.0) jekyll-paginate (1.1.0) - jekyll-readme-index (0.2.0) - jekyll (~> 3.0) - jekyll-redirect-from (0.12.1) - jekyll (~> 3.3) - jekyll-relative-links (0.5.2) - jekyll (~> 3.3) - jekyll-remote-theme (0.2.3) - jekyll (~> 3.5) - rubyzip (>= 1.2.1, < 3.0) - typhoeus (>= 0.7, < 2.0) - jekyll-sass-converter (1.5.0) + jekyll-readme-index (0.3.0) + jekyll (>= 3.0, < 5.0) + jekyll-redirect-from (0.16.0) + jekyll (>= 3.3, < 5.0) + jekyll-relative-links (0.6.1) + jekyll (>= 3.3, < 5.0) + jekyll-remote-theme (0.4.3) + addressable (~> 2.0) + jekyll (>= 3.5, < 5.0) + jekyll-sass-converter (>= 1.0, <= 3.0.0, != 2.0.0) + rubyzip (>= 1.3.0, < 3.0) + jekyll-sass-converter (1.5.2) sass (~> 3.4) - jekyll-seo-tag (2.3.0) - jekyll (~> 3.3) - jekyll-sitemap (1.1.1) - jekyll (~> 3.3) - jekyll-swiss (0.4.0) - jekyll-theme-architect (0.1.0) - jekyll (~> 3.5) + jekyll-seo-tag (2.8.0) + jekyll (>= 3.8, < 5.0) + jekyll-sitemap (1.4.0) + jekyll (>= 3.7, < 5.0) + jekyll-swiss (1.0.0) + jekyll-theme-architect (0.2.0) + jekyll (> 3.5, < 5.0) jekyll-seo-tag (~> 2.0) - jekyll-theme-cayman (0.1.0) - jekyll (~> 3.5) + jekyll-theme-cayman (0.2.0) + jekyll (> 3.5, < 5.0) jekyll-seo-tag (~> 2.0) - jekyll-theme-dinky (0.1.0) - jekyll (~> 3.5) + jekyll-theme-dinky (0.2.0) + jekyll (> 3.5, < 5.0) jekyll-seo-tag (~> 2.0) - jekyll-theme-hacker (0.1.0) - jekyll (~> 3.5) + jekyll-theme-hacker (0.2.0) + jekyll (> 3.5, < 5.0) jekyll-seo-tag (~> 2.0) - jekyll-theme-leap-day (0.1.0) - jekyll (~> 3.5) + jekyll-theme-leap-day (0.2.0) + jekyll (> 3.5, < 5.0) jekyll-seo-tag (~> 2.0) - jekyll-theme-merlot (0.1.0) - jekyll (~> 3.5) + jekyll-theme-merlot (0.2.0) + jekyll (> 3.5, < 5.0) jekyll-seo-tag (~> 2.0) - jekyll-theme-midnight (0.1.0) - jekyll (~> 3.5) + jekyll-theme-midnight (0.2.0) + jekyll (> 3.5, < 5.0) jekyll-seo-tag (~> 2.0) - jekyll-theme-minimal (0.1.0) - jekyll (~> 3.5) + jekyll-theme-minimal (0.2.0) + jekyll (> 3.5, < 5.0) jekyll-seo-tag (~> 2.0) - jekyll-theme-modernist (0.1.0) - jekyll (~> 3.5) + jekyll-theme-modernist (0.2.0) + jekyll (> 3.5, < 5.0) jekyll-seo-tag (~> 2.0) - jekyll-theme-primer (0.5.2) - jekyll (~> 3.5) + jekyll-theme-primer (0.6.0) + jekyll (> 3.5, < 5.0) jekyll-github-metadata (~> 2.9) - jekyll-seo-tag (~> 2.2) - jekyll-theme-slate (0.1.0) - jekyll (~> 3.5) - jekyll-seo-tag (~> 2.0) - jekyll-theme-tactile (0.1.0) - jekyll (~> 3.5) - jekyll-seo-tag (~> 2.0) - jekyll-theme-time-machine (0.1.0) - jekyll (~> 3.5) - jekyll-seo-tag (~> 2.0) - jekyll-titles-from-headings (0.5.0) - jekyll (~> 3.3) - jekyll-watch (1.5.1) + jekyll-seo-tag (~> 2.0) + jekyll-theme-slate (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-tactile (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-time-machine (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-titles-from-headings (0.5.3) + jekyll (>= 3.3, < 5.0) + jekyll-watch (2.2.1) listen (~> 3.0) - jemoji (0.8.1) - activesupport (~> 4.0, >= 4.2.9) + jemoji (0.12.0) gemoji (~> 3.0) html-pipeline (~> 2.2) - jekyll (>= 3.0) - kramdown (1.16.2) - liquid (4.0.0) - listen (3.0.6) - rb-fsevent (>= 0.9.3) - rb-inotify (>= 0.9.7) + jekyll (>= 3.0, < 5.0) + kramdown (2.3.2) + rexml + kramdown-parser-gfm (1.1.0) + kramdown (~> 2.0) + liquid (4.0.4) + listen (3.8.0) + rb-fsevent (~> 0.10, >= 0.10.3) + rb-inotify (~> 0.9, >= 0.9.10) mercenary (0.3.6) - mini_portile2 (2.8.0) - minima (2.1.1) - jekyll (~> 3.3) - minitest (5.11.3) - multipart-post (2.1.1) - net-dns (0.9.0) - nokogiri (1.13.3) - mini_portile2 (~> 2.8.0) + minima (2.5.1) + jekyll (>= 3.5, < 5.0) + jekyll-feed (~> 0.9) + jekyll-seo-tag (~> 2.1) + minitest (5.18.0) + nokogiri (1.14.2-x86_64-linux) racc (~> 1.4) - octokit (4.21.0) - faraday (>= 0.9) - sawyer (~> 0.8.0, >= 0.5.3) + octokit (4.25.1) + faraday (>= 1, < 3) + sawyer (~> 0.9) pathutil (0.16.2) forwardable-extended (~> 2.6) - public_suffix (2.0.5) - racc (1.6.0) - rb-fsevent (0.11.0) + public_suffix (4.0.7) + racc (1.6.2) + rb-fsevent (0.11.2) rb-inotify (0.10.1) ffi (~> 1.0) - rouge (2.2.1) - ruby-enum (0.7.2) - i18n - ruby2_keywords (0.0.4) - rubyzip (2.0.0) + rexml (3.2.5) + rouge (3.26.0) + ruby2_keywords (0.0.5) + rubyzip (2.3.2) safe_yaml (1.0.5) sass (3.7.4) sass-listen (~> 4.0.0) sass-listen (4.0.0) rb-fsevent (~> 0.9, >= 0.9.4) rb-inotify (~> 0.9, >= 0.9.7) - sawyer (0.8.2) + sawyer (0.9.2) addressable (>= 2.3.5) - faraday (> 0.8, < 2.0) + faraday (>= 0.17.3, < 3) + simpleidn (0.2.1) + unf (~> 0.1.4) terminal-table (1.8.0) unicode-display_width (~> 1.1, >= 1.1.1) - thread_safe (0.3.6) - typhoeus (0.8.0) - ethon (>= 0.8.0) - tzinfo (1.2.5) - thread_safe (~> 0.1) - unicode-display_width (1.3.0) + typhoeus (1.4.0) + ethon (>= 0.9.0) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + unf (0.1.4) + unf_ext + unf_ext (0.0.8.2) + unicode-display_width (1.8.0) + webrick (1.8.1) PLATFORMS ruby DEPENDENCIES github-pages + webrick BUNDLED WITH - 1.16.1 + 2.4.8 diff -Nru tree-sitter-0.20.7/docs/index.md tree-sitter-0.20.8/docs/index.md --- tree-sitter-0.20.7/docs/index.md 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/docs/index.md 2023-04-04 09:15:07.000000000 +0000 @@ -9,93 +9,142 @@ * **General** enough to parse any programming language * **Fast** enough to parse on every keystroke in a text editor * **Robust** enough to provide useful results even in the presence of syntax errors -* **Dependency-free** so that the runtime library (which is written in pure C) can be embedded in any application +* **Dependency-free** so that the runtime library (which is written in pure [C](https://github.com/tree-sitter/tree-sitter/tree/master/lib)) can be embedded in any application ### Language Bindings There are currently bindings that allow Tree-sitter to be used from the following languages: +* [Go](https://github.com/smacker/go-tree-sitter) * [Haskell](https://github.com/tree-sitter/haskell-tree-sitter) * [JavaScript (Node.js)](https://github.com/tree-sitter/node-tree-sitter) * [JavaScript (Wasm)](https://github.com/tree-sitter/tree-sitter/tree/master/lib/binding_web) * [Lua](https://github.com/euclidianAce/ltreesitter) * [OCaml](https://github.com/returntocorp/ocaml-tree-sitter-core) +* [Perl](https://metacpan.org/pod/Text::Treesitter) * [Python](https://github.com/tree-sitter/py-tree-sitter) * [Ruby](https://github.com/tree-sitter/ruby-tree-sitter) +* [Ruby](https://github.com/calicoday/ruby-tree-sitter-ffi) * [Rust](https://github.com/tree-sitter/tree-sitter/tree/master/lib/binding_rust) * [Swift](https://github.com/ChimeHQ/SwiftTreeSitter) * [Kotlin](https://github.com/oxisto/kotlintree) * [Java](https://github.com/serenadeai/java-tree-sitter) -### Available Parsers - -Parsers for these languages are fairly complete: +### Parsers +* [Ada](https://github.com/briot/tree-sitter-ada) +* [Agda](https://github.com/tree-sitter/tree-sitter-agda) +* [Apex](https://github.com/aheber/tree-sitter-sfapex) * [Bash](https://github.com/tree-sitter/tree-sitter-bash) +* [Beancount](https://github.com/zwpaper/tree-sitter-beancount) +* [Cap'n Proto](https://github.com/amaanq/tree-sitter-capnp) * [C](https://github.com/tree-sitter/tree-sitter-c) -* [C#](https://github.com/tree-sitter/tree-sitter-c-sharp) * [C++](https://github.com/tree-sitter/tree-sitter-cpp) +* [C#](https://github.com/tree-sitter/tree-sitter-c-sharp) +* [Clojure](https://github.com/sogaiu/tree-sitter-clojure) +* [CMake](https://github.com/uyha/tree-sitter-cmake) +* [Comment](https://github.com/stsewd/tree-sitter-comment) * [Common Lisp](https://github.com/theHamsta/tree-sitter-commonlisp) * [CSS](https://github.com/tree-sitter/tree-sitter-css) * [CUDA](https://github.com/theHamsta/tree-sitter-cuda) +* [Dart](https://github.com/UserNobody14/tree-sitter-dart) +* [D](https://github.com/gdamore/tree-sitter-d) +* [Dockerfile](https://github.com/camdencheek/tree-sitter-dockerfile) * [DOT](https://github.com/rydesun/tree-sitter-dot) +* [Elixir](https://github.com/elixir-lang/tree-sitter-elixir) * [Elm](https://github.com/elm-tooling/tree-sitter-elm) * [Emacs Lisp](https://github.com/Wilfred/tree-sitter-elisp) * [Eno](https://github.com/eno-lang/tree-sitter-eno) * [ERB / EJS](https://github.com/tree-sitter/tree-sitter-embedded-template) +* [Erlang](https://github.com/WhatsApp/tree-sitter-erlang/) * [Fennel](https://github.com/travonted/tree-sitter-fennel) +* [Fish](https://github.com/ram02z/tree-sitter-fish) +* [Formula](https://github.com/siraben/tree-sitter-formula) +* [Fortran](https://github.com/stadelmanma/tree-sitter-fortran) +* [gitattributes](https://github.com/ObserverOfTime/tree-sitter-gitattributes) +* [gitignore](https://github.com/shunsambongi/tree-sitter-gitignore) +* [Gleam](https://github.com/gleam-lang/tree-sitter-gleam) * [GLSL (OpenGL Shading Language)](https://github.com/theHamsta/tree-sitter-glsl) * [Go](https://github.com/tree-sitter/tree-sitter-go) +* [Go mod](https://github.com/camdencheek/tree-sitter-go-mod) +* [Go work](https://github.com/omertuc/tree-sitter-go-work) +* [Graphql](https://github.com/bkegley/tree-sitter-graphql) +* [Hack](https://github.com/slackhq/tree-sitter-hack) +* [Haskell](https://github.com/tree-sitter/tree-sitter-haskell) * [HCL](https://github.com/MichaHoffmann/tree-sitter-hcl) * [HTML](https://github.com/tree-sitter/tree-sitter-html) * [Java](https://github.com/tree-sitter/tree-sitter-java) * [JavaScript](https://github.com/tree-sitter/tree-sitter-javascript) +* [jq](https://github.com/flurie/tree-sitter-jq) +* [JSON5](https://github.com/Joakker/tree-sitter-json5) * [JSON](https://github.com/tree-sitter/tree-sitter-json) +* [Julia](https://github.com/tree-sitter/tree-sitter-julia) +* [Kotlin](https://github.com/fwcd/tree-sitter-kotlin) +* [LALRPOP](https://github.com/traxys/tree-sitter-lalrpop) +* [Latex](https://github.com/latex-lsp/tree-sitter-latex) +* [Lean](https://github.com/Julian/tree-sitter-lean) +* [LLVM](https://github.com/benwilliamgraham/tree-sitter-llvm) +* [LLVM MachineIR](https://github.com/Flakebi/tree-sitter-llvm-mir) +* [LLVM TableGen](https://github.com/Flakebi/tree-sitter-tablegen) * [Lua](https://github.com/Azganoth/tree-sitter-lua) * [Make](https://github.com/alemuller/tree-sitter-make) * [Markdown](https://github.com/ikatyang/tree-sitter-markdown) +* [Markdown](https://github.com/MDeiml/tree-sitter-markdown) +* [Meson](https://github.com/Decodetalkers/tree-sitter-meson) +* [Meson](https://github.com/staysail/tree-sitter-meson) +* [Motorola 68000 Assembly](https://github.com/grahambates/tree-sitter-m68k) +* [Nix](https://github.com/cstrahan/tree-sitter-nix) +* [Objective-C](https://github.com/jiyee/tree-sitter-objc) * [OCaml](https://github.com/tree-sitter/tree-sitter-ocaml) +* [Org](https://github.com/milisims/tree-sitter-org) +* [Pascal](https://github.com/Isopod/tree-sitter-pascal) +* [Perl](https://github.com/ganezdragon/tree-sitter-perl) +* [Perl](https://github.com/tree-sitter-perl/tree-sitter-perl) +* [Perl POD](https://github.com/tree-sitter-perl/tree-sitter-pod) * [PHP](https://github.com/tree-sitter/tree-sitter-php) +* [Portable Game Notation](https://github.com/rolandwalker/tree-sitter-pgn) +* [PowerShell](https://github.com/PowerShell/tree-sitter-PowerShell) +* [Protocol Buffers](https://github.com/mitchellh/tree-sitter-proto) * [Python](https://github.com/tree-sitter/tree-sitter-python) +* [QML](https://github.com/yuja/tree-sitter-qmljs) +* [Racket](https://github.com/6cdh/tree-sitter-racket) +* [Rasi](https://github.com/Fymyte/tree-sitter-rasi) +* [re2c](https://github.com/alemuller/tree-sitter-re2c) +* [Regex](https://github.com/tree-sitter/tree-sitter-regex) +* [Rego](https://github.com/FallenAngel97/tree-sitter-rego) +* [reStructuredText](https://github.com/stsewd/tree-sitter-rst) +* [R](https://github.com/r-lib/tree-sitter-r) * [Ruby](https://github.com/tree-sitter/tree-sitter-ruby) * [Rust](https://github.com/tree-sitter/tree-sitter-rust) -* [R](https://github.com/r-lib/tree-sitter-r) +* [Scala](https://github.com/tree-sitter/tree-sitter-scala) +* [Scheme](https://github.com/6cdh/tree-sitter-scheme) +* [Scss](https://github.com/serenadeai/tree-sitter-scss) * [S-expressions](https://github.com/AbstractMachinesLab/tree-sitter-sexp) +* [Smali](https://github.com/amaanq/tree-sitter-smali) +* [Smali](https://git.sr.ht/~yotam/tree-sitter-smali) +* [Sourcepawn](https://github.com/nilshelmig/tree-sitter-sourcepawn) * [SPARQL](https://github.com/BonaBeavis/tree-sitter-sparql) -* [SystemRDL](https://github.com/SystemRDL/tree-sitter-systemrdl) +* [SQL - BigQuery](https://github.com/takegue/tree-sitter-sql-bigquery) +* [SQL - PostgreSQL](https://github.com/m-novikov/tree-sitter-sql) +* [SQL - SQLite](https://github.com/dhcmrlchtdj/tree-sitter-sqlite) +* [SSH](https://github.com/metio/tree-sitter-ssh-client-config) * [Svelte](https://github.com/Himujjal/tree-sitter-svelte) +* [Swift](https://github.com/alex-pinkus/tree-sitter-swift) +* [SystemRDL](https://github.com/SystemRDL/tree-sitter-systemrdl) +* [Thrift](https://github.com/duskmoon314/tree-sitter-thrift) * [TOML](https://github.com/ikatyang/tree-sitter-toml) +* [Tree-sitter Query](https://github.com/nvim-treesitter/tree-sitter-query) * [Turtle](https://github.com/BonaBeavis/tree-sitter-turtle) +* [Twig](https://github.com/gbprod/tree-sitter-twig) * [TypeScript](https://github.com/tree-sitter/tree-sitter-typescript) * [Verilog](https://github.com/tree-sitter/tree-sitter-verilog) * [VHDL](https://github.com/alemuller/tree-sitter-vhdl) * [Vue](https://github.com/ikatyang/tree-sitter-vue) -* [YAML](https://github.com/ikatyang/tree-sitter-yaml) * [WASM](https://github.com/wasm-lsp/tree-sitter-wasm) * [WGSL WebGPU Shading Language](https://github.com/mehmetoguzderin/tree-sitter-wgsl) - -Parsers for these languages are in development: - -* [Agda](https://github.com/tree-sitter/tree-sitter-agda) -* [Elixir](https://github.com/elixir-lang/tree-sitter-elixir) -* [Erlang](https://github.com/AbstractMachinesLab/tree-sitter-erlang/) -* [Dockerfile](https://github.com/camdencheek/tree-sitter-dockerfile) -* [Go mod](https://github.com/camdencheek/tree-sitter-go-mod) -* [Hack](https://github.com/slackhq/tree-sitter-hack) -* [Haskell](https://github.com/tree-sitter/tree-sitter-haskell) -* [Julia](https://github.com/tree-sitter/tree-sitter-julia) -* [Kotlin](https://github.com/fwcd/tree-sitter-kotlin) -* [Nix](https://github.com/cstrahan/tree-sitter-nix) -* [Objective-C](https://github.com/jiyee/tree-sitter-objc) -* [Org](https://github.com/milisims/tree-sitter-org) -* [Perl](https://github.com/ganezdragon/tree-sitter-perl) -* [Protocol Buffers](https://github.com/mitchellh/tree-sitter-proto) -* [Racket](https://github.com/6cdh/tree-sitter-racket) -* [Scala](https://github.com/tree-sitter/tree-sitter-scala) -* [Sourcepawn](https://github.com/nilshelmig/tree-sitter-sourcepawn) -* [Swift](https://github.com/tree-sitter/tree-sitter-swift) -* [SQL](https://github.com/m-novikov/tree-sitter-sql) - +* [YAML](https://github.com/ikatyang/tree-sitter-yaml) +* [YANG](https://github.com/Hubro/tree-sitter-yang) +* [Zig](https://github.com/maxxnino/tree-sitter-zig) ### Talks on Tree-sitter diff -Nru tree-sitter-0.20.7/docs/section-2-using-parsers.md tree-sitter-0.20.8/docs/section-2-using-parsers.md --- tree-sitter-0.20.7/docs/section-2-using-parsers.md 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/docs/section-2-using-parsers.md 2023-04-04 09:15:07.000000000 +0000 @@ -442,13 +442,13 @@ A _query_ consists of one or more _patterns_, where each pattern is an [S-expression](https://en.wikipedia.org/wiki/S-expression) that matches a certain set of nodes in a syntax tree. The expression to match a given node consists of a pair of parentheses containing two things: the node's type, and optionally, a series of other S-expressions that match the node's children. For example, this pattern would match any `binary_expression` node whose children are both `number_literal` nodes: -``` +``` scheme (binary_expression (number_literal) (number_literal)) ``` Children can also be omitted. For example, this would match any `binary_expression` where at least _one_ of child is a `string_literal` node: -``` +``` scheme (binary_expression (string_literal)) ``` @@ -456,7 +456,7 @@ In general, it's a good idea to make patterns more specific by specifying [field names](#node-field-names) associated with child nodes. You do this by prefixing a child pattern with a field name followed by a colon. For example, this pattern would match an `assignment_expression` node where the `left` child is a `member_expression` whose `object` is a `call_expression`. -``` +``` scheme (assignment_expression left: (member_expression object: (call_expression))) @@ -466,7 +466,7 @@ You can also constrain a pattern so that it only matches nodes that *lack* a certain field. To do this, add a field name prefixed by a `!` within the parent pattern. For example, this pattern would match a class declaration with no type parameters: -``` +``` scheme (class_declaration name: (identifier) @class_name !type_parameters) @@ -476,7 +476,7 @@ The parenthesized syntax for writing nodes only applies to [named nodes](#named-vs-anonymous-nodes). To match specific anonymous nodes, you write their name between double quotes. For example, this pattern would match any `binary_expression` where the operator is `!=` and the right side is `null`: -``` +``` scheme (binary_expression operator: "!=" right: (null)) @@ -488,7 +488,7 @@ For example, this pattern would match any assignment of a `function` to an `identifier`, and it would associate the name `the-function-name` with the identifier: -``` +``` scheme (assignment_expression left: (identifier) @the-function-name right: (function)) @@ -496,7 +496,7 @@ And this pattern would match all method definitions, associating the name `the-method-name` with the method name, `the-class-name` with the containing class name: -``` +``` scheme (class_declaration name: (identifier) @the-class-name body: (class_body @@ -510,13 +510,13 @@ For example, this pattern would match a sequence of one or more comments: -``` +``` scheme (comment)+ ``` This pattern would match a class declaration, capturing all of the decorators if any were present: -``` +``` scheme (class_declaration (decorator)* @the-decorator name: (identifier) @the-name) @@ -524,7 +524,7 @@ You can also mark a node as optional using the `?` operator. For example, this pattern would match all function calls, capturing a string argument if one was present: -``` +``` scheme (call_expression function: (identifier) @the-function arguments: (arguments (string)? @the-string-arg)) @@ -534,7 +534,7 @@ You can also use parentheses for grouping a sequence of _sibling_ nodes. For example, this pattern would match a comment followed by a function declaration: -``` +``` scheme ( (comment) (function_declaration) @@ -543,7 +543,7 @@ Any of the quantification operators mentioned above (`+`, `*`, and `?`) can also be applied to groups. For example, this pattern would match a comma-separated series of numbers: -``` +``` scheme ( (number) ("," (number))* @@ -558,7 +558,7 @@ For example, this pattern would match a call to either a variable or an object property. In the case of a variable, capture it as `@function`, and in the case of a property, capture it as `@method`: -``` +``` scheme (call_expression function: [ (identifier) @function @@ -569,7 +569,7 @@ This pattern would match a set of possible keyword tokens, capturing them as `@keyword`: -``` +``` scheme [ "break" "delete" @@ -592,7 +592,7 @@ For example, this pattern would match any node inside a call: -``` +``` scheme (call (_) @call.inner) ``` @@ -602,7 +602,7 @@ When `.` is placed before the _first_ child within a parent pattern, the child will only match when it is the first named node in the parent. For example, the below pattern matches a given `array` node at most once, assigning the `@the-element` capture to the first `identifier` node in the parent `array`: -``` +``` scheme (array . (identifier) @the-element) ``` @@ -610,13 +610,13 @@ Similarly, an anchor placed after a pattern's _last_ child will cause that child pattern to only match nodes that are the last named child of their parent. The below pattern matches only nodes that are the last named child within a `block`. -``` +``` scheme (block (_) @last-expression .) ``` Finally, an anchor _between_ two child patterns will cause the patterns to only match nodes that are immediate siblings. The pattern below, given a long dotted name like `a.b.c.d`, will only match pairs of consecutive identifiers: `a, b`, `b, c`, and `c, d`. -``` +``` scheme (dotted_name (identifier) @prev-id . @@ -633,7 +633,7 @@ For example, this pattern would match identifier whose names is written in `SCREAMING_SNAKE_CASE`: -``` +``` scheme ( (identifier) @constant (#match? @constant "^[A-Z][A-Z_]+") @@ -642,7 +642,7 @@ And this pattern would match key-value pairs where the `value` is an identifier with the same name as the key: -``` +``` scheme ( (pair key: (property_identifier) @key-name diff -Nru tree-sitter-0.20.7/docs/section-3-creating-parsers.md tree-sitter-0.20.8/docs/section-3-creating-parsers.md --- tree-sitter-0.20.7/docs/section-3-creating-parsers.md 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/docs/section-3-creating-parsers.md 2023-04-04 09:15:07.000000000 +0000 @@ -20,9 +20,9 @@ To create a Tree-sitter parser, you need to use [the `tree-sitter` CLI][tree-sitter-cli]. You can install the CLI in a few different ways: -* Install the `tree-sitter-cli` [Node.js module][node-module] using [`npm`][npm], the Node package manager. This is the recommended approach, and it is discussed further in the next section. +* Build the `tree-sitter-cli` [Rust crate][crate] from source using [`cargo`][cargo], the Rust package manager. This works on any platform. See [the contributing docs](/docs/section-5-contributing.md#developing-tree-sitter) for more information. +* Install the `tree-sitter-cli` [Node.js module][node-module] using [`npm`][npm], the Node package manager. This approach is fast, but is only works on certain platforms, because it relies on pre-built binaries. * Download a binary for your platform from [the latest GitHub release][releases], and put it into a directory on your `PATH`. -* Build the `tree-sitter-cli` [Rust crate][crate] from source using [`cargo`][cargo], the Rust package manager. See [the contributing docs](/docs/section-5-contributing.md#developing-tree-sitter) for more information. ### Project Setup @@ -80,6 +80,11 @@ echo 'hello' > example-file tree-sitter parse example-file ``` +Alternatively, in Windows PowerShell: +```pwsh +"hello" | Out-File example-file -Encoding utf8 +tree-sitter parse example-file +``` This should print the following: @@ -152,6 +157,26 @@ (return_statement (number))))) ``` +* If your language's syntax conflicts with the `===` and `---` test separators, you can optionally add an arbitrary identical suffix (in the below example, `|||`) to disambiguate them: + +``` +==================||| +Basic module +==================||| + +---- MODULE Test ---- +increment(n) == n + 1 +==== + +---||| + +(source_file + (module (identifier) + (operator (identifier) + (parameter_list (identifier)) + (plus (identifier_ref) (number))))) +``` + These tests are important. They serve as the parser's API documentation, and they can be run every time you change the grammar to verify that everything still parses correctly. By default, the `tree-sitter test` command runs all of the tests in your `corpus` or `test/corpus/` folder. To run a particular test, you can use the `-f` flag: @@ -200,7 +225,7 @@ The following is a complete list of built-in functions you can use in your `grammar.js` to define rules. Use-cases for some of these functions will be explained in more detail in later sections. -* **Symbols (the `$` object)** - Every grammar rule is written as a JavaScript function that takes a parameter conventionally called `$`. The syntax `$.identifier` is how you refer to another grammar symbol within a rule. +* **Symbols (the `$` object)** - Every grammar rule is written as a JavaScript function that takes a parameter conventionally called `$`. The syntax `$.identifier` is how you refer to another grammar symbol within a rule. Names starting with `$.MISSING` or `$.UNEXPECTED` should be avoided as they have special meaning for the `tree-sitter test` command. * **String and Regex literals** - The terminal symbols in a grammar are described using JavaScript strings and regular expressions. Of course during parsing, Tree-sitter does not actually use JavaScript's regex engine to evaluate these regexes; it generates its own regex-matching logic as part of each parser. Regex literals are just used as a convenient way of writing regular expressions in your grammar. * **Sequences : `seq(rule1, rule2, ...)`** - This function creates a rule that matches any number of other rules, one after another. It is analogous to simply writing multiple symbols next to each other in [EBNF notation][ebnf]. * **Alternatives : `choice(rule1, rule2, ...)`** - This function creates a rule that matches *one* of a set of possible rules. The order of the arguments does not matter. This is analogous to the `|` (pipe) operator in EBNF notation. @@ -222,6 +247,7 @@ * **`inline`** - an array of rule names that should be automatically *removed* from the grammar by replacing all of their usages with a copy of their definition. This is useful for rules that are used in multiple places but for which you *don't* want to create syntax tree nodes at runtime. * **`conflicts`** - an array of arrays of rule names. Each inner array represents a set of rules that's involved in an *LR(1) conflict* that is *intended to exist* in the grammar. When these conflicts occur at runtime, Tree-sitter will use the GLR algorithm to explore all of the possible interpretations. If *multiple* parses end up succeeding, Tree-sitter will pick the subtree whose corresponding rule has the highest total *dynamic precedence*. * **`externals`** - an array of token names which can be returned by an [*external scanner*](#external-scanners). External scanners allow you to write custom C code which runs during the lexing process in order to handle lexical rules (e.g. Python's indentation tokens) that cannot be described by regular expressions. +* **`precedences`** - an array of array of strings, where each array of strings defines named precedence levels in descending order. These names can be used in the `prec` functions to define precedence relative only to other names in the array, rather than globally. Can only be used with parse precedence, not lexical precedence. * **`word`** - the name of a token that will match keywords for the purpose of the [keyword extraction](#keyword-extraction) optimization. * **`supertypes`** an array of hidden rule names which should be considered to be 'supertypes' in the generated [*node types* file][static-node-types]. @@ -498,17 +524,31 @@ ### Conflicting Tokens -Grammars often contain multiple tokens that can match the same characters. For example, a grammar might contain the tokens (`"if"` and `/[a-z]+/`). Tree-sitter differentiates between these conflicting tokens in a few ways: +Grammars often contain multiple tokens that can match the same characters. For example, a grammar might contain the tokens (`"if"` and `/[a-z]+/`). Tree-sitter differentiates between these conflicting tokens in a few ways. -1. **Context-aware Lexing** - Tree-sitter performs lexing on-demand, during the parsing process. At any given position in a source document, the lexer only tries to recognize tokens that are *valid* at that position in the document. +1. **External Scanning** - If your grammar has an external scanner and one or more tokens in your `externals` array are valid at the current location, your external scanner will always be called first to determine whether those tokens are present. -2. **Lexical Precedence** - When the precedence functions described [above](#the-grammar-dsl) are used within the `token` function, the given precedence values serve as instructions to the lexer. If there are two valid tokens that match the characters at a given position in the document, Tree-sitter will select the one with the higher precedence. +1. **Context-Aware Lexing** - Tree-sitter performs lexing on-demand, during the parsing process. At any given position in a source document, the lexer only tries to recognize tokens that are *valid* at that position in the document. -3. **Match Length** - If multiple valid tokens with the same precedence match the characters at a given position in a document, Tree-sitter will select the token that matches the [longest sequence of characters][longest-match]. +1. **Earliest Starting Position** - Tree-sitter will prefer tokens with an earlier starting position. This is most often seen with very permissive regular expressions similar to `/.*/`, which are greedy and will consume as much text as possible. In this example the regex would consume all text until hitting a newline - even if text on that line could be interpreted as a different token. -4. **Match Specificity** - If there are two valid tokens with the same precedence and which both match the same number of characters, Tree-sitter will prefer a token that is specified in the grammar as a `String` over a token specified as a `RegExp`. +1. **Explicit Lexical Precedence** - When the precedence functions described [above](#the-grammar-dsl) are used within the `token` function, the given precedence values serve as instructions to the lexer. If there are two valid tokens that match the characters at a given position in the document, Tree-sitter will select the one with the higher precedence. -5. **Rule Order** - If none of the above criteria can be used to select one token over another, Tree-sitter will prefer the token that appears earlier in the grammar. +1. **Match Length** - If multiple valid tokens with the same precedence match the characters at a given position in a document, Tree-sitter will select the token that matches the [longest sequence of characters][longest-match]. + +1. **Match Specificity** - If there are two valid tokens with the same precedence and which both match the same number of characters, Tree-sitter will prefer a token that is specified in the grammar as a `String` over a token specified as a `RegExp`. + +1. **Rule Order** - If none of the above criteria can be used to select one token over another, Tree-sitter will prefer the token that appears earlier in the grammar. + +### Lexical Precedence vs. Parse Precedence + +One common mistake involves not distinguishing lexical precedence from parse precedence. +Parse precedence determines which rule is chosen to interpret a given sequence of tokens. +Lexical precedence determines which token is chosen to interpret a given section of text. +It is a lower-level operation that is done first. +The above list fully capture tree-sitter's lexical precedence rules, and you will probably refer back to this section of the documentation more often than any other. +Most of the time when you really get stuck, you're dealing with a lexical precedence problem. +Pay particular attention to the difference in meaning between using `prec` inside the `token` function versus outside of it. ### Keywords @@ -590,9 +630,9 @@ }); ``` -Then, add another C or C++ source file to your project. Currently, its path must be `src/scanner.c` or `src/scanner.cc` for the CLI to recognize it. Be sure to add this file to the `sources` section of your `binding.gyp` file so that it will be included when your project is compiled by Node.js. +Then, add another C or C++ source file to your project. Currently, its path must be `src/scanner.c` or `src/scanner.cc` for the CLI to recognize it. Be sure to add this file to the `sources` section of your `binding.gyp` file so that it will be included when your project is compiled by Node.js and uncomment the appropriate block in your `bindings/rust/build.rs` file so that it will be included in your Rust crate. -In this new source file, define an [`enum`][enum] type containing the names of all of your external tokens. The ordering of this enum must match the order in your grammar's `externals` array. +In this new source file, define an [`enum`][enum] type containing the names of all of your external tokens. The ordering of this enum must match the order in your grammar's `externals` array; the actual names do not matter. ```c #include @@ -655,6 +695,7 @@ ``` This function should *restore* the state of your scanner based the bytes that were previously written by the `serialize` function. It is called with a pointer to your scanner, a pointer to the buffer of bytes, and the number of bytes that should be read. +It is good practice to explicitly erase your scanner state variables at the start of this function, before restoring their values from the byte buffer. #### Scan @@ -672,10 +713,11 @@ * **`int32_t lookahead`** - The current next character in the input stream, represented as a 32-bit unicode code point. * **`TSSymbol result_symbol`** - The symbol that was recognized. Your scan function should *assign* to this field one of the values from the `TokenType` enum, described above. -* **`void (*advance)(TSLexer *, bool skip)`** - A function for advancing to the next character. If you pass `true` for the second argument, the current character will be treated as whitespace. +* **`void (*advance)(TSLexer *, bool skip)`** - A function for advancing to the next character. If you pass `true` for the second argument, the current character will be treated as whitespace; whitespace won't be included in the text range associated with tokens emitted by the external scanner. * **`void (*mark_end)(TSLexer *)`** - A function for marking the end of the recognized token. This allows matching tokens that require multiple characters of lookahead. By default (if you don't call `mark_end`), any character that you moved past using the `advance` function will be included in the size of the token. But once you call `mark_end`, then any later calls to `advance` will *not* increase the size of the returned token. You can call `mark_end` multiple times to increase the size of the token. * **`uint32_t (*get_column)(TSLexer *)`** - A function for querying the current column position of the lexer. It returns the number of codepoints since the start of the current line. The codepoint position is recalculated on every call to this function by reading from the start of the line. -* **`bool (*is_at_included_range_start)(TSLexer *)`** - A function for checking if the parser has just skipped some characters in the document. When parsing an embedded document using the `ts_parser_set_included_ranges` function (described in the [multi-language document section][multi-language-section]), your scanner may want to apply some special behavior when moving to a disjoint part of the document. For example, in [EJS documents][ejs], the JavaScript parser uses this function to enable inserting automatic semicolon tokens in between the code directives, delimited by `<%` and `%>`. +* **`bool (*is_at_included_range_start)(const TSLexer *)`** - A function for checking whether the parser has just skipped some characters in the document. When parsing an embedded document using the `ts_parser_set_included_ranges` function (described in the [multi-language document section][multi-language-section]), your scanner may want to apply some special behavior when moving to a disjoint part of the document. For example, in [EJS documents][ejs], the JavaScript parser uses this function to enable inserting automatic semicolon tokens in between the code directives, delimited by `<%` and `%>`. +* **`bool (*eof)(const TSLexer *)`** - A function for determining whether the lexer is at the end of the file. The value of `lookahead` will be `0` at the end of a file, but this function should be used instead of checking for that value because the `0` or "NUL" value is also a valid character that could be present in the file being parsed. The third argument to the `scan` function is an array of booleans that indicates which of your external tokens are currently expected by the parser. You should only look for a given token if it is valid according to this array. At the same time, you cannot backtrack, so you may need to combine certain pieces of logic. @@ -694,6 +736,21 @@ } ``` +#### Other External Scanner Details + +If a token in your `externals` array is valid at the current position in the parse, your external scanner will be called first before anything else is done. +This means your external scanner functions as a powerful override of tree-sitter's lexing behavior, and can be used to solve problems that can't be cracked with ordinary lexical, parse, or dynamic precedence. + +If a syntax error is encountered during regular parsing, tree-sitter's first action during error recovery will be to call your external scanner's `scan` function with all tokens marked valid. +Your scanner should detect this case and handle it appropriately. +One simple method of detection is to add an unused token to the end of your `externals` array, for example `externals: $ => [$.token1, $.token2, $.error_sentinel]`, then check whether that token is marked valid to determine whether tree-sitter is in error correction mode. + +If you put terminal keywords in your `externals` array, for example `externals: $ => ['if', 'then', 'else']`, then any time those terminals are present in your grammar they will be tokenized by your external scanner. +It is equivalent to writing `externals: [$.if_keyword, $.then_keyword, $.else_keyword]` then using `alias($.if_keyword, 'if')` in your grammar. + +External scanners are a common cause of infinite loops. +Be very careful when emitting zero-width tokens from your external scanner, and if you consume characters in a loop be sure use the `eof` function to check whether you are at the end of the file. + [ambiguous-grammar]: https://en.wikipedia.org/wiki/Ambiguous_grammar [antlr]: http://www.antlr.org/ [bison-dprec]: https://www.gnu.org/software/bison/manual/html_node/Generalized-LR-Parsing.html diff -Nru tree-sitter-0.20.7/.gitattributes tree-sitter-0.20.8/.gitattributes --- tree-sitter-0.20.7/.gitattributes 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/.gitattributes 2023-04-04 09:15:07.000000000 +0000 @@ -1,2 +1,5 @@ /lib/src/unicode/*.h linguist-vendored /lib/src/unicode/LICENSE linguist-vendored + +/cli/src/generate/prepare_grammar/*.json -diff +Cargo.lock -diff diff -Nru tree-sitter-0.20.7/.github/scripts/cross.sh tree-sitter-0.20.8/.github/scripts/cross.sh --- tree-sitter-0.20.7/.github/scripts/cross.sh 1970-01-01 00:00:00.000000000 +0000 +++ tree-sitter-0.20.8/.github/scripts/cross.sh 2023-04-04 09:15:07.000000000 +0000 @@ -0,0 +1,10 @@ +#!/bin/bash + +set -x +set -e + +if [ "$CROSS" != 1 ]; then + exit 111 +fi + +docker run --rm -v /home/runner:/home/runner -w "$PWD" "$CROSS_IMAGE" "$@" diff -Nru tree-sitter-0.20.7/.github/scripts/make.sh tree-sitter-0.20.8/.github/scripts/make.sh --- tree-sitter-0.20.7/.github/scripts/make.sh 1970-01-01 00:00:00.000000000 +0000 +++ tree-sitter-0.20.8/.github/scripts/make.sh 2023-04-04 09:15:07.000000000 +0000 @@ -0,0 +1,19 @@ +#!/bin/bash + +set -x +set -e + +if [ "$CROSS" = 1 ]; then + if [ -z "$CC" ]; then + echo "make.sh: CC is not set" >&2 + exit 111 + fi + if [ -z "$AR" ]; then + echo "make.sh: AR is not set" >&2 + exit 111 + fi + + cross.sh make CC=$CC AR=$AR "$@" +else + make "$@" +fi diff -Nru tree-sitter-0.20.7/.github/scripts/tree-sitter.sh tree-sitter-0.20.8/.github/scripts/tree-sitter.sh --- tree-sitter-0.20.7/.github/scripts/tree-sitter.sh 1970-01-01 00:00:00.000000000 +0000 +++ tree-sitter-0.20.8/.github/scripts/tree-sitter.sh 2023-04-04 09:15:07.000000000 +0000 @@ -0,0 +1,12 @@ +#!/bin/bash + +set -x +set -e + +tree_sitter="$ROOT"/target/"$TARGET"/release/tree-sitter + +if [ "$CROSS" = 1 ]; then + cross.sh $CROSS_RUNNER "$tree_sitter" "$@" +else + "$tree_sitter" "$@" +fi diff -Nru tree-sitter-0.20.7/.github/workflows/build.yml tree-sitter-0.20.8/.github/workflows/build.yml --- tree-sitter-0.20.7/.github/workflows/build.yml 1970-01-01 00:00:00.000000000 +0000 +++ tree-sitter-0.20.8/.github/workflows/build.yml 2023-04-04 09:15:07.000000000 +0000 @@ -0,0 +1,171 @@ +name: Build & Test + +env: + CARGO_TERM_COLOR: always + RUSTFLAGS: "-D warnings" + CROSS_DEBUG: 1 + +on: + workflow_call: + inputs: + ref: + default: ${{ github.ref }} + type: string + +jobs: + build: + name: ${{ matrix.job.name }} (${{ matrix.job.target }}) (${{ matrix.job.os }}) + runs-on: ${{ matrix.job.os }} + strategy: + fail-fast: false + matrix: + job: + - { name: linux-aarch64 , target: aarch64-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } + - { name: linux-arm , target: arm-unknown-linux-gnueabihf , os: ubuntu-latest , use-cross: true } + - { name: linux-x64 , target: x86_64-unknown-linux-gnu , os: ubuntu-latest } + - { name: linux-x86 , target: i686-unknown-linux-gnu , os: ubuntu-latest , use-cross: true } + - { name: windows-x64 , target: x86_64-pc-windows-msvc , os: windows-latest } + - { name: windows-x86 , target: i686-pc-windows-msvc , os: windows-latest } + - { name: macos-x64 , target: x86_64-apple-darwin , os: macos-latest } + + env: + BUILD_CMD: cargo + + defaults: + run: + shell: bash + + steps: + - name: Checkout source code + uses: actions/checkout@v3 + with: + ref: ${{ inputs.ref }} + + - name: Read Emscripten version + run: | + echo "EMSCRIPTEN_VERSION=$(cat cli/emscripten-version)" >> $GITHUB_ENV + + - name: Install Emscripten + uses: mymindstorm/setup-emsdk@v12 + with: + version: ${{ env.EMSCRIPTEN_VERSION }} + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.job.target }} + + - name: Install cross + if: matrix.job.use-cross + uses: taiki-e/install-action@v2 + with: + tool: cross + + - name: Build custom cross image + if: ${{ matrix.job.use-cross && matrix.job.os == 'ubuntu-latest' }} + run: | + cd .. + + target="${{ matrix.job.target }}" + image=ghcr.io/cross-rs/$target:custom + echo "CROSS_IMAGE=$image" >> $GITHUB_ENV + + echo "[target.$target]" >> Cross.toml + echo "image = \"$image\"" >> Cross.toml + echo "CROSS_CONFIG=$PWD/Cross.toml" >> $GITHUB_ENV + + echo "FROM ghcr.io/cross-rs/$target:edge" >> Dockerfile + echo "ENV DEBIAN_FRONTEND=noninteractive" >> Dockerfile + echo "RUN apt-get update && apt-get install -y nodejs" >> Dockerfile + docker build -t $image . + docker images + docker run --rm $image env + + cd - + + - name: Setup extra env + run: | + PATH="$PWD/.github/scripts:$PATH" + echo "PATH=$PATH" >> $GITHUB_ENV + echo "ROOT=$PWD" >> $GITHUB_ENV + echo "TREE_SITTER=tree-sitter.sh" >> $GITHUB_ENV + + export TARGET=${{ matrix.job.target }} + echo "TARGET=$TARGET" >> $GITHUB_ENV + + USE_CROSS="${{ matrix.job.use-cross }}" + + if [ "$USE_CROSS" == "true" ]; then + echo "BUILD_CMD=cross" >> $GITHUB_ENV + + export CROSS=1; echo "CROSS=$CROSS" >> $GITHUB_ENV + + runner=$(cross.sh bash -c "env | sed -nr '/^CARGO_TARGET_.*_RUNNER=/s///p'") + [ -n "$runner" ] && echo "CROSS_RUNNER=$runner" >> $GITHUB_ENV + echo "runner: $runner" + + case "$TARGET" in + i686-unknown-linux-gnu) CC=i686-linux-gnu-gcc AR=i686-linux-gnu-ar ;; + aarch64-unknown-linux-gnu) CC=aarch64-linux-gnu-gcc AR=aarch64-linux-gnu-ar ;; + arm-unknown-linux-gnueabihf) CC=arm-unknown-linux-gnueabihf-gcc AR=arm-unknown-linux-gnueabihf-gcc-ar ;; + esac + + [ -n "$CC" ] && echo "CC=$CC" >> $GITHUB_ENV + [ -n "$AR" ] && echo "AR=$AR" >> $GITHUB_ENV + fi + + case "$TARGET" in + *-windows-*) + echo "RUST_TEST_THREADS=1" >> $GITHUB_ENV # See #2041 tree-sitter issue + ;; + esac + + - name: Build C library + if: "!contains(matrix.job.os, 'windows')" # Requires an additional adapted Makefile for `cl.exe` compiler + run: make.sh CFLAGS="-Werror" -j + + - name: Build wasm library + run: script/build-wasm + + - name: Build CLI + run: $BUILD_CMD build --release --target=${{ matrix.job.target }} + + - name: Fetch fixtures + run: script/fetch-fixtures + + - name: Generate fixtures + run: script/generate-fixtures + + - name: Generate WASM fixtures + if: "!matrix.job.use-cross" + run: script/generate-fixtures-wasm + + - name: Run main tests + run: $BUILD_CMD test --target=${{ matrix.job.target }} + + - name: Run wasm tests + if: "!matrix.job.use-cross" # TODO: Install Emscripten into custom cross images + run: script/test-wasm + + - name: Run benchmarks + if: "!matrix.job.use-cross" # It doesn't make sense to benchmark something in an emulator + run: $BUILD_CMD bench benchmark -p tree-sitter-cli --target=${{ matrix.job.target }} + + - name: Upload CLI artifact + uses: actions/upload-artifact@v3 + with: + name: tree-sitter.${{ matrix.job.name }} + path: target/${{ matrix.job.target }}/release/tree-sitter${{ contains(matrix.job.target, 'windows') && '.exe' || '' }} + if-no-files-found: error + retention-days: 7 + + - name: Upload WASM artifacts + if: ${{ matrix.job.name == 'linux-x64' }} + uses: actions/upload-artifact@v3 + with: + name: tree-sitter.wasm + path: | + lib/binding_web/tree-sitter.js + lib/binding_web/tree-sitter.wasm + if-no-files-found: error + retention-days: 7 diff -Nru tree-sitter-0.20.7/.github/workflows/CICD.yml tree-sitter-0.20.8/.github/workflows/CICD.yml --- tree-sitter-0.20.7/.github/workflows/CICD.yml 1970-01-01 00:00:00.000000000 +0000 +++ tree-sitter-0.20.8/.github/workflows/CICD.yml 2023-04-04 09:15:07.000000000 +0000 @@ -0,0 +1,69 @@ +name: CICD + +on: + workflow_dispatch: + pull_request: + push: + branches: + - master + - check/* + +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true + +jobs: + init: + name: Init + runs-on: ubuntu-latest + steps: + - name: Get PR head ref + if: ${{ github.event_name == 'pull_request' }} + id: ref + run: | + echo "ref=refs/pull/${{ github.event.pull_request.number }}/head" >> $GITHUB_OUTPUT + outputs: + ref: >- + ${{ + (github.event_name == 'pull_request' && startsWith(github.head_ref, 'release/v')) + && steps.ref.outputs.ref + || github.ref + }} + + fast_checks: + name: Fast checks + uses: ./.github/workflows/fast_checks.yml + + full_checks: + name: Full Rust checks + needs: fast_checks + uses: ./.github/workflows/full_rust_checks.yml + + min_version: + name: Minimum supported rust version + needs: fast_checks + uses: ./.github/workflows/msrv.yml + with: + package: tree-sitter-cli + + build: + name: Build & Test + needs: [init, fast_checks] + uses: ./.github/workflows/build.yml + with: + ref: ${{ needs.init.outputs.ref }} + + release: + name: Release + needs: [init, fast_checks, full_checks, min_version, build] + if: > + github.event.pull_request.head.repo.full_name == github.repository && + startsWith(github.head_ref, 'release/v') + uses: ./.github/workflows/release.yml + with: + ref: ${{ needs.init.outputs.ref }} + + publish: + name: Publish + needs: release + uses: ./.github/workflows/publish.yml diff -Nru tree-sitter-0.20.7/.github/workflows/ci.yml tree-sitter-0.20.8/.github/workflows/ci.yml --- tree-sitter-0.20.7/.github/workflows/ci.yml 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/.github/workflows/ci.yml 1970-01-01 00:00:00.000000000 +0000 @@ -1,149 +0,0 @@ -name: CI - -on: - push: - branches: - - master - tags: - - v* - pull_request: - branches: - - "**" - -env: - CARGO_TERM_COLOR: always - CARGO_INCREMENTAL: 0 - -jobs: - unix-tests: - name: Unix tests - runs-on: ${{ matrix.os }} - strategy: - fail-fast: true - matrix: - os: - - macos-latest - - ubuntu-latest - steps: - - name: Checkout repo - uses: actions/checkout@v2 - - # Work around https://github.com/actions/cache/issues/403. - - name: Use GNU tar - if: matrix.os == 'macos-latest' - run: | - echo PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH" >> $GITHUB_ENV - - - name: Read Emscripten version - run: | - printf 'EMSCRIPTEN_VERSION=%s\n' "$(cat cli/emscripten-version)" >> $GITHUB_ENV - - - name: Cache artifacts - id: cache - uses: actions/cache@v2 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }}-emscripten-${{ env.EMSCRIPTEN_VERSION }} - - - name: Install rust - if: steps.cache.outputs.cache-hit != 'true' - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - profile: minimal - - - name: Check Rust code formatting - run: cargo fmt -- --check - - - name: Install emscripten - uses: mymindstorm/setup-emsdk@v10 - with: - version: ${{ env.EMSCRIPTEN_VERSION }} - - - name: Build C library - run: make - - - name: Build wasm library - run: script/build-wasm - - - name: Build CLI - run: | - RUSTFLAGS="-D warnings" - cargo build --release - - - name: Set up fixture parsers - run: | - script/fetch-fixtures - script/generate-fixtures - script/generate-fixtures-wasm - - - name: Run main tests - run: cargo test - - - name: Run wasm tests - run: script/test-wasm - - - name: Run benchmarks - run: script/benchmark - - - name: Compress CLI binary - if: startsWith(github.ref, 'refs/tags/v') - run: | - cp target/release/tree-sitter . - export platform=$(echo ${{ runner.os }} | awk '{print tolower($0)}') - gzip --suffix "-${platform}-x64.gz" tree-sitter - - - name: Release - uses: softprops/action-gh-release@v1 - if: startsWith(github.ref, 'refs/tags/v') - with: - draft: true - files: | - tree-sitter-*.gz - lib/binding_web/tree-sitter.js - lib/binding_web/tree-sitter.wasm - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - windows-tests: - name: Windows tests - runs-on: windows-latest - steps: - - name: Checkout repo - uses: actions/checkout@v2 - - - name: Cache artifacts - id: cache - uses: actions/cache@v2 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }} - - - name: Install rust - if: steps.cache.outputs.cache-hit != 'true' - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - profile: minimal - - - name: Check Rust code formatting - run: cargo fmt -- --check - - - name: Build CLI - run: | - $env:RUSTFLAGS="-D warnings" - cargo build --release - - - name: Set up fixture parsers - run: | - script/fetch-fixtures.cmd - script/generate-fixtures.cmd - - - name: Run main tests - run: script/test diff -Nru tree-sitter-0.20.7/.github/workflows/fast_checks.yml tree-sitter-0.20.8/.github/workflows/fast_checks.yml --- tree-sitter-0.20.7/.github/workflows/fast_checks.yml 1970-01-01 00:00:00.000000000 +0000 +++ tree-sitter-0.20.8/.github/workflows/fast_checks.yml 2023-04-04 09:15:07.000000000 +0000 @@ -0,0 +1,31 @@ +name: Fast checks to fail fast on any simple code issues + +env: + CARGO_TERM_COLOR: always + RUSTFLAGS: "-D warnings" + +on: + workflow_call: + +jobs: + check_rust_formatting: + name: Check Rust formating + runs-on: ubuntu-latest + steps: + + - name: Checkout source code + uses: actions/checkout@v3 + + - name: Run cargo fmt + run: cargo fmt -- --check + + check_c_warnings: + name: Check C warnings + runs-on: ubuntu-latest + steps: + + - name: Checkout source code + uses: actions/checkout@v3 + + - name: Make C library to check that it's able to compile without warnings + run: make -j CFLAGS="-Werror" diff -Nru tree-sitter-0.20.7/.github/workflows/full_rust_checks.yml tree-sitter-0.20.8/.github/workflows/full_rust_checks.yml --- tree-sitter-0.20.7/.github/workflows/full_rust_checks.yml 1970-01-01 00:00:00.000000000 +0000 +++ tree-sitter-0.20.8/.github/workflows/full_rust_checks.yml 2023-04-04 09:15:07.000000000 +0000 @@ -0,0 +1,32 @@ +name: Full Rust codebase checks + +env: + CARGO_TERM_COLOR: always + RUSTFLAGS: "-D warnings" + +on: + workflow_call: + +jobs: + run: + name: Run checks + runs-on: ubuntu-latest + steps: + + - name: Checkout source code + uses: actions/checkout@v3 + + - name: Install rust toolchain + uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + + - name: Run cargo fmt + run: cargo fmt -- --check + + # - name: Run clippy + # run: cargo clippy --all-targets + + - name: Run cargo check + run: cargo check --workspace --examples --tests --benches --bins diff -Nru tree-sitter-0.20.7/.github/workflows/msrv.yml tree-sitter-0.20.8/.github/workflows/msrv.yml --- tree-sitter-0.20.7/.github/workflows/msrv.yml 1970-01-01 00:00:00.000000000 +0000 +++ tree-sitter-0.20.8/.github/workflows/msrv.yml 2023-04-04 09:15:07.000000000 +0000 @@ -0,0 +1,42 @@ +name: Minimum supported rust version + +env: + CARGO_TERM_COLOR: always + RUSTFLAGS: "-D warnings" + +on: + workflow_call: + inputs: + package: + description: Target cargo package name + required: true + type: string + + +jobs: + run: + name: Run checks + runs-on: ubuntu-latest + steps: + + - name: Checkout source code + uses: actions/checkout@v3 + + - name: Get the MSRV from the package metadata + id: msrv + run: cargo metadata --no-deps --format-version 1 | jq -r '"version=" + (.packages[] | select(.name == "${{ inputs.package }}").rust_version)' >> $GITHUB_OUTPUT + + - name: Install rust toolchain (v${{ steps.msrv.outputs.version }}) + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ steps.msrv.outputs.version }} + components: clippy, rustfmt + + - name: Run cargo fmt + run: cargo fmt -- --check + + # - name: Run clippy (on minimum supported rust version to prevent warnings we can't fix) + # run: cargo clippy --all-targets + + # - name: Run main tests + # run: cargo test diff -Nru tree-sitter-0.20.7/.github/workflows/publish.yml tree-sitter-0.20.8/.github/workflows/publish.yml --- tree-sitter-0.20.7/.github/workflows/publish.yml 1970-01-01 00:00:00.000000000 +0000 +++ tree-sitter-0.20.8/.github/workflows/publish.yml 2023-04-04 09:15:07.000000000 +0000 @@ -0,0 +1,21 @@ +name: Publish to registries + +on: + workflow_call: + +jobs: + crates_io: + name: Publish to Crates.io + runs-on: ubuntu-latest + steps: + - name: Publish packages + run: | + echo "::warning::TODO: add a Crates.io publish logic" + + npm: + name: Publish to npmjs.com + runs-on: ubuntu-latest + steps: + - name: Publish packages + run: | + echo "::warning::TODO: add a npmjs.com publish logic" diff -Nru tree-sitter-0.20.7/.github/workflows/release.yml tree-sitter-0.20.8/.github/workflows/release.yml --- tree-sitter-0.20.7/.github/workflows/release.yml 1970-01-01 00:00:00.000000000 +0000 +++ tree-sitter-0.20.8/.github/workflows/release.yml 2023-04-04 09:15:07.000000000 +0000 @@ -0,0 +1,101 @@ +name: Release + +on: + workflow_call: + inputs: + ref: + default: ${{ github.ref }} + type: string + +jobs: + permissions: + name: Check permissions + runs-on: ubuntu-latest + outputs: + release_allowed: ${{ steps.maintainer.outputs.is_maintainer == 'true' }} + steps: + + - name: Is maintainer + id: maintainer + env: + GH_TOKEN: ${{ github.token }} + repo: ${{ github.repository }} + actor: ${{ github.actor }} + run: | + maintainer=$( + gh api "/repos/${repo}/collaborators" | + jq ".[] | {login, maintainer: .permissions | .maintain} | select(.login == \"${actor}\") | .maintainer" + ); + if [ "$maintainer" == "true" ]; then + echo "@${actor} has maintainer level permissions :rocket:" >> $GITHUB_STEP_SUMMARY; + echo "is_maintainer=true" >> $GITHUB_OUTPUT + fi + + release: + name: Release + needs: permissions + if: needs.permissions.outputs.release_allowed + runs-on: ubuntu-latest + permissions: + contents: write + steps: + + - name: Checkout source code + uses: actions/checkout@v3 + with: + ref: ${{ inputs.ref }} + + - name: Download build artifacts + uses: actions/download-artifact@v3 + with: + path: artifacts + + - name: Display structure of downloaded files + run: ls -lR + working-directory: artifacts + + - name: Prepare release artifacts + run: | + mkdir -p target + mv artifacts/tree-sitter.wasm/* target/ + rm -r artifacts/tree-sitter.wasm + for platform in $(cd artifacts; ls | sed 's/^tree-sitter\.//'); do + exe=$(ls artifacts/tree-sitter.$platform/tree-sitter*) + gzip --stdout --name $exe > target/tree-sitter-$platform.gz + done + rm -rf artifacts + ls -l target/ + + - name: Get tag name from a release/v* branch name + id: tag_name + env: + tag: ${{ github.head_ref }} + run: echo "tag=${tag#release/}" >> $GITHUB_OUTPUT + + - name: Add a release tag + env: + ref: ${{ inputs.ref }} + tag: ${{ steps.tag_name.outputs.tag }} + message: "Release ${{ steps.tag_name.outputs.tag }}" + run: | + git config user.name "${GITHUB_ACTOR}" + git config user.email "${GITHUB_ACTOR}@users.noreply.github.com" + git tag -a "$tag" HEAD -m "$message" + git push origin "$tag" + + - name: Create release + uses: softprops/action-gh-release@v1 + with: + name: ${{ steps.tag_name.outputs.tag }} + tag_name: ${{ steps.tag_name.outputs.tag }} + fail_on_unmatched_files: true + files: | + target/tree-sitter-*.gz + target/tree-sitter.wasm + target/tree-sitter.js + + - name: Merge release PR + env: + GH_TOKEN: ${{ github.token }} + run: | + gh pr merge ${{ github.event.pull_request.html_url }} --match-head-commit $(git rev-parse HEAD) --merge --delete-branch diff -Nru tree-sitter-0.20.7/highlight/Cargo.toml tree-sitter-0.20.8/highlight/Cargo.toml --- tree-sitter-0.20.7/highlight/Cargo.toml 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/highlight/Cargo.toml 2023-04-04 09:15:07.000000000 +0000 @@ -12,6 +12,7 @@ keywords = ["incremental", "parsing", "syntax", "highlighting"] categories = ["parsing", "text-editors"] repository = "https://github.com/tree-sitter/tree-sitter" +rust-version.workspace = true [lib] crate-type = ["lib", "staticlib"] diff -Nru tree-sitter-0.20.7/highlight/README.md tree-sitter-0.20.8/highlight/README.md --- tree-sitter-0.20.7/highlight/README.md 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/highlight/README.md 2023-04-04 09:15:07.000000000 +0000 @@ -1,7 +1,5 @@ # `tree-sitter-highlight` -[![Build Status](https://travis-ci.org/tree-sitter/tree-sitter.svg?branch=master)](https://travis-ci.org/tree-sitter/tree-sitter) -[![Build status](https://ci.appveyor.com/api/projects/status/vtmbd6i92e97l55w/branch/master?svg=true)](https://ci.appveyor.com/project/maxbrunsfeld/tree-sitter/branch/master) [![Crates.io](https://img.shields.io/crates/v/tree-sitter-highlight.svg)](https://crates.io/crates/tree-sitter-highlight) ### Usage diff -Nru tree-sitter-0.20.7/lib/binding_rust/bindings.rs tree-sitter-0.20.8/lib/binding_rust/bindings.rs --- tree-sitter-0.20.7/lib/binding_rust/bindings.rs 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/lib/binding_rust/bindings.rs 2023-04-04 09:15:07.000000000 +0000 @@ -346,6 +346,12 @@ pub fn ts_tree_language(arg1: *const TSTree) -> *const TSLanguage; } extern "C" { + #[doc = " Get the array of included ranges that was used to parse the syntax tree."] + #[doc = ""] + #[doc = " The returned pointer must be freed by the caller."] + pub fn ts_tree_included_ranges(arg1: *const TSTree, length: *mut u32) -> *mut TSRange; +} +extern "C" { #[doc = " Edit the syntax tree to keep it in sync with source code that has been"] #[doc = " edited."] #[doc = ""] @@ -373,6 +379,10 @@ ) -> *mut TSRange; } extern "C" { + #[doc = " Write a DOT graph describing the syntax tree to the given file."] + pub fn ts_tree_print_dot_graph(arg1: *const TSTree, file_descriptor: ::std::os::raw::c_int); +} +extern "C" { #[doc = " Get the node's type as a null-terminated string."] pub fn ts_node_type(arg1: TSNode) -> *const ::std::os::raw::c_char; } @@ -668,6 +678,9 @@ ) -> *const TSQueryPredicateStep; } extern "C" { + pub fn ts_query_is_pattern_non_local(self_: *const TSQuery, pattern_index: u32) -> bool; +} +extern "C" { pub fn ts_query_is_pattern_rooted(self_: *const TSQuery, pattern_index: u32) -> bool; } extern "C" { diff -Nru tree-sitter-0.20.7/lib/binding_rust/lib.rs tree-sitter-0.20.8/lib/binding_rust/lib.rs --- tree-sitter-0.20.7/lib/binding_rust/lib.rs 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/lib/binding_rust/lib.rs 2023-04-04 09:15:07.000000000 +0000 @@ -38,7 +38,7 @@ /// An opaque object that defines how to parse a particular language. The code for each /// `Language` is generated by the Tree-sitter CLI. #[doc(alias = "TSLanguage")] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] #[repr(transparent)] pub struct Language(*const ffi::TSLanguage); @@ -763,6 +763,28 @@ util::CBufferIter::new(ptr, count as usize).map(|r| r.into()) } } + + /// Get the included ranges that were used to parse the syntax tree. + pub fn included_ranges(&self) -> Vec { + let mut count = 0u32; + unsafe { + let ptr = ffi::ts_tree_included_ranges(self.0.as_ptr(), &mut count as *mut u32); + let ranges = slice::from_raw_parts(ptr, count as usize); + let result = ranges.iter().copied().map(|range| range.into()).collect(); + (FREE_FN)(ptr as *mut c_void); + result + } + } + + /// Print a graph of the tree to the given file descriptor. + /// The graph is formatted in the DOT language. You may want to pipe this graph + /// directly to a `dot(1)` process in order to generate SVG output. + #[cfg(unix)] + #[doc(alias = "ts_tree_print_dot_graph")] + pub fn print_dot_graph(&self, file: &impl AsRawFd) { + let fd = file.as_raw_fd(); + unsafe { ffi::ts_tree_print_dot_graph(self.0.as_ptr(), fd) } + } } impl fmt::Debug for Tree { @@ -1714,11 +1736,17 @@ } /// Check if a given pattern within a query has a single root node. - #[doc(alias = "ts_query_is_pattern_guaranteed_at_step")] + #[doc(alias = "ts_query_is_pattern_rooted")] pub fn is_pattern_rooted(&self, index: usize) -> bool { unsafe { ffi::ts_query_is_pattern_rooted(self.ptr.as_ptr(), index as u32) } } + /// Check if a given pattern within a query has a single root node. + #[doc(alias = "ts_query_is_pattern_non_local")] + pub fn is_pattern_non_local(&self, index: usize) -> bool { + unsafe { ffi::ts_query_is_pattern_non_local(self.ptr.as_ptr(), index as u32) } + } + /// Check if a given step in a query is 'definite'. /// /// A query step is 'definite' if its parent pattern will be guaranteed to match diff -Nru tree-sitter-0.20.7/lib/binding_rust/README.md tree-sitter-0.20.8/lib/binding_rust/README.md --- tree-sitter-0.20.7/lib/binding_rust/README.md 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/lib/binding_rust/README.md 2023-04-04 09:15:07.000000000 +0000 @@ -1,7 +1,5 @@ # Rust Tree-sitter -[![Build Status](https://travis-ci.org/tree-sitter/tree-sitter.svg?branch=master)](https://travis-ci.org/tree-sitter/tree-sitter) -[![Build status](https://ci.appveyor.com/api/projects/status/vtmbd6i92e97l55w/branch/master?svg=true)](https://ci.appveyor.com/project/maxbrunsfeld/tree-sitter/branch/master) [![Crates.io](https://img.shields.io/crates/v/tree-sitter.svg)](https://crates.io/crates/tree-sitter) Rust bindings to the [Tree-sitter][] parsing library. diff -Nru tree-sitter-0.20.7/lib/binding_web/exports.json tree-sitter-0.20.8/lib/binding_web/exports.json --- tree-sitter-0.20.7/lib/binding_web/exports.json 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/lib/binding_web/exports.json 2023-04-04 09:15:07.000000000 +0000 @@ -4,16 +4,18 @@ "_malloc", "_realloc", - "__ZNKSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE4copyEPcmm", "__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE6__initEPKcm", - "__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7reserveEm", "__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE9__grow_byEmmmmmm", "__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE9push_backEc", - "__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEED2Ev", "__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE9push_backEw", + "__ZNKSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE4copyEPcmm", + "__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7reserveEm", + "__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE6resizeEmw", + "__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEED2Ev", "__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEED2Ev", "__ZdlPv", "__Znwm", + "___cxa_atexit", "_abort", "_iswalnum", "_iswalpha", @@ -24,6 +26,7 @@ "_memcmp", "_memcpy", "_memmove", + "_memset", "_strlen", "_towupper", diff -Nru tree-sitter-0.20.7/lib/binding_web/package.json tree-sitter-0.20.8/lib/binding_web/package.json --- tree-sitter-0.20.7/lib/binding_web/package.json 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/lib/binding_web/package.json 2023-04-04 09:15:07.000000000 +0000 @@ -27,8 +27,8 @@ }, "homepage": "https://github.com/tree-sitter/tree-sitter/tree/master/lib/binding_web", "devDependencies": { - "chai": "^4.2.0", - "mocha": "^6.1.4", - "terser": "^3.17.0" + "chai": "^4.3.7", + "mocha": "^10.2.0", + "terser": "^5.16.6" } } diff -Nru tree-sitter-0.20.7/lib/binding_web/README.md tree-sitter-0.20.8/lib/binding_web/README.md --- tree-sitter-0.20.7/lib/binding_web/README.md 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/lib/binding_web/README.md 2023-04-04 09:15:07.000000000 +0000 @@ -1,8 +1,6 @@ Web Tree-sitter =============== -[![Build Status](https://travis-ci.org/tree-sitter/tree-sitter.svg?branch=master)](https://travis-ci.org/tree-sitter/tree-sitter) - WebAssembly bindings to the [Tree-sitter](https://github.com/tree-sitter/tree-sitter) parsing library. ### Setup diff -Nru tree-sitter-0.20.7/lib/binding_web/test/parser-test.js tree-sitter-0.20.8/lib/binding_web/test/parser-test.js --- tree-sitter-0.20.7/lib/binding_web/test/parser-test.js 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/lib/binding_web/test/parser-test.js 2023-04-04 09:15:07.000000000 +0000 @@ -153,7 +153,7 @@ 'type: (primitive_type) ' + 'declarator: (init_declarator ' + 'declarator: (pointer_declarator declarator: (identifier)) ' + - 'value: (raw_string_literal))))' + 'value: (raw_string_literal delimiter: (raw_string_delimiter) (raw_string_content) (raw_string_delimiter)))))' ); }).timeout(5000); diff -Nru tree-sitter-0.20.7/lib/Cargo.toml tree-sitter-0.20.8/lib/Cargo.toml --- tree-sitter-0.20.7/lib/Cargo.toml 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/lib/Cargo.toml 2023-04-04 09:15:07.000000000 +0000 @@ -1,14 +1,15 @@ [package] name = "tree-sitter" description = "Rust bindings to the Tree-sitter parsing library" -version = "0.20.9" +version = "0.20.10" authors = ["Max Brunsfeld "] -edition = "2018" +edition = "2021" license = "MIT" readme = "binding_rust/README.md" keywords = ["incremental", "parsing"] categories = ["api-bindings", "parsing", "text-editors"] repository = "https://github.com/tree-sitter/tree-sitter" +rust-version.workspace = true build = "binding_rust/build.rs" diff -Nru tree-sitter-0.20.7/lib/include/tree_sitter/api.h tree-sitter-0.20.8/lib/include/tree_sitter/api.h --- tree-sitter-0.20.7/lib/include/tree_sitter/api.h 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/lib/include/tree_sitter/api.h 2023-04-04 09:15:07.000000000 +0000 @@ -382,6 +382,13 @@ const TSLanguage *ts_tree_language(const TSTree *); /** + * Get the array of included ranges that was used to parse the syntax tree. + * + * The returned pointer must be freed by the caller. + */ +TSRange *ts_tree_included_ranges(const TSTree *, uint32_t *length); + +/** * Edit the syntax tree to keep it in sync with source code that has been * edited. * @@ -413,7 +420,7 @@ /** * Write a DOT graph describing the syntax tree to the given file. */ -void ts_tree_print_dot_graph(const TSTree *, FILE *); +void ts_tree_print_dot_graph(const TSTree *, int file_descriptor); /******************/ /* Section - Node */ @@ -743,15 +750,26 @@ uint32_t *length ); -bool ts_query_is_pattern_rooted( - const TSQuery *self, - uint32_t pattern_index -); +/* + * Check if the given pattern in the query has a single root node. + */ +bool ts_query_is_pattern_rooted(const TSQuery *self, uint32_t pattern_index); -bool ts_query_is_pattern_guaranteed_at_step( - const TSQuery *self, - uint32_t byte_offset -); +/* + * Check if the given pattern in the query is 'non local'. + * + * A non-local pattern has multiple root nodes and can match within a + * repeating sequence of nodes, as specified by the grammar. Non-local + * patterns disable certain optimizations that would otherwise be possible + * when executing a query on a specific range of a syntax tree. + */ +bool ts_query_is_pattern_non_local(const TSQuery *self, uint32_t pattern_index); + +/* + * Check if a given pattern is guaranteed to match once a given step is reached. + * The step is specified by its byte offset in the query's source code. + */ +bool ts_query_is_pattern_guaranteed_at_step(const TSQuery *self, uint32_t byte_offset); /** * Get the name and length of one of the query's captures, or one of the diff -Nru tree-sitter-0.20.7/lib/src/array.h tree-sitter-0.20.8/lib/src/array.h --- tree-sitter-0.20.7/lib/src/array.h 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/lib/src/array.h 2023-04-04 09:15:07.000000000 +0000 @@ -170,10 +170,10 @@ *self = swap; } -static inline void array__grow(VoidArray *self, size_t count, size_t element_size) { - size_t new_size = self->size + count; +static inline void array__grow(VoidArray *self, uint32_t count, size_t element_size) { + uint32_t new_size = self->size + count; if (new_size > self->capacity) { - size_t new_capacity = self->capacity * 2; + uint32_t new_capacity = self->capacity * 2; if (new_capacity < 8) new_capacity = 8; if (new_capacity < new_size) new_capacity = new_size; array__reserve(self, element_size, new_capacity); diff -Nru tree-sitter-0.20.7/lib/src/clock.h tree-sitter-0.20.8/lib/src/clock.h --- tree-sitter-0.20.7/lib/src/clock.h 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/lib/src/clock.h 2023-04-04 09:15:07.000000000 +0000 @@ -1,6 +1,7 @@ #ifndef TREE_SITTER_CLOCK_H_ #define TREE_SITTER_CLOCK_H_ +#include #include typedef uint64_t TSDuration; @@ -82,6 +83,10 @@ TSClock result = base; result.tv_sec += duration / 1000000; result.tv_nsec += (duration % 1000000) * 1000; + if (result.tv_nsec >= 1000000000) { + result.tv_nsec -= 1000000000; + ++(result.tv_sec); + } return result; } diff -Nru tree-sitter-0.20.7/lib/src/lexer.c tree-sitter-0.20.8/lib/src/lexer.c --- tree-sitter-0.20.7/lib/src/lexer.c 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/lib/src/lexer.c 2023-04-04 09:15:07.000000000 +0000 @@ -104,13 +104,16 @@ static void ts_lexer_goto(Lexer *self, Length position) { self->current_position = position; - bool found_included_range = false; // Move to the first valid position at or after the given position. + bool found_included_range = false; for (unsigned i = 0; i < self->included_range_count; i++) { TSRange *included_range = &self->included_ranges[i]; - if (included_range->end_byte > position.bytes) { - if (included_range->start_byte >= position.bytes) { + if ( + included_range->end_byte > self->current_position.bytes && + included_range->end_byte > included_range->start_byte + ) { + if (included_range->start_byte >= self->current_position.bytes) { self->current_position = (Length) { .bytes = included_range->start_byte, .extent = included_range->start_point, @@ -127,8 +130,8 @@ // If the current position is outside of the current chunk of text, // then clear out the current chunk of text. if (self->chunk && ( - position.bytes < self->chunk_start || - position.bytes >= self->chunk_start + self->chunk_size + self->current_position.bytes < self->chunk_start || + self->current_position.bytes >= self->chunk_start + self->chunk_size )) { ts_lexer__clear_chunk(self); } @@ -164,27 +167,31 @@ } } - const TSRange *current_range = NULL; - if (self->current_included_range_index < self->included_range_count) { - current_range = &self->included_ranges[self->current_included_range_index]; - if (self->current_position.bytes == current_range->end_byte) { - self->current_included_range_index++; - if (self->current_included_range_index < self->included_range_count) { - current_range++; - self->current_position = (Length) { - current_range->start_byte, - current_range->start_point, - }; - } else { - current_range = NULL; - } + const TSRange *current_range = &self->included_ranges[self->current_included_range_index]; + while ( + self->current_position.bytes >= current_range->end_byte || + current_range->end_byte == current_range->start_byte + ) { + self->current_included_range_index++; + if (self->current_included_range_index < self->included_range_count) { + current_range++; + self->current_position = (Length) { + current_range->start_byte, + current_range->start_point, + }; + } else { + current_range = NULL; + break; } } if (skip) self->token_start_position = self->current_position; if (current_range) { - if (self->current_position.bytes >= self->chunk_start + self->chunk_size) { + if ( + self->current_position.bytes < self->chunk_start || + self->current_position.bytes >= self->chunk_start + self->chunk_size + ) { ts_lexer__get_chunk(self); } ts_lexer__get_lookahead(self); @@ -339,6 +346,13 @@ ts_lexer__mark_end(&self->data); } + // If the token ended at an included range boundary, then its end position + // will have been reset to the end of the preceding range. Reset the start + // position to match. + if (self->token_end_position.bytes < self->token_start_position.bytes) { + self->token_start_position = self->token_end_position; + } + uint32_t current_lookahead_end_byte = self->current_position.bytes + 1; // In order to determine that a byte sequence is invalid UTF8 or UTF16, diff -Nru tree-sitter-0.20.7/lib/src/parser.c tree-sitter-0.20.8/lib/src/parser.c --- tree-sitter-0.20.7/lib/src/parser.c 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/lib/src/parser.c 2023-04-04 09:15:07.000000000 +0000 @@ -447,8 +447,14 @@ // avoid infinite loops which could otherwise occur, because the lexer is // looking for any possible token, instead of looking for the specific set of // tokens that are valid in some parse state. + // + // Note that it's possible that the token end position may be *before* the + // original position of the lexer because of the way that tokens are positioned + // at included range boundaries: when a token is terminated at the start of + // an included range, it is marked as ending at the *end* of the preceding + // included range. if ( - self->lexer.token_end_position.bytes == current_position.bytes && + self->lexer.token_end_position.bytes <= current_position.bytes && (error_mode || !ts_stack_has_advanced_since_error(self->stack, version)) && !external_scanner_state_changed ) { @@ -525,10 +531,6 @@ self->language ); } else { - if (self->lexer.token_end_position.bytes < self->lexer.token_start_position.bytes) { - self->lexer.token_start_position = self->lexer.token_end_position; - } - bool is_keyword = false; TSSymbol symbol = self->lexer.data.result_symbol; Length padding = length_sub(self->lexer.token_start_position, start_position); @@ -605,7 +607,7 @@ static void ts_parser__set_cached_token( TSParser *self, - size_t byte_index, + uint32_t byte_index, Subtree last_external_token, Subtree token ) { @@ -1461,7 +1463,9 @@ ((self->cancellation_flag && atomic_load(self->cancellation_flag)) || (!clock_is_null(self->end_clock) && clock_is_gt(clock_now(), self->end_clock))) ) { - ts_subtree_release(&self->tree_pool, lookahead); + if (lookahead.ptr) { + ts_subtree_release(&self->tree_pool, lookahead); + } return false; } @@ -1937,8 +1941,16 @@ } } + // After advancing each version of the stack, re-sort the versions by their cost, + // removing any versions that are no longer worth pursuing. unsigned min_error_cost = ts_parser__condense_stack(self); + + // If there's already a finished parse tree that's better than any in-progress version, + // then terminate parsing. Clear the parse stack to remove any extra references to subtrees + // within the finished tree, ensuring that these subtrees can be safely mutated in-place + // for rebalancing. if (self->finished_tree.ptr && ts_subtree_error_cost(self->finished_tree) < min_error_cost) { + ts_stack_clear(self->stack); break; } diff -Nru tree-sitter-0.20.7/lib/src/query.c tree-sitter-0.20.8/lib/src/query.c --- tree-sitter-0.20.7/lib/src/query.c 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/lib/src/query.c 2023-04-04 09:15:07.000000000 +0000 @@ -146,6 +146,7 @@ Slice steps; Slice predicate_steps; uint32_t start_byte; + bool is_non_local; } QueryPattern; typedef struct { @@ -228,11 +229,20 @@ AnalysisStateEntry stack[MAX_ANALYSIS_STATE_DEPTH]; uint16_t depth; uint16_t step_index; + TSSymbol root_symbol; } AnalysisState; typedef Array(AnalysisState *) AnalysisStateSet; -typedef Array(AnalysisState *) AnalysisStatePool; +typedef struct { + AnalysisStateSet states; + AnalysisStateSet next_states; + AnalysisStateSet deeper_states; + AnalysisStateSet state_pool; + Array(uint16_t) final_step_indices; + Array(TSSymbol) finished_parent_symbols; + bool did_abort; +} QueryAnalysis; /* * AnalysisSubgraph - A subset of the states in the parse table that are used @@ -253,6 +263,8 @@ Array(AnalysisSubgraphNode) nodes; } AnalysisSubgraph; +typedef Array(AnalysisSubgraph) AnalysisSubgraphArray; + /* * StatePredecessorMap - A map that stores the predecessors of each parse state. * This is used during query analysis to determine which parse states can lead @@ -269,8 +281,8 @@ */ struct TSQuery { SymbolTable captures; - Array(CaptureQuantifiers) capture_quantifiers; SymbolTable predicate_values; + Array(CaptureQuantifiers) capture_quantifiers; Array(QueryStep) steps; Array(PatternEntry) pattern_map; Array(TSQueryPredicateStep) predicate_steps; @@ -278,6 +290,7 @@ Array(StepOffset) step_offsets; Array(TSFieldId) negated_fields; Array(char) string_buffer; + Array(TSSymbol) repeat_symbols_with_rootless_patterns; const TSLanguage *language; uint16_t wildcard_root_pattern_count; }; @@ -297,6 +310,7 @@ TSPoint start_point; TSPoint end_point; uint32_t next_state_id; + bool on_visible_node; bool ascending; bool halted; bool did_exceed_match_limit; @@ -934,30 +948,23 @@ return false; } -static inline AnalysisState *analysis_state__clone(AnalysisState const *self) { - AnalysisState *new_state = ts_malloc(sizeof(AnalysisState)); - *new_state = *self; - return new_state; -} - -/**************** +/****************** * AnalysisStateSet - ****************/ + ******************/ // Obtains an `AnalysisState` instance, either by consuming one from this set's object pool, or by // cloning one from scratch. static inline AnalysisState *analysis_state_pool__clone_or_reuse( - AnalysisStatePool *self, + AnalysisStateSet *self, AnalysisState *borrowed_item ) { AnalysisState *new_item; if (self->size) { new_item = array_pop(self); - *new_item = *borrowed_item; } else { - new_item = analysis_state__clone(borrowed_item); + new_item = ts_malloc(sizeof(AnalysisState)); } - + *new_item = *borrowed_item; return new_item; } @@ -967,9 +974,9 @@ // // The caller retains ownership of the passed-in memory. However, the clone that is created by this // function will be managed by the state set. -static inline void analysis_state_set__insert_sorted_by_clone( +static inline void analysis_state_set__insert_sorted( AnalysisStateSet *self, - AnalysisStatePool *pool, + AnalysisStateSet *pool, AnalysisState *borrowed_item ) { unsigned index, exists; @@ -988,9 +995,9 @@ // // The caller retains ownership of the passed-in memory. However, the clone that is created by this // function will be managed by the state set. -static inline void analysis_state_set__push_by_clone( +static inline void analysis_state_set__push( AnalysisStateSet *self, - AnalysisStatePool *pool, + AnalysisStateSet *pool, AnalysisState *borrowed_item ) { AnalysisState *new_item = analysis_state_pool__clone_or_reuse(pool, borrowed_item); @@ -998,7 +1005,7 @@ } // Removes all items from this set, returning it to an empty state. -static inline void analysis_state_set__clear(AnalysisStateSet *self, AnalysisStatePool *pool) { +static inline void analysis_state_set__clear(AnalysisStateSet *self, AnalysisStateSet *pool) { array_push_all(pool, self); array_clear(self); } @@ -1012,6 +1019,31 @@ array_delete(self); } +/**************** + * QueryAnalyzer + ****************/ + +static inline QueryAnalysis query_analysis__new() { + return (QueryAnalysis) { + .states = array_new(), + .next_states = array_new(), + .deeper_states = array_new(), + .state_pool = array_new(), + .final_step_indices = array_new(), + .finished_parent_symbols = array_new(), + .did_abort = false, + }; +} + +static inline void query_analysis__delete(QueryAnalysis *self) { + analysis_state_set__delete(&self->states); + analysis_state_set__delete(&self->next_states); + analysis_state_set__delete(&self->deeper_states); + analysis_state_set__delete(&self->state_pool); + array_delete(&self->final_step_indices); + array_delete(&self->finished_parent_symbols); +} + /*********************** * AnalysisSubgraphNode ***********************/ @@ -1113,7 +1145,322 @@ array_insert(&self->pattern_map, index, new_entry); } +// Walk the subgraph for this non-terminal, tracking all of the possible +// sequences of progress within the pattern. +static void ts_query__perform_analysis( + TSQuery *self, + const AnalysisSubgraphArray *subgraphs, + QueryAnalysis *analysis +) { + unsigned recursion_depth_limit = 0; + unsigned prev_final_step_count = 0; + array_clear(&analysis->final_step_indices); + array_clear(&analysis->finished_parent_symbols); + + for (unsigned iteration = 0;; iteration++) { + if (iteration == MAX_ANALYSIS_ITERATION_COUNT) { + analysis->did_abort = true; + break; + } + + #ifdef DEBUG_ANALYZE_QUERY + printf("Iteration: %u. Final step indices:", iteration); + for (unsigned j = 0; j < analysis->final_step_indices.size; j++) { + printf(" %4u", analysis->final_step_indices.contents[j]); + } + printf("\n"); + for (unsigned j = 0; j < analysis->states.size; j++) { + AnalysisState *state = analysis->states.contents[j]; + printf(" %3u: step: %u, stack: [", j, state->step_index); + for (unsigned k = 0; k < state->depth; k++) { + printf( + " {%s, child: %u, state: %4u", + self->language->symbol_names[state->stack[k].parent_symbol], + state->stack[k].child_index, + state->stack[k].parse_state + ); + if (state->stack[k].field_id) printf(", field: %s", self->language->field_names[state->stack[k].field_id]); + if (state->stack[k].done) printf(", DONE"); + printf("}"); + } + printf(" ]\n"); + } + #endif + + // If no further progress can be made within the current recursion depth limit, then + // bump the depth limit by one, and continue to process the states the exceeded the + // limit. But only allow this if progress has been made since the last time the depth + // limit was increased. + if (analysis->states.size == 0) { + if ( + analysis->deeper_states.size > 0 && + analysis->final_step_indices.size > prev_final_step_count + ) { + #ifdef DEBUG_ANALYZE_QUERY + printf("Increase recursion depth limit to %u\n", recursion_depth_limit + 1); + #endif + + prev_final_step_count = analysis->final_step_indices.size; + recursion_depth_limit++; + AnalysisStateSet _states = analysis->states; + analysis->states = analysis->deeper_states; + analysis->deeper_states = _states; + continue; + } + + break; + } + + analysis_state_set__clear(&analysis->next_states, &analysis->state_pool); + for (unsigned j = 0; j < analysis->states.size; j++) { + AnalysisState * const state = analysis->states.contents[j]; + + // For efficiency, it's important to avoid processing the same analysis state more + // than once. To achieve this, keep the states in order of ascending position within + // their hypothetical syntax trees. In each iteration of this loop, start by advancing + // the states that have made the least progress. Avoid advancing states that have already + // made more progress. + if (analysis->next_states.size > 0) { + int comparison = analysis_state__compare_position( + &state, + array_back(&analysis->next_states) + ); + if (comparison == 0) { + analysis_state_set__insert_sorted(&analysis->next_states, &analysis->state_pool, state); + continue; + } else if (comparison > 0) { + #ifdef DEBUG_ANALYZE_QUERY + printf("Terminate iteration at state %u\n", j); + #endif + while (j < analysis->states.size) { + analysis_state_set__push( + &analysis->next_states, + &analysis->state_pool, + analysis->states.contents[j] + ); + j++; + } + break; + } + } + + const TSStateId parse_state = analysis_state__top(state)->parse_state; + const TSSymbol parent_symbol = analysis_state__top(state)->parent_symbol; + const TSFieldId parent_field_id = analysis_state__top(state)->field_id; + const unsigned child_index = analysis_state__top(state)->child_index; + const QueryStep * const step = &self->steps.contents[state->step_index]; + + unsigned subgraph_index, exists; + array_search_sorted_by(subgraphs, .symbol, parent_symbol, &subgraph_index, &exists); + if (!exists) continue; + const AnalysisSubgraph *subgraph = &subgraphs->contents[subgraph_index]; + + // Follow every possible path in the parse table, but only visit states that + // are part of the subgraph for the current symbol. + LookaheadIterator lookahead_iterator = ts_language_lookaheads(self->language, parse_state); + while (ts_lookahead_iterator_next(&lookahead_iterator)) { + TSSymbol sym = lookahead_iterator.symbol; + + AnalysisSubgraphNode successor = { + .state = parse_state, + .child_index = child_index, + }; + if (lookahead_iterator.action_count) { + const TSParseAction *action = &lookahead_iterator.actions[lookahead_iterator.action_count - 1]; + if (action->type == TSParseActionTypeShift) { + if (!action->shift.extra) { + successor.state = action->shift.state; + successor.child_index++; + } + } else { + continue; + } + } else if (lookahead_iterator.next_state != 0) { + successor.state = lookahead_iterator.next_state; + successor.child_index++; + } else { + continue; + } + + unsigned node_index; + array_search_sorted_with( + &subgraph->nodes, + analysis_subgraph_node__compare, &successor, + &node_index, &exists + ); + while (node_index < subgraph->nodes.size) { + AnalysisSubgraphNode *node = &subgraph->nodes.contents[node_index++]; + if (node->state != successor.state || node->child_index != successor.child_index) break; + + // Use the subgraph to determine what alias and field will eventually be applied + // to this child node. + TSSymbol alias = ts_language_alias_at(self->language, node->production_id, child_index); + TSSymbol visible_symbol = alias + ? alias + : self->language->symbol_metadata[sym].visible + ? self->language->public_symbol_map[sym] + : 0; + TSFieldId field_id = parent_field_id; + if (!field_id) { + const TSFieldMapEntry *field_map, *field_map_end; + ts_language_field_map(self->language, node->production_id, &field_map, &field_map_end); + for (; field_map != field_map_end; field_map++) { + if (!field_map->inherited && field_map->child_index == child_index) { + field_id = field_map->field_id; + break; + } + } + } + + // Create a new state that has advanced past this hypothetical subtree. + AnalysisState next_state = *state; + AnalysisStateEntry *next_state_top = analysis_state__top(&next_state); + next_state_top->child_index = successor.child_index; + next_state_top->parse_state = successor.state; + if (node->done) next_state_top->done = true; + + // Determine if this hypothetical child node would match the current step + // of the query pattern. + bool does_match = false; + if (visible_symbol) { + does_match = true; + if (step->symbol == WILDCARD_SYMBOL) { + if ( + step->is_named && + !self->language->symbol_metadata[visible_symbol].named + ) does_match = false; + } else if (step->symbol != visible_symbol) { + does_match = false; + } + if (step->field && step->field != field_id) { + does_match = false; + } + if ( + step->supertype_symbol && + !analysis_state__has_supertype(state, step->supertype_symbol) + ) does_match = false; + } + + // If this child is hidden, then descend into it and walk through its children. + // If the top entry of the stack is at the end of its rule, then that entry can + // be replaced. Otherwise, push a new entry onto the stack. + else if (sym >= self->language->token_count) { + if (!next_state_top->done) { + if (next_state.depth + 1 >= MAX_ANALYSIS_STATE_DEPTH) { + #ifdef DEBUG_ANALYZE_QUERY + printf("Exceeded depth limit for state %u\n", j); + #endif + + analysis->did_abort = true; + continue; + } + + next_state.depth++; + next_state_top = analysis_state__top(&next_state); + } + + *next_state_top = (AnalysisStateEntry) { + .parse_state = parse_state, + .parent_symbol = sym, + .child_index = 0, + .field_id = field_id, + .done = false, + }; + + if (analysis_state__recursion_depth(&next_state) > recursion_depth_limit) { + analysis_state_set__insert_sorted( + &analysis->deeper_states, + &analysis->state_pool, + &next_state + ); + continue; + } + } + + // Pop from the stack when this state reached the end of its current syntax node. + while (next_state.depth > 0 && next_state_top->done) { + next_state.depth--; + next_state_top = analysis_state__top(&next_state); + } + + // If this hypothetical child did match the current step of the query pattern, + // then advance to the next step at the current depth. This involves skipping + // over any descendant steps of the current child. + const QueryStep *next_step = step; + if (does_match) { + for (;;) { + next_state.step_index++; + next_step = &self->steps.contents[next_state.step_index]; + if ( + next_step->depth == PATTERN_DONE_MARKER || + next_step->depth <= step->depth + ) break; + } + } else if (successor.state == parse_state) { + continue; + } + + for (;;) { + // Skip pass-through states. Although these states have alternatives, they are only + // used to implement repetitions, and query analysis does not need to process + // repetitions in order to determine whether steps are possible and definite. + if (next_step->is_pass_through) { + next_state.step_index++; + next_step++; + continue; + } + + // If the pattern is finished or hypothetical parent node is complete, then + // record that matching can terminate at this step of the pattern. Otherwise, + // add this state to the list of states to process on the next iteration. + if (!next_step->is_dead_end) { + bool did_finish_pattern = self->steps.contents[next_state.step_index].depth != step->depth; + if (did_finish_pattern) { + array_insert_sorted_by(&analysis->finished_parent_symbols, , state->root_symbol); + } else if (next_state.depth == 0) { + array_insert_sorted_by(&analysis->final_step_indices, , next_state.step_index); + } else { + analysis_state_set__insert_sorted(&analysis->next_states, &analysis->state_pool, &next_state); + } + } + + // If the state has advanced to a step with an alternative step, then add another state + // at that alternative step. This process is simpler than the process of actually matching a + // pattern during query execution, because for the purposes of query analysis, there is no + // need to process repetitions. + if ( + does_match && + next_step->alternative_index != NONE && + next_step->alternative_index > next_state.step_index + ) { + next_state.step_index = next_step->alternative_index; + next_step = &self->steps.contents[next_state.step_index]; + } else { + break; + } + } + } + } + } + + AnalysisStateSet _states = analysis->states; + analysis->states = analysis->next_states; + analysis->next_states = _states; + } +} + static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { + Array(uint16_t) non_rooted_pattern_start_steps = array_new(); + for (unsigned i = 0; i < self->pattern_map.size; i++) { + PatternEntry *pattern = &self->pattern_map.contents[i]; + if (!pattern->is_rooted) { + QueryStep *step = &self->steps.contents[pattern->step_index]; + if (step->symbol != WILDCARD_SYMBOL) { + array_push(&non_rooted_pattern_start_steps, i); + } + } + } + // Walk forward through all of the steps in the query, computing some // basic information about each step. Mark all of the steps that contain // captures, and record the indices of all of the steps that have child steps. @@ -1158,7 +1505,7 @@ // of the hidden symbols in the grammar, because these might occur within // one of the parent nodes, such that their children appear to belong to the // parent. - Array(AnalysisSubgraph) subgraphs = array_new(); + AnalysisSubgraphArray subgraphs = array_new(); for (unsigned i = 0; i < parent_step_indices.size; i++) { uint32_t parent_step_index = parent_step_indices.contents[i]; TSSymbol parent_symbol = self->steps.contents[parent_step_index].symbol; @@ -1320,11 +1667,7 @@ // For each non-terminal pattern, determine if the pattern can successfully match, // and identify all of the possible children within the pattern where matching could fail. bool all_patterns_are_valid = true; - AnalysisStateSet states = array_new(); - AnalysisStateSet next_states = array_new(); - AnalysisStateSet deeper_states = array_new(); - AnalysisStatePool state_pool = array_new(); - Array(uint16_t) final_step_indices = array_new(); + QueryAnalysis analysis = query_analysis__new(); for (unsigned i = 0; i < parent_step_indices.size; i++) { uint16_t parent_step_index = parent_step_indices.contents[i]; uint16_t parent_depth = self->steps.contents[parent_step_index].depth; @@ -1348,11 +1691,11 @@ // Initialize an analysis state at every parse state in the table where // this parent symbol can occur. AnalysisSubgraph *subgraph = &subgraphs.contents[subgraph_index]; - analysis_state_set__clear(&states, &state_pool); - analysis_state_set__clear(&deeper_states, &state_pool); + analysis_state_set__clear(&analysis.states, &analysis.state_pool); + analysis_state_set__clear(&analysis.deeper_states, &analysis.state_pool); for (unsigned j = 0; j < subgraph->start_states.size; j++) { TSStateId parse_state = subgraph->start_states.contents[j]; - analysis_state_set__push_by_clone(&states, &state_pool, &((AnalysisState) { + analysis_state_set__push(&analysis.states, &analysis.state_pool, &((AnalysisState) { .step_index = parent_step_index + 1, .stack = { [0] = { @@ -1364,312 +1707,23 @@ }, }, .depth = 1, + .root_symbol = parent_symbol, })); } - // Walk the subgraph for this non-terminal, tracking all of the possible - // sequences of progress within the pattern. - bool can_finish_pattern = false; - bool did_abort_analysis = false; - unsigned recursion_depth_limit = 0; - unsigned prev_final_step_count = 0; - array_clear(&final_step_indices); - for (unsigned iteration = 0;; iteration++) { - if (iteration == MAX_ANALYSIS_ITERATION_COUNT) { - did_abort_analysis = true; - break; - } - - #ifdef DEBUG_ANALYZE_QUERY - printf("Iteration: %u. Final step indices:", iteration); - for (unsigned j = 0; j < final_step_indices.size; j++) { - printf(" %4u", final_step_indices.contents[j]); - } - printf("\nWalk states for %u %s:\n", i, ts_language_symbol_name(self->language, parent_symbol)); - for (unsigned j = 0; j < states.size; j++) { - AnalysisState *state = states.contents[j]; - printf(" %3u: step: %u, stack: [", j, state->step_index); - for (unsigned k = 0; k < state->depth; k++) { - printf( - " {%s, child: %u, state: %4u", - self->language->symbol_names[state->stack[k].parent_symbol], - state->stack[k].child_index, - state->stack[k].parse_state - ); - if (state->stack[k].field_id) printf(", field: %s", self->language->field_names[state->stack[k].field_id]); - if (state->stack[k].done) printf(", DONE"); - printf("}"); - } - printf(" ]\n"); - } - #endif - - // If no further progress can be made within the current recursion depth limit, then - // bump the depth limit by one, and continue to process the states the exceeded the - // limit. But only allow this if progress has been made since the last time the depth - // limit was increased. - if (states.size == 0) { - if ( - deeper_states.size > 0 - && final_step_indices.size > prev_final_step_count - ) { - #ifdef DEBUG_ANALYZE_QUERY - printf("Increase recursion depth limit to %u\n", recursion_depth_limit + 1); - #endif - - prev_final_step_count = final_step_indices.size; - recursion_depth_limit++; - AnalysisStateSet _states = states; - states = deeper_states; - deeper_states = _states; - continue; - } - - break; - } - - analysis_state_set__clear(&next_states, &state_pool); - for (unsigned j = 0; j < states.size; j++) { - AnalysisState * const state = states.contents[j]; - - // For efficiency, it's important to avoid processing the same analysis state more - // than once. To achieve this, keep the states in order of ascending position within - // their hypothetical syntax trees. In each iteration of this loop, start by advancing - // the states that have made the least progress. Avoid advancing states that have already - // made more progress. - if (next_states.size > 0) { - int comparison = analysis_state__compare_position( - &state, - array_back(&next_states) - ); - if (comparison == 0) { - #ifdef DEBUG_ANALYZE_QUERY - printf("Skip iteration for state %u\n", j); - #endif - analysis_state_set__insert_sorted_by_clone(&next_states, &state_pool, state); - continue; - } else if (comparison > 0) { - #ifdef DEBUG_ANALYZE_QUERY - printf("Terminate iteration at state %u\n", j); - #endif - while (j < states.size) { - analysis_state_set__push_by_clone( - &next_states, - &state_pool, - states.contents[j] - ); - j++; - } - break; - } - } - - const TSStateId parse_state = analysis_state__top(state)->parse_state; - const TSSymbol parent_symbol = analysis_state__top(state)->parent_symbol; - const TSFieldId parent_field_id = analysis_state__top(state)->field_id; - const unsigned child_index = analysis_state__top(state)->child_index; - const QueryStep * const step = &self->steps.contents[state->step_index]; - - unsigned subgraph_index, exists; - array_search_sorted_by(&subgraphs, .symbol, parent_symbol, &subgraph_index, &exists); - if (!exists) continue; - const AnalysisSubgraph *subgraph = &subgraphs.contents[subgraph_index]; - - // Follow every possible path in the parse table, but only visit states that - // are part of the subgraph for the current symbol. - LookaheadIterator lookahead_iterator = ts_language_lookaheads(self->language, parse_state); - while (ts_lookahead_iterator_next(&lookahead_iterator)) { - TSSymbol sym = lookahead_iterator.symbol; - - AnalysisSubgraphNode successor = { - .state = parse_state, - .child_index = child_index, - }; - if (lookahead_iterator.action_count) { - const TSParseAction *action = &lookahead_iterator.actions[lookahead_iterator.action_count - 1]; - if (action->type == TSParseActionTypeShift) { - if (!action->shift.extra) { - successor.state = action->shift.state; - successor.child_index++; - } - } else { - continue; - } - } else if (lookahead_iterator.next_state != 0) { - successor.state = lookahead_iterator.next_state; - successor.child_index++; - } else { - continue; - } - - unsigned node_index; - array_search_sorted_with( - &subgraph->nodes, - analysis_subgraph_node__compare, &successor, - &node_index, &exists - ); - while (node_index < subgraph->nodes.size) { - AnalysisSubgraphNode *node = &subgraph->nodes.contents[node_index++]; - if (node->state != successor.state || node->child_index != successor.child_index) break; - - // Use the subgraph to determine what alias and field will eventually be applied - // to this child node. - TSSymbol alias = ts_language_alias_at(self->language, node->production_id, child_index); - TSSymbol visible_symbol = alias - ? alias - : self->language->symbol_metadata[sym].visible - ? self->language->public_symbol_map[sym] - : 0; - TSFieldId field_id = parent_field_id; - if (!field_id) { - const TSFieldMapEntry *field_map, *field_map_end; - ts_language_field_map(self->language, node->production_id, &field_map, &field_map_end); - for (; field_map != field_map_end; field_map++) { - if (!field_map->inherited && field_map->child_index == child_index) { - field_id = field_map->field_id; - break; - } - } - } - - // Create a new state that has advanced past this hypothetical subtree. - AnalysisState next_state = *state; - AnalysisStateEntry *next_state_top = analysis_state__top(&next_state); - next_state_top->child_index = successor.child_index; - next_state_top->parse_state = successor.state; - if (node->done) next_state_top->done = true; - - // Determine if this hypothetical child node would match the current step - // of the query pattern. - bool does_match = false; - if (visible_symbol) { - does_match = true; - if (step->symbol == WILDCARD_SYMBOL) { - if ( - step->is_named && - !self->language->symbol_metadata[visible_symbol].named - ) does_match = false; - } else if (step->symbol != visible_symbol) { - does_match = false; - } - if (step->field && step->field != field_id) { - does_match = false; - } - if ( - step->supertype_symbol && - !analysis_state__has_supertype(state, step->supertype_symbol) - ) does_match = false; - } - - // If this child is hidden, then descend into it and walk through its children. - // If the top entry of the stack is at the end of its rule, then that entry can - // be replaced. Otherwise, push a new entry onto the stack. - else if (sym >= self->language->token_count) { - if (!next_state_top->done) { - if (next_state.depth + 1 >= MAX_ANALYSIS_STATE_DEPTH) { - #ifdef DEBUG_ANALYZE_QUERY - printf("Exceeded depth limit for state %u\n", j); - #endif - - did_abort_analysis = true; - continue; - } - - next_state.depth++; - next_state_top = analysis_state__top(&next_state); - } - - *next_state_top = (AnalysisStateEntry) { - .parse_state = parse_state, - .parent_symbol = sym, - .child_index = 0, - .field_id = field_id, - .done = false, - }; - - if (analysis_state__recursion_depth(&next_state) > recursion_depth_limit) { - analysis_state_set__insert_sorted_by_clone( - &deeper_states, - &state_pool, - &next_state - ); - continue; - } - } - - // Pop from the stack when this state reached the end of its current syntax node. - while (next_state.depth > 0 && next_state_top->done) { - next_state.depth--; - next_state_top = analysis_state__top(&next_state); - } - - // If this hypothetical child did match the current step of the query pattern, - // then advance to the next step at the current depth. This involves skipping - // over any descendant steps of the current child. - const QueryStep *next_step = step; - if (does_match) { - for (;;) { - next_state.step_index++; - next_step = &self->steps.contents[next_state.step_index]; - if ( - next_step->depth == PATTERN_DONE_MARKER || - next_step->depth <= parent_depth + 1 - ) break; - } - } else if (successor.state == parse_state) { - continue; - } - - for (;;) { - // Skip pass-through states. Although these states have alternatives, they are only - // used to implement repetitions, and query analysis does not need to process - // repetitions in order to determine whether steps are possible and definite. - if (next_step->is_pass_through) { - next_state.step_index++; - next_step++; - continue; - } - - // If the pattern is finished or hypothetical parent node is complete, then - // record that matching can terminate at this step of the pattern. Otherwise, - // add this state to the list of states to process on the next iteration. - if (!next_step->is_dead_end) { - bool did_finish_pattern = self->steps.contents[next_state.step_index].depth != parent_depth + 1; - if (did_finish_pattern) can_finish_pattern = true; - if (did_finish_pattern || next_state.depth == 0) { - array_insert_sorted_by(&final_step_indices, , next_state.step_index); - } else { - analysis_state_set__insert_sorted_by_clone(&next_states, &state_pool, &next_state); - } - } - - // If the state has advanced to a step with an alternative step, then add another state - // at that alternative step. This process is simpler than the process of actually matching a - // pattern during query execution, because for the purposes of query analysis, there is no - // need to process repetitions. - if ( - does_match && - next_step->alternative_index != NONE && - next_step->alternative_index > next_state.step_index - ) { - next_state.step_index = next_step->alternative_index; - next_step = &self->steps.contents[next_state.step_index]; - } else { - break; - } - } - } - } - } + #ifdef DEBUG_ANALYZE_QUERY + printf( + "\nWalk states for %s:\n", + ts_language_symbol_name(self->language, analysis.states.contents[0]->stack[0].parent_symbol) + ); + #endif - AnalysisStateSet _states = states; - states = next_states; - next_states = _states; - } + analysis.did_abort = false; + ts_query__perform_analysis(self, &subgraphs, &analysis); // If this pattern could not be fully analyzed, then every step should // be considered fallible. - if (did_abort_analysis) { + if (analysis.did_abort) { for (unsigned j = parent_step_index + 1; j < self->steps.size; j++) { QueryStep *step = &self->steps.contents[j]; if ( @@ -1686,9 +1740,9 @@ // If this pattern cannot match, store the pattern index so that it can be // returned to the caller. - if (!can_finish_pattern) { - assert(final_step_indices.size > 0); - uint16_t impossible_step_index = *array_back(&final_step_indices); + if (analysis.finished_parent_symbols.size == 0) { + assert(analysis.final_step_indices.size > 0); + uint16_t impossible_step_index = *array_back(&analysis.final_step_indices); uint32_t i, exists; array_search_sorted_by(&self->step_offsets, .step_index, impossible_step_index, &i, &exists); if (i >= self->step_offsets.size) i = self->step_offsets.size - 1; @@ -1699,8 +1753,8 @@ // Mark as fallible any step where a match terminated. // Later, this property will be propagated to all of the step's predecessors. - for (unsigned j = 0; j < final_step_indices.size; j++) { - uint32_t final_step_index = final_step_indices.contents[j]; + for (unsigned j = 0; j < analysis.final_step_indices.size; j++) { + uint32_t final_step_index = analysis.final_step_indices.contents[j]; QueryStep *step = &self->steps.contents[final_step_index]; if ( step->depth != PATTERN_DONE_MARKER && @@ -1810,21 +1864,81 @@ } #endif + // Determine which repetition symbols in this language have the possibility + // of matching non-rooted patterns in this query. These repetition symbols + // prevent certain optimizations with range restrictions. + analysis.did_abort = false; + for (uint32_t i = 0; i < non_rooted_pattern_start_steps.size; i++) { + uint16_t pattern_entry_index = non_rooted_pattern_start_steps.contents[i]; + PatternEntry *pattern_entry = &self->pattern_map.contents[pattern_entry_index]; + + analysis_state_set__clear(&analysis.states, &analysis.state_pool); + analysis_state_set__clear(&analysis.deeper_states, &analysis.state_pool); + for (unsigned j = 0; j < subgraphs.size; j++) { + AnalysisSubgraph *subgraph = &subgraphs.contents[j]; + TSSymbolMetadata metadata = ts_language_symbol_metadata(self->language, subgraph->symbol); + if (metadata.visible || metadata.named) continue; + + for (uint32_t k = 0; k < subgraph->start_states.size; k++) { + TSStateId parse_state = subgraph->start_states.contents[k]; + analysis_state_set__push(&analysis.states, &analysis.state_pool, &((AnalysisState) { + .step_index = pattern_entry->step_index, + .stack = { + [0] = { + .parse_state = parse_state, + .parent_symbol = subgraph->symbol, + .child_index = 0, + .field_id = 0, + .done = false, + }, + }, + .root_symbol = subgraph->symbol, + .depth = 1, + })); + } + } + + #ifdef DEBUG_ANALYZE_QUERY + printf("\nWalk states for rootless pattern step %u:\n", step_index); + #endif + + ts_query__perform_analysis( + self, + &subgraphs, + &analysis + ); + + if (analysis.finished_parent_symbols.size > 0) { + self->patterns.contents[pattern_entry->pattern_index].is_non_local = true; + } + + for (unsigned k = 0; k < analysis.finished_parent_symbols.size; k++) { + TSSymbol symbol = analysis.finished_parent_symbols.contents[k]; + array_insert_sorted_by(&self->repeat_symbols_with_rootless_patterns, , symbol); + } + } + + #ifdef DEBUG_ANALYZE_QUERY + if (self->repeat_symbols_with_rootless_patterns.size > 0) { + printf("\nRepetition symbols with rootless patterns:\n"); + printf("aborted analysis: %d\n", analysis.did_abort); + for (unsigned i = 0; i < self->repeat_symbols_with_rootless_patterns.size; i++) { + TSSymbol symbol = self->repeat_symbols_with_rootless_patterns.contents[i]; + printf(" %u, %s\n", symbol, ts_language_symbol_name(self->language, symbol)); + } + printf("\n"); + } + #endif + // Cleanup for (unsigned i = 0; i < subgraphs.size; i++) { array_delete(&subgraphs.contents[i].start_states); array_delete(&subgraphs.contents[i].nodes); } array_delete(&subgraphs); - for (unsigned i = 0; i < state_pool.size; i++) { - ts_free(state_pool.contents[i]); - } - array_delete(&state_pool); + query_analysis__delete(&analysis); array_delete(&next_nodes); - analysis_state_set__delete(&states); - analysis_state_set__delete(&next_states); - analysis_state_set__delete(&deeper_states); - array_delete(&final_step_indices); + array_delete(&non_rooted_pattern_start_steps); array_delete(&parent_step_indices); array_delete(&predicate_capture_ids); state_predecessor_map_delete(&predecessor_map); @@ -1918,11 +2032,11 @@ prev_position = stream->input + stream->next_size; } else { if (stream->next == '\\') { - array_extend(&self->string_buffer, (stream->input - prev_position), prev_position); + array_extend(&self->string_buffer, (uint32_t)(stream->input - prev_position), prev_position); prev_position = stream->input + 1; is_escaped = true; } else if (stream->next == '"') { - array_extend(&self->string_buffer, (stream->input - prev_position), prev_position); + array_extend(&self->string_buffer, (uint32_t)(stream->input - prev_position), prev_position); stream_advance(stream); return TSQueryErrorNone; } else if (stream->next == '\n') { @@ -2571,6 +2685,7 @@ .step_offsets = array_new(), .string_buffer = array_new(), .negated_fields = array_new(), + .repeat_symbols_with_rootless_patterns = array_new(), .wildcard_root_pattern_count = 0, .language = language, }; @@ -2588,6 +2703,7 @@ .steps = (Slice) {.offset = start_step_index}, .predicate_steps = (Slice) {.offset = start_predicate_step_index}, .start_byte = stream_offset(&stream), + .is_non_local = false, })); CaptureQuantifiers capture_quantifiers = capture_quantifiers_new(); *error_type = ts_query__parse_pattern(self, &stream, 0, false, &capture_quantifiers); @@ -2685,6 +2801,7 @@ array_delete(&self->step_offsets); array_delete(&self->string_buffer); array_delete(&self->negated_fields); + array_delete(&self->repeat_symbols_with_rootless_patterns); symbol_table_delete(&self->captures); symbol_table_delete(&self->predicate_values); for (uint32_t index = 0; index < self->capture_quantifiers.size; index++) { @@ -2766,6 +2883,17 @@ return true; } +bool ts_query_is_pattern_non_local( + const TSQuery *self, + uint32_t pattern_index +) { + if (pattern_index < self->patterns.size) { + return self->patterns.contents[pattern_index].is_non_local; + } else { + return false; + } +} + bool ts_query_is_pattern_guaranteed_at_step( const TSQuery *self, uint32_t byte_offset @@ -2880,6 +3008,7 @@ array_clear(&self->finished_states); ts_tree_cursor_reset(&self->cursor, node); capture_list_pool_reset(&self->capture_list_pool); + self->on_visible_node = true; self->next_state_id = 0; self->depth = 0; self->ascending = false; @@ -3214,6 +3343,50 @@ return &self->states.contents[state_index + 1]; } +static inline bool ts_query_cursor__should_descend_outside_of_range( + TSQueryCursor *self +) { + // If there are in-progress matches whose remaining steps occur + // deeper in the tree, then descend. + for (unsigned i = 0; i < self->states.size; i++) { + QueryState *state = &self->states.contents[i];; + QueryStep *next_step = &self->query->steps.contents[state->step_index]; + if ( + next_step->depth != PATTERN_DONE_MARKER && + state->start_depth + next_step->depth > self->depth + ) { + return true; + } + } + + // If the current node is hidden, then a non-rooted pattern might match + // one if its roots inside of this node, and match another of its roots + // as part of a sibling node, so we may need to descend. + if (!self->on_visible_node) { + // Descending into a repetition node outside of the range can be + // expensive, because these nodes can have many visible children. + // Avoid descending into repetition nodes unless we have already + // determined that this query can match rootless patterns inside + // of this type of repetition node. + Subtree subtree = ts_tree_cursor_current_subtree(&self->cursor); + if (ts_subtree_is_repetition(subtree)) { + bool exists; + uint32_t index; + array_search_sorted_by( + &self->query->repeat_symbols_with_rootless_patterns,, + ts_subtree_symbol(subtree), + &index, + &exists + ); + return exists; + } + + return true; + } + + return false; +} + // Walk the tree, processing patterns until at least one pattern finishes, // If one or more patterns finish, return `true` and store their states in the // `finished_states` array. Multiple patterns can finish on the same node. If @@ -3238,61 +3411,80 @@ // Exit the current node. if (self->ascending) { - LOG( - "leave node. depth:%u, type:%s\n", - self->depth, - ts_node_type(ts_tree_cursor_current_node(&self->cursor)) - ); + if (self->on_visible_node) { + LOG( + "leave node. depth:%u, type:%s\n", + self->depth, + ts_node_type(ts_tree_cursor_current_node(&self->cursor)) + ); + } // Leave this node by stepping to its next sibling or to its parent. - if (ts_tree_cursor_goto_next_sibling(&self->cursor)) { - self->ascending = false; - } else if (ts_tree_cursor_goto_parent(&self->cursor)) { - self->depth--; - } else { - LOG("halt at root\n"); - self->halted = true; + switch (ts_tree_cursor_goto_next_sibling_internal(&self->cursor)) { + case TreeCursorStepVisible: + if (!self->on_visible_node) { + self->depth++; + self->on_visible_node = true; + } + self->ascending = false; + break; + case TreeCursorStepHidden: + if (self->on_visible_node) { + self->depth--; + self->on_visible_node = false; + } + self->ascending = false; + break; + default: + if (ts_tree_cursor_goto_parent(&self->cursor)) { + self->depth--; + } else { + LOG("halt at root\n"); + self->halted = true; + } } - // After leaving a node, remove any states that cannot make further progress. - uint32_t deleted_count = 0; - for (unsigned i = 0, n = self->states.size; i < n; i++) { - QueryState *state = &self->states.contents[i]; - QueryStep *step = &self->query->steps.contents[state->step_index]; - - // If a state completed its pattern inside of this node, but was deferred from finishing - // in order to search for longer matches, mark it as finished. - if (step->depth == PATTERN_DONE_MARKER) { - if (state->start_depth > self->depth || self->halted) { - LOG(" finish pattern %u\n", state->pattern_index); - array_push(&self->finished_states, *state); - did_match = true; + if (self->on_visible_node) { + // After leaving a node, remove any states that cannot make further progress. + uint32_t deleted_count = 0; + for (unsigned i = 0, n = self->states.size; i < n; i++) { + QueryState *state = &self->states.contents[i]; + QueryStep *step = &self->query->steps.contents[state->step_index]; + + // If a state completed its pattern inside of this node, but was deferred from finishing + // in order to search for longer matches, mark it as finished. + if (step->depth == PATTERN_DONE_MARKER) { + if (state->start_depth > self->depth || self->halted) { + LOG(" finish pattern %u\n", state->pattern_index); + array_push(&self->finished_states, *state); + did_match = true; + deleted_count++; + continue; + } + } + + // If a state needed to match something within this node, then remove that state + // as it has failed to match. + else if ((uint32_t)state->start_depth + (uint32_t)step->depth > self->depth) { + LOG( + " failed to match. pattern:%u, step:%u\n", + state->pattern_index, + state->step_index + ); + capture_list_pool_release( + &self->capture_list_pool, + state->capture_list_id + ); deleted_count++; continue; } - } - // If a state needed to match something within this node, then remove that state - // as it has failed to match. - else if ((uint32_t)state->start_depth + (uint32_t)step->depth > self->depth) { - LOG( - " failed to match. pattern:%u, step:%u\n", - state->pattern_index, - state->step_index - ); - capture_list_pool_release( - &self->capture_list_pool, - state->capture_list_id - ); - deleted_count++; - continue; - } - - if (deleted_count > 0) { - self->states.contents[i - deleted_count] = *state; + if (deleted_count > 0) { + self->states.contents[i - deleted_count] = *state; + } } + self->states.size -= deleted_count; } - self->states.size -= deleted_count; } // Enter a new node. @@ -3300,413 +3492,410 @@ // Get the properties of the current node. TSNode node = ts_tree_cursor_current_node(&self->cursor); TSNode parent_node = ts_tree_cursor_parent_node(&self->cursor); - TSSymbol symbol = ts_node_symbol(node); - bool is_named = ts_node_is_named(node); - bool has_later_siblings; - bool has_later_named_siblings; - bool can_have_later_siblings_with_this_field; - TSFieldId field_id = 0; - TSSymbol supertypes[8] = {0}; - unsigned supertype_count = 8; - ts_tree_cursor_current_status( - &self->cursor, - &field_id, - &has_later_siblings, - &has_later_named_siblings, - &can_have_later_siblings_with_this_field, - supertypes, - &supertype_count + bool parent_precedes_range = !ts_node_is_null(parent_node) && ( + ts_node_end_byte(parent_node) <= self->start_byte || + point_lte(ts_node_end_point(parent_node), self->start_point) ); - LOG( - "enter node. depth:%u, type:%s, field:%s, row:%u state_count:%u, finished_state_count:%u\n", - self->depth, - ts_node_type(node), - ts_language_field_name_for_id(self->query->language, field_id), - ts_node_start_point(node).row, - self->states.size, - self->finished_states.size + bool parent_follows_range = !ts_node_is_null(parent_node) && ( + ts_node_start_byte(parent_node) >= self->end_byte || + point_gte(ts_node_start_point(parent_node), self->end_point) ); - - bool node_intersects_range = ( - ts_node_end_byte(node) > self->start_byte && - ts_node_start_byte(node) < self->end_byte && - point_gt(ts_node_end_point(node), self->start_point) && - point_lt(ts_node_start_point(node), self->end_point) + bool node_precedes_range = parent_precedes_range || ( + ts_node_end_byte(node) <= self->start_byte || + point_lte(ts_node_end_point(node), self->start_point) ); - bool parent_intersects_range = ts_node_is_null(parent_node) || ( - ts_node_end_byte(parent_node) > self->start_byte && - ts_node_start_byte(parent_node) < self->end_byte && - point_gt(ts_node_end_point(parent_node), self->start_point) && - point_lt(ts_node_start_point(parent_node), self->end_point) + bool node_follows_range = parent_follows_range || ( + ts_node_start_byte(node) >= self->end_byte || + point_gte(ts_node_start_point(node), self->end_point) ); - bool node_is_error = symbol == ts_builtin_sym_error; - bool parent_is_error = - !ts_node_is_null(parent_node) && - ts_node_symbol(parent_node) == ts_builtin_sym_error; - - // Add new states for any patterns whose root node is a wildcard. - if (!node_is_error) { - for (unsigned i = 0; i < self->query->wildcard_root_pattern_count; i++) { - PatternEntry *pattern = &self->query->pattern_map.contents[i]; - - // If this node matches the first step of the pattern, then add a new - // state at the start of this pattern. - QueryStep *step = &self->query->steps.contents[pattern->step_index]; - if ( - (pattern->is_rooted ? - node_intersects_range : - (parent_intersects_range && !parent_is_error)) && - (!step->field || field_id == step->field) && - (!step->supertype_symbol || supertype_count > 0) - ) { - ts_query_cursor__add_state(self, pattern); - } - } - } + bool parent_intersects_range = !parent_precedes_range && !parent_follows_range; + bool node_intersects_range = !node_precedes_range && !node_follows_range; - // Add new states for any patterns whose root node matches this node. - unsigned i; - if (ts_query__pattern_map_search(self->query, symbol, &i)) { - PatternEntry *pattern = &self->query->pattern_map.contents[i]; - - QueryStep *step = &self->query->steps.contents[pattern->step_index]; - do { - // If this node matches the first step of the pattern, then add a new - // state at the start of this pattern. - if ( - (pattern->is_rooted ? - node_intersects_range : - (parent_intersects_range && !parent_is_error)) && - (!step->field || field_id == step->field) - ) { - ts_query_cursor__add_state(self, pattern); - } + if (self->on_visible_node) { + TSSymbol symbol = ts_node_symbol(node); + bool is_named = ts_node_is_named(node); + bool has_later_siblings; + bool has_later_named_siblings; + bool can_have_later_siblings_with_this_field; + TSFieldId field_id = 0; + TSSymbol supertypes[8] = {0}; + unsigned supertype_count = 8; + ts_tree_cursor_current_status( + &self->cursor, + &field_id, + &has_later_siblings, + &has_later_named_siblings, + &can_have_later_siblings_with_this_field, + supertypes, + &supertype_count + ); + LOG( + "enter node. depth:%u, type:%s, field:%s, row:%u state_count:%u, finished_state_count:%u\n", + self->depth, + ts_node_type(node), + ts_language_field_name_for_id(self->query->language, field_id), + ts_node_start_point(node).row, + self->states.size, + self->finished_states.size + ); - // Advance to the next pattern whose root node matches this node. - i++; - if (i == self->query->pattern_map.size) break; - pattern = &self->query->pattern_map.contents[i]; - step = &self->query->steps.contents[pattern->step_index]; - } while (step->symbol == symbol); - } - - // Update all of the in-progress states with current node. - for (unsigned i = 0, copy_count = 0; i < self->states.size; i += 1 + copy_count) { - QueryState *state = &self->states.contents[i]; - QueryStep *step = &self->query->steps.contents[state->step_index]; - state->has_in_progress_alternatives = false; - copy_count = 0; - - // Check that the node matches all of the criteria for the next - // step of the pattern. - if ((uint32_t)state->start_depth + (uint32_t)step->depth != self->depth) continue; - - // Determine if this node matches this step of the pattern, and also - // if this node can have later siblings that match this step of the - // pattern. - bool node_does_match = false; - if (step->symbol == WILDCARD_SYMBOL) { - node_does_match = !node_is_error && (is_named || !step->is_named); - } else { - node_does_match = symbol == step->symbol; - } - bool later_sibling_can_match = has_later_siblings; - if ((step->is_immediate && is_named) || state->seeking_immediate_match) { - later_sibling_can_match = false; - } - if (step->is_last_child && has_later_named_siblings) { - node_does_match = false; - } - if (step->supertype_symbol) { - bool has_supertype = false; - for (unsigned j = 0; j < supertype_count; j++) { - if (supertypes[j] == step->supertype_symbol) { - has_supertype = true; - break; + bool node_is_error = symbol == ts_builtin_sym_error; + bool parent_is_error = + !ts_node_is_null(parent_node) && + ts_node_symbol(parent_node) == ts_builtin_sym_error; + + // Add new states for any patterns whose root node is a wildcard. + if (!node_is_error) { + for (unsigned i = 0; i < self->query->wildcard_root_pattern_count; i++) { + PatternEntry *pattern = &self->query->pattern_map.contents[i]; + + // If this node matches the first step of the pattern, then add a new + // state at the start of this pattern. + QueryStep *step = &self->query->steps.contents[pattern->step_index]; + if ( + (pattern->is_rooted ? + node_intersects_range : + (parent_intersects_range && !parent_is_error)) && + (!step->field || field_id == step->field) && + (!step->supertype_symbol || supertype_count > 0) + ) { + ts_query_cursor__add_state(self, pattern); } } - if (!has_supertype) node_does_match = false; } - if (step->field) { - if (step->field == field_id) { - if (!can_have_later_siblings_with_this_field) { - later_sibling_can_match = false; - } + + // Add new states for any patterns whose root node matches this node. + unsigned i; + if (ts_query__pattern_map_search(self->query, symbol, &i)) { + PatternEntry *pattern = &self->query->pattern_map.contents[i]; + + QueryStep *step = &self->query->steps.contents[pattern->step_index]; + do { + // If this node matches the first step of the pattern, then add a new + // state at the start of this pattern. + if ( + (pattern->is_rooted ? + node_intersects_range : + (parent_intersects_range && !parent_is_error)) && + (!step->field || field_id == step->field) + ) { + ts_query_cursor__add_state(self, pattern); + } + + // Advance to the next pattern whose root node matches this node. + i++; + if (i == self->query->pattern_map.size) break; + pattern = &self->query->pattern_map.contents[i]; + step = &self->query->steps.contents[pattern->step_index]; + } while (step->symbol == symbol); + } + + // Update all of the in-progress states with current node. + for (unsigned i = 0, copy_count = 0; i < self->states.size; i += 1 + copy_count) { + QueryState *state = &self->states.contents[i]; + QueryStep *step = &self->query->steps.contents[state->step_index]; + state->has_in_progress_alternatives = false; + copy_count = 0; + + // Check that the node matches all of the criteria for the next + // step of the pattern. + if ((uint32_t)state->start_depth + (uint32_t)step->depth != self->depth) continue; + + // Determine if this node matches this step of the pattern, and also + // if this node can have later siblings that match this step of the + // pattern. + bool node_does_match = false; + if (step->symbol == WILDCARD_SYMBOL) { + node_does_match = !node_is_error && (is_named || !step->is_named); } else { + node_does_match = symbol == step->symbol; + } + bool later_sibling_can_match = has_later_siblings; + if ((step->is_immediate && is_named) || state->seeking_immediate_match) { + later_sibling_can_match = false; + } + if (step->is_last_child && has_later_named_siblings) { node_does_match = false; } - } - - if (step->negated_field_list_id) { - TSFieldId *negated_field_ids = &self->query->negated_fields.contents[step->negated_field_list_id]; - for (;;) { - TSFieldId negated_field_id = *negated_field_ids; - if (negated_field_id) { - negated_field_ids++; - if (ts_node_child_by_field_id(node, negated_field_id).id) { - node_does_match = false; + if (step->supertype_symbol) { + bool has_supertype = false; + for (unsigned j = 0; j < supertype_count; j++) { + if (supertypes[j] == step->supertype_symbol) { + has_supertype = true; break; } + } + if (!has_supertype) node_does_match = false; + } + if (step->field) { + if (step->field == field_id) { + if (!can_have_later_siblings_with_this_field) { + later_sibling_can_match = false; + } } else { - break; + node_does_match = false; } } - } - // Remove states immediately if it is ever clear that they cannot match. - if (!node_does_match) { - if (!later_sibling_can_match) { - LOG( - " discard state. pattern:%u, step:%u\n", - state->pattern_index, - state->step_index - ); - capture_list_pool_release( - &self->capture_list_pool, - state->capture_list_id - ); - array_erase(&self->states, i); - i--; + if (step->negated_field_list_id) { + TSFieldId *negated_field_ids = &self->query->negated_fields.contents[step->negated_field_list_id]; + for (;;) { + TSFieldId negated_field_id = *negated_field_ids; + if (negated_field_id) { + negated_field_ids++; + if (ts_node_child_by_field_id(node, negated_field_id).id) { + node_does_match = false; + break; + } + } else { + break; + } + } } - continue; - } - // Some patterns can match their root node in multiple ways, capturing different - // children. If this pattern step could match later children within the same - // parent, then this query state cannot simply be updated in place. It must be - // split into two states: one that matches this node, and one which skips over - // this node, to preserve the possibility of matching later siblings. - if (later_sibling_can_match && ( - step->contains_captures || - ts_query__step_is_fallible(self->query, state->step_index) - )) { - if (ts_query_cursor__copy_state(self, &state)) { - LOG( - " split state for capture. pattern:%u, step:%u\n", - state->pattern_index, - state->step_index - ); - copy_count++; + // Remove states immediately if it is ever clear that they cannot match. + if (!node_does_match) { + if (!later_sibling_can_match) { + LOG( + " discard state. pattern:%u, step:%u\n", + state->pattern_index, + state->step_index + ); + capture_list_pool_release( + &self->capture_list_pool, + state->capture_list_id + ); + array_erase(&self->states, i); + i--; + } + continue; } - } - // If this pattern started with a wildcard, such that the pattern map - // actually points to the *second* step of the pattern, then check - // that the node has a parent, and capture the parent node if necessary. - if (state->needs_parent) { - TSNode parent = ts_tree_cursor_parent_node(&self->cursor); - if (ts_node_is_null(parent)) { - LOG(" missing parent node\n"); - state->dead = true; - } else { - state->needs_parent = false; - QueryStep *skipped_wildcard_step = step; - do { - skipped_wildcard_step--; - } while ( - skipped_wildcard_step->is_dead_end || - skipped_wildcard_step->is_pass_through || - skipped_wildcard_step->depth > 0 - ); - if (skipped_wildcard_step->capture_ids[0] != NONE) { - LOG(" capture wildcard parent\n"); - ts_query_cursor__capture( - self, - state, - skipped_wildcard_step, - parent + // Some patterns can match their root node in multiple ways, capturing different + // children. If this pattern step could match later children within the same + // parent, then this query state cannot simply be updated in place. It must be + // split into two states: one that matches this node, and one which skips over + // this node, to preserve the possibility of matching later siblings. + if (later_sibling_can_match && ( + step->contains_captures || + ts_query__step_is_fallible(self->query, state->step_index) + )) { + if (ts_query_cursor__copy_state(self, &state)) { + LOG( + " split state for capture. pattern:%u, step:%u\n", + state->pattern_index, + state->step_index ); + copy_count++; } } - } - // If the current node is captured in this pattern, add it to the capture list. - if (step->capture_ids[0] != NONE) { - ts_query_cursor__capture(self, state, step, node); - } + // If this pattern started with a wildcard, such that the pattern map + // actually points to the *second* step of the pattern, then check + // that the node has a parent, and capture the parent node if necessary. + if (state->needs_parent) { + TSNode parent = ts_tree_cursor_parent_node(&self->cursor); + if (ts_node_is_null(parent)) { + LOG(" missing parent node\n"); + state->dead = true; + } else { + state->needs_parent = false; + QueryStep *skipped_wildcard_step = step; + do { + skipped_wildcard_step--; + } while ( + skipped_wildcard_step->is_dead_end || + skipped_wildcard_step->is_pass_through || + skipped_wildcard_step->depth > 0 + ); + if (skipped_wildcard_step->capture_ids[0] != NONE) { + LOG(" capture wildcard parent\n"); + ts_query_cursor__capture( + self, + state, + skipped_wildcard_step, + parent + ); + } + } + } - if (state->dead) { - array_erase(&self->states, i); - i--; - continue; - } + // If the current node is captured in this pattern, add it to the capture list. + if (step->capture_ids[0] != NONE) { + ts_query_cursor__capture(self, state, step, node); + } - // Advance this state to the next step of its pattern. - state->step_index++; - state->seeking_immediate_match = false; - LOG( - " advance state. pattern:%u, step:%u\n", - state->pattern_index, - state->step_index - ); + if (state->dead) { + array_erase(&self->states, i); + i--; + continue; + } - QueryStep *next_step = &self->query->steps.contents[state->step_index]; - if (stop_on_definite_step && next_step->root_pattern_guaranteed) did_match = true; + // Advance this state to the next step of its pattern. + state->step_index++; + state->seeking_immediate_match = false; + LOG( + " advance state. pattern:%u, step:%u\n", + state->pattern_index, + state->step_index + ); - // If this state's next step has an alternative step, then copy the state in order - // to pursue both alternatives. The alternative step itself may have an alternative, - // so this is an interactive process. - unsigned end_index = i + 1; - for (unsigned j = i; j < end_index; j++) { - QueryState *state = &self->states.contents[j]; QueryStep *next_step = &self->query->steps.contents[state->step_index]; - if (next_step->alternative_index != NONE) { - // A "dead-end" step exists only to add a non-sequential jump into the step sequence, - // via its alternative index. When a state reaches a dead-end step, it jumps straight - // to the step's alternative. - if (next_step->is_dead_end) { - state->step_index = next_step->alternative_index; - j--; - continue; - } + if (stop_on_definite_step && next_step->root_pattern_guaranteed) did_match = true; - // A "pass-through" step exists only to add a branch into the step sequence, - // via its alternative_index. When a state reaches a pass-through step, it splits - // in order to process the alternative step, and then it advances to the next step. - if (next_step->is_pass_through) { - state->step_index++; - j--; - } + // If this state's next step has an alternative step, then copy the state in order + // to pursue both alternatives. The alternative step itself may have an alternative, + // so this is an interactive process. + unsigned end_index = i + 1; + for (unsigned j = i; j < end_index; j++) { + QueryState *state = &self->states.contents[j]; + QueryStep *next_step = &self->query->steps.contents[state->step_index]; + if (next_step->alternative_index != NONE) { + // A "dead-end" step exists only to add a non-sequential jump into the step sequence, + // via its alternative index. When a state reaches a dead-end step, it jumps straight + // to the step's alternative. + if (next_step->is_dead_end) { + state->step_index = next_step->alternative_index; + j--; + continue; + } - QueryState *copy = ts_query_cursor__copy_state(self, &state); - if (copy) { - LOG( - " split state for branch. pattern:%u, from_step:%u, to_step:%u, immediate:%d, capture_count: %u\n", - copy->pattern_index, - copy->step_index, - next_step->alternative_index, - next_step->alternative_is_immediate, - capture_list_pool_get(&self->capture_list_pool, copy->capture_list_id)->size - ); - end_index++; - copy_count++; - copy->step_index = next_step->alternative_index; - if (next_step->alternative_is_immediate) { - copy->seeking_immediate_match = true; + // A "pass-through" step exists only to add a branch into the step sequence, + // via its alternative_index. When a state reaches a pass-through step, it splits + // in order to process the alternative step, and then it advances to the next step. + if (next_step->is_pass_through) { + state->step_index++; + j--; + } + + QueryState *copy = ts_query_cursor__copy_state(self, &state); + if (copy) { + LOG( + " split state for branch. pattern:%u, from_step:%u, to_step:%u, immediate:%d, capture_count: %u\n", + copy->pattern_index, + copy->step_index, + next_step->alternative_index, + next_step->alternative_is_immediate, + capture_list_pool_get(&self->capture_list_pool, copy->capture_list_id)->size + ); + end_index++; + copy_count++; + copy->step_index = next_step->alternative_index; + if (next_step->alternative_is_immediate) { + copy->seeking_immediate_match = true; + } } } } } - } - for (unsigned i = 0; i < self->states.size; i++) { - QueryState *state = &self->states.contents[i]; - if (state->dead) { - array_erase(&self->states, i); - i--; - continue; - } + for (unsigned i = 0; i < self->states.size; i++) { + QueryState *state = &self->states.contents[i]; + if (state->dead) { + array_erase(&self->states, i); + i--; + continue; + } - // Enfore the longest-match criteria. When a query pattern contains optional or - // repeated nodes, this is necessary to avoid multiple redundant states, where - // one state has a strict subset of another state's captures. - bool did_remove = false; - for (unsigned j = i + 1; j < self->states.size; j++) { - QueryState *other_state = &self->states.contents[j]; - - // Query states are kept in ascending order of start_depth and pattern_index. - // Since the longest-match criteria is only used for deduping matches of the same - // pattern and root node, we only need to perform pairwise comparisons within a - // small slice of the states array. - if ( - other_state->start_depth != state->start_depth || - other_state->pattern_index != state->pattern_index - ) break; - - bool left_contains_right, right_contains_left; - ts_query_cursor__compare_captures( - self, - state, - other_state, - &left_contains_right, - &right_contains_left - ); - if (left_contains_right) { - if (state->step_index == other_state->step_index) { - LOG( - " drop shorter state. pattern: %u, step_index: %u\n", - state->pattern_index, - state->step_index - ); - capture_list_pool_release(&self->capture_list_pool, other_state->capture_list_id); - array_erase(&self->states, j); - j--; - continue; + // Enfore the longest-match criteria. When a query pattern contains optional or + // repeated nodes, this is necessary to avoid multiple redundant states, where + // one state has a strict subset of another state's captures. + bool did_remove = false; + for (unsigned j = i + 1; j < self->states.size; j++) { + QueryState *other_state = &self->states.contents[j]; + + // Query states are kept in ascending order of start_depth and pattern_index. + // Since the longest-match criteria is only used for deduping matches of the same + // pattern and root node, we only need to perform pairwise comparisons within a + // small slice of the states array. + if ( + other_state->start_depth != state->start_depth || + other_state->pattern_index != state->pattern_index + ) break; + + bool left_contains_right, right_contains_left; + ts_query_cursor__compare_captures( + self, + state, + other_state, + &left_contains_right, + &right_contains_left + ); + if (left_contains_right) { + if (state->step_index == other_state->step_index) { + LOG( + " drop shorter state. pattern: %u, step_index: %u\n", + state->pattern_index, + state->step_index + ); + capture_list_pool_release(&self->capture_list_pool, other_state->capture_list_id); + array_erase(&self->states, j); + j--; + continue; + } + other_state->has_in_progress_alternatives = true; } - other_state->has_in_progress_alternatives = true; - } - if (right_contains_left) { - if (state->step_index == other_state->step_index) { - LOG( - " drop shorter state. pattern: %u, step_index: %u\n", - state->pattern_index, - state->step_index - ); - capture_list_pool_release(&self->capture_list_pool, state->capture_list_id); - array_erase(&self->states, i); - i--; - did_remove = true; - break; + if (right_contains_left) { + if (state->step_index == other_state->step_index) { + LOG( + " drop shorter state. pattern: %u, step_index: %u\n", + state->pattern_index, + state->step_index + ); + capture_list_pool_release(&self->capture_list_pool, state->capture_list_id); + array_erase(&self->states, i); + i--; + did_remove = true; + break; + } + state->has_in_progress_alternatives = true; } - state->has_in_progress_alternatives = true; } - } - // If the state is at the end of its pattern, remove it from the list - // of in-progress states and add it to the list of finished states. - if (!did_remove) { - LOG( - " keep state. pattern: %u, start_depth: %u, step_index: %u, capture_count: %u\n", - state->pattern_index, - state->start_depth, - state->step_index, - capture_list_pool_get(&self->capture_list_pool, state->capture_list_id)->size - ); - QueryStep *next_step = &self->query->steps.contents[state->step_index]; - if (next_step->depth == PATTERN_DONE_MARKER) { - if (state->has_in_progress_alternatives) { - LOG(" defer finishing pattern %u\n", state->pattern_index); - } else { - LOG(" finish pattern %u\n", state->pattern_index); - array_push(&self->finished_states, *state); - array_erase(&self->states, state - self->states.contents); - did_match = true; - i--; + // If the state is at the end of its pattern, remove it from the list + // of in-progress states and add it to the list of finished states. + if (!did_remove) { + LOG( + " keep state. pattern: %u, start_depth: %u, step_index: %u, capture_count: %u\n", + state->pattern_index, + state->start_depth, + state->step_index, + capture_list_pool_get(&self->capture_list_pool, state->capture_list_id)->size + ); + QueryStep *next_step = &self->query->steps.contents[state->step_index]; + if (next_step->depth == PATTERN_DONE_MARKER) { + if (state->has_in_progress_alternatives) { + LOG(" defer finishing pattern %u\n", state->pattern_index); + } else { + LOG(" finish pattern %u\n", state->pattern_index); + array_push(&self->finished_states, *state); + array_erase(&self->states, (uint32_t)(state - self->states.contents)); + did_match = true; + i--; + } } } } } - // When the current node ends prior to the desired start offset, - // only descend for the purpose of continuing in-progress matches. - bool should_descend = node_intersects_range; - if (!should_descend) { - for (unsigned i = 0; i < self->states.size; i++) { - QueryState *state = &self->states.contents[i];; - QueryStep *next_step = &self->query->steps.contents[state->step_index]; - if ( - next_step->depth != PATTERN_DONE_MARKER && - state->start_depth + next_step->depth > self->depth - ) { - should_descend = true; + bool should_descend = + node_intersects_range || + ts_query_cursor__should_descend_outside_of_range(self); + if (should_descend) { + switch (ts_tree_cursor_goto_first_child_internal(&self->cursor)) { + case TreeCursorStepVisible: + self->depth++; + self->on_visible_node = true; + continue; + case TreeCursorStepHidden: + self->on_visible_node = false; + continue; + default: break; - } } } - if (!should_descend) { - LOG( - " not descending. node end byte: %u, start byte: %u\n", - ts_node_end_byte(node), - self->start_byte - ); - } - - if (should_descend && ts_tree_cursor_goto_first_child(&self->cursor)) { - self->depth++; - } else { - self->ascending = true; - } + self->ascending = true; } } } diff -Nru tree-sitter-0.20.7/lib/src/stack.c tree-sitter-0.20.8/lib/src/stack.c --- tree-sitter-0.20.7/lib/src/stack.c 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/lib/src/stack.c 2023-04-04 09:15:07.000000000 +0000 @@ -326,7 +326,7 @@ bool include_subtrees = false; if (goal_subtree_count >= 0) { include_subtrees = true; - array_reserve(&iterator.subtrees, ts_subtree_alloc_size(goal_subtree_count) / sizeof(Subtree)); + array_reserve(&iterator.subtrees, (uint32_t)ts_subtree_alloc_size(goal_subtree_count) / sizeof(Subtree)); } array_push(&self->iterators, iterator); diff -Nru tree-sitter-0.20.7/lib/src/subtree.h tree-sitter-0.20.8/lib/src/subtree.h --- tree-sitter-0.20.7/lib/src/subtree.h 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/lib/src/subtree.h 2023-04-04 09:15:07.000000000 +0000 @@ -114,7 +114,7 @@ Length size; uint32_t lookahead_bytes; uint32_t error_cost; - uint16_t child_count; + uint32_t child_count; TSSymbol symbol; TSStateId parse_state; @@ -291,6 +291,12 @@ return self.data.is_inline ? 0 : self.ptr->repeat_depth; } +static inline uint32_t ts_subtree_is_repetition(Subtree self) { + return self.data.is_inline + ? 0 + : !self.ptr->named && !self.ptr->visible && self.ptr->child_count != 0; +} + static inline uint32_t ts_subtree_node_count(Subtree self) { return (self.data.is_inline || self.ptr->child_count == 0) ? 1 : self.ptr->node_count; } diff -Nru tree-sitter-0.20.7/lib/src/tree.c tree-sitter-0.20.8/lib/src/tree.c --- tree-sitter-0.20.7/lib/src/tree.c 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/lib/src/tree.c 2023-04-04 09:15:07.000000000 +0000 @@ -66,17 +66,23 @@ range->end_point = POINT_MAX; } } - if (range->start_byte >= edit->old_end_byte) { - range->start_byte = edit->new_end_byte + (range->start_byte - edit->old_end_byte); - range->start_point = point_add( - edit->new_end_point, - point_sub(range->start_point, edit->old_end_point) - ); - if (range->start_byte < edit->new_end_byte) { - range->start_byte = UINT32_MAX; - range->start_point = POINT_MAX; - } + } else if (range->end_byte > edit->start_byte) { + range->end_byte = edit->start_byte; + range->end_point = edit->start_point; + } + if (range->start_byte >= edit->old_end_byte) { + range->start_byte = edit->new_end_byte + (range->start_byte - edit->old_end_byte); + range->start_point = point_add( + edit->new_end_point, + point_sub(range->start_point, edit->old_end_point) + ); + if (range->start_byte < edit->new_end_byte) { + range->start_byte = UINT32_MAX; + range->start_point = POINT_MAX; } + } else if (range->start_byte > edit->start_byte) { + range->start_byte = edit->start_byte; + range->start_point = edit->start_point; } } @@ -85,6 +91,13 @@ ts_subtree_pool_delete(&pool); } +TSRange *ts_tree_included_ranges(const TSTree *self, uint32_t *length) { + *length = self->included_range_count; + TSRange *ranges = ts_calloc(self->included_range_count, sizeof(TSRange)); + memcpy(ranges, self->included_ranges, self->included_range_count * sizeof(TSRange)); + return ranges; +} + TSRange *ts_tree_get_changed_ranges(const TSTree *self, const TSTree *other, uint32_t *count) { TreeCursor cursor1 = {NULL, array_new()}; TreeCursor cursor2 = {NULL, array_new()}; @@ -110,6 +123,21 @@ return result; } -void ts_tree_print_dot_graph(const TSTree *self, FILE *file) { +#ifdef _WIN32 + +void ts_tree_print_dot_graph(const TSTree *self, int fd) { + (void)self; + (void)fd; +} + +#else + +#include + +void ts_tree_print_dot_graph(const TSTree *self, int fd) { + FILE *file = fdopen(dup(fd), "a"); ts_subtree_print_dot_graph(self->root, self->language, file); + fclose(file); } + +#endif diff -Nru tree-sitter-0.20.7/lib/src/tree_cursor.c tree-sitter-0.20.8/lib/src/tree_cursor.c --- tree-sitter-0.20.7/lib/src/tree_cursor.c 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/lib/src/tree_cursor.c 2023-04-04 09:15:07.000000000 +0000 @@ -98,34 +98,43 @@ // TSTreeCursor - walking the tree -bool ts_tree_cursor_goto_first_child(TSTreeCursor *_self) { +TreeCursorStep ts_tree_cursor_goto_first_child_internal(TSTreeCursor *_self) { TreeCursor *self = (TreeCursor *)_self; + bool visible; + TreeCursorEntry entry; + CursorChildIterator iterator = ts_tree_cursor_iterate_children(self); + while (ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible)) { + if (visible) { + array_push(&self->stack, entry); + return TreeCursorStepVisible; + } + if (ts_subtree_visible_child_count(*entry.subtree) > 0) { + array_push(&self->stack, entry); + return TreeCursorStepHidden; + } + } + return TreeCursorStepNone; +} - bool did_descend; - do { - did_descend = false; - - bool visible; - TreeCursorEntry entry; - CursorChildIterator iterator = ts_tree_cursor_iterate_children(self); - while (ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible)) { - if (visible) { - array_push(&self->stack, entry); +bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) { + for (;;) { + switch (ts_tree_cursor_goto_first_child_internal(self)) { + case TreeCursorStepHidden: + continue; + case TreeCursorStepVisible: return true; - } - - if (ts_subtree_visible_child_count(*entry.subtree) > 0) { - array_push(&self->stack, entry); - did_descend = true; - break; - } + default: + return false; } - } while (did_descend); - + } return false; } -int64_t ts_tree_cursor_goto_first_child_for_byte(TSTreeCursor *_self, uint32_t goal_byte) { +static inline int64_t ts_tree_cursor_goto_first_child_for_byte_and_point( + TSTreeCursor *_self, + uint32_t goal_byte, + TSPoint goal_point +) { TreeCursor *self = (TreeCursor *)_self; uint32_t initial_size = self->stack.size; uint32_t visible_child_index = 0; @@ -138,16 +147,14 @@ TreeCursorEntry entry; CursorChildIterator iterator = ts_tree_cursor_iterate_children(self); while (ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible)) { - uint32_t end_byte = entry.position.bytes + ts_subtree_size(*entry.subtree).bytes; - bool at_goal = end_byte >= goal_byte; + Length entry_end = length_add(entry.position, ts_subtree_size(*entry.subtree)); + bool at_goal = entry_end.bytes >= goal_byte && point_gte(entry_end.extent, goal_point); uint32_t visible_child_count = ts_subtree_visible_child_count(*entry.subtree); - if (at_goal) { if (visible) { array_push(&self->stack, entry); return visible_child_index; } - if (visible_child_count > 0) { array_push(&self->stack, entry); did_descend = true; @@ -165,45 +172,15 @@ return -1; } -int64_t ts_tree_cursor_goto_first_child_for_point(TSTreeCursor *_self, TSPoint goal_point) { - TreeCursor *self = (TreeCursor *)_self; - uint32_t initial_size = self->stack.size; - uint32_t visible_child_index = 0; - - bool did_descend; - do { - did_descend = false; - - bool visible; - TreeCursorEntry entry; - CursorChildIterator iterator = ts_tree_cursor_iterate_children(self); - while (ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible)) { - TSPoint end_point = point_add(entry.position.extent, ts_subtree_size(*entry.subtree).extent); - bool at_goal = point_gte(end_point, goal_point); - uint32_t visible_child_count = ts_subtree_visible_child_count(*entry.subtree); - if (at_goal) { - if (visible) { - array_push(&self->stack, entry); - return visible_child_index; - } - if (visible_child_count > 0) { - array_push(&self->stack, entry); - did_descend = true; - break; - } - } else if (visible) { - visible_child_index++; - } else { - visible_child_index += visible_child_count; - } - } - } while (did_descend); +int64_t ts_tree_cursor_goto_first_child_for_byte(TSTreeCursor *self, uint32_t goal_byte) { + return ts_tree_cursor_goto_first_child_for_byte_and_point(self, goal_byte, POINT_ZERO); +} - self->stack.size = initial_size; - return -1; +int64_t ts_tree_cursor_goto_first_child_for_point(TSTreeCursor *self, TSPoint goal_point) { + return ts_tree_cursor_goto_first_child_for_byte_and_point(self, 0, goal_point); } -bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *_self) { +TreeCursorStep ts_tree_cursor_goto_next_sibling_internal(TSTreeCursor *_self) { TreeCursor *self = (TreeCursor *)_self; uint32_t initial_size = self->stack.size; @@ -221,19 +198,30 @@ while (ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible)) { if (visible) { array_push(&self->stack, entry); - return true; + return TreeCursorStepVisible; } if (ts_subtree_visible_child_count(*entry.subtree)) { array_push(&self->stack, entry); - ts_tree_cursor_goto_first_child(_self); - return true; + return TreeCursorStepHidden; } } } self->stack.size = initial_size; - return false; + return TreeCursorStepNone; +} + +bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self) { + switch (ts_tree_cursor_goto_next_sibling_internal(self)) { + case TreeCursorStepHidden: + ts_tree_cursor_goto_first_child(self); + return true; + case TreeCursorStepVisible: + return true; + default: + return false; + } } bool ts_tree_cursor_goto_parent(TSTreeCursor *_self) { diff -Nru tree-sitter-0.20.7/lib/src/tree_cursor.h tree-sitter-0.20.8/lib/src/tree_cursor.h --- tree-sitter-0.20.7/lib/src/tree_cursor.h 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/lib/src/tree_cursor.h 2023-04-04 09:15:07.000000000 +0000 @@ -15,6 +15,12 @@ Array(TreeCursorEntry) stack; } TreeCursor; +typedef enum { + TreeCursorStepNone, + TreeCursorStepHidden, + TreeCursorStepVisible, +} TreeCursorStep; + void ts_tree_cursor_init(TreeCursor *, TSNode); void ts_tree_cursor_current_status( const TSTreeCursor *, @@ -26,6 +32,15 @@ unsigned * ); +TreeCursorStep ts_tree_cursor_goto_first_child_internal(TSTreeCursor *); +TreeCursorStep ts_tree_cursor_goto_next_sibling_internal(TSTreeCursor *); + +static inline Subtree ts_tree_cursor_current_subtree(const TSTreeCursor *_self) { + const TreeCursor *self = (const TreeCursor *)_self; + TreeCursorEntry *last_entry = array_back(&self->stack); + return *last_entry->subtree; +} + TSNode ts_tree_cursor_parent_node(const TSTreeCursor *); #endif // TREE_SITTER_TREE_CURSOR_H_ diff -Nru tree-sitter-0.20.7/lib/src/tree.h tree-sitter-0.20.8/lib/src/tree.h --- tree-sitter-0.20.7/lib/src/tree.h 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/lib/src/tree.h 2023-04-04 09:15:07.000000000 +0000 @@ -1,6 +1,8 @@ #ifndef TREE_SITTER_TREE_H_ #define TREE_SITTER_TREE_H_ +#include "./subtree.h" + #ifdef __cplusplus extern "C" { #endif diff -Nru tree-sitter-0.20.7/Makefile tree-sitter-0.20.8/Makefile --- tree-sitter-0.20.7/Makefile 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/Makefile 2023-04-04 09:15:07.000000000 +0000 @@ -1,4 +1,4 @@ -VERSION := 0.6.3 +VERSION := 0.20.9 # install directory layout PREFIX ?= /usr/local diff -Nru tree-sitter-0.20.7/README.md tree-sitter-0.20.8/README.md --- tree-sitter-0.20.7/README.md 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/README.md 2023-04-04 09:15:07.000000000 +0000 @@ -1,7 +1,6 @@ # tree-sitter -[![Build Status](https://github.com/tree-sitter/tree-sitter/workflows/CI/badge.svg)](https://github.com/tree-sitter/tree-sitter/actions) -[![Build status](https://ci.appveyor.com/api/projects/status/vtmbd6i92e97l55w/branch/master?svg=true)](https://ci.appveyor.com/project/maxbrunsfeld/tree-sitter/branch/master) +[![CICD](https://github.com/tree-sitter/tree-sitter/actions/workflows/CICD.yml/badge.svg)](https://github.com/tree-sitter/tree-sitter/actions/workflows/CICD.yml) [![DOI](https://zenodo.org/badge/14164618.svg)](https://zenodo.org/badge/latestdoi/14164618) Tree-sitter is a parser generator tool and an incremental parsing library. It can build a concrete syntax tree for a source file and efficiently update the syntax tree as the source file is edited. Tree-sitter aims to be: diff -Nru tree-sitter-0.20.7/script/build-wasm tree-sitter-0.20.8/script/build-wasm --- tree-sitter-0.20.7/script/build-wasm 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/script/build-wasm 2023-04-04 09:15:07.000000000 +0000 @@ -71,12 +71,18 @@ emscripten/emsdk:$emscripen_version \ emcc" else - echo 'You must have either `docker` or `emcc` on your PATH to run this script' + if [[ "$force_docker" == "1" ]]; then + echo 'You must have `docker` on your PATH to run this script with --docker' + else + echo 'You must have either `docker` or `emcc` on your PATH to run this script' + fi exit 1 fi mkdir -p target/scratch +runtime_methods='stringToUTF16','AsciiToString' + # Use emscripten to generate `tree-sitter.js` and `tree-sitter.wasm` # in the `target/scratch` directory $emcc \ @@ -88,6 +94,7 @@ -s NODEJS_CATCH_EXIT=0 \ -s NODEJS_CATCH_REJECTION=0 \ -s EXPORTED_FUNCTIONS=@${web_dir}/exports.json \ + -s EXPORTED_RUNTIME_METHODS=$runtime_methods \ $emscripten_flags \ -fno-exceptions \ -std=c99 \ diff -Nru tree-sitter-0.20.7/script/generate-bindings tree-sitter-0.20.8/script/generate-bindings --- tree-sitter-0.20.7/script/generate-bindings 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/script/generate-bindings 2023-04-04 09:15:07.000000000 +0000 @@ -7,10 +7,7 @@ --no-layout-tests \ --whitelist-type '^TS.*' \ --whitelist-function '^ts_.*' \ - --opaque-type FILE \ - --blocklist-type FILE \ --blocklist-type '^__.*' \ - --blocklist-function ts_tree_print_dot_graph \ --size_t-is-usize \ $header_path > $output_path diff -Nru tree-sitter-0.20.7/script/generate-fixtures tree-sitter-0.20.8/script/generate-fixtures --- tree-sitter-0.20.7/script/generate-fixtures 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/script/generate-fixtures 2023-04-04 09:15:07.000000000 +0000 @@ -2,12 +2,18 @@ set -e -cargo build --release +root_dir=$PWD + +if [ "$CI" == true ]; then + set -x + tree_sitter="$TREE_SITTER" +else + cargo build --release + tree_sitter=${root_dir}/target/release/tree-sitter +fi filter_grammar_name=$1 -root_dir=$PWD -tree_sitter=${root_dir}/target/release/tree-sitter grammars_dir=${root_dir}/test/fixtures/grammars grammar_files=$(find $grammars_dir -name grammar.js | grep -v node_modules) diff -Nru tree-sitter-0.20.7/script/generate-fixtures-wasm tree-sitter-0.20.8/script/generate-fixtures-wasm --- tree-sitter-0.20.7/script/generate-fixtures-wasm 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/script/generate-fixtures-wasm 2023-04-04 09:15:07.000000000 +0000 @@ -2,7 +2,15 @@ set -e -cargo build --release +root_dir=$PWD + +if [ "$CI" == true ]; then + set -x + tree_sitter="$TREE_SITTER" +else + cargo build --release + tree_sitter=${root_dir}/target/release/tree-sitter +fi build_wasm_args= if [[ $1 == "--docker" ]]; then @@ -12,8 +20,6 @@ filter_grammar_name=$1 -root_dir=$PWD -tree_sitter=${root_dir}/target/release/tree-sitter grammars_dir=${root_dir}/test/fixtures/grammars grammar_files=$(find $grammars_dir -name grammar.js | grep -v node_modules) diff -Nru tree-sitter-0.20.7/tags/Cargo.toml tree-sitter-0.20.8/tags/Cargo.toml --- tree-sitter-0.20.7/tags/Cargo.toml 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/tags/Cargo.toml 2023-04-04 09:15:07.000000000 +0000 @@ -12,6 +12,7 @@ keywords = ["incremental", "parsing", "syntax", "tagging"] categories = ["parsing", "text-editors"] repository = "https://github.com/tree-sitter/tree-sitter" +rust-version.workspace = true [lib] crate-type = ["lib", "staticlib"] diff -Nru tree-sitter-0.20.7/test/fixtures/error_corpus/python_errors.txt tree-sitter-0.20.8/test/fixtures/error_corpus/python_errors.txt --- tree-sitter-0.20.7/test/fixtures/error_corpus/python_errors.txt 2022-09-02 22:00:47.000000000 +0000 +++ tree-sitter-0.20.8/test/fixtures/error_corpus/python_errors.txt 2023-04-04 09:15:07.000000000 +0000 @@ -89,7 +89,8 @@ parameters: (parameters) (ERROR (identifier)) body: (block - (expression_statement (string))))) + (expression_statement (string + string_content: (string_content)))))) =========================================== incomplete definition in class definition @@ -108,4 +109,4 @@ (ERROR) body: (block)) (expression_statement - (identifier))) \ No newline at end of file + (identifier))) diff -Nru tree-sitter-0.20.7/test/fixtures/template_corpus/readme.md tree-sitter-0.20.8/test/fixtures/template_corpus/readme.md --- tree-sitter-0.20.7/test/fixtures/template_corpus/readme.md 1970-01-01 00:00:00.000000000 +0000 +++ tree-sitter-0.20.8/test/fixtures/template_corpus/readme.md 2023-04-04 09:15:07.000000000 +0000 @@ -0,0 +1,6 @@ +The Template Corpus +=================== + +This directory contains corpus tests that exercise parsing a set of disjoint ranges within a file. + +Each of these input files contains source code surrounded by the delimiters `<%` and `%>`. The content outside of these delimiters is meant to be ignored. \ No newline at end of file diff -Nru tree-sitter-0.20.7/test/fixtures/template_corpus/ruby_templates.txt tree-sitter-0.20.8/test/fixtures/template_corpus/ruby_templates.txt --- tree-sitter-0.20.7/test/fixtures/template_corpus/ruby_templates.txt 1970-01-01 00:00:00.000000000 +0000 +++ tree-sitter-0.20.8/test/fixtures/template_corpus/ruby_templates.txt 2023-04-04 09:15:07.000000000 +0000 @@ -0,0 +1,78 @@ +============================== +Templates with errors +============================== + +
+ <% if notice.present? %> +

<% notice %>

+ <% end %> +
+

Foods

+
+ <% link_to 'New food', new_food_path, class: "block font-medium" %> + <% link_to 'Search Database', database_foods_search_path, class: "block font-medium" %> +
+
+ + <% . render partial: "form", locals: { food: @new_food } %> + + <% form_with url: "/search", method: :get do |form| %> + <% form.label :previous_query, 'Search previous foods:' %> + <% form.text_field :previous_query %> + <% form.submit "Search" %> + <% end %> + +
+ <% render @foods %> +
+
+ +--- + +(program + (if + (call (identifier) (identifier)) + (then (identifier))) + (call + (identifier) + (argument_list + (string (string_content)) + (identifier) + (pair (hash_key_symbol) (string (string_content))))) + (call + (identifier) + (argument_list + (string (string_content)) + (identifier) + (pair (hash_key_symbol) (string (string_content))))) + (ERROR) + (call + (identifier) + (argument_list + (pair (hash_key_symbol) (string (string_content))) + (pair (hash_key_symbol) (hash (pair (hash_key_symbol) (instance_variable)))))) + (call + (identifier) + (argument_list + (pair (hash_key_symbol) (string (string_content))) + (pair (hash_key_symbol) (simple_symbol))) + (do_block + (block_parameters + (identifier)) + (body_statement + (call + (identifier) + (identifier) + (argument_list (simple_symbol) (string (string_content)))) + (call + (identifier) + (identifier) + (argument_list + (simple_symbol))) + (call + (identifier) + (identifier) + (argument_list (string (string_content))))))) + (call + (identifier) + (argument_list (instance_variable)))) \ No newline at end of file