diff --git a/.appveyor.yml b/.appveyor.yml index 610ac134..d463b7a2 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -22,7 +22,7 @@ test_script: # Fetch and regenerate the fixture parsers - script\fetch-fixtures.cmd - cargo build --release - - script\regenerate-fixtures.cmd + - script\generate-fixtures.cmd # Run tests - script\test.cmd diff --git a/.gitignore b/.gitignore index 360390b1..ed31e54a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,10 @@ fuzz-results test/fixtures/grammars/* !test/fixtures/grammars/.gitkeep +package-lock.json +node_modules + +docs/assets/js/tree-sitter.js /target *.rs.bk @@ -15,3 +19,4 @@ test/fixtures/grammars/* *.obj *.exp *.lib +*.wasm diff --git a/.travis.yml b/.travis.yml index 06c71b34..9fa65759 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,21 +2,33 @@ language: rust rust: - stable -os: - - linux - - osx +matrix: + include: + - os: osx + - os: linux + services: docker + env: TEST_WASM=1 + +before_install: + # Install node + - if [ -n "$TEST_WASM" ]; then nvm install 10 && nvm use 10; fi script: # Fetch and regenerate the fixture parsers - script/fetch-fixtures - cargo build --release - - script/regenerate-fixtures + - script/generate-fixtures - # Run tests + # Run the tests - export TREE_SITTER_STATIC_ANALYSIS=1 - script/test - script/benchmark + # Build and test the WASM binding + - if [ -n "$TEST_WASM" ]; then script/build-wasm; fi + - if [ -n "$TEST_WASM" ]; then script/generate-fixtures-wasm; fi + - if [ -n "$TEST_WASM" ]; then script/test-wasm; fi + branches: only: - master @@ -31,7 +43,10 @@ deploy: api_key: secure: "cAd2mQP+Q55v3zedo5ZyOVc3hq3XKMW93lp5LuXV6CYKYbIhkyfym4qfs+C9GJQiIP27cnePYM7B3+OMIFwSPIgXHWWSsuloMtDgYSc/PAwb2dZnJqAyog3BohW/QiGTSnvbVlxPF6P9RMQU6+JP0HJzEJy6QBTa4Und/j0jm24=" file_glob: true - file: "tree-sitter-*.gz" + file: + - "tree-sitter-*.gz" + - "target/release/tree-sitter.js" + - "target/release/tree-sitter.wasm" draft: true overwrite: true skip_cleanup: true diff --git a/cli/npm/package-lock.json b/cli/npm/package-lock.json deleted file mode 100644 index ab7209ec..00000000 --- a/cli/npm/package-lock.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "tree-sitter-cli", - "version": "0.15.1", - "lockfileVersion": 1 -} diff --git a/cli/src/generate/mod.rs b/cli/src/generate/mod.rs index 14c13aa4..48e747c9 100644 --- a/cli/src/generate/mod.rs +++ b/cli/src/generate/mod.rs @@ -15,7 +15,7 @@ mod grammars; mod nfa; mod node_types; mod npm_files; -mod parse_grammar; +pub mod parse_grammar; mod prepare_grammar; mod render; mod rules; diff --git a/cli/src/generate/parse_grammar.rs b/cli/src/generate/parse_grammar.rs index ce4b881a..feb560a9 100644 --- a/cli/src/generate/parse_grammar.rs +++ b/cli/src/generate/parse_grammar.rs @@ -64,8 +64,8 @@ enum RuleJSON { } #[derive(Deserialize)] -struct GrammarJSON { - name: String, +pub(crate) struct GrammarJSON { + pub(crate) name: String, rules: Map, conflicts: Option>>, externals: Option>, diff --git a/cli/src/lib.rs b/cli/src/lib.rs index 19b82194..5d026cde 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -8,6 +8,7 @@ pub mod parse; pub mod properties; pub mod test; pub mod util; +pub mod wasm; #[cfg(test)] mod tests; diff --git a/cli/src/main.rs b/cli/src/main.rs index a8b4fd8b..9decd720 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -5,7 +5,7 @@ use std::path::Path; use std::process::exit; use std::{u64, usize}; use tree_sitter_cli::{ - config, error, generate, highlight, loader, logger, parse, properties, test, + config, error, generate, highlight, loader, logger, parse, properties, test, wasm, }; fn main() { @@ -90,6 +90,11 @@ fn run() -> error::Result<()> { .arg(Arg::with_name("html").long("html").short("h")) .arg(Arg::with_name("time").long("time").short("t")), ) + .subcommand( + SubCommand::with_name("build-wasm") + .about("Compile a parser to WASM") + .arg(Arg::with_name("path").index(1).multiple(true)), + ) .get_matches(); let home_dir = dirs::home_dir().expect("Failed to read home directory"); @@ -237,6 +242,9 @@ fn run() -> error::Result<()> { ))); } } + } else if let Some(matches) = matches.subcommand_matches("build-wasm") { + let grammar_path = current_dir.join(matches.value_of("path").unwrap_or("")); + wasm::compile_language_to_wasm(&grammar_path)?; } Ok(()) diff --git a/cli/src/tests/node_test.rs b/cli/src/tests/node_test.rs index ebe404cd..a5fcaa3d 100644 --- a/cli/src/tests/node_test.rs +++ b/cli/src/tests/node_test.rs @@ -265,42 +265,90 @@ fn test_node_descendant_for_range() { let tree = parse_json_example(); let array_node = tree.root_node().child(0).unwrap(); + // Leaf node starts and ends at the given bounds - byte query let colon_index = JSON_EXAMPLE.find(":").unwrap(); - let node1 = array_node - .descendant_for_byte_range(colon_index, colon_index) + let colon_node = array_node + .descendant_for_byte_range(colon_index, colon_index + 1) .unwrap(); - assert_eq!(node1.kind(), ":"); - assert_eq!(node1.start_byte(), colon_index); - assert_eq!(node1.end_byte(), colon_index + 1); - assert_eq!(node1.start_position(), Point::new(6, 7)); - assert_eq!(node1.end_position(), Point::new(6, 8)); + assert_eq!(colon_node.kind(), ":"); + assert_eq!(colon_node.start_byte(), colon_index); + assert_eq!(colon_node.end_byte(), colon_index + 1); + assert_eq!(colon_node.start_position(), Point::new(6, 7)); + assert_eq!(colon_node.end_position(), Point::new(6, 8)); + // Leaf node starts and ends at the given bounds - point query + let colon_node = array_node + .descendant_for_point_range(Point::new(6, 7), Point::new(6, 8)) + .unwrap(); + assert_eq!(colon_node.kind(), ":"); + assert_eq!(colon_node.start_byte(), colon_index); + assert_eq!(colon_node.end_byte(), colon_index + 1); + assert_eq!(colon_node.start_position(), Point::new(6, 7)); + assert_eq!(colon_node.end_position(), Point::new(6, 8)); + + // Leaf node starts at the lower bound, ends after the upper bound - byte query let string_index = JSON_EXAMPLE.find("\"x\"").unwrap(); - let node2 = array_node + let string_node = array_node + .descendant_for_byte_range(string_index, string_index + 2) + .unwrap(); + assert_eq!(string_node.kind(), "string"); + assert_eq!(string_node.start_byte(), string_index); + assert_eq!(string_node.end_byte(), string_index + 3); + assert_eq!(string_node.start_position(), Point::new(6, 4)); + assert_eq!(string_node.end_position(), Point::new(6, 7)); + + // Leaf node starts at the lower bound, ends after the upper bound - point query + let string_node = array_node + .descendant_for_point_range(Point::new(6, 4), Point::new(6, 6)) + .unwrap(); + assert_eq!(string_node.kind(), "string"); + assert_eq!(string_node.start_byte(), string_index); + assert_eq!(string_node.end_byte(), string_index + 3); + assert_eq!(string_node.start_position(), Point::new(6, 4)); + assert_eq!(string_node.end_position(), Point::new(6, 7)); + + // Leaf node starts before the lower bound, ends at the upper bound - byte query + let null_index = JSON_EXAMPLE.find("null").unwrap(); + let null_node = array_node + .descendant_for_byte_range(null_index + 1, null_index + 4) + .unwrap(); + assert_eq!(null_node.kind(), "null"); + assert_eq!(null_node.start_byte(), null_index); + assert_eq!(null_node.end_byte(), null_index + 4); + assert_eq!(null_node.start_position(), Point::new(6, 9)); + assert_eq!(null_node.end_position(), Point::new(6, 13)); + + // Leaf node starts before the lower bound, ends at the upper bound - point query + let null_node = array_node + .descendant_for_point_range(Point::new(6, 11), Point::new(6, 13)) + .unwrap(); + assert_eq!(null_node.kind(), "null"); + assert_eq!(null_node.start_byte(), null_index); + assert_eq!(null_node.end_byte(), null_index + 4); + assert_eq!(null_node.start_position(), Point::new(6, 9)); + assert_eq!(null_node.end_position(), Point::new(6, 13)); + + // The bounds span multiple leaf nodes - return the smallest node that does span it. + let pair_node = array_node .descendant_for_byte_range(string_index + 2, string_index + 4) .unwrap(); - assert_eq!(node2.kind(), "pair"); - assert_eq!(node2.start_byte(), string_index); - assert_eq!(node2.end_byte(), string_index + 9); - assert_eq!(node2.start_position(), Point::new(6, 4)); - assert_eq!(node2.end_position(), Point::new(6, 13)); + assert_eq!(pair_node.kind(), "pair"); + assert_eq!(pair_node.start_byte(), string_index); + assert_eq!(pair_node.end_byte(), string_index + 9); + assert_eq!(pair_node.start_position(), Point::new(6, 4)); + assert_eq!(pair_node.end_position(), Point::new(6, 13)); - assert_eq!(node1.parent(), Some(node2)); - - let node3 = array_node - .named_descendant_for_byte_range(string_index, string_index + 2) - .unwrap(); - assert_eq!(node3.kind(), "string"); - assert_eq!(node3.start_byte(), string_index); - assert_eq!(node3.end_byte(), string_index + 3); + assert_eq!(colon_node.parent(), Some(pair_node)); // no leaf spans the given range - return the smallest node that does span it. - let node4 = array_node - .named_descendant_for_byte_range(string_index, string_index + 3) + let pair_node = array_node + .named_descendant_for_point_range(Point::new(6, 6), Point::new(6, 8)) .unwrap(); - assert_eq!(node4.kind(), "pair"); - assert_eq!(node4.start_byte(), string_index); - assert_eq!(node4.end_byte(), string_index + 9); + assert_eq!(pair_node.kind(), "pair"); + assert_eq!(pair_node.start_byte(), string_index); + assert_eq!(pair_node.end_byte(), string_index + 9); + assert_eq!(pair_node.start_position(), Point::new(6, 4)); + assert_eq!(pair_node.end_position(), Point::new(6, 13)); } #[test] diff --git a/cli/src/wasm.rs b/cli/src/wasm.rs new file mode 100644 index 00000000..1aaa1f61 --- /dev/null +++ b/cli/src/wasm.rs @@ -0,0 +1,99 @@ +use super::error::{Error, Result}; +use super::generate::parse_grammar::GrammarJSON; +use std::ffi::{OsStr, OsString}; +use std::fs; +use std::path::Path; +use std::process::Command; + +pub fn compile_language_to_wasm(language_dir: &Path) -> Result<()> { + let src_dir = language_dir.join("src"); + + // Parse the grammar.json to find out the language name. + let grammar_json_path = src_dir.join("grammar.json"); + let grammar_json = fs::read_to_string(&grammar_json_path).map_err(|e| { + format!( + "Failed to read grammar file {:?} - {}", + grammar_json_path, e + ) + })?; + let grammar: GrammarJSON = serde_json::from_str(&grammar_json).map_err(|e| { + format!( + "Failed to parse grammar file {:?} - {}", + grammar_json_path, e + ) + })?; + let output_filename = format!("tree-sitter-{}.wasm", grammar.name); + + // Get the current user id so that files created in the docker container will have + // the same owner. + let user_id_output = Command::new("id") + .arg("-u") + .output() + .map_err(|e| format!("Failed to get get current user id {}", e))?; + let user_id = String::from_utf8_lossy(&user_id_output.stdout); + let user_id = user_id.trim(); + + // Use `emscripten-slim` docker image with the parser directory mounted as a volume. + let mut command = Command::new("docker"); + let mut volume_string = OsString::from(language_dir); + volume_string.push(":/src"); + command.args(&["run", "--rm"]); + command.args(&[OsStr::new("--volume"), &volume_string]); + command.args(&["--user", user_id, "trzeci/emscripten-slim"]); + + // Run emscripten in the container + command.args(&[ + "emcc", + "-o", + &output_filename, + "-Os", + "-s", + "WASM=1", + "-s", + "SIDE_MODULE=1", + "-s", + &format!("EXPORTED_FUNCTIONS=[\"_tree_sitter_{}\"]", grammar.name), + "-I", + "src", + ]); + + // Find source files to pass to emscripten + let src_entries = fs::read_dir(&src_dir) + .map_err(|e| format!("Failed to read source directory {:?} - {}", src_dir, e))?; + + for entry in src_entries { + let entry = entry?; + let file_name = entry.file_name(); + + // Do not compile the node.js binding file. + if file_name + .to_str() + .map_or(false, |s| s.starts_with("binding")) + { + continue; + } + + // Compile any .c, .cc, or .cpp files + if let Some(extension) = Path::new(&file_name).extension().and_then(|s| s.to_str()) { + if extension == "c" || extension == "cc" || extension == "cpp" { + command.arg(Path::new("src").join(entry.file_name())); + } + } + } + + let output = command + .output() + .map_err(|e| format!("Failed to run docker emcc command - {}", e))?; + if !output.status.success() { + return Err(Error::from(format!( + "emcc command failed - {}", + String::from_utf8_lossy(&output.stderr) + ))); + } + + // Move the created `.wasm` file into the current working directory. + fs::rename(&language_dir.join(&output_filename), &output_filename) + .map_err(|e| format!("Couldn't find output file {:?} - {}", output_filename, e))?; + + Ok(()) +} diff --git a/docs/_layouts/default.html b/docs/_layouts/default.html index d000ad37..25be9c2d 100644 --- a/docs/_layouts/default.html +++ b/docs/_layouts/default.html @@ -130,7 +130,7 @@ } }); - $('h1, h2, h3').filter('[id]').each(function() { + $('h2, h3').filter('[id]').each(function() { $(this).html('' + $(this).text() + ''); }); diff --git a/docs/assets/css/style.scss b/docs/assets/css/style.scss index 269cab34..57fe291d 100644 --- a/docs/assets/css/style.scss +++ b/docs/assets/css/style.scss @@ -116,3 +116,43 @@ body { } } } + +#playground-container { + > .CodeMirror { + height: auto; + max-height: 350px; + border: 1px solid #aaa; + } + + .CodeMirror-scroll { + height: auto; + max-height: 350px; + } + + h4, select, .field { + display: inline-block; + margin-right: 20px; + } + + #logging-checkbox { + height: 15px; + } + + .CodeMirror div.CodeMirror-cursor { + border-left: 3px solid red; + } +} + +#output-container { + position: relative; + margin-top: 0; + overflow: auto; + padding: 20px 10px; + max-height: 350px; + border: 1px solid #aaa; +} + +.tree-link.highlighted { + background-color: #ddd; + text-decoration: underline; +} diff --git a/docs/assets/js/playground.js b/docs/assets/js/playground.js new file mode 100644 index 00000000..0b7e8127 --- /dev/null +++ b/docs/assets/js/playground.js @@ -0,0 +1,189 @@ +const codeInput = document.getElementById('code-input'); +const languageSelect = document.getElementById('language-select'); +const loggingCheckbox = document.getElementById('logging-checkbox'); +const outputContainer = document.getElementById('output-container'); +const updateTimeSpan = document.getElementById('update-time'); +const demoContainer = document.getElementById('playground-container'); + +const languagesByName = {}; + +let tree; +let parser; +let codeEditor; +let queryEditor; +let languageName; +let highlightedNodeLink; + +main(); + +async function main() { + await TreeSitter.init(); + parser = new TreeSitter(); + codeEditor = CodeMirror.fromTextArea(codeInput, { + lineNumbers: true, + showCursorWhenSelecting: true + }); + + languageName = languageSelect.value; + codeEditor.on('changes', handleCodeChange); + codeEditor.on('cursorActivity', handleCursorMovement); + loggingCheckbox.addEventListener('change', handleLoggingChange); + languageSelect.addEventListener('change', handleLanguageChange); + outputContainer.addEventListener('click', handleTreeClick); + + await handleLanguageChange(); + demoContainer.style.visibility = 'visible'; +} + +async function handleLanguageChange() { + const newLanguageName = languageSelect.value; + if (!languagesByName[newLanguageName]) { + const url = `${LANGUAGE_BASE_URL}/tree-sitter-${newLanguageName}.wasm` + languageSelect.disabled = true; + try { + languagesByName[newLanguageName] = await TreeSitter.Language.load(url); + } catch (e) { + console.error(e); + languageSelect.value = languageName; + return + } finally { + languageSelect.disabled = false; + } + } + + tree = null; + languageName = newLanguageName; + parser.setLanguage(languagesByName[newLanguageName]); + handleCodeChange(); +} + +function handleLoggingChange() { + if (loggingCheckbox.checked) { + parser.setLogger(console.log); + } else { + parser.setLogger(null); + } +} + +function handleCodeChange(editor, changes) { + let start; + if (tree && changes) { + start = performance.now(); + for (const change of changes) { + const edit = treeEditForEditorChange(change); + tree.edit(edit); + } + } else { + start = performance.now(); + } + const newTree = parser.parse(codeEditor.getValue() + '\n', tree); + tree && tree.delete(); + tree = newTree; + updateTimeSpan.innerText = `${(performance.now() - start).toFixed(1)} ms`; + renderTree(); +} + +function handleTreeClick(event) { + if (event.target.className === 'tree-link') { + event.preventDefault(); + const row = parseInt(event.target.dataset.row); + const column = parseInt(event.target.dataset.column); + codeEditor.setCursor({line: row, ch: column}); + codeEditor.focus(); + } +} + +function treeEditForEditorChange(change) { + const oldLineCount = change.removed.length; + const newLineCount = change.text.length; + const lastLineLength = change.text[newLineCount - 1].length; + + const startPosition = {row: change.from.line, column: change.from.ch}; + const oldEndPosition = {row: change.to.line, column: change.to.ch}; + const newEndPosition = { + row: startPosition.row + newLineCount - 1, + column: newLineCount === 1 + ? startPosition.column + lastLineLength + : lastLineLength + }; + + const startIndex = codeEditor.indexFromPos(change.from); + let newEndIndex = startIndex + newLineCount - 1; + let oldEndIndex = startIndex + oldLineCount - 1; + for (let i = 0; i < newLineCount; i++) newEndIndex += change.text[i].length; + for (let i = 0; i < oldLineCount; i++) oldEndIndex += change.removed[i].length; + + return { + startIndex, oldEndIndex, newEndIndex, + startPosition, oldEndPosition, newEndPosition + }; +} + +var handleCursorMovement = debounce(() => { + if (highlightedNodeLink) { + highlightedNodeLink.classList.remove('highlighted'); + highlightedNodeLink = null; + } + + const selection = codeEditor.getDoc().listSelections()[0]; + let start = {row: selection.anchor.line, column: selection.anchor.ch}; + let end = {row: selection.head.line, column: selection.head.ch}; + if ( + start.row > end.row || + ( + start.row === end.row && + start.column > end.column + ) + ) { + let swap = end; + end = start; + start = swap; + } + const node = tree.rootNode.namedDescendantForPosition(start, end); + const link = document.querySelector(`.tree-link[data-id="${node[0]}"]`); + link.classList.add('highlighted'); + highlightedNodeLink = link; + + $(outputContainer).animate({ + scrollTop: Math.max(0, link.offsetTop - outputContainer.clientHeight / 2) + }, 200); +}, 300); + +var renderTree = debounce(() => { + let result = ""; + renderNode(tree.rootNode, 0); + function renderNode(node, indentLevel) { + let space = ' '.repeat(indentLevel); + const type = node.type; + const start = node.startPosition; + const end = node.endPosition; + result += space; + result += "(${type}` + result += `[${start.row + 1}, ${start.column}] - [${end.row + 1}, ${end.column}]`; + if (node.namedChildren.length > 0) { + for (let i = 0, n = node.namedChildren.length; i < n; i++) { + result += '\n'; + renderNode(node.namedChildren[i], indentLevel + 1); + } + } + result += ')'; + } + + outputContainer.innerHTML = result; +}, 200); + +function debounce(func, wait, immediate) { + var timeout; + return function() { + var context = this, args = arguments; + var later = function() { + timeout = null; + if (!immediate) func.apply(context, args); + }; + var callNow = immediate && !timeout; + clearTimeout(timeout); + timeout = setTimeout(later, wait); + if (callNow) func.apply(context, args); + }; +}; diff --git a/docs/section-5-playground.html b/docs/section-5-playground.html new file mode 100644 index 00000000..5a2bcdaf --- /dev/null +++ b/docs/section-5-playground.html @@ -0,0 +1,60 @@ +--- +layout: default +title: Playground +permalink: playground +--- + + + +

Playground

+ + + + + +{% if jekyll.environment == "development" %} + + +{% else %} + + +{% endif %} + + diff --git a/lib/src/node.c b/lib/src/node.c index e9796768..85139048 100644 --- a/lib/src/node.c +++ b/lib/src/node.c @@ -319,7 +319,7 @@ static inline TSNode ts_node__descendant_for_byte_range(TSNode self, uint32_t mi TSNode child; NodeChildIterator iterator = ts_node_iterate_children(&node); while (ts_node_child_iterator_next(&iterator, &child)) { - if (iterator.position.bytes > max) { + if (max <= iterator.position.bytes) { if (ts_node_start_byte(child) > min) break; node = child; if (ts_node__is_relevant(node, include_anonymous)) { @@ -348,7 +348,7 @@ static inline TSNode ts_node__descendant_for_point_range(TSNode self, TSPoint mi TSNode child; NodeChildIterator iterator = ts_node_iterate_children(&node); while (ts_node_child_iterator_next(&iterator, &child)) { - if (point_gt(iterator.position.extent, max)) { + if (point_lte(max, iterator.position.extent)) { if (point_gt(ts_node_start_point(child), min)) break; node = child; if (ts_node__is_relevant(node, include_anonymous)) { diff --git a/lib/web/binding.c b/lib/web/binding.c new file mode 100644 index 00000000..748c91a6 --- /dev/null +++ b/lib/web/binding.c @@ -0,0 +1,343 @@ +#include +#include +#include + +/*****************************/ +/* Section - Data marshaling */ +/*****************************/ + +static const uint32_t INPUT_BUFFER_SIZE = 10 * 1024; + +const void *TRANSFER_BUFFER[12] = { + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, +}; + +void *ts_init() { + TRANSFER_BUFFER[0] = (const void *)TREE_SITTER_LANGUAGE_VERSION; + TRANSFER_BUFFER[1] = (const void *)TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION; + return TRANSFER_BUFFER; +} + +static uint32_t code_unit_to_byte(uint32_t unit) { + return unit << 1; +} + +static uint32_t byte_to_code_unit(uint32_t byte) { + return byte >> 1; +} + +static void marshal_node(const void **buffer, TSNode node) { + buffer[0] = (const void *)node.id; + buffer[1] = (const void *)node.context[0]; + buffer[2] = (const void *)node.context[1]; + buffer[3] = (const void *)node.context[2]; + buffer[4] = (const void *)node.context[3]; +} + +static TSNode unmarshal_node(const TSTree *tree) { + TSNode node; + node.id = TRANSFER_BUFFER[0]; + node.context[0] = (uint32_t)TRANSFER_BUFFER[1]; + node.context[1] = (uint32_t)TRANSFER_BUFFER[2]; + node.context[2] = (uint32_t)TRANSFER_BUFFER[3]; + node.context[3] = (uint32_t)TRANSFER_BUFFER[4]; + node.tree = tree; + return node; +} + +static void marshal_point(TSPoint point) { + TRANSFER_BUFFER[0] = (const void *)point.row; + TRANSFER_BUFFER[1] = (const void *)byte_to_code_unit(point.column); +} + +static TSPoint unmarshal_point(const void **address) { + TSPoint point; + point.row = (uint32_t)address[0]; + point.column = code_unit_to_byte((uint32_t)address[1]); + return point; +} + +static TSInputEdit unmarshal_edit() { + TSInputEdit edit; + const void **address = TRANSFER_BUFFER; + edit.start_point = unmarshal_point(address); address += 2; + edit.old_end_point = unmarshal_point(address); address += 2; + edit.new_end_point = unmarshal_point(address); address += 2; + edit.start_byte = code_unit_to_byte((uint32_t)*address); address += 1; + edit.old_end_byte = code_unit_to_byte((uint32_t)*address); address += 1; + edit.new_end_byte = code_unit_to_byte((uint32_t)*address); address += 1; + return edit; +} + +/********************/ +/* Section - Parser */ +/********************/ + +extern void tree_sitter_parse_callback( + char *input_buffer, + uint32_t index, + uint32_t row, + uint32_t column, + uint32_t *length_read +); + +extern void tree_sitter_log_callback( + void *payload, + TSLogType log_type, + const char *message +); + +void ts_parser_new_wasm() { + TSParser *parser = ts_parser_new(); + char *input_buffer = calloc(INPUT_BUFFER_SIZE, sizeof(char)); + TRANSFER_BUFFER[0] = parser; + TRANSFER_BUFFER[1] = input_buffer; +} + +static const char *call_parse_callback( + void *payload, + uint32_t byte, + TSPoint position, + uint32_t *bytes_read +) { + char *buffer = (char *)payload; + tree_sitter_parse_callback( + buffer, + byte_to_code_unit(byte), + position.row, + byte_to_code_unit(position.column), + bytes_read + ); + *bytes_read = code_unit_to_byte(*bytes_read); + if (*bytes_read >= INPUT_BUFFER_SIZE) { + *bytes_read = INPUT_BUFFER_SIZE - 2; + } + return buffer; +} + +void ts_parser_enable_logger_wasm(TSParser *self, bool should_log) { + TSLogger logger = {self, should_log ? tree_sitter_log_callback : NULL}; + ts_parser_set_logger(self, logger); +} + +TSTree *ts_parser_parse_wasm( + TSParser *self, + char *input_buffer, + const TSTree *old_tree, + TSRange *ranges, + uint32_t range_count +) { + TSInput input = { + input_buffer, + call_parse_callback, + TSInputEncodingUTF16 + }; + if (range_count) { + for (unsigned i = 0; i < range_count; i++) { + TSRange *range = &ranges[i]; + range->start_byte = code_unit_to_byte(range->start_byte); + range->end_byte = code_unit_to_byte(range->end_byte); + range->start_point.column = code_unit_to_byte(range->start_point.column); + range->end_point.column = code_unit_to_byte(range->end_point.column); + } + ts_parser_set_included_ranges(self, ranges, range_count); + free(ranges); + } else { + ts_parser_set_included_ranges(self, NULL, 0); + } + return ts_parser_parse(self, old_tree, input); +} + +/******************/ +/* Section - Tree */ +/******************/ + +void ts_tree_root_node_wasm(const TSTree *tree) { + marshal_node(TRANSFER_BUFFER, ts_tree_root_node(tree)); +} + +void ts_tree_edit_wasm(TSTree *tree) { + TSInputEdit edit = unmarshal_edit(); + ts_tree_edit(tree, &edit); +} + +/******************/ +/* Section - Node */ +/******************/ + +static TSTreeCursor scratch_cursor = {0}; + +uint16_t ts_node_symbol_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + return ts_node_symbol(node); +} + +uint32_t ts_node_child_count_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + return ts_node_child_count(node); +} + +uint32_t ts_node_named_child_count_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + return ts_node_named_child_count(node); +} + +void ts_node_child_wasm(const TSTree *tree, uint32_t index) { + TSNode node = unmarshal_node(tree); + marshal_node(TRANSFER_BUFFER, ts_node_child(node, index)); +} + +void ts_node_named_child_wasm(const TSTree *tree, uint32_t index) { + TSNode node = unmarshal_node(tree); + marshal_node(TRANSFER_BUFFER, ts_node_named_child(node, index)); +} + +void ts_node_next_sibling_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + marshal_node(TRANSFER_BUFFER, ts_node_next_sibling(node)); +} + +void ts_node_prev_sibling_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + marshal_node(TRANSFER_BUFFER, ts_node_prev_sibling(node)); +} + +void ts_node_next_named_sibling_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + marshal_node(TRANSFER_BUFFER, ts_node_next_named_sibling(node)); +} + +void ts_node_prev_named_sibling_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + marshal_node(TRANSFER_BUFFER, ts_node_prev_named_sibling(node)); +} + +void ts_node_parent_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + marshal_node(TRANSFER_BUFFER, ts_node_parent(node)); +} + +void ts_node_descendant_for_index_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + const void **address = TRANSFER_BUFFER + 5; + uint32_t start = code_unit_to_byte((uint32_t)address[0]); + uint32_t end = code_unit_to_byte((uint32_t)address[1]); + marshal_node(TRANSFER_BUFFER, ts_node_descendant_for_byte_range(node, start, end)); +} + +void ts_node_named_descendant_for_index_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + const void **address = TRANSFER_BUFFER + 5; + uint32_t start = code_unit_to_byte((uint32_t)address[0]); + uint32_t end = code_unit_to_byte((uint32_t)address[1]); + marshal_node(TRANSFER_BUFFER, ts_node_named_descendant_for_byte_range(node, start, end)); +} + +void ts_node_descendant_for_position_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + const void **address = TRANSFER_BUFFER + 5; + TSPoint start = unmarshal_point(address); address += 2; + TSPoint end = unmarshal_point(address); + marshal_node(TRANSFER_BUFFER, ts_node_descendant_for_point_range(node, start, end)); +} + +void ts_node_named_descendant_for_position_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + const void **address = TRANSFER_BUFFER + 5; + TSPoint start = unmarshal_point(address); address += 2; + TSPoint end = unmarshal_point(address); + marshal_node(TRANSFER_BUFFER, ts_node_named_descendant_for_point_range(node, start, end)); +} + +void ts_node_start_point_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + marshal_point(ts_node_start_point(node)); +} + +void ts_node_end_point_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + marshal_point(ts_node_end_point(node)); +} + +uint32_t ts_node_start_index_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + return byte_to_code_unit(ts_node_start_byte(node)); +} + +uint32_t ts_node_end_index_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + return byte_to_code_unit(ts_node_end_byte(node)); +} + +char *ts_node_to_string_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + return ts_node_string(node); +} + +void ts_node_children_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + uint32_t count = ts_node_child_count(node); + const void **result = NULL; + if (count > 0) { + result = calloc(sizeof(void *), 5 * count); + const void **address = result; + ts_tree_cursor_reset(&scratch_cursor, node); + ts_tree_cursor_goto_first_child(&scratch_cursor); + marshal_node(address, ts_tree_cursor_current_node(&scratch_cursor)); + for (uint32_t i = 1; i < count; i++) { + address += 5; + ts_tree_cursor_goto_next_sibling(&scratch_cursor); + TSNode child = ts_tree_cursor_current_node(&scratch_cursor); + marshal_node(address, child); + } + } + TRANSFER_BUFFER[0] = (const void *)count; + TRANSFER_BUFFER[1] = result; +} + +void ts_node_named_children_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + uint32_t count = ts_node_named_child_count(node); + const void **result = NULL; + if (count > 0) { + result = calloc(sizeof(void *), 5 * count); + const void **address = result; + ts_tree_cursor_reset(&scratch_cursor, node); + ts_tree_cursor_goto_first_child(&scratch_cursor); + uint32_t i = 0; + for (;;) { + TSNode child = ts_tree_cursor_current_node(&scratch_cursor); + if (ts_node_is_named(child)) { + marshal_node(address, child); + address += 5; + i++; + if (i == count) break; + } + if (!ts_tree_cursor_goto_next_sibling(&scratch_cursor)) break; + } + } + TRANSFER_BUFFER[0] = (const void *)count; + TRANSFER_BUFFER[1] = result; +} + +int ts_node_is_named_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + return ts_node_is_named(node); +} + +int ts_node_has_changes_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + return ts_node_has_changes(node); +} + +int ts_node_has_error_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + return ts_node_has_error(node); +} + +int ts_node_is_missing_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + return ts_node_is_missing(node); +} diff --git a/lib/web/binding.js b/lib/web/binding.js new file mode 100644 index 00000000..85bf1cb6 --- /dev/null +++ b/lib/web/binding.js @@ -0,0 +1,544 @@ +const C = Module; +const INTERNAL = {}; +const SIZE_OF_INT = 4; +const SIZE_OF_NODE = 5 * SIZE_OF_INT; +const SIZE_OF_POINT = 2 * SIZE_OF_INT; +const SIZE_OF_RANGE = 2 * SIZE_OF_INT + 2 * SIZE_OF_POINT; + +var VERSION; +var MIN_COMPATIBLE_VERSION; +var TRANSFER_BUFFER; +var currentParseCallback; +var currentLogCallback; +var initPromise; + +class Parser { + static init() { + if (!initPromise) { + initPromise = new Promise(resolve => { + Module.onRuntimeInitialized = resolve + }).then(() => { + TRANSFER_BUFFER = C._ts_init(); + VERSION = getValue(TRANSFER_BUFFER, 'i32'); + MIN_COMPATIBLE_VERSION = getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); + }); + } + return initPromise; + } + + constructor() { + C._ts_parser_new_wasm(); + this[0] = getValue(TRANSFER_BUFFER, 'i32'); + this[1] = getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); + } + + delete() { + C._ts_parser_delete(this[0]); + C._free(this[1]); + } + + setLanguage(language) { + let address; + if (!language) { + address = 0; + language = null; + } else if (language.constructor === Language) { + address = language[0]; + const version = C._ts_language_version(address); + if (version < MIN_COMPATIBLE_VERSION || VERSION < version) { + throw new Error( + `Incompatible language version ${version}. ` + + `Compatibility range ${MIN_COMPATIBLE_VERSION} through ${VERSION}.` + ); + } + } else { + throw new Error('Argument must be a Language'); + } + this.language = language; + C._ts_parser_set_language(this[0], address); + return this; + } + + getLanguage() { + return this.language + } + + parse(callback, oldTree, options) { + if (typeof callback === 'string') { + currentParseCallback = index => callback.slice(index); + } else if (typeof callback === 'function') { + currentParseCallback = callback; + } else { + throw new Error("Argument must be a string or a function"); + } + + if (this.logCallback) { + currentLogCallback = this.logCallback; + C._ts_parser_enable_logger_wasm(this[0], 1); + } else { + currentLogCallback = null; + C._ts_parser_enable_logger_wasm(this[0], 0); + } + + let rangeCount = 0; + let rangeAddress = 0; + if (options && options.includedRanges) { + rangeCount = options.includedRanges.length; + rangeAddress = C._calloc(rangeCount, SIZE_OF_RANGE); + let address = rangeAddress; + for (let i = 0; i < rangeCount; i++) { + marshalRange(address, options.includedRanges[i]); + address += SIZE_OF_RANGE; + } + } + + const treeAddress = C._ts_parser_parse_wasm( + this[0], + this[1], + oldTree ? oldTree[0] : 0, + rangeAddress, + rangeCount + ); + + if (!treeAddress) { + currentParseCallback = null; + currentLogCallback = null; + throw new Error('Parsing failed'); + } + + const result = new Tree(INTERNAL, treeAddress, this.language, currentParseCallback); + currentParseCallback = null; + currentLogCallback = null; + return result; + } + + reset() { + C._ts_parser_parse_wasm(this[0]); + } + + setTimeoutMicros(timeout) { + C._ts_parser_set_timeout_micros(this[0], timeout); + } + + getTimeoutMicros(timeout) { + C._ts_parser_timeout_micros(this[0]); + } + + setLogger(callback) { + if (!callback) { + callback = null; + } else if (typeof callback !== "function") { + throw new Error("Logger callback must be a function"); + } + this.logCallback = callback; + return this; + } + + getLogger() { + return this.logCallback; + } +} + +class Tree { + constructor(internal, address, language, textCallback) { + if (internal !== INTERNAL) { + throw new Error('Illegal constructor') + } + this[0] = address; + this.language = language; + this.textCallback = textCallback; + } + + copy() { + const address = C._ts_tree_copy(this[0]); + return new Tree(INTERNAL, address, this.language, this.textCallback); + } + + delete() { + C._ts_tree_delete(this[0]); + } + + edit(edit) { + marshalEdit(edit); + C._ts_tree_edit_wasm(this[0]); + } + + get rootNode() { + C._ts_tree_root_node_wasm(this[0]); + return unmarshalNode(this); + } + + getLanguage() { + return this.language; + } +} + +class Node { + constructor(internal, tree) { + if (internal !== INTERNAL) { + throw new Error('Illegal constructor') + } + this.tree = tree; + } + + get id() { + return this[0]; + } + + get typeId() { + marshalNode(this); + return C._ts_node_symbol_wasm(this.tree); + } + + get type() { + return this.tree.language.types[this.typeId] || 'ERROR'; + } + + get startPosition() { + marshalNode(this); + C._ts_node_start_point_wasm(this.tree[0]); + return unmarshalPoint(TRANSFER_BUFFER); + } + + get endPosition() { + marshalNode(this); + C._ts_node_end_point_wasm(this.tree[0]); + return unmarshalPoint(TRANSFER_BUFFER); + } + + get startIndex() { + marshalNode(this); + return C._ts_node_start_index_wasm(this.tree[0]); + } + + get endIndex() { + marshalNode(this); + return C._ts_node_end_index_wasm(this.tree[0]); + } + + get text() { + const startIndex = this.startIndex; + const length = this.endIndex - startIndex; + let result = this.tree.textCallback(startIndex); + while (result.length < length) { + result += this.tree.textCallback(startIndex + result.length); + } + return result.slice(0, length); + } + + isNamed() { + marshalNode(this); + return C._ts_node_is_named_wasm(this.tree[0]) === 1; + } + + hasError() { + marshalNode(this); + return C._ts_node_has_error_wasm(this.tree[0]) === 1; + } + + hasChanges() { + marshalNode(this); + return C._ts_node_has_changes_wasm(this.tree[0]) === 1; + } + + isMissing() { + marshalNode(this); + return C._ts_node_is_missing_wasm(this.tree[0]) === 1; + } + + equals(other) { + if (this === other) return true; + for (let i = 0; i < 5; i++) { + if (this[i] !== other[i]) return false; + } + return true; + } + + child(index) { + marshalNode(this); + C._ts_node_child_wasm(this.tree[0], index); + return unmarshalNode(this.tree); + } + + namedChild(index) { + marshalNode(this); + C._ts_node_named_child_wasm(this.tree[0], index); + return unmarshalNode(this.tree); + } + + get childCount() { + marshalNode(this); + return C._ts_node_child_count_wasm(this.tree[0]); + } + + get namedChildCount() { + marshalNode(this); + return C._ts_node_named_child_count_wasm(this.tree[0]); + } + + get firstChild() { + return this.child(0); + } + + get firstNamedChild() { + return this.namedChild(0); + } + + get lastChild() { + return this.child(this.childCount - 1); + } + + get lastNamedChild() { + return this.namedChild(this.namedChildCount - 1); + } + + get children() { + if (!this._children) { + marshalNode(this); + C._ts_node_children_wasm(this.tree[0]); + const count = getValue(TRANSFER_BUFFER, 'i32'); + const buffer = getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); + this._children = new Array(count); + if (count > 0) { + let address = buffer; + for (let i = 0; i < count; i++) { + this._children[i] = unmarshalNode(this.tree, address); + address += SIZE_OF_NODE; + } + C._free(buffer); + } + } + return this._children; + } + + get namedChildren() { + if (!this._namedChildren) { + marshalNode(this); + C._ts_node_named_children_wasm(this.tree[0]); + const count = getValue(TRANSFER_BUFFER, 'i32'); + const buffer = getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); + this._namedChildren = new Array(count); + if (count > 0) { + let address = buffer; + for (let i = 0; i < count; i++) { + this._namedChildren[i] = unmarshalNode(this.tree, address); + address += SIZE_OF_NODE; + } + C._free(buffer); + } + } + return this._namedChildren; + } + + get nextSibling() { + marshalNode(this); + C._ts_node_next_sibling_wasm(this.tree[0]); + return unmarshalNode(this.tree); + } + + get previousSibling() { + marshalNode(this); + C._ts_node_prev_sibling_wasm(this.tree[0]); + return unmarshalNode(this.tree); + } + + get nextNamedSibling() { + marshalNode(this); + C._ts_node_next_named_sibling_wasm(this.tree[0]); + return unmarshalNode(this.tree); + } + + get previousNamedSibling() { + marshalNode(this); + C._ts_node_prev_named_sibling_wasm(this.tree[0]); + return unmarshalNode(this.tree); + } + + get parent() { + marshalNode(this); + C._ts_node_parent_wasm(this.tree[0]); + return unmarshalNode(this.tree); + } + + descendantForIndex(start, end = start) { + if (typeof start !== 'number' || typeof end !== 'number') { + throw new Error('Arguments must be numbers'); + } + + marshalNode(this); + let address = TRANSFER_BUFFER + SIZE_OF_NODE; + setValue(address, start, 'i32'); + setValue(address + SIZE_OF_INT, end, 'i32'); + C._ts_node_descendant_for_index_wasm(this.tree[0]); + return unmarshalNode(this.tree); + } + + namedDescendantForIndex(start, end = start) { + if (typeof start !== 'number' || typeof end !== 'number') { + throw new Error('Arguments must be numbers'); + } + + marshalNode(this); + let address = TRANSFER_BUFFER + SIZE_OF_NODE; + setValue(address, start, 'i32'); + setValue(address + SIZE_OF_INT, end, 'i32'); + C._ts_node_named_descendant_for_index_wasm(this.tree[0]); + return unmarshalNode(this.tree); + } + + descendantForPosition(start, end = start) { + if (!isPoint(start) || !isPoint(end)) { + throw new Error('Arguments must be {row, column} objects'); + } + + marshalNode(this); + let address = TRANSFER_BUFFER + SIZE_OF_NODE; + marshalPoint(address, start); + marshalPoint(address + SIZE_OF_POINT, end); + C._ts_node_descendant_for_position_wasm(this.tree[0]); + return unmarshalNode(this.tree); + } + + namedDescendantForPosition(start, end = start) { + if (!isPoint(start) || !isPoint(end)) { + throw new Error('Arguments must be {row, column} objects'); + } + + marshalNode(this); + let address = TRANSFER_BUFFER + SIZE_OF_NODE; + marshalPoint(address, start); + marshalPoint(address + SIZE_OF_POINT, end); + C._ts_node_named_descendant_for_position_wasm(this.tree[0]); + return unmarshalNode(this.tree); + } + + toString() { + marshalNode(this); + const address = C._ts_node_to_string_wasm(this.tree[0]); + const result = AsciiToString(address); + C._free(address); + return result; + } +} + +class Language { + constructor(internal, address) { + if (internal !== INTERNAL) { + throw new Error('Illegal constructor') + } + this[0] = address; + this.types = new Array(C._ts_language_symbol_count(this[0])); + for (let i = 0, n = this.types.length; i < n; i++) { + if (C._ts_language_symbol_type(this[0], i) < 2) { + this.types[i] = UTF8ToString(C._ts_language_symbol_name(this[0], i)); + } + } + } + + get version() { + return C._ts_language_version(this[0]); + } + + static load(url) { + let bytes; + if ( + typeof require === 'function' && + require('url').parse(url).protocol == null + ) { + const fs = require('fs'); + bytes = Promise.resolve(fs.readFileSync(url)); + } else { + bytes = fetch(url) + .then(response => response.arrayBuffer() + .then(buffer => { + if (response.ok) { + return new Uint8Array(buffer); + } else { + const body = new TextDecoder('utf-8').decode(buffer); + throw new Error(`Language.load failed with status ${response.status}.\n\n${body}`) + } + })); + } + + return bytes + .then(bytes => loadWebAssemblyModule(bytes, {loadAsync: true})) + .then(mod => { + const functionName = Object.keys(mod).find(key => key.includes("tree_sitter_")); + const languageAddress = mod[functionName](); + return new Language(INTERNAL, languageAddress); + }); + } +} + +function isPoint(point) { + return ( + point && + typeof point.row === 'number' && + typeof point.row === 'number' + ); +} + +function marshalNode(node) { + let address = TRANSFER_BUFFER; + for (let i = 0; i < 5; i++) { + setValue(address, node[i], 'i32'); + address += SIZE_OF_INT; + } +} + +function unmarshalNode(tree, address = TRANSFER_BUFFER) { + const id = getValue(address, 'i32'); + if (id === 0) return null; + const result = new Node(INTERNAL, tree); + result[0] = id; + address += SIZE_OF_INT; + for (let i = 1; i < 5; i++) { + result[i] = getValue(address, 'i32'); + address += SIZE_OF_INT; + } + return result; +} + +function marshalPoint(address, point) { + setValue(address, point.row, 'i32') + setValue(address + SIZE_OF_INT, point.column, 'i32') +} + +function unmarshalPoint(address) { + return { + row: getValue(address, 'i32'), + column: getValue(address + SIZE_OF_INT, 'i32') + } +} + +function marshalRange(address, range) { + marshalPoint(address, range.startPosition); address += SIZE_OF_POINT; + marshalPoint(address, range.endPosition); address += SIZE_OF_POINT; + setValue(address, range.startIndex, 'i32'); address += SIZE_OF_INT; + setValue(address, range.endIndex, 'i32'); address += SIZE_OF_INT; +} + +function unmarshalRange(address) { + const result = {}; + result.startPosition = unmarshalPoint(address); address += SIZE_OF_POINT; + result.endPosition = unmarshalPoint(address); address += SIZE_OF_POINT; + result.startIndex = getValue(address, 'i32'); address += SIZE_OF_INT; + result.endIndex = getValue(address, 'i32'); + return result; +} + +function marshalEdit(edit) { + let address = TRANSFER_BUFFER; + marshalPoint(address, edit.startPosition); address += SIZE_OF_POINT; + marshalPoint(address, edit.oldEndPosition); address += SIZE_OF_POINT; + marshalPoint(address, edit.newEndPosition); address += SIZE_OF_POINT; + setValue(address, edit.startIndex, 'i32'); address += SIZE_OF_INT; + setValue(address, edit.oldEndIndex, 'i32'); address += SIZE_OF_INT; + setValue(address, edit.newEndIndex, 'i32'); address += SIZE_OF_INT; +} + +Parser.Language = Language; + +return Parser; + +})); diff --git a/lib/web/imports.js b/lib/web/imports.js new file mode 100644 index 00000000..ea34926f --- /dev/null +++ b/lib/web/imports.js @@ -0,0 +1,25 @@ +mergeInto(LibraryManager.library, { + tree_sitter_parse_callback: function( + inputBufferAddress, + index, + row, + column, + lengthAddress + ) { + var INPUT_BUFFER_SIZE = 10 * 1024; + var string = currentParseCallback(index, {row: row, column: column}); + if (typeof string === 'string') { + setValue(lengthAddress, string.length, 'i32'); + stringToUTF16(string, inputBufferAddress, INPUT_BUFFER_SIZE); + } else { + setValue(lengthAddress, 0, 'i32'); + } + }, + + tree_sitter_log_callback: function(_payload, isLexMessage, messageAddress) { + if (currentLogCallback) { + const message = UTF8ToString(messageAddress); + currentLogCallback(message, isLexMessage !== 0); + } + } +}); diff --git a/lib/web/package.json b/lib/web/package.json new file mode 100644 index 00000000..9f0b60bc --- /dev/null +++ b/lib/web/package.json @@ -0,0 +1,31 @@ +{ + "name": "tree-sitter.wasm", + "version": "0.0.1", + "description": "Tree-sitter bindings for the web", + "main": "index.js", + "directories": { + "test": "test" + }, + "scripts": { + "test": "mocha" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/tree-sitter/tree-sitter.git" + }, + "keywords": [ + "incremental", + "parsing" + ], + "author": "Max Brunsfeld", + "license": "MIT", + "bugs": { + "url": "https://github.com/tree-sitter/tree-sitter/issues" + }, + "homepage": "https://github.com/tree-sitter/tree-sitter#readme", + "devDependencies": { + "chai": "^4.2.0", + "mocha": "^6.1.4", + "terser": "^3.17.0" + } +} diff --git a/lib/web/prefix.js b/lib/web/prefix.js new file mode 100644 index 00000000..ccd94fa4 --- /dev/null +++ b/lib/web/prefix.js @@ -0,0 +1,11 @@ +(function (root, factory) { + if (typeof define === 'function' && define.amd) { + define([], factory); + } else if (typeof exports === 'object') { + module.exports = factory(); + // module.exports.init(); + // delete module.exports.init; + } else { + window.TreeSitter = factory(); + } +}(this, function () { diff --git a/lib/web/test/helper.js b/lib/web/test/helper.js new file mode 100644 index 00000000..6586f247 --- /dev/null +++ b/lib/web/test/helper.js @@ -0,0 +1,8 @@ +const release = '../../../target/release' +const Parser = require(`${release}/tree-sitter.js`); +const JavaScript = require.resolve(`${release}/tree-sitter-javascript.wasm`); + +module.exports = Parser.init().then(async () => ({ + Parser, + JavaScript: await Parser.Language.load(JavaScript), +})); diff --git a/lib/web/test/node-test.js b/lib/web/test/node-test.js new file mode 100644 index 00000000..5d5716f9 --- /dev/null +++ b/lib/web/test/node-test.js @@ -0,0 +1,358 @@ +const {assert} = require('chai'); +let Parser, JavaScript; + +describe("Node", () => { + let parser, tree; + + before(async () => + ({Parser, JavaScript} = await require('./helper')) + ); + + beforeEach(() => { + tree = null; + parser = new Parser().setLanguage(JavaScript); + }); + + afterEach(() => { + parser.delete(); + tree.delete(); + }); + + describe(".children", () => { + it("returns an array of child nodes", () => { + tree = parser.parse("x10 + 1000"); + assert.equal(1, tree.rootNode.children.length); + const sumNode = tree.rootNode.firstChild.firstChild; + assert.deepEqual( + sumNode.children.map(child => child.type), + ["identifier", "+", "number"] + ); + }); + }); + + describe(".namedChildren", () => { + it("returns an array of named child nodes", () => { + tree = parser.parse("x10 + 1000"); + const sumNode = tree.rootNode.firstChild.firstChild; + assert.equal(1, tree.rootNode.namedChildren.length); + assert.deepEqual( + ["identifier", "number"], + sumNode.namedChildren.map(child => child.type) + ); + }); + }); + + describe(".startIndex and .endIndex", () => { + it("returns the character index where the node starts/ends in the text", () => { + tree = parser.parse("a👍👎1 / b👎c👎"); + const quotientNode = tree.rootNode.firstChild.firstChild; + + assert.equal(0, quotientNode.startIndex); + assert.equal(15, quotientNode.endIndex); + assert.deepEqual( + [0, 7, 9], + quotientNode.children.map(child => child.startIndex) + ); + assert.deepEqual( + [6, 8, 15], + quotientNode.children.map(child => child.endIndex) + ); + }); + }); + + describe(".startPosition and .endPosition", () => { + it("returns the row and column where the node starts/ends in the text", () => { + tree = parser.parse("x10 + 1000"); + const sumNode = tree.rootNode.firstChild.firstChild; + assert.equal("binary_expression", sumNode.type); + + assert.deepEqual({ row: 0, column: 0 }, sumNode.startPosition); + assert.deepEqual({ row: 0, column: 10 }, sumNode.endPosition); + assert.deepEqual( + [{ row: 0, column: 0 }, { row: 0, column: 4 }, { row: 0, column: 6 }], + sumNode.children.map(child => child.startPosition) + ); + assert.deepEqual( + [{ row: 0, column: 3 }, { row: 0, column: 5 }, { row: 0, column: 10 }], + sumNode.children.map(child => child.endPosition) + ); + }); + + it("handles characters that occupy two UTF16 code units", () => { + tree = parser.parse("a👍👎1 /\n b👎c👎"); + const sumNode = tree.rootNode.firstChild.firstChild; + assert.deepEqual( + [ + [{ row: 0, column: 0 }, { row: 0, column: 6 }], + [{ row: 0, column: 7 }, { row: 0, column: 8 }], + [{ row: 1, column: 1 }, { row: 1, column: 7 }] + ], + sumNode.children.map(child => [child.startPosition, child.endPosition]) + ); + }); + }); + + describe(".parent", () => { + it("returns the node's parent", () => { + tree = parser.parse("x10 + 1000"); + const sumNode = tree.rootNode.firstChild; + const variableNode = sumNode.firstChild; + assert.notEqual(sumNode.id, variableNode.id); + assert.equal(sumNode.id, variableNode.parent.id); + assert.equal(tree.rootNode.id, sumNode.parent.id); + }); + }); + + describe('.child(), .firstChild, .lastChild', () => { + it('returns null when the node has no children', () => { + tree = parser.parse("x10 + 1000"); + const sumNode = tree.rootNode.firstChild.firstChild; + const variableNode = sumNode.firstChild; + assert.equal(variableNode.firstChild, null); + assert.equal(variableNode.lastChild, null); + assert.equal(variableNode.firstNamedChild, null); + assert.equal(variableNode.lastNamedChild, null); + assert.equal(variableNode.child(1), null); + }) + }); + + describe(".nextSibling and .previousSibling", () => { + it("returns the node's next and previous sibling", () => { + tree = parser.parse("x10 + 1000"); + const sumNode = tree.rootNode.firstChild.firstChild; + assert.equal(sumNode.children[1].id, sumNode.children[0].nextSibling.id); + assert.equal(sumNode.children[2].id, sumNode.children[1].nextSibling.id); + assert.equal( + sumNode.children[0].id, + sumNode.children[1].previousSibling.id + ); + assert.equal( + sumNode.children[1].id, + sumNode.children[2].previousSibling.id + ); + }); + }); + + describe(".nextNamedSibling and .previousNamedSibling", () => { + it("returns the node's next and previous named sibling", () => { + tree = parser.parse("x10 + 1000"); + const sumNode = tree.rootNode.firstChild.firstChild; + assert.equal( + sumNode.namedChildren[1].id, + sumNode.namedChildren[0].nextNamedSibling.id + ); + assert.equal( + sumNode.namedChildren[0].id, + sumNode.namedChildren[1].previousNamedSibling.id + ); + }); + }); + + describe(".descendantForIndex(min, max)", () => { + it("returns the smallest node that spans the given range", () => { + tree = parser.parse("x10 + 1000"); + const sumNode = tree.rootNode.firstChild.firstChild; + assert.equal("identifier", sumNode.descendantForIndex(1, 2).type); + assert.equal("+", sumNode.descendantForIndex(4, 4).type); + + assert.throws(() => { + sumNode.descendantForIndex(1, {}); + }, "Arguments must be numbers"); + + assert.throws(() => { + sumNode.descendantForIndex(); + }, "Arguments must be numbers"); + }); + }); + + describe(".namedDescendantForIndex", () => { + it("returns the smallest node that spans the given range", () => { + tree = parser.parse("x10 + 1000"); + const sumNode = tree.rootNode.firstChild; + assert.equal("identifier", sumNode.descendantForIndex(1, 2).type); + assert.equal("+", sumNode.descendantForIndex(4, 4).type); + }); + }); + + describe(".descendantForPosition(min, max)", () => { + it("returns the smallest node that spans the given range", () => { + tree = parser.parse("x10 + 1000"); + const sumNode = tree.rootNode.firstChild; + + assert.equal( + "identifier", + sumNode.descendantForPosition( + { row: 0, column: 1 }, + { row: 0, column: 2 } + ).type + ); + + assert.equal( + "+", + sumNode.descendantForPosition({ row: 0, column: 4 }).type + ); + + assert.throws(() => { + sumNode.descendantForPosition(1, {}); + }, "Arguments must be {row, column} objects"); + + assert.throws(() => { + sumNode.descendantForPosition(); + }, "Arguments must be {row, column} objects"); + }); + }); + + describe(".namedDescendantForPosition(min, max)", () => { + it("returns the smallest named node that spans the given range", () => { + tree = parser.parse("x10 + 1000"); + const sumNode = tree.rootNode.firstChild; + + assert.equal( + sumNode.namedDescendantForPosition( + { row: 0, column: 1 }, + { row: 0, column: 2 } + ).type, + "identifier", + ); + + assert.equal( + sumNode.namedDescendantForPosition({ row: 0, column: 4 }).type, + 'binary_expression' + ); + }); + }); + + describe(".hasError()", () => { + it("returns true if the node contains an error", () => { + tree = parser.parse("1 + 2 * * 3"); + const node = tree.rootNode; + assert.equal( + node.toString(), + '(program (expression_statement (binary_expression (number) (binary_expression (number) (ERROR) (number)))))' + ); + + const sum = node.firstChild.firstChild; + assert(sum.hasError()); + assert(!sum.children[0].hasError()); + assert(!sum.children[1].hasError()); + assert(sum.children[2].hasError()); + }); + }); + + describe(".isMissing()", () => { + it("returns true if the node is missing from the source and was inserted via error recovery", () => { + tree = parser.parse("(2 ||)"); + const node = tree.rootNode; + assert.equal( + node.toString(), + "(program (expression_statement (parenthesized_expression (binary_expression (number) (MISSING identifier)))))" + ); + + const sum = node.firstChild.firstChild.firstNamedChild; + assert.equal(sum.type, 'binary_expression') + assert(sum.hasError()); + assert(!sum.children[0].isMissing()); + assert(!sum.children[1].isMissing()); + assert(sum.children[2].isMissing()); + }); + }); + + describe(".text", () => { + const text = "α0 / b👎c👎"; + + Object.entries({ + '.parse(String)': text, + '.parse(Function)': offset => text.substr(offset, 4) + }).forEach(([method, parse]) => + it(`returns the text of a node generated by ${method}`, async () => { + const [numeratorSrc, denominatorSrc] = text.split(/\s*\/\s+/) + tree = await parser.parse(text) + const quotientNode = tree.rootNode.firstChild.firstChild; + const [numerator, slash, denominator] = quotientNode.children; + + assert.equal(text, tree.rootNode.text, 'root node text'); + assert.equal(denominatorSrc, denominator.text, 'denominator text'); + assert.equal(text, quotientNode.text, 'quotient text'); + assert.equal(numeratorSrc, numerator.text, 'numerator text'); + assert.equal('/', slash.text, '"/" text'); + }) + ); + }); + + describe.skip('.descendantsOfType(type, min, max)', () => { + it('finds all of the descendants of the given type in the given range', () => { + tree = parser.parse("a + 1 * b * 2 + c + 3"); + const outerSum = tree.rootNode.firstChild.firstChild; + let descendants = outerSum.descendantsOfType('number', {row: 0, column: 2}, {row: 0, column: 15}) + assert.deepEqual( + descendants.map(node => node.startIndex), + [4, 12] + ); + + descendants = outerSum.descendantsOfType('identifier', {row: 0, column: 2}, {row: 0, column: 15}) + assert.deepEqual( + descendants.map(node => node.startIndex), + [8] + ); + + descendants = outerSum.descendantsOfType('identifier', {row: 0, column: 0}, {row: 0, column: 30}) + assert.deepEqual( + descendants.map(node => node.startIndex), + [0, 8, 16] + ); + + descendants = outerSum.descendantsOfType('number', {row: 0, column: 0}, {row: 0, column: 30}) + assert.deepEqual( + descendants.map(node => node.startIndex), + [4, 12, 20] + ); + + descendants = outerSum.descendantsOfType( + ['identifier', 'number'], + {row: 0, column: 0}, + {row: 0, column: 30} + ) + assert.deepEqual( + descendants.map(node => node.startIndex), + [0, 4, 8, 12, 16, 20] + ); + + descendants = outerSum.descendantsOfType('number') + assert.deepEqual( + descendants.map(node => node.startIndex), + [4, 12, 20] + ); + + descendants = outerSum.firstChild.descendantsOfType('number', {row: 0, column: 0}, {row: 0, column: 30}) + assert.deepEqual( + descendants.map(node => node.startIndex), + [4, 12] + ); + }) + }); + + describe.skip('.closest(type)', () => { + it('returns the closest ancestor of the given type', () => { + tree = parser.parse("a(b + -d.e)"); + const property = tree.rootNode.descendantForIndex("a(b + -d.".length); + assert.equal(property.type, 'property_identifier'); + + const unary = property.closest('unary_expression') + assert.equal(unary.type, 'unary_expression') + assert.equal(unary.startIndex, 'a(b + '.length) + assert.equal(unary.endIndex, 'a(b + -d.e'.length) + + const sum = property.closest(['binary_expression', 'call_expression']) + assert.equal(sum.type, 'binary_expression') + assert.equal(sum.startIndex, 2) + assert.equal(sum.endIndex, 'a(b + -d.e'.length) + }); + + it('throws an exception when an invalid argument is given', () => { + tree = parser.parse("a + 1 * b * 2 + c + 3"); + const number = tree.rootNode.descendantForIndex(4) + + assert.throws(() => number.closest({a: 1}), /Argument must be a string or array of strings/) + }); + }); +}); diff --git a/lib/web/test/parser-test.js b/lib/web/test/parser-test.js new file mode 100644 index 00000000..991efc73 --- /dev/null +++ b/lib/web/test/parser-test.js @@ -0,0 +1,158 @@ +const {assert} = require('chai'); +let Parser, JavaScript; + +describe("Parser", () => { + let parser; + + before(async () => + ({Parser, JavaScript} = await require('./helper')) + ); + + beforeEach(() => { + parser = new Parser(); + }); + + afterEach(() => { + parser.delete() + }); + + describe(".setLanguage", () => { + it("allows setting the language to null", () => { + assert.equal(parser.getLanguage(), null); + parser.setLanguage(JavaScript); + assert.equal(parser.getLanguage(), JavaScript); + parser.setLanguage(null); + assert.equal(parser.getLanguage(), null); + }); + + it("throws an exception when the given object is not a tree-sitter language", () => { + assert.throws(() => parser.setLanguage({}), /Argument must be a Language/); + assert.throws(() => parser.setLanguage(1), /Argument must be a Language/); + }); + }); + + describe(".setLogger", () => { + beforeEach(() => { + parser.setLanguage(JavaScript) + }); + + it("calls the given callback for each parse event", () => { + const debugMessages = []; + parser.setLogger((message) => debugMessages.push(message)); + parser.parse("a + b + c"); + assert.includeMembers(debugMessages, [ + "skip character:' '", + "consume character:'b'", + "reduce sym:program, child_count:1", + "accept" + ]); + }); + + it("allows the callback to be retrieved later", () => { + const callback = () => {} + parser.setLogger(callback); + assert.equal(parser.getLogger(), callback); + parser.setLogger(false); + assert.equal(parser.getLogger(), null); + }); + + it("disables debugging when given a falsy value", () => { + const debugMessages = []; + parser.setLogger((message) => debugMessages.push(message)); + parser.setLogger(false); + parser.parse("a + b * c"); + assert.equal(debugMessages.length, 0); + }); + + it("throws an error when given a truthy value that isn't a function ", () => { + assert.throws( + () => parser.setLogger("5"), + "Logger callback must be a function" + ); + }); + + it("rethrows errors thrown by the logging callback", () => { + const error = new Error("The error message"); + parser.setLogger((msg, params) => { throw error; }); + assert.throws( + () => parser.parse("ok;"), + "The error message" + ); + }); + }); + + describe(".parse", () => { + let tree; + + beforeEach(() => { + tree = null; + parser.setLanguage(JavaScript) + }); + + afterEach(() => { + if (tree) tree.delete(); + }); + + it("reads from the given input", () => { + const parts = ["first", "_", "second", "_", "third"]; + tree = parser.parse(() => parts.shift()); + assert.equal(tree.rootNode.toString(), "(program (expression_statement (identifier)))"); + }); + + it("stops reading when the input callback return something that's not a string", () => { + const parts = ["abc", "def", "ghi", {}, {}, {}, "second-word", " "]; + tree = parser.parse(() => parts.shift()); + assert.equal( + tree.rootNode.toString(), + "(program (expression_statement (identifier)))" + ); + assert.equal(tree.rootNode.endIndex, 9); + assert.equal(parts.length, 2); + }); + + it("throws an exception when the given input is not a function", () => { + assert.throws(() => parser.parse(null), "Argument must be a string or a function"); + assert.throws(() => parser.parse(5), "Argument must be a string or a function"); + assert.throws(() => parser.parse({}), "Argument must be a string or a function"); + }); + + it("handles long input strings", () => { + const repeatCount = 10000; + const inputString = "[" + "0,".repeat(repeatCount) + "]"; + + tree = parser.parse(inputString); + assert.equal(tree.rootNode.type, "program"); + assert.equal(tree.rootNode.firstChild.firstChild.namedChildCount, repeatCount); + }); + + it('parses only the text within the `includedRanges` if they are specified', () => { + const sourceCode = "<% foo() %> <% bar %>"; + + const start1 = sourceCode.indexOf('foo'); + const end1 = start1 + 5 + const start2 = sourceCode.indexOf('bar'); + const end2 = start2 + 3 + + const tree = parser.parse(sourceCode, null, { + includedRanges: [ + { + startIndex: start1, + endIndex: end1, + startPosition: {row: 0, column: start1}, + endPosition: {row: 0, column: end1} + }, + { + startIndex: start2, + endIndex: end2, + startPosition: {row: 0, column: start2}, + endPosition: {row: 0, column: end2} + }, + ] + }); + + assert.equal( + tree.rootNode.toString(), + '(program (expression_statement (call_expression (identifier) (arguments))) (expression_statement (identifier)))' + ); + }) + });}); diff --git a/lib/web/test/tree-test.js b/lib/web/test/tree-test.js new file mode 100644 index 00000000..ea7f085f --- /dev/null +++ b/lib/web/test/tree-test.js @@ -0,0 +1,112 @@ +const {assert} = require('chai'); +let Parser, JavaScript; + +describe("Tree", () => { + let parser, tree; + + before(async () => + ({Parser, JavaScript} = await require('./helper')) + ); + + beforeEach(() => { + parser = new Parser().setLanguage(JavaScript); + }); + + afterEach(() => { + parser.delete(); + tree.delete(); + }); + + describe('.edit', () => { + let input, edit + + it('updates the positions of nodes', () => { + parser.setLanguage(JavaScript) + + input = 'abc + cde'; + tree = parser.parse(input); + assert.equal( + tree.rootNode.toString(), + "(program (expression_statement (binary_expression (identifier) (identifier))))" + ); + + let sumNode = tree.rootNode.firstChild.firstChild; + let variableNode1 = sumNode.firstChild; + let variableNode2 = sumNode.lastChild; + assert.equal(variableNode1.startIndex, 0); + assert.equal(variableNode1.endIndex, 3); + assert.equal(variableNode2.startIndex, 6); + assert.equal(variableNode2.endIndex, 9); + + ([input, edit] = spliceInput(input, input.indexOf('bc'), 0, ' * ')); + assert.equal(input, 'a * bc + cde'); + tree.edit(edit); + + sumNode = tree.rootNode.firstChild.firstChild; + variableNode1 = sumNode.firstChild; + variableNode2 = sumNode.lastChild; + assert.equal(variableNode1.startIndex, 0); + assert.equal(variableNode1.endIndex, 6); + assert.equal(variableNode2.startIndex, 9); + assert.equal(variableNode2.endIndex, 12); + + tree = parser.parse(input, tree); + assert.equal( + tree.rootNode.toString(), + "(program (expression_statement (binary_expression (binary_expression (identifier) (identifier)) (identifier))))" + ); + }); + + it("handles non-ascii characters", () => { + input = 'αβδ + cde'; + + tree = parser.parse(input); + assert.equal( + tree.rootNode.toString(), + "(program (expression_statement (binary_expression (identifier) (identifier))))" + ); + + let variableNode = tree.rootNode.firstChild.firstChild.lastChild; + + ([input, edit] = spliceInput(input, input.indexOf('δ'), 0, '👍 * ')); + assert.equal(input, 'αβ👍 * δ + cde'); + tree.edit(edit); + + variableNode = tree.rootNode.firstChild.firstChild.lastChild; + assert.equal(variableNode.startIndex, input.indexOf('cde')); + + tree = parser.parse(input, tree); + assert.equal( + tree.rootNode.toString(), + "(program (expression_statement (binary_expression (binary_expression (identifier) (identifier)) (identifier))))" + ); + }); + }); +}); + +function spliceInput(input, startIndex, lengthRemoved, newText) { + const oldEndIndex = startIndex + lengthRemoved; + const newEndIndex = startIndex + newText.length; + const startPosition = getExtent(input.slice(0, startIndex)); + const oldEndPosition = getExtent(input.slice(0, oldEndIndex)); + input = input.slice(0, startIndex) + newText + input.slice(oldEndIndex); + const newEndPosition = getExtent(input.slice(0, newEndIndex)); + return [ + input, + { + startIndex, startPosition, + oldEndIndex, oldEndPosition, + newEndIndex, newEndPosition + } + ]; +} + +function getExtent(text) { + let row = 0 + let index; + for (index = 0; index != -1; index = text.indexOf('\n', index)) { + index++ + row++; + } + return {row, column: text.length - index}; +} diff --git a/script/build-wasm b/script/build-wasm new file mode 100755 index 00000000..819c601f --- /dev/null +++ b/script/build-wasm @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +set -e + +export EMCC_FORCE_STDLIBS=libc++ + +args="-Os" +if [[ "$1" == "--debug" ]]; then + args="-s ASSERTIONS=1 SAFE_HEAP=1 -O0" +fi + +mkdir -p target/scratch target/release + +docker run \ + --rm \ + -v $(pwd):/src \ + -u $(id -u) \ + trzeci/emscripten-slim \ + \ + emcc \ + -s WASM=1 \ + -s ALLOW_MEMORY_GROWTH \ + -s MAIN_MODULE=1 \ + -s EXPORT_ALL=1 \ + $args \ + -std=c99 \ + -D 'fprintf(...)=' \ + -I lib/src \ + -I lib/include \ + -I lib/utf8proc \ + --js-library lib/web/imports.js \ + --pre-js lib/web/prefix.js \ + --post-js lib/web/binding.js \ + lib/src/lib.c \ + lib/web/binding.c \ + -o target/scratch/tree-sitter.js + +if [ ! -d lib/web/node_modules/terser ]; then + ( + cd lib/web + npm install + ) +fi + +lib/web/node_modules/.bin/terser \ + --compress \ + --mangle \ + --keep-fnames \ + --keep-classnames \ + -- target/scratch/tree-sitter.js \ + > target/release/tree-sitter.js + +mv target/scratch/tree-sitter.wasm target/release/tree-sitter.wasm diff --git a/script/regenerate-fixtures b/script/generate-fixtures similarity index 52% rename from script/regenerate-fixtures rename to script/generate-fixtures index c47c53f9..6296d8ab 100755 --- a/script/regenerate-fixtures +++ b/script/generate-fixtures @@ -7,27 +7,16 @@ cargo build --release root_dir=$PWD tree_sitter=${root_dir}/target/release/tree-sitter grammars_dir=${root_dir}/test/fixtures/grammars - -grammar_names=( - bash - c - cpp - embedded-template - go - html - javascript - json - python - rust -) +grammar_names=$(ls $grammars_dir) if [[ "$#" > 0 ]]; then grammar_names=($1) fi -for grammar_name in "${grammar_names[@]}"; do +for grammar_name in $grammar_names; do echo "Regenerating ${grammar_name} parser" - cd ${grammars_dir}/${grammar_name} - $tree_sitter generate src/grammar.json - cd $PWD + ( + cd ${grammars_dir}/${grammar_name} + $tree_sitter generate src/grammar.json + ) done diff --git a/script/generate-fixtures-wasm b/script/generate-fixtures-wasm new file mode 100755 index 00000000..cbcf88bd --- /dev/null +++ b/script/generate-fixtures-wasm @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -e + +cargo build --release + +root_dir=$PWD +tree_sitter=${root_dir}/target/release/tree-sitter +grammars_dir=${root_dir}/test/fixtures/grammars +grammar_names=( + c + javascript + python +) + +if [[ "$#" > 0 ]]; then + grammar_names=($1) +fi + +for grammar_name in ${grammar_names[@]}; do + echo "Compiling ${grammar_name} parser to wasm" + $tree_sitter build-wasm ${grammars_dir}/${grammar_name} +done + +mv tree-sitter-*.wasm target/release/ diff --git a/script/regenerate-fixtures.cmd b/script/generate-fixtures.cmd similarity index 100% rename from script/regenerate-fixtures.cmd rename to script/generate-fixtures.cmd diff --git a/script/serve-docs b/script/serve-docs index b7776fd1..f3a80a15 100755 --- a/script/serve-docs +++ b/script/serve-docs @@ -1,4 +1,28 @@ #!/bin/bash +root=$PWD cd docs -bundle exec jekyll serve "$@" + +bundle exec jekyll serve "$@" & + +bundle exec ruby <