From a2d6048226ceb1b09a7e4cf330d75ced0d3f27a3 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 30 Nov 2020 14:28:27 -0800 Subject: [PATCH 1/7] Get the wasm build working w/ latest emscripten --- lib/binding_web/binding.c | 27 +++++++++++++++++---------- lib/binding_web/binding.js | 2 +- lib/binding_web/imports.js | 2 +- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/lib/binding_web/binding.c b/lib/binding_web/binding.c index eb463b26..9180f405 100644 --- a/lib/binding_web/binding.c +++ b/lib/binding_web/binding.c @@ -115,18 +115,10 @@ extern void tree_sitter_parse_callback( ); extern void tree_sitter_log_callback( - void *payload, - TSLogType log_type, + bool is_lex_message, const char *message ); -void ts_parser_new_wasm() { - TSParser *parser = ts_parser_new(); - char *input_buffer = calloc(INPUT_BUFFER_SIZE, sizeof(char)); - TRANSFER_BUFFER[0] = parser; - TRANSFER_BUFFER[1] = input_buffer; -} - static const char *call_parse_callback( void *payload, uint32_t byte, @@ -148,8 +140,23 @@ static const char *call_parse_callback( return buffer; } +static void call_log_callback( + void *payload, + TSLogType log_type, + const char *message +) { + tree_sitter_log_callback(log_type == TSLogTypeLex, message); +} + +void ts_parser_new_wasm() { + TSParser *parser = ts_parser_new(); + char *input_buffer = calloc(INPUT_BUFFER_SIZE, sizeof(char)); + TRANSFER_BUFFER[0] = parser; + TRANSFER_BUFFER[1] = input_buffer; +} + void ts_parser_enable_logger_wasm(TSParser *self, bool should_log) { - TSLogger logger = {self, should_log ? tree_sitter_log_callback : NULL}; + TSLogger logger = {self, should_log ? call_log_callback : NULL}; ts_parser_set_logger(self, logger); } diff --git a/lib/binding_web/binding.js b/lib/binding_web/binding.js index 15b07116..95bfa828 100644 --- a/lib/binding_web/binding.js +++ b/lib/binding_web/binding.js @@ -880,7 +880,7 @@ class Language { } return bytes - .then(bytes => loadWebAssemblyModule(bytes, {loadAsync: true})) + .then(bytes => loadSideModule(bytes, {loadAsync: true})) .then(mod => { const symbolNames = Object.keys(mod) const functionName = symbolNames.find(key => diff --git a/lib/binding_web/imports.js b/lib/binding_web/imports.js index ea34926f..a76c42ac 100644 --- a/lib/binding_web/imports.js +++ b/lib/binding_web/imports.js @@ -16,7 +16,7 @@ mergeInto(LibraryManager.library, { } }, - tree_sitter_log_callback: function(_payload, isLexMessage, messageAddress) { + tree_sitter_log_callback: function(isLexMessage, messageAddress) { if (currentLogCallback) { const message = UTF8ToString(messageAddress); currentLogCallback(message, isLexMessage !== 0); From b118e7d7505d5f5621b7cf269a1b155e6f20588e Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 30 Nov 2020 15:28:21 -0800 Subject: [PATCH 2/7] Make binding.js syntactically valid Put the end of the surrounding closure into a separate file, suffix.js. --- lib/binding_web/binding.js | 5 ----- lib/binding_web/suffix.js | 2 ++ script/build-wasm | 1 + 3 files changed, 3 insertions(+), 5 deletions(-) create mode 100644 lib/binding_web/suffix.js diff --git a/lib/binding_web/binding.js b/lib/binding_web/binding.js index 95bfa828..b060715f 100644 --- a/lib/binding_web/binding.js +++ b/lib/binding_web/binding.js @@ -7,7 +7,6 @@ const SIZE_OF_RANGE = 2 * SIZE_OF_INT + 2 * SIZE_OF_POINT; const ZERO_POINT = {row: 0, column: 0}; const QUERY_WORD_REGEX = /[\w-.]*/g; -const PREDICATE_STEP_TYPE_DONE = 0; const PREDICATE_STEP_TYPE_CAPTURE = 1; const PREDICATE_STEP_TYPE_STRING = 2; @@ -1140,7 +1139,3 @@ function marshalEdit(edit) { } Parser.Language = Language; - -return Parser; - -})); diff --git a/lib/binding_web/suffix.js b/lib/binding_web/suffix.js new file mode 100644 index 00000000..0e9fe021 --- /dev/null +++ b/lib/binding_web/suffix.js @@ -0,0 +1,2 @@ +return Parser; +})); diff --git a/script/build-wasm b/script/build-wasm index 63ec4fe0..b139f6c2 100755 --- a/script/build-wasm +++ b/script/build-wasm @@ -95,6 +95,7 @@ $emcc \ --js-library ${web_dir}/imports.js \ --pre-js ${web_dir}/prefix.js \ --post-js ${web_dir}/binding.js \ + --post-js ${web_dir}/suffix.js \ lib/src/lib.c \ ${web_dir}/binding.c \ -o target/scratch/tree-sitter.js From 751ffd2ee13ef7b29de60585fec3a52dab1f5b4e Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 30 Nov 2020 16:25:01 -0800 Subject: [PATCH 3/7] Use new emscripten when building with docker --- cli/src/wasm.rs | 2 +- script/build-wasm | 10 +++++----- script/generate-fixtures-wasm | 8 +++++++- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/cli/src/wasm.rs b/cli/src/wasm.rs index 47cea90a..8bbcfbdf 100644 --- a/cli/src/wasm.rs +++ b/cli/src/wasm.rs @@ -57,7 +57,7 @@ pub fn compile_language_to_wasm(language_dir: &Path, force_docker: bool) -> Resu } // Run `emcc` in a container using the `emscripten-slim` image - command.args(&["trzeci/emscripten-slim", "emcc"]); + command.args(&["emscripten/emsdk", "emcc"]); } else { return Error::err( "You must have either emcc or docker on your PATH to run this command".to_string(), diff --git a/script/build-wasm b/script/build-wasm index b139f6c2..75c6a7d1 100755 --- a/script/build-wasm +++ b/script/build-wasm @@ -64,11 +64,11 @@ emcc= if which emcc > /dev/null && [[ "$force_docker" == "0" ]]; then emcc=emcc elif which docker > /dev/null; then - emcc="docker run \ - --rm \ - -v $(pwd):/src:Z \ - -u $(id -u) \ - trzeci/emscripten-slim \ + emcc="docker run \ + --rm \ + -v $(pwd):/src:Z \ + -u $(id -u) \ + emscripten/emsdk \ emcc" else echo 'You must have either `docker` or `emcc` on your PATH to run this script' diff --git a/script/generate-fixtures-wasm b/script/generate-fixtures-wasm index a987e31a..9d44b58c 100755 --- a/script/generate-fixtures-wasm +++ b/script/generate-fixtures-wasm @@ -4,6 +4,12 @@ set -e cargo build --release +build_wasm_args= +if [[ $1 == "--docker" ]]; then + build_wasm_args="--docker" + shift +fi + filter_grammar_name=$1 root_dir=$PWD @@ -20,7 +26,7 @@ while read -r grammar_file; do fi echo "Compiling ${grammar_name} parser to wasm" - "$tree_sitter" build-wasm $grammar_dir + "$tree_sitter" build-wasm $build_wasm_args $grammar_dir done <<< "$grammar_files" mv tree-sitter-*.wasm target/release/ From 18980b7b99757e4ffa262a49501ae07ad7a8d986 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 30 Nov 2020 16:25:17 -0800 Subject: [PATCH 4/7] wasm: Avoid registering uncaught exception/rejection handlers --- script/build-wasm | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/script/build-wasm b/script/build-wasm index 75c6a7d1..1b5e48ec 100755 --- a/script/build-wasm +++ b/script/build-wasm @@ -30,7 +30,6 @@ EOF set -e web_dir=lib/binding_web -exports=$(cat ${web_dir}/exports.json) emscripten_flags="-O3" minify_js=1 force_docker=0 @@ -79,25 +78,27 @@ mkdir -p target/scratch # Use emscripten to generate `tree-sitter.js` and `tree-sitter.wasm` # in the `target/scratch` directory -$emcc \ - -s WASM=1 \ - -s TOTAL_MEMORY=33554432 \ - -s ALLOW_MEMORY_GROWTH=1 \ - -s MAIN_MODULE=2 \ - -s NO_FILESYSTEM=1 \ - -s "EXPORTED_FUNCTIONS=${exports}" \ - $emscripten_flags \ - -std=c99 \ - -D 'fprintf(...)=' \ - -D NDEBUG= \ - -I lib/src \ - -I lib/include \ - --js-library ${web_dir}/imports.js \ - --pre-js ${web_dir}/prefix.js \ - --post-js ${web_dir}/binding.js \ - --post-js ${web_dir}/suffix.js \ - lib/src/lib.c \ - ${web_dir}/binding.c \ +$emcc \ + -s WASM=1 \ + -s TOTAL_MEMORY=33554432 \ + -s ALLOW_MEMORY_GROWTH=1 \ + -s MAIN_MODULE=2 \ + -s NO_FILESYSTEM=1 \ + -s NODEJS_CATCH_EXIT=0 \ + -s NODEJS_CATCH_REJECTION=0 \ + -s EXPORTED_FUNCTIONS=@${web_dir}/exports.json \ + $emscripten_flags \ + -std=c99 \ + -D 'fprintf(...)=' \ + -D NDEBUG= \ + -I lib/src \ + -I lib/include \ + --js-library ${web_dir}/imports.js \ + --pre-js ${web_dir}/prefix.js \ + --post-js ${web_dir}/binding.js \ + --post-js ${web_dir}/suffix.js \ + lib/src/lib.c \ + ${web_dir}/binding.c \ -o target/scratch/tree-sitter.js # Use terser to write a minified version of `tree-sitter.js` into From 2699c01ab1c588da81f9d86c97488876c0a0b6c4 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 30 Nov 2020 16:45:45 -0800 Subject: [PATCH 5/7] Use latest emscripten on CI --- script/fetch-emscripten | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/fetch-emscripten b/script/fetch-emscripten index d16c857e..c1b072ad 100755 --- a/script/fetch-emscripten +++ b/script/fetch-emscripten @@ -2,7 +2,7 @@ set -e -EMSCRIPTEN_VERSION=1.39.15 +EMSCRIPTEN_VERSION=2.0.9 mkdir -p target EMSDK_DIR="./target/emsdk" From 591a2c62495ca81109e061846a7a6eb8e66ecfac Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 30 Nov 2020 16:46:02 -0800 Subject: [PATCH 6/7] Remove web binding paths from travis config --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7205ae03..79d84d13 100644 --- a/.travis.yml +++ b/.travis.yml @@ -58,8 +58,6 @@ deploy: file_glob: true file: - "tree-sitter-*.gz" - - "target/release/tree-sitter.js" - - "target/release/tree-sitter.wasm" draft: true overwrite: true skip_cleanup: true From d3f30e298b9caed7058c95e9535f5a33a4be6648 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 30 Nov 2020 20:36:36 -0800 Subject: [PATCH 7/7] Use node 12 on travis --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 79d84d13..282ba02d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,8 +14,8 @@ matrix: before_install: # Install node - - nvm install 10 - - nvm use 10 + - nvm install 12 + - nvm use 12 # Download emscripten and create a shorthand for adding it to the PATH. # Don't add it to the path globally because it overrides the default