From ad43b211f4930404563a3ec65ba1a75e0948c9fc Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 14 May 2019 11:12:56 -0700 Subject: [PATCH] Allow building the wasm libs with native emscripten instead of docker And build them on the mac CI as well as the linux CI --- .travis.yml | 24 ++++++--- cli/src/main.rs | 13 +++-- cli/src/wasm.rs | 46 +++++++++------- cli/src/web_ui.html | 39 ++++++++++---- cli/src/web_ui.rs | 15 ++++-- script/build-wasm | 116 ++++++++++++++++++++++++++-------------- script/fetch-emscripten | 30 +++++++++++ 7 files changed, 200 insertions(+), 83 deletions(-) create mode 100755 script/fetch-emscripten diff --git a/.travis.yml b/.travis.yml index 9fa65759..fb55014b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,30 +5,36 @@ rust: matrix: include: - os: osx + env: INSTALL_EMSCRIPTEN=1 - os: linux services: docker - env: TEST_WASM=1 before_install: # Install node - - if [ -n "$TEST_WASM" ]; then nvm install 10 && nvm use 10; fi + - nvm install 10 + - nvm use 10 + + # Download emscripten and add it to the PATH + - if [ -n "$INSTALL_EMSCRIPTEN" ]; then export wasm_env="$(script/fetch-emscripten)"; fi script: + # Build the WASM binding + - (eval "$wasm_env" && script/build-wasm) + + # Build the CLI + - cargo build --release + # Fetch and regenerate the fixture parsers - script/fetch-fixtures - - cargo build --release - script/generate-fixtures + - (eval "$wasm_env" && script/generate-fixtures-wasm) # Run the tests - export TREE_SITTER_STATIC_ANALYSIS=1 - script/test + - script/test-wasm - script/benchmark - # Build and test the WASM binding - - if [ -n "$TEST_WASM" ]; then script/build-wasm; fi - - if [ -n "$TEST_WASM" ]; then script/generate-fixtures-wasm; fi - - if [ -n "$TEST_WASM" ]; then script/test-wasm; fi - branches: only: - master @@ -56,4 +62,6 @@ deploy: cache: cargo: true directories: + - target/emsdk - test/fixtures/grammars + - /home/travis/.emscripten_cache diff --git a/cli/src/main.rs b/cli/src/main.rs index 78f79cc1..e98a1a00 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -93,9 +93,16 @@ fn run() -> error::Result<()> { .subcommand( SubCommand::with_name("build-wasm") .about("Compile a parser to WASM") + .arg( + Arg::with_name("docker") + .long("docker") + .help("Run emscripten via docker even if it is installed locally"), + ) .arg(Arg::with_name("path").index(1).multiple(true)), ) - .subcommand(SubCommand::with_name("ui").about("Test a parser interactively in the browser")) + .subcommand( + SubCommand::with_name("web-ui").about("Test a parser interactively in the browser"), + ) .get_matches(); let home_dir = dirs::home_dir().expect("Failed to read home directory"); @@ -245,8 +252,8 @@ fn run() -> error::Result<()> { } } else if let Some(matches) = matches.subcommand_matches("build-wasm") { let grammar_path = current_dir.join(matches.value_of("path").unwrap_or("")); - wasm::compile_language_to_wasm(&grammar_path)?; - } else if matches.subcommand_matches("ui").is_some() { + wasm::compile_language_to_wasm(&grammar_path, matches.is_present("docker"))?; + } else if matches.subcommand_matches("web-ui").is_some() { web_ui::serve(¤t_dir); } diff --git a/cli/src/wasm.rs b/cli/src/wasm.rs index 3fa38c65..61e608b5 100644 --- a/cli/src/wasm.rs +++ b/cli/src/wasm.rs @@ -22,31 +22,39 @@ pub fn get_grammar_name(src_dir: &Path) -> Result { Ok(grammar.name) } -pub fn compile_language_to_wasm(language_dir: &Path) -> Result<()> { +pub fn compile_language_to_wasm(language_dir: &Path, force_docker: bool) -> Result<()> { let src_dir = language_dir.join("src"); let grammar_name = get_grammar_name(&src_dir)?; let output_filename = format!("tree-sitter-{}.wasm", grammar_name); - // Get the current user id so that files created in the docker container will have - // the same owner. - let user_id_output = Command::new("id") - .arg("-u") - .output() - .map_err(|e| format!("Failed to get get current user id {}", e))?; - let user_id = String::from_utf8_lossy(&user_id_output.stdout); - let user_id = user_id.trim(); + let mut command; + if !force_docker && Command::new("emcc").output().is_ok() { + command = Command::new("emcc"); + command.current_dir(&language_dir); + } else { + command = Command::new("docker"); + command.args(&["run", "--rm"]); - // Use `emscripten-slim` docker image with the parser directory mounted as a volume. - let mut command = Command::new("docker"); - let mut volume_string = OsString::from(language_dir); - volume_string.push(":/src"); - command.args(&["run", "--rm"]); - command.args(&[OsStr::new("--volume"), &volume_string]); - command.args(&["--user", user_id, "trzeci/emscripten-slim"]); + // Mount the parser directory as a volume + let mut volume_string = OsString::from(language_dir); + volume_string.push(":/src"); + command.args(&[OsStr::new("--volume"), &volume_string]); + + // Get the current user id so that files created in the docker container will have + // the same owner. + let user_id_output = Command::new("id") + .arg("-u") + .output() + .map_err(|e| format!("Failed to get get current user id {}", e))?; + let user_id = String::from_utf8_lossy(&user_id_output.stdout); + let user_id = user_id.trim(); + command.args(&["--user", user_id]); + + // Run `emcc` in a container using the `emscripten-slim` image + command.args(&["trzeci/emscripten-slim", "emcc"]); + } - // Run emscripten in the container command.args(&[ - "emcc", "-o", &output_filename, "-Os", @@ -89,7 +97,7 @@ pub fn compile_language_to_wasm(language_dir: &Path) -> Result<()> { let output = command .output() - .map_err(|e| format!("Failed to run docker emcc command - {}", e))?; + .map_err(|e| format!("Failed to run emcc command - {}", e))?; if !output.status.success() { return Err(Error::from(format!( "emcc command failed - {}", diff --git a/cli/src/web_ui.html b/cli/src/web_ui.html index 58ab0e7f..2422a3d8 100644 --- a/cli/src/web_ui.html +++ b/cli/src/web_ui.html @@ -1,19 +1,25 @@ - Tree-sitter + tree-sitter THE_LANGUAGE_NAME + +
- + THE_LANGUAGE_NAME +
+ +
+
- +
@@ -44,6 +50,10 @@ diff --git a/cli/src/web_ui.rs b/cli/src/web_ui.rs index 2ea0b4ab..4870b3ea 100644 --- a/cli/src/web_ui.rs +++ b/cli/src/web_ui.rs @@ -9,7 +9,7 @@ use webbrowser; const PLAYGROUND_JS: &'static [u8] = include_bytes!("../../docs/assets/js/playground.js"); const LIB_JS: &'static [u8] = include_bytes!("../../lib/binding_web/tree-sitter.js"); const LIB_WASM: &'static [u8] = include_bytes!("../../lib/binding_web/tree-sitter.wasm"); -const HTML: &'static [u8] = include_bytes!("./web_ui.html"); +const HTML: &'static str = include_str!("./web_ui.html"); pub fn serve(grammar_path: &Path) { let port = get_available_port().expect("Couldn't find an available port"); @@ -18,19 +18,28 @@ pub fn serve(grammar_path: &Path) { let grammar_name = wasm::get_grammar_name(&grammar_path.join("src")) .map_err(|e| format!("Failed to get wasm filename: {:?}", e)) .unwrap(); - let language_wasm = fs::read(format!("./tree-sitter-{}.wasm", grammar_name)).unwrap(); + let wasm_filename = format!("tree-sitter-{}.wasm", grammar_name); + let language_wasm = fs::read(grammar_path.join(&wasm_filename)) + .map_err(|_| { + format!( + "Failed to read '{}'. Run `tree-sitter build-wasm` first.", + wasm_filename + ) + }) + .unwrap(); webbrowser::open(&format!("http://127.0.0.1:{}", port)) .map_err(|e| format!("Failed to open '{}' in a web browser. Error: {}", url, e)) .unwrap(); + let html = HTML.replace("THE_LANGUAGE_NAME", &grammar_name); let html_header = Header::from_str("Content-type: text/html").unwrap(); let js_header = Header::from_str("Content-type: application/javascript").unwrap(); let wasm_header = Header::from_str("Content-type: application/wasm").unwrap(); for request in server.incoming_requests() { let (body, header) = match request.url() { - "/" => (HTML, &html_header), + "/" => (html.as_ref(), &html_header), "/playground.js" => (PLAYGROUND_JS, &js_header), "/tree-sitter.js" => (LIB_JS, &js_header), "/tree-sitter.wasm" => (LIB_WASM, &wasm_header), diff --git a/script/build-wasm b/script/build-wasm index 300a1141..f84086bb 100755 --- a/script/build-wasm +++ b/script/build-wasm @@ -1,12 +1,10 @@ #!/usr/bin/env bash -set -e - -if [[ "$1" == "--help" || "$1" == "-h" ]]; then +usage() { cat < 0 ]]; do + case "$1" in + --debug) + minify_js=0 + emscripten_flags="-s ASSERTIONS=1 -s SAFE_HEAP=1 -Os" + ;; + + --help) + usage + exit 0 + ;; + + --docker) + force_docker=1 + ;; + + *) + usage + echo "Unrecognized argument '$1'" + exit 1 + ;; + esac + shift +done + +emcc= +if which emcc > /dev/null && [[ "$force_docker" == "0" ]]; then + export EMCC_FORCE_STDLIBS=libc++ + emcc=emcc +elif which docker > /dev/null; then + emcc="docker run \ + --rm \ + -v $(pwd):/src \ + -u $(id -u) \ + -e EMCC_FORCE_STDLIBS=libc++ \ + trzeci/emscripten-slim \ + emcc" +else + echo 'You must have either `docker` or `emcc` on your PATH to run this script' + exit 1 fi mkdir -p target/scratch -docker run \ - --rm \ - -v $(pwd):/src \ - -u $(id -u) \ - -e EMCC_FORCE_STDLIBS=libc++ \ - trzeci/emscripten-slim \ - \ - emcc \ - -s WASM=1 \ - -s TOTAL_MEMORY=33554432 \ - -s ALLOW_MEMORY_GROWTH \ - -s MAIN_MODULE=2 \ - -s NO_FILESYSTEM=1 \ +# Use emscripten to generate `tree-sitter.js` and `tree-sitter.wasm` +# in the `target/scratch` directory +$emcc \ + -s WASM=1 \ + -s TOTAL_MEMORY=33554432 \ + -s ALLOW_MEMORY_GROWTH \ + -s MAIN_MODULE=2 \ + -s NO_FILESYSTEM=1 \ -s "EXPORTED_FUNCTIONS=${exports}" \ - $args \ - -std=c99 \ - -D 'fprintf(...)=' \ - -I lib/src \ - -I lib/include \ - -I lib/utf8proc \ - --js-library ${web_dir}/imports.js \ - --pre-js ${web_dir}/prefix.js \ - --post-js ${web_dir}/binding.js \ - lib/src/lib.c \ - ${web_dir}/binding.c \ + $emscripten_flags \ + -std=c99 \ + -D 'fprintf(...)=' \ + -I lib/src \ + -I lib/include \ + -I lib/utf8proc \ + --js-library ${web_dir}/imports.js \ + --pre-js ${web_dir}/prefix.js \ + --post-js ${web_dir}/binding.js \ + lib/src/lib.c \ + ${web_dir}/binding.c \ -o target/scratch/tree-sitter.js - -if [[ "$minify" == "1" ]]; then +# Use terser to write a minified version of `tree-sitter.js` into +# the `lib/binding_web` directory. +if [[ "$minify_js" == "1" ]]; then if [ ! -d ${web_dir}/node_modules/terser ]; then ( cd ${web_dir} diff --git a/script/fetch-emscripten b/script/fetch-emscripten new file mode 100755 index 00000000..0af6dea7 --- /dev/null +++ b/script/fetch-emscripten @@ -0,0 +1,30 @@ +#!/bin/bash + +set -e + +mkdir -p target +EMSDK_DIR="./target/emsdk" + +( + if [ ! -f "$EMSDK_DIR/emsdk" ]; then + echo 'Downloading emscripten SDK...' + git clone https://github.com/emscripten-core/emsdk.git $EMSDK_DIR + fi + + cd $EMSDK_DIR + + echo 'Updating emscripten SDK...' + git pull + ./emsdk list + + echo 'Installing latest emscripten...' + ./emsdk install latest + + echo 'Activating latest emscripten...' + ./emsdk activate latest +) >&2 + +( + source "$EMSDK_DIR/emsdk_env.sh" > /dev/null + declare -px +)