Allow building the wasm libs with native emscripten instead of docker

And build them on the mac CI as well as the linux CI

.travis.yml

@@ -5,30 +5,36 @@ rust:
 matrix:
   include:
     - os: osx
+      env: INSTALL_EMSCRIPTEN=1
     - os: linux
       services: docker
-      env: TEST_WASM=1
 
 before_install:
   # Install node
-  - if [ -n "$TEST_WASM" ]; then nvm install 10 && nvm use 10; fi
+  - nvm install 10
+  - nvm use 10
+
+  # Download emscripten and add it to the PATH
+  - if [ -n "$INSTALL_EMSCRIPTEN" ]; then export wasm_env="$(script/fetch-emscripten)"; fi
 
 script:
+  # Build the WASM binding
+  - (eval "$wasm_env" && script/build-wasm)
+
+  # Build the CLI
+  - cargo build --release
+
   # Fetch and regenerate the fixture parsers
   - script/fetch-fixtures
-  - cargo build --release
   - script/generate-fixtures
+  - (eval "$wasm_env" && script/generate-fixtures-wasm)
 
   # Run the tests
   - export TREE_SITTER_STATIC_ANALYSIS=1
   - script/test
+  - script/test-wasm
   - script/benchmark
 
-  # Build and test the WASM binding
-  - if [ -n "$TEST_WASM" ]; then script/build-wasm; fi
-  - if [ -n "$TEST_WASM" ]; then script/generate-fixtures-wasm; fi
-  - if [ -n "$TEST_WASM" ]; then script/test-wasm; fi
-
 branches:
   only:
   - master
@@ -56,4 +62,6 @@ deploy:
 cache:
   cargo: true
   directories:
+    - target/emsdk
     - test/fixtures/grammars
+    - /home/travis/.emscripten_cache

cli/src/main.rs

@@ -93,9 +93,16 @@ fn run() -> error::Result<()> {
         .subcommand(
             SubCommand::with_name("build-wasm")
                 .about("Compile a parser to WASM")
+                .arg(
+                    Arg::with_name("docker")
+                        .long("docker")
+                        .help("Run emscripten via docker even if it is installed locally"),
+                )
                 .arg(Arg::with_name("path").index(1).multiple(true)),
         )
-        .subcommand(SubCommand::with_name("ui").about("Test a parser interactively in the browser"))
+        .subcommand(
+            SubCommand::with_name("web-ui").about("Test a parser interactively in the browser"),
+        )
         .get_matches();
 
     let home_dir = dirs::home_dir().expect("Failed to read home directory");
@@ -245,8 +252,8 @@ fn run() -> error::Result<()> {
         }
     } else if let Some(matches) = matches.subcommand_matches("build-wasm") {
         let grammar_path = current_dir.join(matches.value_of("path").unwrap_or(""));
-        wasm::compile_language_to_wasm(&grammar_path)?;
-    } else if matches.subcommand_matches("ui").is_some() {
+        wasm::compile_language_to_wasm(&grammar_path, matches.is_present("docker"))?;
+    } else if matches.subcommand_matches("web-ui").is_some() {
         web_ui::serve(&current_dir);
     }
 

cli/src/wasm.rs

@@ -22,31 +22,39 @@ pub fn get_grammar_name(src_dir: &Path) -> Result<String> {
     Ok(grammar.name)
 }
 
-pub fn compile_language_to_wasm(language_dir: &Path) -> Result<()> {
+pub fn compile_language_to_wasm(language_dir: &Path, force_docker: bool) -> Result<()> {
     let src_dir = language_dir.join("src");
     let grammar_name = get_grammar_name(&src_dir)?;
     let output_filename = format!("tree-sitter-{}.wasm", grammar_name);
 
-    // Get the current user id so that files created in the docker container will have
-    // the same owner.
-    let user_id_output = Command::new("id")
-        .arg("-u")
-        .output()
-        .map_err(|e| format!("Failed to get get current user id {}", e))?;
-    let user_id = String::from_utf8_lossy(&user_id_output.stdout);
-    let user_id = user_id.trim();
+    let mut command;
+    if !force_docker && Command::new("emcc").output().is_ok() {
+        command = Command::new("emcc");
+        command.current_dir(&language_dir);
+    } else {
+        command = Command::new("docker");
+        command.args(&["run", "--rm"]);
 
-    // Use `emscripten-slim` docker image with the parser directory mounted as a volume.
-    let mut command = Command::new("docker");
-    let mut volume_string = OsString::from(language_dir);
-    volume_string.push(":/src");
-    command.args(&["run", "--rm"]);
-    command.args(&[OsStr::new("--volume"), &volume_string]);
-    command.args(&["--user", user_id, "trzeci/emscripten-slim"]);
+        // Mount the parser directory as a volume
+        let mut volume_string = OsString::from(language_dir);
+        volume_string.push(":/src");
+        command.args(&[OsStr::new("--volume"), &volume_string]);
+
+        // Get the current user id so that files created in the docker container will have
+        // the same owner.
+        let user_id_output = Command::new("id")
+            .arg("-u")
+            .output()
+            .map_err(|e| format!("Failed to get get current user id {}", e))?;
+        let user_id = String::from_utf8_lossy(&user_id_output.stdout);
+        let user_id = user_id.trim();
+        command.args(&["--user", user_id]);
+
+        // Run `emcc` in a container using the `emscripten-slim` image
+        command.args(&["trzeci/emscripten-slim", "emcc"]);
+    }
 
-    // Run emscripten in the container
     command.args(&[
-        "emcc",
         "-o",
         &output_filename,
         "-Os",
@@ -89,7 +97,7 @@ pub fn compile_language_to_wasm(language_dir: &Path) -> Result<()> {
 
     let output = command
         .output()
-        .map_err(|e| format!("Failed to run docker emcc command - {}", e))?;
+        .map_err(|e| format!("Failed to run emcc command - {}", e))?;
     if !output.status.success() {
         return Err(Error::from(format!(
             "emcc command failed - {}",

cli/src/web_ui.html

@@ -1,19 +1,25 @@
 <head>
-  <title>Tree-sitter</title>
+  <title>tree-sitter THE_LANGUAGE_NAME</title>
   <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.45.0/codemirror.min.css">
   <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/clusterize.js/0.18.0/clusterize.min.css">
+  <link rel="icon" type="image/png" href="http://tree-sitter.github.io/tree-sitter/assets/images/favicon-32x32.png" sizes="32x32" />
+  <link rel="icon" type="image/png" href="http://tree-sitter.github.io/tree-sitter/assets/images/favicon-16x16.png" sizes="16x16" />
 </head>
 
 <body>
   <div id="playground-container">
     <header>
       <div class=header-item>
-        <label for="logging-checkbox">Log</label>
+        <bold>THE_LANGUAGE_NAME</bold>
+      </div>
+
+      <div class=header-item>
+        <label for="logging-checkbox">log</label>
         <input id="logging-checkbox" type="checkbox"></input>
       </div>
 
       <div class=header-item>
-        <label for="update-time">Update time: </label>
+        <label for="update-time">parse time: </label>
         <span id="update-time"></span>
       </div>
     </header>
@@ -44,6 +50,10 @@
   <script src=playground.js></script>
 
   <style>
+    body {
+      font: Sans Serif;
+    }
+
     #playground-container {
       position: absolute;
       top: 0;
@@ -69,7 +79,7 @@
     }
 
     .header-item {
-      margin-right: 20px;
+      margin-right: 30px;
     }
 
     .CodeMirror {
@@ -90,21 +100,28 @@
       margin: 0;
     }
 
-    h4, select, .field {
-      display: inline-block;
-      margin-right: 20px;
-    }
-
     #logging-checkbox {
-      height: 15px;
+      vertical-align: middle;
     }
 
     .CodeMirror div.CodeMirror-cursor {
       border-left: 3px solid red;
     }
 
+    a {
+      text-decoration: none;
+      color: #040404;
+      padding: 2px;
+    }
+
+    a:hover {
+      text-decoration: underline;
+    }
+
     a.highlighted {
-      background-color: #ddd;
+      background-color: #d9d9d9;
+      color: red;
+      border-radius: 3px;
       text-decoration: underline;
     }
   </style>

cli/src/web_ui.rs

@@ -9,7 +9,7 @@ use webbrowser;
 const PLAYGROUND_JS: &'static [u8] = include_bytes!("../../docs/assets/js/playground.js");
 const LIB_JS: &'static [u8] = include_bytes!("../../lib/binding_web/tree-sitter.js");
 const LIB_WASM: &'static [u8] = include_bytes!("../../lib/binding_web/tree-sitter.wasm");
-const HTML: &'static [u8] = include_bytes!("./web_ui.html");
+const HTML: &'static str = include_str!("./web_ui.html");
 
 pub fn serve(grammar_path: &Path) {
     let port = get_available_port().expect("Couldn't find an available port");
@@ -18,19 +18,28 @@ pub fn serve(grammar_path: &Path) {
     let grammar_name = wasm::get_grammar_name(&grammar_path.join("src"))
         .map_err(|e| format!("Failed to get wasm filename: {:?}", e))
         .unwrap();
-    let language_wasm = fs::read(format!("./tree-sitter-{}.wasm", grammar_name)).unwrap();
+    let wasm_filename = format!("tree-sitter-{}.wasm", grammar_name);
+    let language_wasm = fs::read(grammar_path.join(&wasm_filename))
+        .map_err(|_| {
+            format!(
+                "Failed to read '{}'. Run `tree-sitter build-wasm` first.",
+                wasm_filename
+            )
+        })
+        .unwrap();
 
     webbrowser::open(&format!("http://127.0.0.1:{}", port))
         .map_err(|e| format!("Failed to open '{}' in a web browser. Error: {}", url, e))
         .unwrap();
 
+    let html = HTML.replace("THE_LANGUAGE_NAME", &grammar_name);
     let html_header = Header::from_str("Content-type: text/html").unwrap();
     let js_header = Header::from_str("Content-type: application/javascript").unwrap();
     let wasm_header = Header::from_str("Content-type: application/wasm").unwrap();
 
     for request in server.incoming_requests() {
         let (body, header) = match request.url() {
-            "/" => (HTML, &html_header),
+            "/" => (html.as_ref(), &html_header),
             "/playground.js" => (PLAYGROUND_JS, &js_header),
             "/tree-sitter.js" => (LIB_JS, &js_header),
             "/tree-sitter.wasm" => (LIB_WASM, &wasm_header),

script/build-wasm

@@ -1,12 +1,10 @@
 #!/usr/bin/env bash
 
-set -e
-
-if [[ "$1" == "--help" || "$1" == "-h" ]]; then
+usage() {
   cat <<EOF
 USAGE
 
-  $0 [--debug]
+  $0 [--help] [--debug] [--docker]
 
 SUMMARY
 
@@ -15,57 +13,97 @@ SUMMARY
 
 REQUIREMENTS
 
-  You must have the \`docker\` command on your PATH for this to work.
+  You must have either the \`emcc\` command or the \`docker\` command
+  on your PATH for this to work.
 
 OPTIONS
 
-  --debug: Compile the library more quickly, with fewer optimizations and more runtime assertions.
+  --help:   Display this message.
+  --debug:  Compile the library more quickly, with fewer optimizations
+            and more runtime assertions.
+  --docker: Run emscripten using docker, even if \`emcc\` is installed.
+            By default, \`emcc\` will be run directly when available.
 
 EOF
-  exit 0
-fi
+}
+
+set -e
 
 web_dir=lib/binding_web
 exports=$(cat ${web_dir}/exports.json)
-args="-Os"
-minify=1
+emscripten_flags="-Os"
+minify_js=1
+force_docker=0
 
-if [[ "$1" == "--debug" ]]; then
-  minify=0
-  args="-s ASSERTIONS=1 -s SAFE_HEAP=1 -Os"
+while [[ $# > 0 ]]; do
+  case "$1" in
+    --debug)
+      minify_js=0
+      emscripten_flags="-s ASSERTIONS=1 -s SAFE_HEAP=1 -Os"
+      ;;
+
+    --help)
+      usage
+      exit 0
+      ;;
+
+    --docker)
+      force_docker=1
+      ;;
+
+    *)
+      usage
+      echo "Unrecognized argument '$1'"
+      exit 1
+      ;;
+  esac
+  shift
+done
+
+emcc=
+if which emcc > /dev/null && [[ "$force_docker" == "0" ]]; then
+  export EMCC_FORCE_STDLIBS=libc++
+  emcc=emcc
+elif which docker > /dev/null; then
+  emcc="docker run               \
+    --rm                         \
+    -v $(pwd):/src               \
+    -u $(id -u)                  \
+    -e EMCC_FORCE_STDLIBS=libc++ \
+    trzeci/emscripten-slim       \
+    emcc"
+else
+  echo 'You must have either `docker` or `emcc` on your PATH to run this script'
+  exit 1
 fi
 
 mkdir -p target/scratch
 
-docker run                          \
-  --rm                              \
-  -v $(pwd):/src                    \
-  -u $(id -u)                       \
-  -e EMCC_FORCE_STDLIBS=libc++      \
-  trzeci/emscripten-slim            \
-                                    \
-  emcc                              \
-  -s WASM=1                         \
-  -s TOTAL_MEMORY=33554432          \
-  -s ALLOW_MEMORY_GROWTH            \
-  -s MAIN_MODULE=2                  \
-  -s NO_FILESYSTEM=1                \
+# Use emscripten to generate `tree-sitter.js` and `tree-sitter.wasm`
+# in the `target/scratch` directory
+$emcc                                \
+  -s WASM=1                          \
+  -s TOTAL_MEMORY=33554432           \
+  -s ALLOW_MEMORY_GROWTH             \
+  -s MAIN_MODULE=2                   \
+  -s NO_FILESYSTEM=1                 \
   -s "EXPORTED_FUNCTIONS=${exports}" \
-  $args                             \
-  -std=c99                          \
-  -D 'fprintf(...)='                \
-  -I lib/src                        \
-  -I lib/include                    \
-  -I lib/utf8proc                   \
-  --js-library ${web_dir}/imports.js   \
-  --pre-js ${web_dir}/prefix.js        \
-  --post-js ${web_dir}/binding.js      \
-  lib/src/lib.c                     \
-  ${web_dir}/binding.c                 \
+  $emscripten_flags                  \
+  -std=c99                           \
+  -D 'fprintf(...)='                 \
+  -I lib/src                         \
+  -I lib/include                     \
+  -I lib/utf8proc                    \
+  --js-library ${web_dir}/imports.js \
+  --pre-js ${web_dir}/prefix.js      \
+  --post-js ${web_dir}/binding.js    \
+  lib/src/lib.c                      \
+  ${web_dir}/binding.c               \
   -o target/scratch/tree-sitter.js
 
-
-if [[ "$minify" == "1" ]]; then
+# Use terser to write a minified version of `tree-sitter.js` into
+# the `lib/binding_web` directory.
+if [[ "$minify_js" == "1" ]]; then
   if [ ! -d ${web_dir}/node_modules/terser ]; then
     (
       cd ${web_dir}

script/fetch-emscripten

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -e
+
+mkdir -p target
+EMSDK_DIR="./target/emsdk"
+
+(
+  if [ ! -f "$EMSDK_DIR/emsdk" ]; then
+    echo 'Downloading emscripten SDK...'
+    git clone https://github.com/emscripten-core/emsdk.git $EMSDK_DIR
+  fi
+
+  cd $EMSDK_DIR
+
+  echo 'Updating emscripten SDK...'
+  git pull
+  ./emsdk list
+
+  echo 'Installing latest emscripten...'
+  ./emsdk install latest
+
+  echo 'Activating latest emscripten...'
+  ./emsdk activate latest
+) >&2
+
+(
+  source "$EMSDK_DIR/emsdk_env.sh" > /dev/null
+  declare -px
+)