Build and test wasm on CI

.appveyor.yml

@@ -22,7 +22,7 @@ test_script:
   # Fetch and regenerate the fixture parsers
   - script\fetch-fixtures.cmd
   - cargo build --release
-  - script\regenerate-fixtures.cmd
+  - script\generate-fixtures.cmd
 
   # Run tests
   - script\test.cmd

.gitignore

@@ -7,6 +7,8 @@ fuzz-results
 
 test/fixtures/grammars/*
 !test/fixtures/grammars/.gitkeep
+package-lock.json
+node_modules
 
 docs/assets/js/tree-sitter.js
 

.travis.yml

@@ -2,21 +2,33 @@ language: rust
 rust:
   - stable
 
-os:
-  - linux
-  - osx
+matrix:
+  include:
+    - os: osx
+    - os: linux
+      services: docker
+      env: TEST_WASM=1
+
+before_install:
+  # Install node
+  - if [ -n "$TEST_WASM" ]; then nvm install 10 && nvm use 10; fi
 
 script:
   # Fetch and regenerate the fixture parsers
   - script/fetch-fixtures
   - cargo build --release
-  - script/regenerate-fixtures
+  - script/generate-fixtures
 
-  # Run tests
+  # Run the tests
   - export TREE_SITTER_STATIC_ANALYSIS=1
   - script/test
   - script/benchmark
 
+  # Build and test the WASM binding
+  - if [ -n "$TEST_WASM" ]; then script/build-wasm; fi
+  - if [ -n "$TEST_WASM" ]; then script/generate-fixtures-wasm; fi
+  - if [ -n "$TEST_WASM" ]; then script/test-wasm; fi
+
 branches:
   only:
   - master
@@ -31,7 +43,10 @@ deploy:
   api_key:
     secure: "cAd2mQP+Q55v3zedo5ZyOVc3hq3XKMW93lp5LuXV6CYKYbIhkyfym4qfs+C9GJQiIP27cnePYM7B3+OMIFwSPIgXHWWSsuloMtDgYSc/PAwb2dZnJqAyog3BohW/QiGTSnvbVlxPF6P9RMQU6+JP0HJzEJy6QBTa4Und/j0jm24="
   file_glob: true
-  file: "tree-sitter-*.gz"
+  file:
+    - "tree-sitter-*.gz"
+    - "target/release/tree-sitter.js"
+    - "target/release/tree-sitter.wasm"
   draft: true
   overwrite: true
   skip_cleanup: true

cli/npm/package-lock.json

@@ -1,5 +0,0 @@
-{
-  "name": "tree-sitter-cli",
-  "version": "0.15.1",
-  "lockfileVersion": 1
-}

cli/src/wasm.rs

@@ -1,11 +1,14 @@
 use super::error::{Error, Result};
 use super::generate::parse_grammar::GrammarJSON;
+use std::ffi::{OsStr, OsString};
 use std::fs;
 use std::path::Path;
 use std::process::Command;
 
 pub fn compile_language_to_wasm(language_dir: &Path) -> Result<()> {
     let src_dir = language_dir.join("src");
+
+    // Parse the grammar.json to find out the language name.
     let grammar_json_path = src_dir.join("grammar.json");
     let grammar_json = fs::read_to_string(&grammar_json_path).map_err(|e| {
         format!(
@@ -19,11 +22,30 @@ pub fn compile_language_to_wasm(language_dir: &Path) -> Result<()> {
             grammar_json_path, e
         )
     })?;
+    let output_filename = format!("tree-sitter-{}.wasm", grammar.name);
 
-    let mut command = Command::new("emcc");
+    // Get the current user id so that files created in the docker container will have
+    // the same owner.
+    let user_id_output = Command::new("id")
+        .arg("-u")
+        .output()
+        .map_err(|e| format!("Failed to get get current user id {}", e))?;
+    let user_id = String::from_utf8_lossy(&user_id_output.stdout);
+    let user_id = user_id.trim();
+
+    // Use `emscripten-slim` docker image with the parser directory mounted as a volume.
+    let mut command = Command::new("docker");
+    let mut volume_string = OsString::from(language_dir);
+    volume_string.push(":/src");
+    command.args(&["run", "--rm"]);
+    command.args(&[OsStr::new("--volume"), &volume_string]);
+    command.args(&["--user", user_id, "trzeci/emscripten-slim"]);
+
+    // Run emscripten in the container
     command.args(&[
+        "emcc",
         "-o",
-        &format!("tree-sitter-{}.wasm", grammar.name),
+        &output_filename,
         "-Os",
         "-s",
         "WASM=1",
@@ -31,14 +53,15 @@ pub fn compile_language_to_wasm(language_dir: &Path) -> Result<()> {
         "SIDE_MODULE=1",
         "-s",
         &format!("EXPORTED_FUNCTIONS=[\"_tree_sitter_{}\"]", grammar.name),
+        "-I",
+        "src",
     ]);
-    command.arg("-I").arg(&src_dir);
 
-    // Find source files to compile
-    let entries = fs::read_dir(&src_dir)
+    // Find source files to pass to emscripten
+    let src_entries = fs::read_dir(&src_dir)
         .map_err(|e| format!("Failed to read source directory {:?} - {}", src_dir, e))?;
 
-    for entry in entries {
+    for entry in src_entries {
         let entry = entry?;
         let file_name = entry.file_name();
 
@@ -53,20 +76,24 @@ pub fn compile_language_to_wasm(language_dir: &Path) -> Result<()> {
         // Compile any .c, .cc, or .cpp files
         if let Some(extension) = Path::new(&file_name).extension().and_then(|s| s.to_str()) {
             if extension == "c" || extension == "cc" || extension == "cpp" {
-                command.arg(entry.path());
+                command.arg(Path::new("src").join(entry.file_name()));
             }
         }
     }
 
     let output = command
         .output()
-        .map_err(|e| format!("Failed to run emcc command - {}", e))?;
-    if output.status.success() {
-        Ok(())
-    } else {
-        Err(Error::from(format!(
+        .map_err(|e| format!("Failed to run docker emcc command - {}", e))?;
+    if !output.status.success() {
+        return Err(Error::from(format!(
             "emcc command failed - {}",
             String::from_utf8_lossy(&output.stderr)
-        )))
+        )));
     }
+
+    // Move the created `.wasm` file into the current working directory.
+    fs::rename(&language_dir.join(&output_filename), &output_filename)
+        .map_err(|e| format!("Couldn't find output file {:?} - {}", output_filename, e))?;
+
+    Ok(())
 }

lib/web/binding.js

@@ -8,14 +8,18 @@ const SIZE_OF_RANGE = 2 * SIZE_OF_INT + 2 * SIZE_OF_POINT;
 var TRANSFER_BUFFER;
 var currentParseCallback;
 var currentLogCallback;
+var initPromise;
 
 class Parser {
   static init() {
-    return new Promise(resolve => {
-      Module.onRuntimeInitialized = resolve
-    }).then(() => {
-      TRANSFER_BUFFER = C._ts_init();
-    });
+    if (!initPromise) {
+      initPromise = new Promise(resolve => {
+        Module.onRuntimeInitialized = resolve
+      }).then(() => {
+        TRANSFER_BUFFER = C._ts_init();
+      });
+    }
+    return initPromise;
   }
 
   constructor() {
@@ -38,6 +42,7 @@ class Parser {
     if (C._ts_parser_language(this[0]) !== language[0]) {
       throw new Error('Incompatible language');
     }
+    return this;
   }
 
   getLanguage() {
@@ -53,6 +58,7 @@ class Parser {
     }
     C._ts_parser_set_included_ranges(self[0], buffer, ranges.length);
     C._free(buffer);
+    return this;
   }
 
   getIncludedRanges() {
@@ -67,9 +73,9 @@ class Parser {
     return result;
   }
 
-  parse(oldTree, callback) {
+  parse(callback, oldTree) {
     if (typeof callback === 'string') {
-      return this.parse(oldTree, index => callback.slice(index))
+      return this.parse(index => callback.slice(index), oldTree);
     }
 
     if (this.logCallback) {
@@ -282,19 +288,31 @@ class Language {
   }
 
   static load(url) {
-    return fetch(url)
-      .then(response => response.arrayBuffer()
-        .then(buffer => {
-          if (response.ok) {
-            return loadWebAssemblyModule(new Uint8Array(buffer), {loadAsync: true});
-          } else {
-            const body = new TextDecoder('utf-8').decode(buffer);
-            throw new Error(`Language.load failed with status ${response.status}.\n\n${body}`)
-          }
-        }))
-      .then(exports => {
-        const functionName = Object.keys(exports).find(key => key.includes("tree_sitter_"));
-        const languageAddress = exports[functionName]();
+    let bytes;
+    if (
+      typeof require === 'function' &&
+      require('url').parse(url).protocol == null
+    ) {
+      const fs = require('fs');
+      bytes = Promise.resolve(fs.readFileSync(url));
+    } else {
+      bytes = fetch(url)
+        .then(response => response.arrayBuffer()
+          .then(buffer => {
+            if (response.ok) {
+              return new Uint8Array(buffer);
+            } else {
+              const body = new TextDecoder('utf-8').decode(buffer);
+              throw new Error(`Language.load failed with status ${response.status}.\n\n${body}`)
+            }
+          }));
+    }
+
+    return bytes
+      .then(bytes => loadWebAssemblyModule(bytes, {loadAsync: true}))
+      .then(mod => {
+        const functionName = Object.keys(mod).find(key => key.includes("tree_sitter_"));
+        const languageAddress = mod[functionName]();
         return new Language(INTERNAL, languageAddress);
       });
   }

lib/web/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "tree-sitter.wasm",
+  "version": "0.0.1",
+  "description": "Tree-sitter bindings for the web",
+  "main": "index.js",
+  "directories": {
+    "test": "test"
+  },
+  "scripts": {
+    "test": "mocha"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/tree-sitter/tree-sitter.git"
+  },
+  "keywords": [
+    "incremental",
+    "parsing"
+  ],
+  "author": "Max Brunsfeld",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/tree-sitter/tree-sitter/issues"
+  },
+  "homepage": "https://github.com/tree-sitter/tree-sitter#readme",
+  "devDependencies": {
+    "chai": "^4.2.0",
+    "mocha": "^6.1.4"
+  }
+}

lib/web/prefix.js

@@ -3,8 +3,8 @@
     define([], factory);
   } else if (typeof exports === 'object') {
     module.exports = factory();
-    module.exports.init();
-    delete module.exports.init;
+    // module.exports.init();
+    // delete module.exports.init;
   } else {
     window.TreeSitter = factory();
   }

lib/web/test/parser-test.js

@@ -0,0 +1,31 @@
+const assert = require('assert');
+const Parser = require('../../../target/release/tree-sitter.js');
+let JavaScript, Python;
+
+before(async () => {
+  await Parser.init();
+  JavaScript = await Parser.Language.load('../../target/scratch/tree-sitter-javascript.wasm');
+  Python = await Parser.Language.load('../../target/scratch/tree-sitter-python.wasm');
+});
+
+describe("Parser", () => {
+  let parser;
+
+  beforeEach(() => {
+    parser = new Parser();
+  });
+
+  afterEach(() => {
+    parser.delete();
+  });
+
+  it("parses strings", () => {
+    const tree = parser
+      .setLanguage(JavaScript)
+      .parse("a('hi')\n");
+    assert.equal(
+      tree.rootNode.toString(),
+      "(program (expression_statement (call_expression (identifier) (arguments (string)))))"
+    );
+  });
+});

script/build-wasm

@@ -11,7 +11,13 @@ fi
 
 mkdir -p $target_dir
 
-emcc                                \
+docker run                          \
+  --rm                              \
+  -v $(pwd):/src                    \
+  -u $(id -u)                       \
+  trzeci/emscripten-slim            \
+                                    \
+  emcc                              \
   -s WASM=1                         \
   -s ALLOW_MEMORY_GROWTH            \
   -s MAIN_MODULE=1                  \

script/generate-fixtures

@@ -7,27 +7,16 @@ cargo build --release
 root_dir=$PWD
 tree_sitter=${root_dir}/target/release/tree-sitter
 grammars_dir=${root_dir}/test/fixtures/grammars
-
-grammar_names=(
-  bash
-  c
-  cpp
-  embedded-template
-  go
-  html
-  javascript
-  json
-  python
-  rust
-)
+grammar_names=$(ls $grammars_dir)
 
 if [[ "$#" > 0 ]]; then
   grammar_names=($1)
 fi
 
-for grammar_name in "${grammar_names[@]}"; do
+for grammar_name in $grammar_names; do
   echo "Regenerating ${grammar_name} parser"
-  cd ${grammars_dir}/${grammar_name}
-  $tree_sitter generate src/grammar.json
-  cd $PWD
+  (
+    cd ${grammars_dir}/${grammar_name}
+    $tree_sitter generate src/grammar.json
+  )
 done

script/generate-fixtures-wasm

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -e
+
+cargo build --release
+
+root_dir=$PWD
+tree_sitter=${root_dir}/target/release/tree-sitter
+grammars_dir=${root_dir}/test/fixtures/grammars
+grammar_names=(
+  javascript
+  python
+)
+
+if [[ "$#" > 0 ]]; then
+  grammar_names=($1)
+fi
+
+for grammar_name in ${grammar_names[@]}; do
+  echo "Compiling ${grammar_name} parser to wasm"
+  $tree_sitter build-wasm ${grammars_dir}/${grammar_name}
+done
+
+mv tree-sitter-*.wasm target/scratch/

script/test-wasm

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -e
+
+cd lib/web
+
+if [ ! -d "node_modules/chai" ] || [ ! -d "node_modules/mocha" ]; then
+  echo "Installing test dependencies..."
+  npm install
+fi
+
+./node_modules/.bin/mocha