From 66e006105cf312587eb68b3271b379a4044d1d21 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 25 Apr 2019 17:27:24 -0700 Subject: [PATCH] Build and test wasm on CI --- .appveyor.yml | 2 +- .gitignore | 2 + .travis.yml | 27 +++++++-- cli/npm/package-lock.json | 5 -- cli/src/wasm.rs | 53 ++++++++++++----- lib/web/binding.js | 58 ++++++++++++------- lib/web/package.json | 30 ++++++++++ lib/web/prefix.js | 4 +- lib/web/test/parser-test.js | 31 ++++++++++ script/build-wasm | 8 ++- ...{regenerate-fixtures => generate-fixtures} | 23 ++------ script/generate-fixtures-wasm | 24 ++++++++ ...ate-fixtures.cmd => generate-fixtures.cmd} | 0 script/test-wasm | 12 ++++ 14 files changed, 214 insertions(+), 65 deletions(-) delete mode 100644 cli/npm/package-lock.json create mode 100644 lib/web/package.json create mode 100644 lib/web/test/parser-test.js rename script/{regenerate-fixtures => generate-fixtures} (52%) create mode 100755 script/generate-fixtures-wasm rename script/{regenerate-fixtures.cmd => generate-fixtures.cmd} (100%) create mode 100755 script/test-wasm diff --git a/.appveyor.yml b/.appveyor.yml index 610ac134..d463b7a2 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -22,7 +22,7 @@ test_script: # Fetch and regenerate the fixture parsers - script\fetch-fixtures.cmd - cargo build --release - - script\regenerate-fixtures.cmd + - script\generate-fixtures.cmd # Run tests - script\test.cmd diff --git a/.gitignore b/.gitignore index 5fc189ca..ed31e54a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,8 @@ fuzz-results test/fixtures/grammars/* !test/fixtures/grammars/.gitkeep +package-lock.json +node_modules docs/assets/js/tree-sitter.js diff --git a/.travis.yml b/.travis.yml index 06c71b34..9fa65759 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,21 +2,33 @@ language: rust rust: - stable -os: - - linux - - osx +matrix: + include: + - os: osx + - os: linux + services: docker + env: TEST_WASM=1 + +before_install: + # Install node + - if [ -n "$TEST_WASM" ]; then nvm install 10 && nvm use 10; fi script: # Fetch and regenerate the fixture parsers - script/fetch-fixtures - cargo build --release - - script/regenerate-fixtures + - script/generate-fixtures - # Run tests + # Run the tests - export TREE_SITTER_STATIC_ANALYSIS=1 - script/test - script/benchmark + # Build and test the WASM binding + - if [ -n "$TEST_WASM" ]; then script/build-wasm; fi + - if [ -n "$TEST_WASM" ]; then script/generate-fixtures-wasm; fi + - if [ -n "$TEST_WASM" ]; then script/test-wasm; fi + branches: only: - master @@ -31,7 +43,10 @@ deploy: api_key: secure: "cAd2mQP+Q55v3zedo5ZyOVc3hq3XKMW93lp5LuXV6CYKYbIhkyfym4qfs+C9GJQiIP27cnePYM7B3+OMIFwSPIgXHWWSsuloMtDgYSc/PAwb2dZnJqAyog3BohW/QiGTSnvbVlxPF6P9RMQU6+JP0HJzEJy6QBTa4Und/j0jm24=" file_glob: true - file: "tree-sitter-*.gz" + file: + - "tree-sitter-*.gz" + - "target/release/tree-sitter.js" + - "target/release/tree-sitter.wasm" draft: true overwrite: true skip_cleanup: true diff --git a/cli/npm/package-lock.json b/cli/npm/package-lock.json deleted file mode 100644 index ab7209ec..00000000 --- a/cli/npm/package-lock.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "tree-sitter-cli", - "version": "0.15.1", - "lockfileVersion": 1 -} diff --git a/cli/src/wasm.rs b/cli/src/wasm.rs index 1177391e..1aaa1f61 100644 --- a/cli/src/wasm.rs +++ b/cli/src/wasm.rs @@ -1,11 +1,14 @@ use super::error::{Error, Result}; use super::generate::parse_grammar::GrammarJSON; +use std::ffi::{OsStr, OsString}; use std::fs; use std::path::Path; use std::process::Command; pub fn compile_language_to_wasm(language_dir: &Path) -> Result<()> { let src_dir = language_dir.join("src"); + + // Parse the grammar.json to find out the language name. let grammar_json_path = src_dir.join("grammar.json"); let grammar_json = fs::read_to_string(&grammar_json_path).map_err(|e| { format!( @@ -19,11 +22,30 @@ pub fn compile_language_to_wasm(language_dir: &Path) -> Result<()> { grammar_json_path, e ) })?; + let output_filename = format!("tree-sitter-{}.wasm", grammar.name); - let mut command = Command::new("emcc"); + // Get the current user id so that files created in the docker container will have + // the same owner. + let user_id_output = Command::new("id") + .arg("-u") + .output() + .map_err(|e| format!("Failed to get get current user id {}", e))?; + let user_id = String::from_utf8_lossy(&user_id_output.stdout); + let user_id = user_id.trim(); + + // Use `emscripten-slim` docker image with the parser directory mounted as a volume. + let mut command = Command::new("docker"); + let mut volume_string = OsString::from(language_dir); + volume_string.push(":/src"); + command.args(&["run", "--rm"]); + command.args(&[OsStr::new("--volume"), &volume_string]); + command.args(&["--user", user_id, "trzeci/emscripten-slim"]); + + // Run emscripten in the container command.args(&[ + "emcc", "-o", - &format!("tree-sitter-{}.wasm", grammar.name), + &output_filename, "-Os", "-s", "WASM=1", @@ -31,14 +53,15 @@ pub fn compile_language_to_wasm(language_dir: &Path) -> Result<()> { "SIDE_MODULE=1", "-s", &format!("EXPORTED_FUNCTIONS=[\"_tree_sitter_{}\"]", grammar.name), + "-I", + "src", ]); - command.arg("-I").arg(&src_dir); - // Find source files to compile - let entries = fs::read_dir(&src_dir) + // Find source files to pass to emscripten + let src_entries = fs::read_dir(&src_dir) .map_err(|e| format!("Failed to read source directory {:?} - {}", src_dir, e))?; - for entry in entries { + for entry in src_entries { let entry = entry?; let file_name = entry.file_name(); @@ -53,20 +76,24 @@ pub fn compile_language_to_wasm(language_dir: &Path) -> Result<()> { // Compile any .c, .cc, or .cpp files if let Some(extension) = Path::new(&file_name).extension().and_then(|s| s.to_str()) { if extension == "c" || extension == "cc" || extension == "cpp" { - command.arg(entry.path()); + command.arg(Path::new("src").join(entry.file_name())); } } } let output = command .output() - .map_err(|e| format!("Failed to run emcc command - {}", e))?; - if output.status.success() { - Ok(()) - } else { - Err(Error::from(format!( + .map_err(|e| format!("Failed to run docker emcc command - {}", e))?; + if !output.status.success() { + return Err(Error::from(format!( "emcc command failed - {}", String::from_utf8_lossy(&output.stderr) - ))) + ))); } + + // Move the created `.wasm` file into the current working directory. + fs::rename(&language_dir.join(&output_filename), &output_filename) + .map_err(|e| format!("Couldn't find output file {:?} - {}", output_filename, e))?; + + Ok(()) } diff --git a/lib/web/binding.js b/lib/web/binding.js index 96d54407..cd9dad8d 100644 --- a/lib/web/binding.js +++ b/lib/web/binding.js @@ -8,14 +8,18 @@ const SIZE_OF_RANGE = 2 * SIZE_OF_INT + 2 * SIZE_OF_POINT; var TRANSFER_BUFFER; var currentParseCallback; var currentLogCallback; +var initPromise; class Parser { static init() { - return new Promise(resolve => { - Module.onRuntimeInitialized = resolve - }).then(() => { - TRANSFER_BUFFER = C._ts_init(); - }); + if (!initPromise) { + initPromise = new Promise(resolve => { + Module.onRuntimeInitialized = resolve + }).then(() => { + TRANSFER_BUFFER = C._ts_init(); + }); + } + return initPromise; } constructor() { @@ -38,6 +42,7 @@ class Parser { if (C._ts_parser_language(this[0]) !== language[0]) { throw new Error('Incompatible language'); } + return this; } getLanguage() { @@ -53,6 +58,7 @@ class Parser { } C._ts_parser_set_included_ranges(self[0], buffer, ranges.length); C._free(buffer); + return this; } getIncludedRanges() { @@ -67,9 +73,9 @@ class Parser { return result; } - parse(oldTree, callback) { + parse(callback, oldTree) { if (typeof callback === 'string') { - return this.parse(oldTree, index => callback.slice(index)) + return this.parse(index => callback.slice(index), oldTree); } if (this.logCallback) { @@ -282,19 +288,31 @@ class Language { } static load(url) { - return fetch(url) - .then(response => response.arrayBuffer() - .then(buffer => { - if (response.ok) { - return loadWebAssemblyModule(new Uint8Array(buffer), {loadAsync: true}); - } else { - const body = new TextDecoder('utf-8').decode(buffer); - throw new Error(`Language.load failed with status ${response.status}.\n\n${body}`) - } - })) - .then(exports => { - const functionName = Object.keys(exports).find(key => key.includes("tree_sitter_")); - const languageAddress = exports[functionName](); + let bytes; + if ( + typeof require === 'function' && + require('url').parse(url).protocol == null + ) { + const fs = require('fs'); + bytes = Promise.resolve(fs.readFileSync(url)); + } else { + bytes = fetch(url) + .then(response => response.arrayBuffer() + .then(buffer => { + if (response.ok) { + return new Uint8Array(buffer); + } else { + const body = new TextDecoder('utf-8').decode(buffer); + throw new Error(`Language.load failed with status ${response.status}.\n\n${body}`) + } + })); + } + + return bytes + .then(bytes => loadWebAssemblyModule(bytes, {loadAsync: true})) + .then(mod => { + const functionName = Object.keys(mod).find(key => key.includes("tree_sitter_")); + const languageAddress = mod[functionName](); return new Language(INTERNAL, languageAddress); }); } diff --git a/lib/web/package.json b/lib/web/package.json new file mode 100644 index 00000000..c8d6277d --- /dev/null +++ b/lib/web/package.json @@ -0,0 +1,30 @@ +{ + "name": "tree-sitter.wasm", + "version": "0.0.1", + "description": "Tree-sitter bindings for the web", + "main": "index.js", + "directories": { + "test": "test" + }, + "scripts": { + "test": "mocha" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/tree-sitter/tree-sitter.git" + }, + "keywords": [ + "incremental", + "parsing" + ], + "author": "Max Brunsfeld", + "license": "MIT", + "bugs": { + "url": "https://github.com/tree-sitter/tree-sitter/issues" + }, + "homepage": "https://github.com/tree-sitter/tree-sitter#readme", + "devDependencies": { + "chai": "^4.2.0", + "mocha": "^6.1.4" + } +} diff --git a/lib/web/prefix.js b/lib/web/prefix.js index b380d814..ccd94fa4 100644 --- a/lib/web/prefix.js +++ b/lib/web/prefix.js @@ -3,8 +3,8 @@ define([], factory); } else if (typeof exports === 'object') { module.exports = factory(); - module.exports.init(); - delete module.exports.init; + // module.exports.init(); + // delete module.exports.init; } else { window.TreeSitter = factory(); } diff --git a/lib/web/test/parser-test.js b/lib/web/test/parser-test.js new file mode 100644 index 00000000..1bf0095c --- /dev/null +++ b/lib/web/test/parser-test.js @@ -0,0 +1,31 @@ +const assert = require('assert'); +const Parser = require('../../../target/release/tree-sitter.js'); +let JavaScript, Python; + +before(async () => { + await Parser.init(); + JavaScript = await Parser.Language.load('../../target/scratch/tree-sitter-javascript.wasm'); + Python = await Parser.Language.load('../../target/scratch/tree-sitter-python.wasm'); +}); + +describe("Parser", () => { + let parser; + + beforeEach(() => { + parser = new Parser(); + }); + + afterEach(() => { + parser.delete(); + }); + + it("parses strings", () => { + const tree = parser + .setLanguage(JavaScript) + .parse("a('hi')\n"); + assert.equal( + tree.rootNode.toString(), + "(program (expression_statement (call_expression (identifier) (arguments (string)))))" + ); + }); +}); diff --git a/script/build-wasm b/script/build-wasm index 09860e5d..a579ebfe 100755 --- a/script/build-wasm +++ b/script/build-wasm @@ -11,7 +11,13 @@ fi mkdir -p $target_dir -emcc \ +docker run \ + --rm \ + -v $(pwd):/src \ + -u $(id -u) \ + trzeci/emscripten-slim \ + \ + emcc \ -s WASM=1 \ -s ALLOW_MEMORY_GROWTH \ -s MAIN_MODULE=1 \ diff --git a/script/regenerate-fixtures b/script/generate-fixtures similarity index 52% rename from script/regenerate-fixtures rename to script/generate-fixtures index c47c53f9..6296d8ab 100755 --- a/script/regenerate-fixtures +++ b/script/generate-fixtures @@ -7,27 +7,16 @@ cargo build --release root_dir=$PWD tree_sitter=${root_dir}/target/release/tree-sitter grammars_dir=${root_dir}/test/fixtures/grammars - -grammar_names=( - bash - c - cpp - embedded-template - go - html - javascript - json - python - rust -) +grammar_names=$(ls $grammars_dir) if [[ "$#" > 0 ]]; then grammar_names=($1) fi -for grammar_name in "${grammar_names[@]}"; do +for grammar_name in $grammar_names; do echo "Regenerating ${grammar_name} parser" - cd ${grammars_dir}/${grammar_name} - $tree_sitter generate src/grammar.json - cd $PWD + ( + cd ${grammars_dir}/${grammar_name} + $tree_sitter generate src/grammar.json + ) done diff --git a/script/generate-fixtures-wasm b/script/generate-fixtures-wasm new file mode 100755 index 00000000..0713de17 --- /dev/null +++ b/script/generate-fixtures-wasm @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -e + +cargo build --release + +root_dir=$PWD +tree_sitter=${root_dir}/target/release/tree-sitter +grammars_dir=${root_dir}/test/fixtures/grammars +grammar_names=( + javascript + python +) + +if [[ "$#" > 0 ]]; then + grammar_names=($1) +fi + +for grammar_name in ${grammar_names[@]}; do + echo "Compiling ${grammar_name} parser to wasm" + $tree_sitter build-wasm ${grammars_dir}/${grammar_name} +done + +mv tree-sitter-*.wasm target/scratch/ diff --git a/script/regenerate-fixtures.cmd b/script/generate-fixtures.cmd similarity index 100% rename from script/regenerate-fixtures.cmd rename to script/generate-fixtures.cmd diff --git a/script/test-wasm b/script/test-wasm new file mode 100755 index 00000000..380ccab7 --- /dev/null +++ b/script/test-wasm @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -e + +cd lib/web + +if [ ! -d "node_modules/chai" ] || [ ! -d "node_modules/mocha" ]; then + echo "Installing test dependencies..." + npm install +fi + +./node_modules/.bin/mocha