From a6a50a6320cbb6db6377f8363680aa721d73fc94 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 1 May 2019 11:29:35 -0700 Subject: [PATCH] Avoid some bloat in wasm build --- cli/src/wasm.rs | 1 + lib/web/exports.json | 85 +++++++++++++++++++++++++++++++ lib/web/test/helper.js | 7 ++- lib/web/test/parser-test.js | 56 +++++++++++++++++--- script/build-wasm | 9 ++-- script/generate-fixtures-wasm | 8 +-- script/generate-wasm-exports-list | 19 +++++++ 7 files changed, 165 insertions(+), 20 deletions(-) create mode 100644 lib/web/exports.json create mode 100755 script/generate-wasm-exports-list diff --git a/cli/src/wasm.rs b/cli/src/wasm.rs index 7cf77c93..782b9a43 100644 --- a/cli/src/wasm.rs +++ b/cli/src/wasm.rs @@ -55,6 +55,7 @@ pub fn compile_language_to_wasm(language_dir: &Path) -> Result<()> { "TOTAL_MEMORY=33554432", "-s", &format!("EXPORTED_FUNCTIONS=[\"_tree_sitter_{}\"]", grammar.name), + "-fno-exceptions", "-I", "src", ]); diff --git a/lib/web/exports.json b/lib/web/exports.json new file mode 100644 index 00000000..ce95bbd4 --- /dev/null +++ b/lib/web/exports.json @@ -0,0 +1,85 @@ +[ + "_calloc", + "_free", + "_malloc", + + "__ZNKSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE4copyEPcmm", + "__ZNKSt3__220__vector_base_commonILb1EE20__throw_length_errorEv", + "__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE6__initEPKcm", + "__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7reserveEm", + "__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE9__grow_byEmmmmmm", + "__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE9push_backEc", + "__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEC1ERKS5_", + "__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEED2Ev", + "__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE9push_backEw", + "__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEED2Ev", + "__ZdlPv", + "__Znwm", + "___assert_fail", + "_abort", + "_iswalnum", + "_iswalpha", + "_iswdigit", + "_iswlower", + "_iswspace", + "_memchr", + "_memcmp", + "_memcpy", + "_strlen", + "_towupper", + "abort", + + "_ts_init", + "_ts_language_symbol_count", + "_ts_language_symbol_name", + "_ts_language_symbol_type", + "_ts_language_version", + "_ts_node_child_count_wasm", + "_ts_node_child_wasm", + "_ts_node_children_wasm", + "_ts_node_descendant_for_index_wasm", + "_ts_node_descendant_for_position_wasm", + "_ts_node_end_index_wasm", + "_ts_node_end_point_wasm", + "_ts_node_has_changes_wasm", + "_ts_node_has_error_wasm", + "_ts_node_is_missing_wasm", + "_ts_node_is_named_wasm", + "_ts_node_named_child_count_wasm", + "_ts_node_named_child_wasm", + "_ts_node_named_children_wasm", + "_ts_node_named_descendant_for_index_wasm", + "_ts_node_named_descendant_for_position_wasm", + "_ts_node_next_named_sibling_wasm", + "_ts_node_next_sibling_wasm", + "_ts_node_parent_wasm", + "_ts_node_prev_named_sibling_wasm", + "_ts_node_prev_sibling_wasm", + "_ts_node_start_index_wasm", + "_ts_node_start_point_wasm", + "_ts_node_symbol_wasm", + "_ts_node_to_string_wasm", + "_ts_parser_delete", + "_ts_parser_enable_logger_wasm", + "_ts_parser_new_wasm", + "_ts_parser_parse_wasm", + "_ts_parser_set_language", + "_ts_tree_cursor_current_node_id_wasm", + "_ts_tree_cursor_current_node_is_missing_wasm", + "_ts_tree_cursor_current_node_is_named_wasm", + "_ts_tree_cursor_current_node_type_id_wasm", + "_ts_tree_cursor_current_node_wasm", + "_ts_tree_cursor_delete_wasm", + "_ts_tree_cursor_end_index_wasm", + "_ts_tree_cursor_end_position_wasm", + "_ts_tree_cursor_goto_first_child_wasm", + "_ts_tree_cursor_goto_next_sibling_wasm", + "_ts_tree_cursor_goto_parent_wasm", + "_ts_tree_cursor_new_wasm", + "_ts_tree_cursor_reset_wasm", + "_ts_tree_cursor_start_index_wasm", + "_ts_tree_cursor_start_position_wasm", + "_ts_tree_delete", + "_ts_tree_edit_wasm", + "_ts_tree_root_node_wasm" +] diff --git a/lib/web/test/helper.js b/lib/web/test/helper.js index 8a58dc97..a3be9530 100644 --- a/lib/web/test/helper.js +++ b/lib/web/test/helper.js @@ -1,10 +1,9 @@ const release = '../../../target/release' const Parser = require(`${release}/tree-sitter.js`); -const JavaScript = require.resolve(`${release}/tree-sitter-javascript.wasm`); -const Python = require.resolve(`${release}/tree-sitter-python.wasm`); +const languageURL = name => require.resolve(`${release}/tree-sitter-${name}.wasm`); module.exports = Parser.init().then(async () => ({ Parser, - JavaScript: await Parser.Language.load(JavaScript), - Python: await Parser.Language.load(Python) + languageURL, + JavaScript: await Parser.Language.load(languageURL('javascript')), })); diff --git a/lib/web/test/parser-test.js b/lib/web/test/parser-test.js index 81847f37..4be05540 100644 --- a/lib/web/test/parser-test.js +++ b/lib/web/test/parser-test.js @@ -1,11 +1,11 @@ const {assert} = require('chai'); -let Parser, JavaScript, Python; +let Parser, JavaScript, languageURL; describe("Parser", () => { let parser; before(async () => - ({Parser, JavaScript, Python} = await require('./helper')) + ({Parser, JavaScript, languageURL} = await require('./helper')) ); beforeEach(() => { @@ -125,14 +125,56 @@ describe("Parser", () => { assert.equal(tree.rootNode.firstChild.firstChild.namedChildCount, repeatCount); }).timeout(5000); - it("can use languages with external scanners written in C++", () => { - parser.setLanguage(Python); - tree = parser.parse("def foo():\n bar()"); + it("can use the bash parser", async () => { + parser.setLanguage(await Parser.Language.load(languageURL('bash'))); + tree = parser.parse("FOO=bar echo < err.txt > hello.txt \nhello\nEOF"); assert.equal( tree.rootNode.toString(), - '(module (function_definition (identifier) (parameters) (expression_statement (call (identifier) (argument_list)))))' + '(program (redirected_statement (command ' + + '(variable_assignment (variable_name) (word)) ' + + '(command_name (word))) ' + + '(heredoc_redirect (heredoc_start)) ' + + '(file_redirect (file_descriptor) (word)) ' + + '(file_redirect (word))) ' + + '(heredoc_body))' ); - }); + }).timeout(5000); + + it("can use the c++ parser", async () => { + parser.setLanguage(await Parser.Language.load(languageURL('cpp'))); + tree = parser.parse("const char *s = R\"EOF(HELLO WORLD)EOF\";"); + assert.equal( + tree.rootNode.toString(), + '(translation_unit (declaration (type_qualifier) (primitive_type) (init_declarator (pointer_declarator (identifier)) (raw_string_literal))))' + ); + }).timeout(5000); + + it("can use the HTML parser", async () => { + parser.setLanguage(await Parser.Language.load(languageURL('html'))); + tree = parser.parse("
"); + assert.equal( + tree.rootNode.toString(), + '(fragment (element (start_tag (tag_name)) (element (start_tag (tag_name)) (element (start_tag (tag_name)) (end_tag (tag_name))) (end_tag (tag_name))) (end_tag (tag_name))))' + ); + }).timeout(5000); + + it("can use the python parser", async () => { + parser.setLanguage(await Parser.Language.load(languageURL('python'))); + tree = parser.parse("class A:\n def b():\n c()"); + assert.equal( + tree.rootNode.toString(), + '(module (class_definition (identifier) (function_definition (identifier) (parameters) (expression_statement (call (identifier) (argument_list))))))' + ); + }).timeout(5000); + + it("can use the rust parser", async () => { + parser.setLanguage(await Parser.Language.load(languageURL('rust'))); + tree = parser.parse("const x: &'static str = r###\"hello\"###;"); + assert.equal( + tree.rootNode.toString(), + '(source_file (const_item (identifier) (reference_type (lifetime (identifier)) (primitive_type)) (raw_string_literal)))' + ); + }).timeout(5000); it('parses only the text within the `includedRanges` if they are specified', () => { const sourceCode = "<% foo() %> <% bar %>"; diff --git a/script/build-wasm b/script/build-wasm index 8238f24c..b6147a37 100755 --- a/script/build-wasm +++ b/script/build-wasm @@ -6,9 +6,11 @@ args="-Os" minify=1 if [[ "$1" == "--debug" ]]; then minify=0 - args="-s ASSERTIONS=1 -s SAFE_HEAP=1 -O0" + args="-s ASSERTIONS=1 -s SAFE_HEAP=1 -Os" fi +exports=$(cat lib/web/exports.json) + mkdir -p target/scratch target/release docker run \ @@ -22,8 +24,9 @@ docker run \ -s WASM=1 \ -s TOTAL_MEMORY=33554432 \ -s ALLOW_MEMORY_GROWTH \ - -s MAIN_MODULE=1 \ - -s EXPORT_ALL=1 \ + -s MAIN_MODULE=2 \ + -s NO_FILESYSTEM=1 \ + -s "EXPORTED_FUNCTIONS=${exports}" \ $args \ -std=c99 \ -D 'fprintf(...)=' \ diff --git a/script/generate-fixtures-wasm b/script/generate-fixtures-wasm index cbcf88bd..75ce4a3c 100755 --- a/script/generate-fixtures-wasm +++ b/script/generate-fixtures-wasm @@ -7,17 +7,13 @@ cargo build --release root_dir=$PWD tree_sitter=${root_dir}/target/release/tree-sitter grammars_dir=${root_dir}/test/fixtures/grammars -grammar_names=( - c - javascript - python -) +grammar_names=$(ls $grammars_dir) if [[ "$#" > 0 ]]; then grammar_names=($1) fi -for grammar_name in ${grammar_names[@]}; do +for grammar_name in $grammar_names; do echo "Compiling ${grammar_name} parser to wasm" $tree_sitter build-wasm ${grammars_dir}/${grammar_name} done diff --git a/script/generate-wasm-exports-list b/script/generate-wasm-exports-list new file mode 100755 index 00000000..25099f7f --- /dev/null +++ b/script/generate-wasm-exports-list @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +root_dir=$PWD +grammars_dir=${root_dir}/test/fixtures/grammars +grammar_names=$(ls $grammars_dir | tr '-' '_') + +symbol_file=$(mktemp) + +for grammar_name in $grammar_names; do + wasm-objdump \ + --details target/release/tree-sitter-${grammar_name}.wasm \ + --section Import \ + | egrep -o '<(\w+)>' \ + | tr -d '<>' \ + >> $symbol_file +done + +sort -u -o $symbol_file $symbol_file +cat $symbol_file