traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #2897 from tree-sitter/wasm-stdlib

Browse source 
Provide a consistent set of standard library function to wasm parsers across environments
This commit is contained in:
Max Brunsfeld 2024-02-05 12:24:17 -08:00 • committed by GitHub
commit 3fe69d629f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 2331 additions and 2276 deletions
Download patch file Download diff file Expand all files Collapse all files

22
Cargo.lock generated
View file

@ -306,7 +306,7 @@ dependencies = [
"itertools",
"log",
"smallvec",
"wasmparser",
"wasmparser 0.118.1",
"wasmtime-types",
]
@ -1273,6 +1273,7 @@ dependencies = [
"tree-sitter-tests-proc-macro",
"unindent",
"walkdir",
"wasmparser 0.121.0",
"webbrowser",
"which",
]
@ -1486,6 +1487,17 @@ dependencies = [
"semver",
]
[[package]]
name = "wasmparser"
version = "0.121.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "953cf6a7606ab31382cb1caa5ae403e77ba70c7f8e12eeda167e7040d42bfda8"
dependencies = [
"bitflags 2.4.1",
"indexmap",
"semver",
]
[[package]]
name = "wasmtime"
version = "16.0.0"
@ -1505,7 +1517,7 @@ dependencies = [
"serde_derive",
"serde_json",
"target-lexicon",
"wasmparser",
"wasmparser 0.118.1",
"wasmtime-cranelift",
"wasmtime-environ",
"wasmtime-jit",
@ -1561,7 +1573,7 @@ dependencies = [
"object",
"target-lexicon",
"thiserror",
"wasmparser",
"wasmparser 0.118.1",
"wasmtime-cranelift-shared",
"wasmtime-environ",
"wasmtime-versioned-export-macros",
@ -1597,7 +1609,7 @@ dependencies = [
"serde_derive",
"target-lexicon",
"thiserror",
"wasmparser",
"wasmparser 0.118.1",
"wasmtime-types",
]
@ -1667,7 +1679,7 @@ dependencies = [
"serde",
"serde_derive",
"thiserror",
"wasmparser",
"wasmparser 0.118.1",
]
[[package]]

1
cli/Cargo.toml
View file

@ -46,6 +46,7 @@ serde_derive = "1.0"
smallbitvec = "2.5.1"
tiny_http = "0.12.0"
walkdir = "2.3.3"
wasmparser = "0.121"
webbrowser = "0.8.10"
which = "4.4.0"

19
cli/src/tests/wasm_language_test.rs
View file

@ -9,6 +9,25 @@ lazy_static! {
static ref ENGINE: Engine = Engine::default();
}
#[test]
fn test_wasm_stdlib_symbols() {
let symbols = tree_sitter::wasm_stdlib_symbols().collect::<Vec<_>>();
assert_eq!(
symbols,
{
let mut symbols = symbols.clone();
symbols.sort();
symbols
},
"symbols aren't sorted"
);
assert!(symbols.contains(&"malloc"));
assert!(symbols.contains(&"free"));
assert!(symbols.contains(&"memset"));
assert!(symbols.contains(&"memcpy"));
}
#[test]
fn test_load_wasm_language() {
allocations::record(|| {

57
cli/src/wasm.rs
View file

@ -1,7 +1,9 @@
use super::generate::parse_grammar::GrammarJSON;
use anyhow::{Context, Result};
use anyhow::{anyhow, Context, Result};
use std::{fs, path::Path};
use tree_sitter::wasm_stdlib_symbols;
use tree_sitter_loader::Loader;
use wasmparser::Parser;
pub fn load_language_wasm_file(language_dir: &Path) -> Result<(String, Vec<u8>)> {
let grammar_name = get_grammar_name(language_dir)
@ -43,5 +45,58 @@ pub fn compile_language_to_wasm(
&output_filename,
force_docker,
)?;
// Exit with an error if the external scanner uses symbols from the
// C or C++ standard libraries that aren't available to wasm parsers.
let stdlib_symbols: Vec<_> = wasm_stdlib_symbols().collect();
let dylink_symbols = [
"__indirect_function_table",
"__memory_base",
"__stack_pointer",
"__table_base",
"__table_base",
"memory",
];
let builtin_symbols = [
"__assert_fail",
"__cxa_atexit",
"abort",
"emscripten_notify_memory_growth",
"proc_exit",
];
let mut missing_symbols = Vec::new();
let wasm_bytes = fs::read(&output_filename)?;
let parser = Parser::new(0);
for payload in parser.parse_all(&wasm_bytes) {
if let wasmparser::Payload::ImportSection(imports) = payload? {
for import in imports {
let import = import?.name;
if !builtin_symbols.contains(&import)
&& !stdlib_symbols.contains(&import)
&& !dylink_symbols.contains(&import)
{
missing_symbols.push(import);
}
}
}
}
if !missing_symbols.is_empty() {
Err(anyhow!(
concat!(
"This external scanner uses a symbol that isn't available to wasm parsers.\n",
"\n",
"Missing symbols:\n",
" {}\n",
"\n",
"Available symbols:\n",
" {}",
),
missing_symbols.join("\n "),
stdlib_symbols.join("\n ")
))?;
}
Ok(())
}

14
lib/binding_rust/build.rs
View file

@ -2,6 +2,8 @@ use std::path::{Path, PathBuf};
use std::{env, fs};
fn main() {
let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
println!("cargo:rerun-if-env-changed=TREE_SITTER_STATIC_ANALYSIS");
if env::var("TREE_SITTER_STATIC_ANALYSIS").is_ok() {
if let (Some(clang_path), Some(scan_build_path)) = (which("clang"), which("scan-build")) {
@ -15,7 +17,13 @@ fn main() {
}
#[cfg(feature = "bindgen")]
generate_bindings();
generate_bindings(&out_dir);
fs::copy(
"src/wasm/stdlib-symbols.txt",
out_dir.join("stdlib-symbols.txt"),
)
.unwrap();
let mut config = cc::Build::new();
@ -44,7 +52,7 @@ fn main() {
}
#[cfg(feature = "bindgen")]
fn generate_bindings() {
fn generate_bindings(out_dir: &PathBuf) {
const HEADER_PATH: &str = "include/tree_sitter/api.h";
println!("cargo:rerun-if-changed={}", HEADER_PATH);
@ -74,9 +82,7 @@ fn generate_bindings() {
.generate()
.expect("Failed to generate bindings");
let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
let bindings_rs = out_dir.join("bindings.rs");
bindings
.write_to_file(&bindings_rs)
.unwrap_or_else(|_| panic!("Failed to write bindings into path: {bindings_rs:?}"));

8
lib/binding_rust/lib.rs
View file

@ -2820,6 +2820,14 @@ impl fmt::Display for QueryError {
}
}
pub fn wasm_stdlib_symbols() -> impl Iterator<Item = &'static str> {
const WASM_STDLIB_SYMBOLS: &str = include_str!(concat!(env!("OUT_DIR"), "/stdlib-symbols.txt"));
WASM_STDLIB_SYMBOLS
.lines()
.map(|s| s.trim_matches(|c| c == '"' || c == ','))
}
extern "C" {
fn free(ptr: *mut c_void);
}

8
lib/binding_rust/wasm_language.rs
View file

@ -8,10 +8,10 @@ use std::{
};
pub use wasmtime;
#[cfg(feature = "wasm")]
pub fn test() {
// Force Cargo to include wasmtime-c-api as a dependency of this crate,
// even though it is only used by the C code.
// Force Cargo to include wasmtime-c-api as a dependency of this crate,
// even though it is only used by the C code.
#[allow(unused)]
fn _use_wasmtime() {
wasmtime_c_api::wasm_engine_new();
}

32
lib/binding_web/check-artifacts-fresh.js
View file

@ -1,22 +1,19 @@
#!/usr/bin/env node
const fs = require('fs');
const path = require('path');
const fs = require("fs");
const path = require("path");
const inputFiles = [
'binding.c',
'binding.js',
'exports.json',
'imports.js',
'prefix.js',
...list('../include/tree_sitter'),
...list('../src')
]
"binding.c",
"binding.js",
"exports.txt",
"imports.js",
"prefix.js",
...list("../include/tree_sitter"),
...list("../src"),
];
const outputFiles = [
'tree-sitter.js',
'tree-sitter.wasm',
]
const outputFiles = ["tree-sitter.js", "tree-sitter.wasm"];
const outputMtime = Math.min(...outputFiles.map(mtime));
@ -28,9 +25,10 @@ for (const inputFile of inputFiles) {
}
function list(dir) {
return fs.readdirSync(path.join(__dirname, dir), 'utf8')
.filter(p => !p.startsWith('.'))
.map(p => path.join(dir, p));
return fs
.readdirSync(path.join(__dirname, dir), "utf8")
.filter((p) => !p.startsWith("."))
.map((p) => path.join(dir, p));
}
function mtime(p) {

134
lib/binding_web/exports.json
View file

@ -1,134 +0,0 @@
[
"_calloc",
"_free",
"_malloc",
"_realloc",
"__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE6__initEPKcm",
"__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE9push_backEc",
"__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE9push_backEw",
"__ZNKSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE4copyEPcmm",
"__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7reserveEm",
"__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE6resizeEmw",
"__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEED2Ev",
"__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEED2Ev",
"__ZdlPv",
"__Znwm",
"___cxa_atexit",
"_abort",
"_isalpha",
"_isspace",
"_iswalnum",
"_iswalpha",
"_iswblank",
"_iswdigit",
"_iswlower",
"_iswupper",
"_iswspace",
"_memchr",
"_memcmp",
"_memcpy",
"_memmove",
"_memset",
"_strlen",
"_strcmp",
"_strncpy",
"_tolower",
"_towlower",
"_towupper",
"_stderr",
"_ts_init",
"_ts_language_field_count",
"_ts_language_field_name_for_id",
"_ts_language_type_is_named_wasm",
"_ts_language_type_is_visible_wasm",
"_ts_language_symbol_count",
"_ts_language_state_count",
"_ts_language_symbol_for_name",
"_ts_language_symbol_name",
"_ts_language_symbol_type",
"_ts_language_version",
"_ts_language_next_state",
"_ts_node_field_name_for_child_wasm",
"_ts_node_child_by_field_id_wasm",
"_ts_node_child_count_wasm",
"_ts_node_child_wasm",
"_ts_node_children_wasm",
"_ts_node_descendant_for_index_wasm",
"_ts_node_descendant_for_position_wasm",
"_ts_node_descendants_of_type_wasm",
"_ts_node_end_index_wasm",
"_ts_node_end_point_wasm",
"_ts_node_has_changes_wasm",
"_ts_node_has_error_wasm",
"_ts_node_is_error_wasm",
"_ts_node_is_missing_wasm",
"_ts_node_is_named_wasm",
"_ts_node_parse_state_wasm",
"_ts_node_next_parse_state_wasm",
"_ts_node_named_child_count_wasm",
"_ts_node_named_child_wasm",
"_ts_node_named_children_wasm",
"_ts_node_named_descendant_for_index_wasm",
"_ts_node_named_descendant_for_position_wasm",
"_ts_node_next_named_sibling_wasm",
"_ts_node_next_sibling_wasm",
"_ts_node_parent_wasm",
"_ts_node_prev_named_sibling_wasm",
"_ts_node_prev_sibling_wasm",
"_ts_node_start_index_wasm",
"_ts_node_start_point_wasm",
"_ts_node_symbol_wasm",
"_ts_node_grammar_symbol_wasm",
"_ts_node_to_string_wasm",
"_ts_parser_delete",
"_ts_parser_enable_logger_wasm",
"_ts_parser_new_wasm",
"_ts_parser_parse_wasm",
"_ts_parser_reset",
"_ts_parser_set_language",
"_ts_parser_set_timeout_micros",
"_ts_parser_timeout_micros",
"_ts_query_capture_count",
"_ts_query_capture_name_for_id",
"_ts_query_captures_wasm",
"_ts_query_delete",
"_ts_query_matches_wasm",
"_ts_query_new",
"_ts_query_pattern_count",
"_ts_query_predicates_for_pattern",
"_ts_query_string_count",
"_ts_query_string_value_for_id",
"_ts_tree_copy",
"_ts_tree_cursor_current_field_id_wasm",
"_ts_tree_cursor_current_node_id_wasm",
"_ts_tree_cursor_current_node_is_missing_wasm",
"_ts_tree_cursor_current_node_is_named_wasm",
"_ts_tree_cursor_current_node_type_id_wasm",
"_ts_tree_cursor_current_node_state_id_wasm",
"_ts_tree_cursor_current_node_wasm",
"_ts_tree_cursor_delete_wasm",
"_ts_tree_cursor_end_index_wasm",
"_ts_tree_cursor_end_position_wasm",
"_ts_tree_cursor_goto_first_child_wasm",
"_ts_tree_cursor_goto_last_child_wasm",
"_ts_tree_cursor_goto_next_sibling_wasm",
"_ts_tree_cursor_goto_previous_sibling_wasm",
"_ts_tree_cursor_goto_parent_wasm",
"_ts_tree_cursor_new_wasm",
"_ts_tree_cursor_reset_wasm",
"_ts_tree_cursor_reset_to_wasm",
"_ts_tree_cursor_start_index_wasm",
"_ts_tree_cursor_start_position_wasm",
"_ts_tree_delete",
"_ts_tree_edit_wasm",
"_ts_tree_get_changed_ranges_wasm",
"_ts_tree_root_node_wasm",
"_ts_lookahead_iterator_new",
"_ts_lookahead_iterator_delete",
"_ts_lookahead_iterator_reset_state",
"_ts_lookahead_iterator_reset",
"_ts_lookahead_iterator_next",
"_ts_lookahead_iterator_current_symbol"
]

93
lib/binding_web/exports.txt Normal file
View file

@ -0,0 +1,93 @@
"ts_init",
"ts_language_field_count",
"ts_language_field_name_for_id",
"ts_language_type_is_named_wasm",
"ts_language_type_is_visible_wasm",
"ts_language_symbol_count",
"ts_language_state_count",
"ts_language_symbol_for_name",
"ts_language_symbol_name",
"ts_language_symbol_type",
"ts_language_version",
"ts_language_next_state",
"ts_node_field_name_for_child_wasm",
"ts_node_child_by_field_id_wasm",
"ts_node_child_count_wasm",
"ts_node_child_wasm",
"ts_node_children_wasm",
"ts_node_descendant_for_index_wasm",
"ts_node_descendant_for_position_wasm",
"ts_node_descendants_of_type_wasm",
"ts_node_end_index_wasm",
"ts_node_end_point_wasm",
"ts_node_has_changes_wasm",
"ts_node_has_error_wasm",
"ts_node_is_error_wasm",
"ts_node_is_missing_wasm",
"ts_node_is_named_wasm",
"ts_node_parse_state_wasm",
"ts_node_next_parse_state_wasm",
"ts_node_named_child_count_wasm",
"ts_node_named_child_wasm",
"ts_node_named_children_wasm",
"ts_node_named_descendant_for_index_wasm",
"ts_node_named_descendant_for_position_wasm",
"ts_node_next_named_sibling_wasm",
"ts_node_next_sibling_wasm",
"ts_node_parent_wasm",
"ts_node_prev_named_sibling_wasm",
"ts_node_prev_sibling_wasm",
"ts_node_start_index_wasm",
"ts_node_start_point_wasm",
"ts_node_symbol_wasm",
"ts_node_grammar_symbol_wasm",
"ts_node_to_string_wasm",
"ts_parser_delete",
"ts_parser_enable_logger_wasm",
"ts_parser_new_wasm",
"ts_parser_parse_wasm",
"ts_parser_reset",
"ts_parser_set_language",
"ts_parser_set_timeout_micros",
"ts_parser_timeout_micros",
"ts_query_capture_count",
"ts_query_capture_name_for_id",
"ts_query_captures_wasm",
"ts_query_delete",
"ts_query_matches_wasm",
"ts_query_new",
"ts_query_pattern_count",
"ts_query_predicates_for_pattern",
"ts_query_string_count",
"ts_query_string_value_for_id",
"ts_tree_copy",
"ts_tree_cursor_current_field_id_wasm",
"ts_tree_cursor_current_node_id_wasm",
"ts_tree_cursor_current_node_is_missing_wasm",
"ts_tree_cursor_current_node_is_named_wasm",
"ts_tree_cursor_current_node_type_id_wasm",
"ts_tree_cursor_current_node_state_id_wasm",
"ts_tree_cursor_current_node_wasm",
"ts_tree_cursor_delete_wasm",
"ts_tree_cursor_end_index_wasm",
"ts_tree_cursor_end_position_wasm",
"ts_tree_cursor_goto_first_child_wasm",
"ts_tree_cursor_goto_last_child_wasm",
"ts_tree_cursor_goto_next_sibling_wasm",
"ts_tree_cursor_goto_previous_sibling_wasm",
"ts_tree_cursor_goto_parent_wasm",
"ts_tree_cursor_new_wasm",
"ts_tree_cursor_reset_wasm",
"ts_tree_cursor_reset_to_wasm",
"ts_tree_cursor_start_index_wasm",
"ts_tree_cursor_start_position_wasm",
"ts_tree_delete",
"ts_tree_edit_wasm",
"ts_tree_get_changed_ranges_wasm",
"ts_tree_root_node_wasm",
"ts_lookahead_iterator_new",
"ts_lookahead_iterator_delete",
"ts_lookahead_iterator_reset_state",
"ts_lookahead_iterator_reset",
"ts_lookahead_iterator_next",
"ts_lookahead_iterator_current_symbol",

61
lib/src/wasm.c
View file

@ -16,46 +16,11 @@
// The following symbols from the C and C++ standard libraries are available
// for external scanners to use.
#define STDLIB_SYMBOL_COUNT 34
const char *STDLIB_SYMBOLS[STDLIB_SYMBOL_COUNT] = {
"_ZNKSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE4copyEPcmm",
"_ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE17__assign_externalEPKcm",
"_ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE17__assign_no_aliasILb0EEERS5_PKcm",
"_ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE17__assign_no_aliasILb1EEERS5_PKcm",
"_ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE25__init_copy_ctor_externalEPKcm",
"_ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE6__initEPKcm",
"_ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7reserveEm",
"_ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE9__grow_byEmmmmmm",
"_ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE9push_backEc",
"_ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEED2Ev",
"_ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE9push_backEw",
"_ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEED2Ev",
"_ZdlPv",
"_Znwm",
"calloc",
"free",
"iswalnum",
"iswalpha",
"iswdigit",
"iswlower",
"iswspace",
"iswupper",
"malloc",
"memchr",
"memcmp",
"memcpy",
"memmove",
"memset",
"realloc",
"strcmp",
"strlen",
"strncpy",
"towlower",
"towupper"
const char *STDLIB_SYMBOLS[] = {
#include "./stdlib-symbols.txt"
};
#define BUILTIN_SYMBOL_COUNT 10
#define MAX_IMPORT_COUNT (BUILTIN_SYMBOL_COUNT + STDLIB_SYMBOL_COUNT)
#define STDLIB_SYMBOL_COUNT (sizeof(STDLIB_SYMBOLS) / sizeof(STDLIB_SYMBOLS[0]))
// The contents of the `dylink.0` custom section of a wasm module,
// as specified by the current WebAssembly dynamic linking ABI proposal.
@ -118,7 +83,7 @@ struct TSWasmStore {
uint32_t current_memory_offset;
uint32_t current_memory_size;
uint32_t current_function_table_offset;
uint16_t fn_indices[STDLIB_SYMBOL_COUNT];
uint16_t *fn_indices;
wasm_globaltype_t *const_i32_type;
wasm_globaltype_t *var_i32_type;
};
@ -582,6 +547,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
wasm_trap_t *trap = NULL;
wasm_message_t message = WASM_EMPTY_VEC;
wasm_exporttype_vec_t export_types = WASM_EMPTY_VEC;
wasmtime_extern_t *imports = NULL;
// Initialize store's memory
wasm_limits_t memory_limits = {.min = INITIAL_MEMORY_SIZE, .max = MAX_MEMORY_SIZE};
@ -677,6 +643,7 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
.language_instances = array_new(),
.function_table = function_table,
.current_memory_offset = 0,
.fn_indices = ts_calloc(STDLIB_SYMBOL_COUNT, sizeof(uint16_t)),
.current_memory_size = 64 * MEMORY_PAGE_SIZE,
.current_function_table_offset = definitions_len,
.const_i32_type = wasm_globaltype_new(wasm_valtype_new_i32(), WASM_CONST),
@ -706,10 +673,9 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
wasmtime_instance_t instance;
wasm_importtype_vec_t import_types = WASM_EMPTY_VEC;
wasmtime_module_imports(stdlib_module, &import_types);
if (import_types.size > MAX_IMPORT_COUNT) goto error;
wasmtime_extern_t imports[MAX_IMPORT_COUNT];
for (unsigned i = 0; i < import_types.size && i < MAX_IMPORT_COUNT; i++) {
imports = ts_calloc(import_types.size, sizeof(wasmtime_extern_t));
for (unsigned i = 0; i < import_types.size; i++) {
wasm_importtype_t *type = import_types.data[i];
const wasm_name_t *import_name = wasm_importtype_name(type);
if (!ts_wasm_store__provide_builtin_import(self, import_name, &imports[i])) {
@ -724,6 +690,8 @@ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) {
}
error = wasmtime_instance_new(context, stdlib_module, imports, import_types.size, &instance, &trap);
ts_free(imports);
imports = NULL;
if (error) {
wasmtime_error_message(error, &message);
wasm_error->kind = TSWasmErrorKindInstantiate;
@ -811,11 +779,13 @@ error:
if (error) wasmtime_error_delete(error);
if (message.size) wasm_byte_vec_delete(&message);
if (export_types.size) wasm_exporttype_vec_delete(&export_types);
if (imports) ts_free(imports);
return NULL;
}
void ts_wasm_store_delete(TSWasmStore *self) {
if (!self) return;
ts_free(self->fn_indices);
wasm_globaltype_delete(self->const_i32_type);
wasm_globaltype_delete(self->var_i32_type);
wasmtime_store_delete(self->store);
@ -852,6 +822,7 @@ static bool ts_wasm_store__instantiate(
wasm_trap_t *trap = NULL;
wasm_message_t message = WASM_EMPTY_VEC;
char *language_function_name = NULL;
wasmtime_extern_t *imports = NULL;
wasmtime_context_t *context = wasmtime_store_context(self->store);
// Grow the function table to make room for the new functions.
@ -886,8 +857,7 @@ static bool ts_wasm_store__instantiate(
// Build the imports list for the module.
wasm_importtype_vec_t import_types = WASM_EMPTY_VEC;
wasmtime_module_imports(module, &import_types);
if (import_types.size > MAX_IMPORT_COUNT) goto error;
wasmtime_extern_t imports[MAX_IMPORT_COUNT];
imports = ts_calloc(import_types.size, sizeof(wasmtime_extern_t));
for (unsigned i = 0; i < import_types.size; i++) {
const wasm_importtype_t *import_type = import_types.data[i];
@ -924,6 +894,8 @@ static bool ts_wasm_store__instantiate(
wasmtime_instance_t instance;
error = wasmtime_instance_new(context, module, imports, import_types.size, &instance, &trap);
wasm_importtype_vec_delete(&import_types);
ts_free(imports);
imports = NULL;
if (error) {
wasmtime_error_message(error, &message);
format(
@ -1025,6 +997,7 @@ error:
if (message.size) wasm_byte_vec_delete(&message);
if (error) wasmtime_error_delete(error);
if (trap) wasm_trap_delete(trap);
if (imports) ts_free(imports);
return false;
}

36
lib/src/wasm/stdlib-symbols.json
View file

@ -1,36 +0,0 @@
[
"__ZNKSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE4copyEPcmm",
"__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE17__assign_externalEPKcm",
"__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE17__assign_no_aliasILb0EEERS5_PKcm",
"__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE17__assign_no_aliasILb1EEERS5_PKcm",
"__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE25__init_copy_ctor_externalEPKcm",
"__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE6__initEPKcm",
"__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7reserveEm",
"__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE9__grow_byEmmmmmm",
"__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE9push_backEc",
"__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEED2Ev",
"__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE9push_backEw",
"__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEED2Ev",
"__ZdlPv",
"__Znwm",
"_calloc",
"_free",
"_iswalnum",
"_iswalpha",
"_iswdigit",
"_iswlower",
"_iswspace",
"_iswupper",
"_malloc",
"_memchr",
"_memcmp",
"_memcpy",
"_memmove",
"_memset",
"_realloc",
"_strcmp",
"_strlen",
"_strncpy",
"_towlower",
"_towupper"
]

36
lib/src/wasm/stdlib-symbols.txt Normal file
View file

@ -0,0 +1,36 @@
"_ZNKSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE4copyEPcmm",
"_ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE17__assign_externalEPKcm",
"_ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE17__assign_no_aliasILb0EEERS5_PKcm",
"_ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE17__assign_no_aliasILb1EEERS5_PKcm",
"_ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE25__init_copy_ctor_externalEPKcm",
"_ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE6__initEPKcm",
"_ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7reserveEm",
"_ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE9__grow_byEmmmmmm",
"_ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE9push_backEc",
"_ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEED2Ev",
"_ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE9push_backEw",
"_ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEED2Ev",
"_ZdlPv",
"_Znwm",
"calloc",
"free",
"iswalnum",
"iswalpha",
"iswblank",
"iswdigit",
"iswlower",
"iswspace",
"iswupper",
"iswxdigit",
"malloc",
"memchr",
"memcmp",
"memcpy",
"memmove",
"memset",
"realloc",
"strcmp",
"strlen",
"strncpy",
"towlower",
"towupper",

4055
lib/src/wasm/wasm-stdlib.h
View file

File diff suppressed because it is too large Load diff

20
script/build-wasm
View file

@ -30,6 +30,7 @@ EOF
set -e
web_dir=lib/binding_web
src_dir=lib/src
emscripten_flags="-O3"
minify_js=1
force_docker=0
@ -98,6 +99,15 @@ mkdir -p target/scratch
runtime_methods='stringToUTF16','AsciiToString'
# Remove quotes, add leading underscores, remove newlines, remove trailing comma.
EXPORTED_FUNCTIONS=$( \
cat ${src_dir}/wasm/stdlib-symbols.txt ${web_dir}/exports.txt | \
sed -e 's/"//g' | \
sed -e 's/^/_/g' | \
tr -d '\n"' | \
sed -e 's/,$//' \
)
# Use emscripten to generate `tree-sitter.js` and `tree-sitter.wasm`
# in the `target/scratch` directory
$emcc \
@ -108,14 +118,14 @@ $emcc \
-s FILESYSTEM=0 \
-s NODEJS_CATCH_EXIT=0 \
-s NODEJS_CATCH_REJECTION=0 \
-s EXPORTED_FUNCTIONS=@${web_dir}/exports.json \
-s EXPORTED_FUNCTIONS=${EXPORTED_FUNCTIONS} \
-s EXPORTED_RUNTIME_METHODS=$runtime_methods \
$emscripten_flags \
-fno-exceptions \
-std=c99 \
-D 'fprintf(...)=' \
-D NDEBUG= \
-I lib/src \
-I ${src_dir} \
-I lib/include \
--js-library ${web_dir}/imports.js \
--pre-js ${web_dir}/prefix.js \
@ -139,9 +149,9 @@ if [[ "$minify_js" == "1" ]]; then
--mangle \
--keep-classnames \
-- target/scratch/tree-sitter.js \
> $web_dir/tree-sitter.js
> ${web_dir}/tree-sitter.js
else
cp target/scratch/tree-sitter.js $web_dir/tree-sitter.js
cp target/scratch/tree-sitter.js ${web_dir}/tree-sitter.js
fi
mv target/scratch/tree-sitter.wasm $web_dir/tree-sitter.wasm
mv target/scratch/tree-sitter.wasm ${web_dir}/tree-sitter.wasm

11
script/build-wasm-stdlib
View file

@ -2,12 +2,21 @@
set -e
# Remove quotes, add leading underscores, remove newlines, remove trailing comma.
EXPORTED_FUNCTIONS=$( \
cat lib/src/wasm/stdlib-symbols.txt | \
sed -e 's/"//g' | \
sed -e 's/^/_/g' | \
tr -d '\n"' | \
sed -e 's/,$//' \
)
emcc \
-o stdlib.wasm \
-Os \
--no-entry \
-s MAIN_MODULE=2 \
-s 'EXPORTED_FUNCTIONS=@lib/src/wasm/stdlib-symbols.json' \
-s "EXPORTED_FUNCTIONS=${EXPORTED_FUNCTIONS}" \
-s 'ALLOW_MEMORY_GROWTH' \
-s 'TOTAL_MEMORY=4MB' \
-fvisibility=hidden \