Merge pull request #1864 from tree-sitter/wasm-language

Add optional WASM feature to the native library, allowing it to run wasm-compiled parsers via wasmtime

cli/src/tests/helpers/dirs.rs

@@ -8,6 +8,7 @@ lazy_static! {
         fs::create_dir_all(&result).unwrap();
         result
     };
+    pub static ref WASM_DIR: PathBuf = ROOT_DIR.join("target").join("release");
     pub static ref SCRATCH_DIR: PathBuf = {
         // https://doc.rust-lang.org/reference/conditional-compilation.html
         let vendor = if cfg!(target_vendor = "apple") {

cli/src/tests/helpers/fixtures.rs

@@ -72,31 +72,28 @@ pub fn get_tags_config(language_name: &str) -> TagsConfiguration {
 }
 
 pub fn get_test_language(name: &str, parser_code: &str, path: Option<&Path>) -> Language {
-    let parser_c_path = SCRATCH_DIR.join(&format!("{}-parser.c", name));
-    if !fs::read_to_string(&parser_c_path)
-        .map(|content| content == parser_code)
-        .unwrap_or(false)
-    {
-        fs::write(&parser_c_path, parser_code).unwrap();
+    let src_dir = SCRATCH_DIR.join("src").join(name);
+    fs::create_dir_all(&src_dir).unwrap();
+
+    let parser_path = src_dir.join("parser.c");
+    if !fs::read_to_string(&parser_path).map_or(false, |content| content == parser_code) {
+        fs::write(&parser_path, parser_code).unwrap();
     }
-    let scanner_path = path.and_then(|p| {
-        let result = p.join("scanner.c");
-        if result.exists() {
-            Some(result)
-        } else {
-            None
+
+    if let Some(path) = path {
+        let scanner_path = path.join("scanner.c");
+        if scanner_path.exists() {
+            let scanner_code = fs::read_to_string(&scanner_path).unwrap();
+            let scanner_copy_path = src_dir.join("scanner.c");
+            if !fs::read_to_string(&scanner_copy_path)
+                .map_or(false, |content| content == scanner_code)
+            {
+                fs::write(&scanner_copy_path, scanner_code).unwrap();
+            }
         }
-    });
+    }
+
     TEST_LOADER
-        .load_language_from_sources(name, &HEADER_DIR, &parser_c_path, scanner_path.as_deref())
+        .load_language_at_path_with_name(&src_dir, &HEADER_DIR, name)
         .unwrap()
 }
-
-pub fn get_test_grammar(name: &str) -> (String, Option<PathBuf>) {
-    let dir = fixtures_dir().join("test_grammars").join(name);
-    let grammar = fs::read_to_string(&dir.join("grammar.json")).expect(&format!(
-        "Can't find grammar.json for test grammar {}",
-        name
-    ));
-    (grammar, Some(dir))
-}

cli/src/tests/mod.rs

@@ -14,3 +14,6 @@ mod test_highlight_test;
 mod test_tags_test;
 mod text_provider_test;
 mod tree_test;
+
+#[cfg(feature = "wasm")]
+mod wasm_language_test;

cli/src/tests/parser_test.rs

@@ -2,13 +2,15 @@ use super::helpers::{
     allocations,
     edits::invert_edit,
     edits::ReadRecorder,
-    fixtures::{get_language, get_test_grammar, get_test_language},
+    fixtures::{get_language, get_test_language},
 };
 use crate::{
     generate::generate_parser_for_grammar,
     parse::{perform_edit, Edit},
+    tests::helpers::fixtures::fixtures_dir,
 };
 use std::{
+    fs,
     sync::atomic::{AtomicUsize, Ordering},
     thread, time,
 };
@@ -427,16 +429,15 @@ fn test_parsing_empty_file_with_reused_tree() {
 
 #[test]
 fn test_parsing_after_editing_tree_that_depends_on_column_values() {
-    let (grammar, path) = get_test_grammar("uses_current_column");
+    let dir = fixtures_dir()
+        .join("test_grammars")
+        .join("uses_current_column");
+    let grammar = fs::read_to_string(&dir.join("grammar.json")).unwrap();
     let (grammar_name, parser_code) = generate_parser_for_grammar(&grammar).unwrap();
 
     let mut parser = Parser::new();
     parser
-        .set_language(get_test_language(
-            &grammar_name,
-            &parser_code,
-            path.as_ref().map(AsRef::as_ref),
-        ))
+        .set_language(get_test_language(&grammar_name, &parser_code, Some(&dir)))
         .unwrap();
 
     let mut code = b"

cli/src/tests/wasm_language_test.rs

@@ -0,0 +1,92 @@
+use crate::tests::helpers::fixtures::WASM_DIR;
+use lazy_static::lazy_static;
+use std::fs;
+use tree_sitter::{wasmtime::Engine, Parser, WasmError, WasmErrorKind, WasmStore};
+
+lazy_static! {
+    static ref ENGINE: Engine = Engine::default();
+}
+
+#[test]
+fn test_load_wasm_language() {
+    let mut store = WasmStore::new(ENGINE.clone()).unwrap();
+    let mut parser = Parser::new();
+
+    let wasm_cpp = fs::read(&WASM_DIR.join(format!("tree-sitter-cpp.wasm"))).unwrap();
+    let wasm_rs = fs::read(&WASM_DIR.join(format!("tree-sitter-rust.wasm"))).unwrap();
+    let wasm_rb = fs::read(&WASM_DIR.join(format!("tree-sitter-ruby.wasm"))).unwrap();
+    let wasm_typescript = fs::read(&WASM_DIR.join(format!("tree-sitter-typescript.wasm"))).unwrap();
+
+    let language_rust = store.load_language("rust", &wasm_rs).unwrap();
+    let language_cpp = store.load_language("cpp", &wasm_cpp).unwrap();
+    let language_ruby = store.load_language("ruby", &wasm_rb).unwrap();
+    let language_typescript = store.load_language("typescript", &wasm_typescript).unwrap();
+    parser.set_wasm_store(store).unwrap();
+
+    let mut parser2 = Parser::new();
+    parser2
+        .set_wasm_store(WasmStore::new(ENGINE.clone()).unwrap())
+        .unwrap();
+
+    for mut parser in [parser, parser2] {
+        for _ in 0..2 {
+            parser.set_language(language_cpp).unwrap();
+            let tree = parser.parse("A<B> c = d();", None).unwrap();
+            assert_eq!(
+                tree.root_node().to_sexp(),
+                "(translation_unit (declaration type: (template_type name: (type_identifier) arguments: (template_argument_list (type_descriptor type: (type_identifier)))) declarator: (init_declarator declarator: (identifier) value: (call_expression function: (identifier) arguments: (argument_list)))))"
+            );
+
+            parser.set_language(language_rust).unwrap();
+            let tree = parser.parse("const A: B = c();", None).unwrap();
+            assert_eq!(
+                tree.root_node().to_sexp(),
+                "(source_file (const_item name: (identifier) type: (type_identifier) value: (call_expression function: (identifier) arguments: (arguments))))"
+            );
+
+            parser.set_language(language_ruby).unwrap();
+            let tree = parser.parse("class A; end", None).unwrap();
+            assert_eq!(
+                tree.root_node().to_sexp(),
+                "(program (class name: (constant)))"
+            );
+
+            parser.set_language(language_typescript).unwrap();
+            let tree = parser.parse("class A {}", None).unwrap();
+            assert_eq!(
+                tree.root_node().to_sexp(),
+                "(program (class_declaration name: (type_identifier) body: (class_body)))"
+            );
+        }
+    }
+}
+
+#[test]
+fn test_load_wasm_errors() {
+    let mut store = WasmStore::new(ENGINE.clone()).unwrap();
+    let wasm = fs::read(&WASM_DIR.join(format!("tree-sitter-rust.wasm"))).unwrap();
+
+    let bad_wasm = &wasm[1..];
+    assert_eq!(
+        store.load_language("rust", &bad_wasm).unwrap_err(),
+        WasmError {
+            kind: WasmErrorKind::Parse,
+            message: "failed to parse dylink section of wasm module".into(),
+        }
+    );
+
+    assert_eq!(
+        store.load_language("not_rust", &wasm).unwrap_err(),
+        WasmError {
+            kind: WasmErrorKind::Instantiate,
+            message: "module did not contain language function: tree_sitter_not_rust".into(),
+        }
+    );
+
+    let mut bad_wasm = wasm.clone();
+    bad_wasm[300..500].iter_mut().for_each(|b| *b = 0);
+    assert_eq!(
+        store.load_language("rust", &bad_wasm).unwrap_err().kind,
+        WasmErrorKind::Compile,
+    );
+}