feat: use compileStreaming when loading wasm parsers

crates/cli/src/main.rs

@@ -908,6 +908,8 @@ impl Build {
             eprintln!("Warning: --docker flag is no longer used, and will be removed in a future release.");
         }
 
+        loader.debug_build(self.debug);
+
         if self.wasm {
             let output_path = self.output.map(|path| current_dir.join(path));
             let root_path = get_root_path(&grammar_path.join("tree-sitter.json"))?;
@@ -946,7 +948,6 @@ impl Build {
                 (false, false) => &[],
             };
 
-            loader.debug_build(self.debug);
             loader.force_rebuild(true);
 
             let config = Config::load(None)?;

crates/loader/src/loader.rs

@@ -1042,7 +1042,7 @@ impl Loader {
             output_name,
             "-fPIC",
             "-shared",
-            "-Os",
+            if self.debug_build { "-g" } else { "-Os" },
             format!("-Wl,--export=tree_sitter_{language_name}").as_str(),
             "-Wl,--allow-undefined",
             "-Wl,--no-entry",

crates/xtask/src/build_wasm.rs

@@ -247,7 +247,7 @@ fn build_wasm(cmd: &mut Command, edit_tsd: bool) -> Result<()> {
                     "undefined, localScope?: any | undefined, handle?: number | undefined): any"
                 ),
                 concat!(
-                    "loadWebAssemblyModule(binary: Uint8Array, flags: Record<string, boolean>,",
+                    "loadWebAssemblyModule(binary: Uint8Array | WebAssembly.Module, flags: Record<string, boolean>,",
                     " libName?: string, localScope?: Record<string, unknown>, handle?: number):",
                     " Promise<Record<string, () => number>>"
                 ),

lib/binding_web/lib/web-tree-sitter.d.ts

@@ -11,13 +11,11 @@ declare namespace RuntimeExports {
      *   maximum number of bytes to read. You can omit this parameter to scan the
      *   string until the first 0 byte. If maxBytesToRead is passed, and the string
      *   at [ptr, ptr+maxBytesToReadr[ contains a null byte in the middle, then the
-     *   string will cut short at that byte index (i.e. maxBytesToRead will not
-     *   produce a string of exact length [ptr, ptr+maxBytesToRead[) N.B. mixing
-     *   frequent uses of UTF8ToString() with and without maxBytesToRead may throw
-     *   JS JIT optimizations off, so it is worth to consider consistently using one
+     *   string will cut short at that byte index.
+     * @param {boolean=} ignoreNul - If true, the function will not stop on a NUL character.
      * @return {string}
      */
-    function UTF8ToString(ptr: number, maxBytesToRead?: number): string;
+    function UTF8ToString(ptr: number, maxBytesToRead?: number | undefined, ignoreNul?: boolean | undefined): string;
     function lengthBytesUTF8(str: string): number;
     function stringToUTF16(str: string, outPtr: number, maxBytesToWrite: number): number;
     /**
@@ -25,7 +23,7 @@ declare namespace RuntimeExports {
      * @param {Object=} localScope
      * @param {number=} handle
      */
-    function loadWebAssemblyModule(binary: Uint8Array, flags: Record<string, boolean>, libName?: string, localScope?: Record<string, unknown>, handle?: number): Promise<Record<string, () => number>>;
+    function loadWebAssemblyModule(binary: Uint8Array | WebAssembly.Module, flags: Record<string, boolean>, libName?: string, localScope?: Record<string, unknown>, handle?: number): Promise<Record<string, () => number>>;
     /**
      * @param {number} ptr
      * @param {string} type

lib/binding_web/src/language.ts

@@ -255,29 +255,33 @@ export class Language {
    * The module can be provided as a path to a file or as a buffer.
    */
   static async load(input: string | Uint8Array): Promise<Language> {
-    let bytes: Promise<Uint8Array>;
+    let binary: Uint8Array | WebAssembly.Module;
     if (input instanceof Uint8Array) {
-      bytes = Promise.resolve(input);
+      binary = input;
+    // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+    } else if (globalThis.process?.versions.node) {
+      const fs: typeof import('fs/promises') = await import('fs/promises');
+      binary = await fs.readFile(input);
     } else {
-      // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-      if (globalThis.process?.versions.node) {
-        const fs: typeof import('fs/promises') = await import('fs/promises');
-        bytes = fs.readFile(input);
-      } else {
-        bytes = fetch(input)
-          .then((response) => response.arrayBuffer()
-            .then((buffer) => {
-              if (response.ok) {
-                return new Uint8Array(buffer);
-              } else {
-                const body = new TextDecoder('utf-8').decode(buffer);
-                throw new Error(`Language.load failed with status ${response.status}.\n\n${body}`);
-              }
-            }));
+      const response = await fetch(input);
+
+      if (!response.ok){
+        const body = await response.text();
+        throw new Error(`Language.load failed with status ${response.status}.\n\n${body}`);
+      }
+
+      const retryResp = response.clone();
+      try {
+        binary = await WebAssembly.compileStreaming(response);
+      } catch (reason) {
+        console.error('wasm streaming compile failed:', reason);
+        console.error('falling back to ArrayBuffer instantiation');
+        // fallback, probably because of bad MIME type
+        binary = new Uint8Array(await retryResp.arrayBuffer())
       }
     }
 
-    const mod = await C.loadWebAssemblyModule(await bytes, { loadAsync: true });
+    const mod = await C.loadWebAssemblyModule(binary, { loadAsync: true });
     const symbolNames = Object.keys(mod);
     const functionName = symbolNames.find((key) => LANGUAGE_FUNCTION_REGEX.test(key) &&
       !key.includes('external_scanner_'));