feat(xtask): give wasm-opt the wasi-sdk treatment

crates/loader/binaryen-version

@@ -0,0 +1 @@
+125

crates/xtask/src/build_wasm.rs

@@ -16,7 +16,7 @@ use notify::{
     EventKind, RecursiveMode,
 };
 use notify_debouncer_full::new_debouncer;
-use tree_sitter_loader::{IoError, LoaderError, WasiSDKClangError};
+use tree_sitter_loader::{IoError, LoaderError, WasmToolError};
 
 use crate::{
     bail_on_err, embed_sources::embed_sources_in_map, watch_wasm, BuildWasm, EMSCRIPTEN_TAG,
@@ -53,6 +53,40 @@ const EXPORTED_RUNTIME_METHODS: [&str; 20] = [
 ];
 
 const WASI_SDK_VERSION: &str = include_str!("../../loader/wasi-sdk-version").trim_ascii();
+const BINARYEN_VERSION: &str = include_str!("../../loader/binaryen-version").trim_ascii();
+
+#[cfg(all(target_os = "macos", target_arch = "aarch64"))]
+const ARCH_OS: Result<&str, LoaderError> = Ok("arm64-macos");
+#[cfg(all(target_os = "macos", target_arch = "x86_64"))]
+const ARCH_OS: Result<&str, LoaderError> = Ok("x86_64-macos");
+#[cfg(all(
+    target_os = "macos",
+    not(any(target_arch = "aarch64", target_arch = "x86_64"))
+))]
+const ARCH_OS: Result<&str, LoaderError> = Err(LoaderError::WasiSDKPlatform);
+
+#[cfg(all(target_os = "windows", target_arch = "aarch64"))]
+const ARCH_OS: Result<&str, LoaderError> = Ok("arm64-windows");
+#[cfg(all(target_os = "windows", target_arch = "x86_64"))]
+const ARCH_OS: Result<&str, LoaderError> = Ok("x86_64-windows");
+#[cfg(all(
+    target_os = "windows",
+    not(any(target_arch = "aarch64", target_arch = "x86_64"))
+))]
+const ARCH_OS: Result<&str, LoaderError> = Err(LoaderError::WasiSDKPlatform);
+
+#[cfg(all(target_os = "linux", target_arch = "aarch64"))]
+const ARCH_OS: Result<&str, LoaderError> = Ok("arm64-linux");
+#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
+const ARCH_OS: Result<&str, LoaderError> = Ok("x86_64-linux");
+#[cfg(all(
+    target_os = "linux",
+    not(any(target_arch = "aarch64", target_arch = "x86_64"))
+))]
+const ARCH_OS: Result<&str, LoaderError> = Err(LoaderError::WasiSDKPlatform);
+
+#[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))]
+const ARCH_OS: Result<&str, LoaderError> = Err(LoaderError::WasiSDKPlatform);
 
 pub fn run_wasm(args: &BuildWasm) -> Result<()> {
     let mut emscripten_flags = if args.debug {
@@ -314,7 +348,7 @@ fn build_wasm(cmd: &mut Command, edit_tsd: bool) -> Result<()> {
     Ok(())
 }
 
-/// This ensures that the wasi-sdk is available, downloading and extracting it if necessary,
+/// This ensures that wasi-sdk is available, downloading and extracting it if necessary,
 /// and returns the path to the `clang` executable.
 ///
 /// If `TREE_SITTER_WASI_SDK_PATH` is set, it will use that path to look for the clang executable.
@@ -335,19 +369,96 @@ pub fn ensure_wasi_sdk_exists() -> Result<PathBuf> {
         vec!["clang", "wasm32-unknown-wasi-clang", "wasm32-wasi-clang"]
     };
 
-    if let Ok(wasi_sdk_path) = std::env::var("TREE_SITTER_WASI_SDK_PATH") {
-        let wasi_sdk_dir = PathBuf::from(wasi_sdk_path);
+    if let Some(path) = get_existing_tool(
+        "clang",
+        "wasi-sdk",
+        &possible_executables,
+        "TREE_SITTER_WASI_SDK_PATH",
+    )? {
+        return Ok(path);
+    }
 
-        for exe in &possible_executables {
-            let clang_exe = wasi_sdk_dir.join("bin").join(exe);
-            if clang_exe.exists() {
-                return Ok(clang_exe);
+    let arch_os = ARCH_OS?;
+    let sdk_filename = format!("wasi-sdk-{WASI_SDK_VERSION}-{arch_os}.tar.gz");
+    let wasi_sdk_major_version = WASI_SDK_VERSION
+        .trim_end_matches(char::is_numeric) // trim minor version...
+        .trim_end_matches('.'); // ...and '.' separator
+    let sdk_url = format!(
+        "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-{wasi_sdk_major_version}/{sdk_filename}",
+    );
+    download_tool(
+        "clang",
+        "wasi-sdk",
+        &sdk_filename,
+        &sdk_url,
+        &possible_executables,
+    )
+}
+
+/// This ensures that binaryen is available, downloading and extracting it if necessary,
+/// and returns the path to the `wasm-opt` executable.
+///
+/// If `TREE_SITTER_BINARYEN_PATH` is set, it will use that path to look for the wasm-opt executable.
+///
+/// Note that this is just a minimially modified version of
+/// `tree_sitter_loader::ensure_binaryen_exists`. In the loader, this functionality is implemented
+/// as a private method of `Loader`. Rather than add this to the public API, we just
+/// re-implement it. Any fixes and/or modifications made to the loader's copy should be reflected
+/// here.
+pub fn ensure_binaryen_exists() -> Result<PathBuf> {
+    let possible_executables = if cfg!(windows) {
+        vec![
+            "wasm-opt.exe",
+            "wasm32-unknown-wasm-opt.exe",
+            "wasm32-wasm-opt.exe",
+        ]
+    } else {
+        vec!["wasm-opt", "wasm32-unknown-wasm-opt", "wasm32-wasm-opt"]
+    };
+    if let Some(path) = get_existing_tool(
+        "wasm-opt",
+        "binaryen",
+        &possible_executables,
+        "TREE_SITTER_BINARYEN_PATH",
+    )? {
+        return Ok(path);
+    }
+
+    let arch_os = ARCH_OS?.replace("arm64-linux", "aarch64-linux");
+    let binaryen_filename = format!("binaryen-version_{BINARYEN_VERSION}-{arch_os}.tar.gz");
+    let binaryen_url = format!(
+       "https://github.com/WebAssembly/binaryen/releases/download/version_{BINARYEN_VERSION}/{binaryen_filename}"
+    );
+    download_tool(
+        "wasm-opt",
+        "binaryen",
+        &binaryen_filename,
+        &binaryen_url,
+        &possible_executables,
+    )
+}
+
+fn get_existing_tool(
+    tool_name: &'static str,
+    toolchain: &'static str,
+    possible_exes: &[&'static str],
+    env_var: &str,
+) -> Result<Option<PathBuf>> {
+    if let Ok(tool_path) = std::env::var(env_var) {
+        let tool_dir = PathBuf::from(tool_path);
+
+        for exe in possible_exes {
+            let tool_exe = tool_dir.join("bin").join(exe);
+            if tool_exe.exists() {
+                return Ok(Some(tool_exe));
             }
         }
 
-        Err(LoaderError::WasiSDKClang(WasiSDKClangError {
-            wasi_sdk_dir: wasi_sdk_dir.to_string_lossy().to_string(),
-            possible_executables: possible_executables.clone(),
+        Err(LoaderError::WasmTool(WasmToolError {
+            exe: tool_name,
+            toolchain,
+            tool_dir: tool_dir.to_string_lossy().to_string(),
+            possible_executables: possible_exes.to_vec(),
             download: false,
         }))?;
     }
@@ -362,82 +473,72 @@ pub fn ensure_wasi_sdk_exists() -> Result<PathBuf> {
         })
     })?;
 
-    let wasi_sdk_dir = cache_dir.join("wasi-sdk");
+    let tool_dir = cache_dir.join(toolchain);
 
-    for exe in &possible_executables {
-        let clang_exe = wasi_sdk_dir.join("bin").join(exe);
-        if clang_exe.exists() {
-            return Ok(clang_exe);
+    for exe in possible_exes {
+        let tool_exe = tool_dir.join("bin").join(exe);
+        if tool_exe.exists() {
+            return Ok(Some(tool_exe));
         }
     }
 
-    fs::create_dir_all(&wasi_sdk_dir).map_err(|error| {
+    Ok(None)
+}
+
+fn download_tool(
+    tool_name: &'static str,
+    toolchain: &'static str,
+    filename: &str,
+    url: &str,
+    possible_exes: &[&'static str],
+) -> Result<PathBuf> {
+    let cache_dir = etcetera::choose_base_strategy()?
+        .cache_dir()
+        .join("tree-sitter");
+    let tool_dir = cache_dir.join(tool_name);
+
+    fs::create_dir_all(&tool_dir).map_err(|error| {
         LoaderError::IO(IoError {
             error,
-            path: Some(wasi_sdk_dir.to_string_lossy().to_string()),
+            path: Some(tool_dir.to_string_lossy().to_string()),
         })
     })?;
 
-    let arch_os = if cfg!(target_os = "macos") {
-        if cfg!(target_arch = "aarch64") {
-            "arm64-macos"
-        } else {
-            "x86_64-macos"
-        }
-    } else if cfg!(target_os = "windows") {
-        if cfg!(target_arch = "aarch64") {
-            "arm64-windows"
-        } else {
-            "x86_64-windows"
-        }
-    } else if cfg!(target_os = "linux") {
-        if cfg!(target_arch = "aarch64") {
-            "arm64-linux"
-        } else {
-            "x86_64-linux"
-        }
-    } else {
-        Err(LoaderError::WasiSDKPlatform)?
-    };
-
-    let sdk_filename = format!("wasi-sdk-{WASI_SDK_VERSION}-{arch_os}.tar.gz");
-    let wasi_sdk_major_version = WASI_SDK_VERSION
-        .trim_end_matches(char::is_numeric) // trim minor version...
-        .trim_end_matches('.'); // ...and '.' separator
-    let sdk_url = format!(
-        "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-{wasi_sdk_major_version}/{sdk_filename}",
-    );
-
-    eprintln!("Downloading wasi-sdk from {sdk_url}...");
-    let temp_tar_path = cache_dir.join(sdk_filename);
+    eprintln!("Downloading {tool_name} from {url}...");
+    let temp_tar_path = cache_dir.join(filename);
 
     let status = Command::new("curl")
         .arg("-f")
         .arg("-L")
         .arg("-o")
         .arg(&temp_tar_path)
-        .arg(&sdk_url)
+        .arg(url)
         .status()
-        .map_err(|e| LoaderError::Curl(sdk_url.clone(), e))?;
+        .map_err(|e| LoaderError::Curl(url.to_string(), e))?;
 
     if !status.success() {
-        Err(LoaderError::WasiSDKDownload(sdk_url))?;
+        Err(LoaderError::WasmToolDownload {
+            tool: tool_name,
+            url: url.to_string(),
+        })?;
     }
 
-    eprintln!("Extracting wasi-sdk to {}...", wasi_sdk_dir.display());
-    extract_tar_gz_with_strip(&temp_tar_path, &wasi_sdk_dir)?;
+    eprintln!("Extracting {tool_name} to {}...", tool_dir.display());
+    extract_tar_gz_with_strip(&temp_tar_path, &tool_dir)?;
 
     fs::remove_file(temp_tar_path).ok();
-    for exe in &possible_executables {
-        let clang_exe = wasi_sdk_dir.join("bin").join(exe);
-        if clang_exe.exists() {
-            return Ok(clang_exe);
+    for exe in possible_exes {
+        let tool_exe = tool_dir.join("bin").join(exe);
+        if tool_exe.exists() {
+            return Ok(tool_exe);
         }
     }
 
-    Err(LoaderError::WasiSDKClang(WasiSDKClangError {
-        wasi_sdk_dir: wasi_sdk_dir.to_string_lossy().to_string(),
-        possible_executables,
+    Err(LoaderError::WasmTool(WasmToolError {
+        exe: tool_name,
+        toolchain,
+        tool_dir: tool_dir.to_string_lossy().to_string(),
+        possible_executables: possible_exes.to_vec(),
         download: true,
     }))?
 }
@@ -471,7 +572,7 @@ pub fn run_wasm_stdlib() -> Result<()> {
 
     let clang_exe = ensure_wasi_sdk_exists()?;
 
-    let output = Command::new(&clang_exe)
+    let compile_output = Command::new(&clang_exe)
         .args([
             "-o",
             "stdlib.wasm",
@@ -494,7 +595,21 @@ pub fn run_wasm_stdlib() -> Result<()> {
         .arg("crates/language/wasm/src/stdlib.c")
         .output()?;
 
-    bail_on_err(&output, "Failed to compile the Tree-sitter Wasm stdlib")?;
+    bail_on_err(
+        &compile_output,
+        "Failed to compile the Tree-sitter Wasm stdlib",
+    )?;
+
+    let wasm_opt_exe = ensure_binaryen_exists()?;
+
+    let opt_output = Command::new(&wasm_opt_exe)
+        .args(["stdlib.wasm", "-Os", "-o", "stdlib.wasm"])
+        .output()?;
+
+    bail_on_err(
+        &opt_output,
+        "Failed to optimize the Tree-sitter Wasm stdlib",
+    )?;
 
     let xxd = Command::new("xxd")
         .args(["-C", "-i", "stdlib.wasm"])

docs/src/6-contributing.md

@@ -93,8 +93,9 @@ cargo xtask build-wasm-stdlib
 
 This command looks for the [Wasi SDK][wasi_sdk] indicated by the `TREE_SITTER_WASI_SDK_PATH`
 environment variable. If you don't have the binary, it can be downloaded from wasi-sdk's [releases][wasi-sdk-releases]
-page. Note that any changes to `crates/language/wasm/**` requires rebuilding the tree-sitter Wasm stdlib via
-`cargo xtask build-wasm-stdlib`.
+page. Similarly, this command also looks for [ the `wasm-opt` tool from binaryen][binaryen] indicated by the `TREE_SITTER_BINARYEN_PATH`
+environment variable. `wasm-opt` and the rest of the binaryen tool suite can be downloaded from the project's [releases][binaryen-releases]
+page. Note that any changes to `crates/language/wasm/**` requires rebuilding the tree-sitter Wasm stdlib via `cargo xtask build-wasm-stdlib`.
 
 ### Debugging
 
@@ -204,6 +205,8 @@ and the tree-sitter module is fetched from [here][js url]. This, along with the
 
 [admonish]: https://github.com/tommilligan/mdbook-admonish
 [admonish reference]: https://tommilligan.github.io/mdbook-admonish/reference.html
+[binaryen]: https://github.com/WebAssembly/binaryen
+[binaryen-releases]: https://github.com/WebAssembly/binaryen/releases
 [cli crate]: https://crates.io/crates/tree-sitter-cli
 [cli package]: https://www.npmjs.com/package/tree-sitter-cli
 [codemirror]: https://codemirror.net