traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: use compileStreaming when loading wasm parsers

Browse source
This commit is contained in:
Boris Verkhovskiy 2025-09-01 20:08:20 -06:00 • committed by GitHub
parent b75196bb81
commit 9b23cd5394
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 30 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

10
lib/binding_web/lib/web-tree-sitter.d.ts generated vendored
View file

@ -11,13 +11,11 @@ declare namespace RuntimeExports {
* maximum number of bytes to read. You can omit this parameter to scan the
* string until the first 0 byte. If maxBytesToRead is passed, and the string
* at [ptr, ptr+maxBytesToReadr[ contains a null byte in the middle, then the
* string will cut short at that byte index (i.e. maxBytesToRead will not
* produce a string of exact length [ptr, ptr+maxBytesToRead[) N.B. mixing
* frequent uses of UTF8ToString() with and without maxBytesToRead may throw
* JS JIT optimizations off, so it is worth to consider consistently using one
* string will cut short at that byte index.
* @param {boolean=} ignoreNul - If true, the function will not stop on a NUL character.
* @return {string}
*/
function UTF8ToString(ptr: number, maxBytesToRead?: number): string;
function UTF8ToString(ptr: number, maxBytesToRead?: number | undefined, ignoreNul?: boolean | undefined): string;
function lengthBytesUTF8(str: string): number;
function stringToUTF16(str: string, outPtr: number, maxBytesToWrite: number): number;
/**
@ -25,7 +23,7 @@ declare namespace RuntimeExports {
* @param {Object=} localScope
* @param {number=} handle
*/
function loadWebAssemblyModule(binary: Uint8Array, flags: Record<string, boolean>, libName?: string, localScope?: Record<string, unknown>, handle?: number): Promise<Record<string, () => number>>;
function loadWebAssemblyModule(binary: Uint8Array | WebAssembly.Module, flags: Record<string, boolean>, libName?: string, localScope?: Record<string, unknown>, handle?: number): Promise<Record<string, () => number>>;
/**
* @param {number} ptr
* @param {string} type

40
lib/binding_web/src/language.ts
View file

@ -255,29 +255,33 @@ export class Language {
* The module can be provided as a path to a file or as a buffer.
*/
static async load(input: string | Uint8Array): Promise<Language> {
let bytes: Promise<Uint8Array>;
let binary: Uint8Array | WebAssembly.Module;
if (input instanceof Uint8Array) {
bytes = Promise.resolve(input);
binary = input;
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
} else if (globalThis.process?.versions.node) {
const fs: typeof import('fs/promises') = await import('fs/promises');
binary = await fs.readFile(input);
} else {
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
if (globalThis.process?.versions.node) {
const fs: typeof import('fs/promises') = await import('fs/promises');
bytes = fs.readFile(input);
} else {
bytes = fetch(input)
.then((response) => response.arrayBuffer()
.then((buffer) => {
if (response.ok) {
return new Uint8Array(buffer);
} else {
const body = new TextDecoder('utf-8').decode(buffer);
throw new Error(`Language.load failed with status ${response.status}.\n\n${body}`);
}
}));
const response = await fetch(input);
if (!response.ok){
const body = await response.text();
throw new Error(`Language.load failed with status ${response.status}.\n\n${body}`);
}
const retryResp = response.clone();
try {
binary = await WebAssembly.compileStreaming(response);
} catch (reason) {
console.error('wasm streaming compile failed:', reason);
console.error('falling back to ArrayBuffer instantiation');
// fallback, probably because of bad MIME type
binary = new Uint8Array(await retryResp.arrayBuffer())
}
}
const mod = await C.loadWebAssemblyModule(await bytes, { loadAsync: true });
const mod = await C.loadWebAssemblyModule(binary, { loadAsync: true });
const symbolNames = Object.keys(mod);
const functionName = symbolNames.find((key) => LANGUAGE_FUNCTION_REGEX.test(key) &&
!key.includes('external_scanner_'));