From 728793a160c42cd2d2c6ecf38df346304ab46ae5 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sat, 9 Mar 2024 01:28:35 -0500 Subject: [PATCH] feat(wasm)!: keep API in-line with upstream and start aligning with node --- lib/binding_web/binding.c | 187 +++++++++++++++++++++--- lib/binding_web/binding.js | 203 ++++++++++++++++++++++----- lib/binding_web/exports.txt | 13 ++ lib/binding_web/test/helper.js | 4 + lib/binding_web/test/node-test.js | 171 +++++++++++++++++++++- lib/binding_web/test/parser-test.js | 99 ++++++++++++- lib/binding_web/test/query-test.js | 53 ++++++- lib/binding_web/tree-sitter-web.d.ts | 120 ++++++++++------ 8 files changed, 743 insertions(+), 107 deletions(-) diff --git a/lib/binding_web/binding.c b/lib/binding_web/binding.c index 4be22a7e..9db2bc5f 100644 --- a/lib/binding_web/binding.c +++ b/lib/binding_web/binding.c @@ -1,9 +1,9 @@ -#include -#include -#include #include "array.h" #include "point.h" +#include +#include + /*****************************/ /* Section - Data marshaling */ /*****************************/ @@ -31,7 +31,7 @@ static uint32_t byte_to_code_unit(uint32_t byte) { } static inline void marshal_node(const void **buffer, TSNode node) { - buffer[0] = (const void *)node.id; + buffer[0] = node.id; buffer[1] = (const void *)byte_to_code_unit(node.context[0]); buffer[2] = (const void *)node.context[1]; buffer[3] = (const void *)byte_to_code_unit(node.context[2]); @@ -50,7 +50,7 @@ static inline TSNode unmarshal_node(const TSTree *tree) { } static inline void marshal_cursor(const TSTreeCursor *cursor) { - TRANSFER_BUFFER[0] = (const void *)cursor->id; + TRANSFER_BUFFER[0] = cursor->id; TRANSFER_BUFFER[1] = (const void *)cursor->context[0]; TRANSFER_BUFFER[2] = (const void *)cursor->context[1]; } @@ -206,11 +206,30 @@ void ts_tree_root_node_wasm(const TSTree *tree) { marshal_node(TRANSFER_BUFFER, ts_tree_root_node(tree)); } +void ts_tree_root_node_with_offset_wasm(const TSTree *tree) { + // read int and point from transfer buffer + const void **address = TRANSFER_BUFFER + 5; + uint32_t offset = code_unit_to_byte((uint32_t)address[0]); + TSPoint extent = unmarshal_point(address + 1); + TSNode node = ts_tree_root_node_with_offset(tree, offset, extent); + marshal_node(TRANSFER_BUFFER, node); +} + void ts_tree_edit_wasm(TSTree *tree) { TSInputEdit edit = unmarshal_edit(); ts_tree_edit(tree, &edit); } +void ts_tree_included_ranges_wasm(const TSTree *tree) { + uint32_t range_count; + TSRange *ranges = ts_tree_included_ranges(tree, &range_count); + for (unsigned i = 0; i < range_count; i++) { + marshal_range(&ranges[i]); + } + TRANSFER_BUFFER[0] = (range_count ? (const void *)range_count : NULL); + TRANSFER_BUFFER[1] = (const void *)ranges; +} + void ts_tree_get_changed_ranges_wasm(TSTree *tree, TSTree *other) { unsigned range_count; TSRange *ranges = ts_tree_get_changed_ranges(tree, other, &range_count); @@ -264,6 +283,24 @@ bool ts_tree_cursor_goto_last_child_wasm(const TSTree *tree) { return result; } +bool ts_tree_cursor_goto_first_child_for_index_wasm(const TSTree *tree) { + TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); + const void **address = TRANSFER_BUFFER + 3; + uint32_t index = code_unit_to_byte((uint32_t)address[0]); + bool result = ts_tree_cursor_goto_first_child_for_byte(&cursor, index); + marshal_cursor(&cursor); + return result; +} + +bool ts_tree_cursor_goto_first_child_for_position_wasm(const TSTree *tree) { + TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); + const void **address = TRANSFER_BUFFER + 3; + TSPoint point = unmarshal_point(address); + bool result = ts_tree_cursor_goto_first_child_for_point(&cursor, point); + marshal_cursor(&cursor); + return result; +} + bool ts_tree_cursor_goto_next_sibling_wasm(const TSTree *tree) { TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); bool result = ts_tree_cursor_goto_next_sibling(&cursor); @@ -278,6 +315,12 @@ bool ts_tree_cursor_goto_previous_sibling_wasm(const TSTree *tree) { return result; } +void ts_tree_cursor_goto_descendant_wasm(const TSTree *tree, uint32_t goal_descendant_index) { + TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); + ts_tree_cursor_goto_descendant(&cursor, goal_descendant_index); + marshal_cursor(&cursor); +} + bool ts_tree_cursor_goto_parent_wasm(const TSTree *tree) { TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); bool result = ts_tree_cursor_goto_parent(&cursor); @@ -309,7 +352,7 @@ bool ts_tree_cursor_current_node_is_missing_wasm(const TSTree *tree) { return ts_node_is_missing(node); } -const uint32_t ts_tree_cursor_current_node_id_wasm(const TSTree *tree) { +uint32_t ts_tree_cursor_current_node_id_wasm(const TSTree *tree) { TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); TSNode node = ts_tree_cursor_current_node(&cursor); return (uint32_t)node.id; @@ -344,6 +387,16 @@ uint32_t ts_tree_cursor_current_field_id_wasm(const TSTree *tree) { return ts_tree_cursor_current_field_id(&cursor); } +uint32_t ts_tree_cursor_current_depth_wasm(const TSTree *tree) { + TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); + return ts_tree_cursor_current_depth(&cursor); +} + +uint32_t ts_tree_cursor_current_descendant_index_wasm(const TSTree *tree) { + TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); + return ts_tree_cursor_current_descendant_index(&cursor); +} + void ts_tree_cursor_current_node_wasm(const TSTree *tree) { TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); marshal_node(TRANSFER_BUFFER, ts_tree_cursor_current_node(&cursor)); @@ -366,6 +419,55 @@ const char *ts_node_field_name_for_child_wasm(const TSTree *tree, uint32_t index return ts_node_field_name_for_child(node, index); } +void ts_node_children_by_field_id_wasm(const TSTree *tree, uint32_t field_id) { + TSNode node = unmarshal_node(tree); + TSTreeCursor cursor = ts_tree_cursor_new(node); + + bool done = field_id == 0; + if (!done) { + ts_tree_cursor_reset(&cursor, node); + ts_tree_cursor_goto_first_child(&cursor); + } + + Array(const void*) result = array_new(); + + while (!done) { + while (ts_tree_cursor_current_field_id(&cursor) != field_id) { + if (!ts_tree_cursor_goto_next_sibling(&cursor)) { + done = true; + break; + } + } + if (done) { + break; + } + TSNode result_node = ts_tree_cursor_current_node(&cursor); + if (!ts_tree_cursor_goto_next_sibling(&cursor)) { + done = true; + } + array_grow_by(&result, 5); + marshal_node(result.contents + result.size - 5, result_node); + } + ts_tree_cursor_delete(&cursor); + + TRANSFER_BUFFER[0] = (const void*)(result.size / 5); + TRANSFER_BUFFER[1] = result.contents; +} + +void ts_node_first_child_for_byte_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + const void** address = TRANSFER_BUFFER + 5; + uint32_t byte = code_unit_to_byte((uint32_t)address[0]); + marshal_node(TRANSFER_BUFFER, ts_node_first_child_for_byte(node, byte)); +} + +void ts_node_first_named_child_for_byte_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + const void** address = TRANSFER_BUFFER + 5; + uint32_t byte = code_unit_to_byte((uint32_t)address[0]); + marshal_node(TRANSFER_BUFFER, ts_node_first_named_child_for_byte(node, byte)); +} + uint16_t ts_node_grammar_symbol_wasm(const TSTree *tree) { TSNode node = unmarshal_node(tree); return ts_node_grammar_symbol(node); @@ -416,6 +518,11 @@ void ts_node_prev_named_sibling_wasm(const TSTree *tree) { marshal_node(TRANSFER_BUFFER, ts_node_prev_named_sibling(node)); } +uint32_t ts_node_descendant_count_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + return ts_node_descendant_count(node); +} + void ts_node_parent_wasm(const TSTree *tree) { TSNode node = unmarshal_node(tree); marshal_node(TRANSFER_BUFFER, ts_node_parent(node)); @@ -515,9 +622,13 @@ void ts_node_named_children_wasm(const TSTree *tree) { marshal_node(address, child); address += 5; i++; - if (i == count) break; + if (i == count) { + break; + } + } + if (!ts_tree_cursor_goto_next_sibling(&scratch_cursor)) { + break; } - if (!ts_tree_cursor_goto_next_sibling(&scratch_cursor)) break; } } TRANSFER_BUFFER[0] = (const void *)count; @@ -526,8 +637,12 @@ void ts_node_named_children_wasm(const TSTree *tree) { bool symbols_contain(const uint32_t *set, uint32_t length, uint32_t value) { for (unsigned i = 0; i < length; i++) { - if (set[i] == value) return true; - if (set[i] > value) break; + if (set[i] == value) { + return true; + } + if (set[i] > value) { + break; + } } return false; } @@ -563,14 +678,18 @@ void ts_node_descendants_of_type_wasm( if (ts_tree_cursor_goto_next_sibling(&scratch_cursor)) { already_visited_children = false; } else { - if (!ts_tree_cursor_goto_parent(&scratch_cursor)) break; + if (!ts_tree_cursor_goto_parent(&scratch_cursor)) { + break; + } already_visited_children = true; } continue; } // If this node is after the selected range, then stop walking. - if (point_lte(end_point, ts_node_start_point(descendant))) break; + if (point_lte(end_point, ts_node_start_point(descendant))) { + break; + } // Add the node to the result if its type matches one of the given // node types. @@ -585,14 +704,18 @@ void ts_node_descendants_of_type_wasm( } else if (ts_tree_cursor_goto_next_sibling(&scratch_cursor)) { already_visited_children = false; } else { - if (!ts_tree_cursor_goto_parent(&scratch_cursor)) break; + if (!ts_tree_cursor_goto_parent(&scratch_cursor)) { + break; + } already_visited_children = true; } } else { if (ts_tree_cursor_goto_next_sibling(&scratch_cursor)) { already_visited_children = false; } else { - if (!ts_tree_cursor_goto_parent(&scratch_cursor)) break; + if (!ts_tree_cursor_goto_parent(&scratch_cursor)) { + break; + } } } } @@ -626,6 +749,11 @@ int ts_node_is_missing_wasm(const TSTree *tree) { return ts_node_is_missing(node); } +int ts_node_is_extra_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + return ts_node_is_extra(node); +} + uint16_t ts_node_parse_state_wasm(const TSTree *tree) { TSNode node = unmarshal_node(tree); return ts_node_parse_state(node); @@ -647,9 +775,14 @@ void ts_query_matches_wasm( uint32_t start_column, uint32_t end_row, uint32_t end_column, - uint32_t match_limit + uint32_t start_index, + uint32_t end_index, + uint32_t match_limit, + uint32_t max_start_depth ) { - if (!scratch_query_cursor) scratch_query_cursor = ts_query_cursor_new(); + if (!scratch_query_cursor) { + scratch_query_cursor = ts_query_cursor_new(); + } if (match_limit == 0) { ts_query_cursor_set_match_limit(scratch_query_cursor, UINT32_MAX); } else { @@ -660,6 +793,9 @@ void ts_query_matches_wasm( TSPoint start_point = {start_row, code_unit_to_byte(start_column)}; TSPoint end_point = {end_row, code_unit_to_byte(end_column)}; ts_query_cursor_set_point_range(scratch_query_cursor, start_point, end_point); + ts_query_cursor_set_byte_range(scratch_query_cursor, start_index, end_index); + ts_query_cursor_set_match_limit(scratch_query_cursor, match_limit); + ts_query_cursor_set_max_start_depth(scratch_query_cursor, max_start_depth); ts_query_cursor_exec(scratch_query_cursor, self, node); uint32_t index = 0; @@ -694,19 +830,24 @@ void ts_query_captures_wasm( uint32_t start_column, uint32_t end_row, uint32_t end_column, - uint32_t match_limit + uint32_t start_index, + uint32_t end_index, + uint32_t match_limit, + uint32_t max_start_depth ) { - if (!scratch_query_cursor) scratch_query_cursor = ts_query_cursor_new(); - if (match_limit == 0) { - ts_query_cursor_set_match_limit(scratch_query_cursor, UINT32_MAX); - } else { - ts_query_cursor_set_match_limit(scratch_query_cursor, match_limit); + if (!scratch_query_cursor) { + scratch_query_cursor = ts_query_cursor_new(); } + ts_query_cursor_set_match_limit(scratch_query_cursor, match_limit); + TSNode node = unmarshal_node(tree); TSPoint start_point = {start_row, code_unit_to_byte(start_column)}; TSPoint end_point = {end_row, code_unit_to_byte(end_column)}; ts_query_cursor_set_point_range(scratch_query_cursor, start_point, end_point); + ts_query_cursor_set_byte_range(scratch_query_cursor, start_index, end_index); + ts_query_cursor_set_match_limit(scratch_query_cursor, match_limit); + ts_query_cursor_set_max_start_depth(scratch_query_cursor, max_start_depth); ts_query_cursor_exec(scratch_query_cursor, self, node); unsigned index = 0; @@ -725,7 +866,7 @@ void ts_query_captures_wasm( array_grow_by(&result, 3 + 6 * match.capture_count); result.contents[index++] = (const void *)(uint32_t)match.pattern_index; result.contents[index++] = (const void *)(uint32_t)match.capture_count; - result.contents[index++] = (const void *)(uint32_t)capture_index; + result.contents[index++] = (const void *)capture_index; for (unsigned i = 0; i < match.capture_count; i++) { const TSQueryCapture *capture = &match.captures[i]; result.contents[index++] = (const void *)capture->index; diff --git a/lib/binding_web/binding.js b/lib/binding_web/binding.js index 5c7f9c81..c870f33d 100644 --- a/lib/binding_web/binding.js +++ b/lib/binding_web/binding.js @@ -69,7 +69,7 @@ class ParserImpl { parse(callback, oldTree, options) { if (typeof callback === 'string') { - currentParseCallback = (index, _, endIndex) => callback.slice(index, endIndex); + currentParseCallback = (index, _) => callback.slice(index); } else if (typeof callback === 'function') { currentParseCallback = callback; } else { @@ -120,14 +120,28 @@ class ParserImpl { C._ts_parser_reset(this[0]); } - setTimeoutMicros(timeout) { - C._ts_parser_set_timeout_micros(this[0], timeout); + getIncludedRanges() { + const count = C._malloc(SIZE_OF_INT); + const rangeAddress = C._ts_parser_included_ranges(this[0], count); + count = getValue(count, 'i32'); + const result = new Array(count); + if (count > 0) { + let address = rangeAddress; + for (let i = 0; i < count; i++) { + result[i] = unmarshalRange(address); + address += SIZE_OF_RANGE; + } + } } getTimeoutMicros() { return C._ts_parser_timeout_micros(this[0]); } + setTimeoutMicros(timeout) { + C._ts_parser_set_timeout_micros(this[0], timeout); + } + setLogger(callback) { if (!callback) { callback = null; @@ -171,6 +185,14 @@ class Tree { return unmarshalNode(this); } + rootNodeWithOffset(offsetBytes, offsetExtent) { + const address = TRANSFER_BUFFER + SIZE_OF_NODE; + setValue(address, offsetBytes, 'i32'); + marshalPoint(address + SIZE_OF_INT, offsetExtent); + C._ts_tree_root_node_with_offset_wasm(this[0]); + return unmarshalNode(this); + } + getLanguage() { return this.language; } @@ -198,6 +220,22 @@ class Tree { } return result; } + + getIncludedRanges() { + C._ts_tree_included_ranges_wasm(this[0]); + const count = getValue(TRANSFER_BUFFER, 'i32'); + const buffer = getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); + const result = new Array(count); + if (count > 0) { + let address = buffer; + for (let i = 0; i < count; i++) { + result[i] = unmarshalRange(address); + address += SIZE_OF_RANGE; + } + C._free(buffer); + } + return result; + } } class Node { @@ -274,6 +312,11 @@ class Node { return C._ts_node_is_missing_wasm(this.tree[0]) === 1; } + get isExtra() { + marshalNode(this); + return C._ts_node_is_extra_wasm(this.tree[0]) === 1; + } + equals(other) { return this.id === other.id; } @@ -284,17 +327,6 @@ class Node { return unmarshalNode(this.tree); } - fieldNameForChild(index) { - marshalNode(this); - const address = C._ts_node_field_name_for_child_wasm(this.tree[0], index); - if (!address) { - return null; - } - const result = AsciiToString(address); - // must not free, the string memory is owned by the language - return result; - } - namedChild(index) { marshalNode(this); C._ts_node_named_child_wasm(this.tree[0], index); @@ -312,6 +344,55 @@ class Node { if (fieldId !== -1) return this.childForFieldId(fieldId); } + fieldNameForChild(index) { + marshalNode(this); + const address = C._ts_node_field_name_for_child_wasm(this.tree[0], index); + if (!address) { + return null; + } + const result = AsciiToString(address); + // must not free, the string memory is owned by the language + return result; + } + + childrenForFieldName(fieldName) { + const fieldId = this.tree.language.fields.indexOf(fieldName); + if (fieldId !== -1 && fieldId !== 0) return this.childrenForFieldId(fieldId); + } + + childrenForFieldId(fieldId) { + marshalNode(this); + C._ts_node_children_by_field_id_wasm(this.tree[0], fieldId); + const count = getValue(TRANSFER_BUFFER, 'i32'); + const buffer = getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32'); + const result = new Array(count); + if (count > 0) { + let address = buffer; + for (let i = 0; i < count; i++) { + result[i] = unmarshalNode(this.tree, address); + address += SIZE_OF_NODE; + } + C._free(buffer); + } + return result; + } + + firstChildForIndex(index) { + marshalNode(this); + const address = TRANSFER_BUFFER + SIZE_OF_NODE; + setValue(address, index, 'i32'); + C._ts_node_first_child_for_byte_wasm(this.tree[0]); + return unmarshalNode(this.tree); + } + + firstNamedChildForIndex(index) { + marshalNode(this); + const address = TRANSFER_BUFFER + SIZE_OF_NODE; + setValue(address, index, 'i32'); + C._ts_node_first_named_child_for_byte_wasm(this.tree[0]); + return unmarshalNode(this.tree); + } + get childCount() { marshalNode(this); return C._ts_node_child_count_wasm(this.tree[0]); @@ -450,6 +531,11 @@ class Node { return unmarshalNode(this.tree); } + get descendantCount() { + marshalNode(this); + return C._ts_node_descendant_count_wasm(this.tree[0]); + } + get parent() { marshalNode(this); C._ts_node_parent_wasm(this.tree[0]); @@ -623,6 +709,16 @@ class TreeCursor { return this.tree.language.fields[this.currentFieldId]; } + get currentDepth() { + marshalTreeCursor(this); + return C._ts_tree_cursor_current_depth_wasm(this.tree[0]); + } + + get currentDescendantIndex() { + marshalTreeCursor(this); + return C._ts_tree_cursor_current_descendant_index_wasm(this.tree[0]); + } + gotoFirstChild() { marshalTreeCursor(this); const result = C._ts_tree_cursor_goto_first_child_wasm(this.tree[0]); @@ -637,6 +733,22 @@ class TreeCursor { return result === 1; } + gotoFirstChildForIndex(goalIndex) { + marshalTreeCursor(this); + setValue(TRANSFER_BUFFER + SIZE_OF_CURSOR, goalIndex, 'i32'); + const result = C._ts_tree_cursor_goto_first_child_for_index_wasm(this.tree[0]); + unmarshalTreeCursor(this); + return result === 1; + } + + gotoFirstChildForPosition(goalPosition) { + marshalTreeCursor(this); + marshalPoint(TRANSFER_BUFFER + SIZE_OF_CURSOR, goalPosition); + const result = C._ts_tree_cursor_goto_first_child_for_position_wasm(this.tree[0]); + unmarshalTreeCursor(this); + return result === 1; + } + gotoNextSibling() { marshalTreeCursor(this); const result = C._ts_tree_cursor_goto_next_sibling_wasm(this.tree[0]); @@ -651,6 +763,12 @@ class TreeCursor { return result === 1; } + gotoDescendant(goalDescendantindex) { + marshalTreeCursor(this); + C._ts_tree_cursor_goto_descendant_wasm(this.tree[0], goalDescendantindex); + unmarshalTreeCursor(this); + } + gotoParent() { marshalTreeCursor(this); const result = C._ts_tree_cursor_goto_parent_wasm(this.tree[0]); @@ -815,6 +933,7 @@ class Language { const refutedProperties = new Array(patternCount); const predicates = new Array(patternCount); const textPredicates = new Array(patternCount); + for (let i = 0; i < patternCount; i++) { const predicatesAddress = C._ts_query_predicates_for_pattern( address, @@ -1141,15 +1260,18 @@ class Query { this[0] = 0; } - matches(node, startPosition, endPosition, options) { - if (!startPosition) startPosition = ZERO_POINT; - if (!endPosition) endPosition = ZERO_POINT; - if (!options) options = {}; - - let matchLimit = options.matchLimit; - if (typeof matchLimit === 'undefined') { - matchLimit = 0; - } else if (typeof matchLimit !== 'number') { + matches( + node, + { + startPosition = ZERO_POINT, + endPosition = ZERO_POINT, + startIndex = 0, + endIndex = 0, + matchLimit = 0xFFFFFFFF, + maxStartDepth = 0xFFFFFFFF, + } = {}, + ) { + if (typeof matchLimit !== 'number') { throw new Error('Arguments must be numbers'); } @@ -1162,7 +1284,10 @@ class Query { startPosition.column, endPosition.row, endPosition.column, + startIndex, + endIndex, matchLimit, + maxStartDepth, ); const rawCount = getValue(TRANSFER_BUFFER, 'i32'); @@ -1198,15 +1323,18 @@ class Query { return result; } - captures(node, startPosition, endPosition, options) { - if (!startPosition) startPosition = ZERO_POINT; - if (!endPosition) endPosition = ZERO_POINT; - if (!options) options = {}; - - let matchLimit = options.matchLimit; - if (typeof matchLimit === 'undefined') { - matchLimit = 0; - } else if (typeof matchLimit !== 'number') { + captures( + node, + { + startPosition = ZERO_POINT, + endPosition = ZERO_POINT, + startIndex = 0, + endIndex = 0, + matchLimit = 0xFFFFFFFF, + maxStartDepth = 0xFFFFFFFF, + } = {}, + ) { + if (typeof matchLimit !== 'number') { throw new Error('Arguments must be numbers'); } @@ -1219,7 +1347,10 @@ class Query { startPosition.column, endPosition.row, endPosition.column, + startIndex, + endIndex, matchLimit, + maxStartDepth, ); const count = getValue(TRANSFER_BUFFER, 'i32'); @@ -1261,6 +1392,14 @@ class Query { return this.predicates[patternIndex]; } + disableCapture(captureName) { + const captureNameLength = lengthBytesUTF8(captureName); + const captureNameAddress = C._malloc(captureNameLength + 1); + stringToUTF8(captureName, captureNameAddress, captureNameLength + 1); + C._ts_query_disable_capture(this[0], captureNameAddress, captureNameLength); + C._free(captureNameAddress); + } + didExceedMatchLimit() { return this.exceededMatchLimit; } diff --git a/lib/binding_web/exports.txt b/lib/binding_web/exports.txt index 0b2e240a..68bbd47e 100644 --- a/lib/binding_web/exports.txt +++ b/lib/binding_web/exports.txt @@ -11,6 +11,9 @@ "ts_language_version", "ts_language_next_state", "ts_node_field_name_for_child_wasm", +"ts_node_children_by_field_id_wasm", +"ts_node_first_child_for_byte_wasm", +"ts_node_first_named_child_for_byte_wasm", "ts_node_child_by_field_id_wasm", "ts_node_child_count_wasm", "ts_node_child_wasm", @@ -24,6 +27,7 @@ "ts_node_has_error_wasm", "ts_node_is_error_wasm", "ts_node_is_missing_wasm", +"ts_node_is_extra_wasm", "ts_node_is_named_wasm", "ts_node_parse_state_wasm", "ts_node_next_parse_state_wasm", @@ -37,6 +41,7 @@ "ts_node_parent_wasm", "ts_node_prev_named_sibling_wasm", "ts_node_prev_sibling_wasm", +"ts_node_descendant_count_wasm", "ts_node_start_index_wasm", "ts_node_start_point_wasm", "ts_node_symbol_wasm", @@ -58,10 +63,13 @@ "ts_query_new", "ts_query_pattern_count", "ts_query_predicates_for_pattern", +"ts_query_disable_capture", "ts_query_string_count", "ts_query_string_value_for_id", "ts_tree_copy", "ts_tree_cursor_current_field_id_wasm", +"ts_tree_cursor_current_depth_wasm", +"ts_tree_cursor_current_descendant_index_wasm", "ts_tree_cursor_current_node_id_wasm", "ts_tree_cursor_current_node_is_missing_wasm", "ts_tree_cursor_current_node_is_named_wasm", @@ -73,8 +81,11 @@ "ts_tree_cursor_end_position_wasm", "ts_tree_cursor_goto_first_child_wasm", "ts_tree_cursor_goto_last_child_wasm", +"ts_tree_cursor_goto_first_child_for_index_wasm", +"ts_tree_cursor_goto_first_child_for_position_wasm", "ts_tree_cursor_goto_next_sibling_wasm", "ts_tree_cursor_goto_previous_sibling_wasm", +"ts_tree_cursor_goto_descendant_wasm", "ts_tree_cursor_goto_parent_wasm", "ts_tree_cursor_new_wasm", "ts_tree_cursor_reset_wasm", @@ -82,9 +93,11 @@ "ts_tree_cursor_start_index_wasm", "ts_tree_cursor_start_position_wasm", "ts_tree_delete", +"ts_tree_included_ranges_wasm", "ts_tree_edit_wasm", "ts_tree_get_changed_ranges_wasm", "ts_tree_root_node_wasm", +"ts_tree_root_node_with_offset_wasm", "ts_lookahead_iterator_new", "ts_lookahead_iterator_delete", "ts_lookahead_iterator_reset_state", diff --git a/lib/binding_web/test/helper.js b/lib/binding_web/test/helper.js index 2cb3a418..5ceba02d 100644 --- a/lib/binding_web/test/helper.js +++ b/lib/binding_web/test/helper.js @@ -7,5 +7,9 @@ function languageURL(name) { module.exports = Parser.init().then(async () => ({ Parser, languageURL, + EmbeddedTemplate: await Parser.Language.load(languageURL('embedded_template')), + HTML: await Parser.Language.load(languageURL('html')), JavaScript: await Parser.Language.load(languageURL('javascript')), + JSON: await Parser.Language.load(languageURL('json')), + Python: await Parser.Language.load(languageURL('python')), })); diff --git a/lib/binding_web/test/node-test.js b/lib/binding_web/test/node-test.js index 9e7e408f..7c3e8f7b 100644 --- a/lib/binding_web/test/node-test.js +++ b/lib/binding_web/test/node-test.js @@ -1,11 +1,41 @@ const {assert} = require('chai'); -let Parser; let JavaScript; +let Parser; let JavaScript; let JSON; let EmbeddedTemplate; let Python; + +const JSON_EXAMPLE = ` + +[ + 123, + false, + { + "x": null + } +] +`; + +function getAllNodes(tree) { + const result = []; + let visitedChildren = false; + const cursor = tree.walk(); + while (true) { + if (!visitedChildren) { + result.push(cursor.currentNode); + if (!cursor.gotoFirstChild()) { + visitedChildren = true; + } + } else if (cursor.gotoNextSibling()) { + visitedChildren = false; + } else if (!cursor.gotoParent()) { + break; + } + } + return result; +} describe('Node', () => { let parser; let tree; before(async () => - ({Parser, JavaScript} = await require('./helper')), + ({Parser, EmbeddedTemplate, JavaScript, JSON, Python} = await require('./helper')), ); beforeEach(() => { @@ -42,6 +72,31 @@ describe('Node', () => { }); }); + describe('.childrenForFieldName', () => { + it('returns an array of child nodes for the given field name', () => { + parser.setLanguage(Python); + const source = ` + if one: + a() + elif two: + b() + elif three: + c() + elif four: + d()`; + + tree = parser.parse(source); + const node = tree.rootNode.firstChild; + assert.equal(node.type, 'if_statement'); + const alternatives = node.childrenForFieldName('alternative'); + const alternativeTexts = alternatives.map((n) => { + const condition = n.childForFieldName('condition'); + return source.slice(condition.startIndex, condition.endIndex); + }); + assert.deepEqual(alternativeTexts, ['two', 'three', 'four']); + }); + }); + describe('.startIndex and .endIndex', () => { it('returns the character index where the node starts/ends in the text', () => { tree = parser.parse('a👍👎1 / b👎c👎'); @@ -304,13 +359,26 @@ describe('Node', () => { }); }); + describe('.isExtra', () => { + it('returns true if the node is an extra node like comments', () => { + tree = parser.parse('foo(/* hi */);'); + const node = tree.rootNode; + const commentNode = node.descendantForIndex(7, 7); + + assert.equal(node.type, 'program'); + assert.equal(commentNode.type, 'comment'); + assert(!node.isExtra); + assert(commentNode.isExtra); + }); + }); + describe('.text', () => { const text = 'α0 / b👎c👎'; Object.entries({ '.parse(String)': text, - '.parse(Function)': (offset) => text.substr(offset, 4), - }).forEach(([method, parse]) => + '.parse(Function)': (offset) => text.slice(offset, 4), + }).forEach(([method, _parse]) => it(`returns the text of a node generated by ${method}`, async () => { const [numeratorSrc, denominatorSrc] = text.split(/\s*\/\s+/); tree = await parser.parse(text); @@ -326,6 +394,78 @@ describe('Node', () => { ); }); + describe('.descendantCount', () => { + it('returns the number of descendants', () => { + parser.setLanguage(JSON); + tree = parser.parse(JSON_EXAMPLE); + const valueNode = tree.rootNode; + const allNodes = getAllNodes(tree); + + assert.equal(valueNode.descendantCount, allNodes.length); + + const cursor = tree.walk(); + for (let i = 0; i < allNodes.length; i++) { + const node = allNodes[i]; + cursor.gotoDescendant(i); + assert.equal(cursor.currentNode.id, node.id, `index ${i}`); + } + + for (let i = allNodes.length - 1; i >= 0; i--) { + const node = allNodes[i]; + cursor.gotoDescendant(i); + assert.equal(cursor.currentNode.id, node.id, `rev index ${i}`); + } + }); + + it('tests a single node tree', () => { + parser.setLanguage(EmbeddedTemplate); + tree = parser.parse('hello'); + + const nodes = getAllNodes(tree); + assert.equal(nodes.length, 2); + assert.equal(tree.rootNode.descendantCount, 2); + + const cursor = tree.walk(); + + cursor.gotoDescendant(0); + assert.equal(cursor.currentDepth, 0); + assert.equal(cursor.currentNode.id, nodes[0].id); + + cursor.gotoDescendant(1); + assert.equal(cursor.currentDepth, 1); + assert.equal(cursor.currentNode.id, nodes[1].id); + }); + }); + + describe('.rootNodeWithOffset', () => { + it('returns the root node of the tree, offset by the given byte offset', () => { + tree = parser.parse(' if (a) b'); + const node = tree.rootNodeWithOffset(6, {row: 2, column: 2}); + assert.equal(node.startIndex, 8); + assert.equal(node.endIndex, 16); + assert.deepEqual(node.startPosition, {row: 2, column: 4}); + assert.deepEqual(node.endPosition, {row: 2, column: 12}); + + let child = node.firstChild.child(2); + assert.equal(child.type, 'expression_statement'); + assert.equal(child.startIndex, 15); + assert.equal(child.endIndex, 16); + assert.deepEqual(child.startPosition, {row: 2, column: 11}); + assert.deepEqual(child.endPosition, {row: 2, column: 12}); + + const cursor = node.walk(); + cursor.gotoFirstChild(); + cursor.gotoFirstChild(); + cursor.gotoNextSibling(); + child = cursor.currentNode; + assert.equal(child.type, 'parenthesized_expression'); + assert.equal(child.startIndex, 11); + assert.equal(child.endIndex, 14); + assert.deepEqual(child.startPosition, {row: 2, column: 7}); + assert.deepEqual(child.endPosition, {row: 2, column: 10}); + }); + }); + describe('.parseState, .nextParseState', () => { const text = '10 / 5'; @@ -435,6 +575,29 @@ describe('Node', () => { }); }); + describe('.firstChildForIndex(index)', () => { + it('returns the first child that extends beyond the given index', () => { + tree = parser.parse('x10 + 1000'); + const sumNode = tree.rootNode.firstChild.firstChild; + + assert.equal('identifier', sumNode.firstChildForIndex(0).type); + assert.equal('identifier', sumNode.firstChildForIndex(1).type); + assert.equal('+', sumNode.firstChildForIndex(3).type); + assert.equal('number', sumNode.firstChildForIndex(5).type); + }); + }); + + describe('.firstNamedChildForIndex(index)', () => { + it('returns the first child that extends beyond the given index', () => { + tree = parser.parse('x10 + 1000'); + const sumNode = tree.rootNode.firstChild.firstChild; + + assert.equal('identifier', sumNode.firstNamedChildForIndex(0).type); + assert.equal('identifier', sumNode.firstNamedChildForIndex(1).type); + assert.equal('number', sumNode.firstNamedChildForIndex(3).type); + }); + }); + describe('.equals(other)', () => { it('returns true if the nodes are the same', () => { tree = parser.parse('1 + 2'); diff --git a/lib/binding_web/test/parser-test.js b/lib/binding_web/test/parser-test.js index 6f25f6e1..3af59e11 100644 --- a/lib/binding_web/test/parser-test.js +++ b/lib/binding_web/test/parser-test.js @@ -1,11 +1,11 @@ const {assert} = require('chai'); -let Parser; let JavaScript; let languageURL; +let Parser; let JavaScript; let HTML; let languageURL; describe('Parser', () => { let parser; before(async () => - ({Parser, JavaScript, languageURL} = await require('./helper')), + ({Parser, JavaScript, HTML, languageURL} = await require('./helper')), ); beforeEach(() => { @@ -73,7 +73,7 @@ describe('Parser', () => { it('rethrows errors thrown by the logging callback', () => { const error = new Error('The error message'); - parser.setLogger((msg, params) => { + parser.setLogger((_msg, _params) => { throw error; }); assert.throws( @@ -83,6 +83,99 @@ describe('Parser', () => { }); }); + describe('multiple included ranges', () => { + it('parses the text within multiple ranges', () => { + parser.setLanguage(JavaScript); + const sourceCode = 'html `
Hello, ${name.toUpperCase()}, it\'s ${now()}.
`'; + const jsTree = parser.parse(sourceCode); + const templateStringNode = jsTree.rootNode.descendantForIndex(sourceCode.indexOf('`<'), sourceCode.indexOf('>`')); + assert.equal(templateStringNode.type, 'template_string'); + + const openQuoteNode = templateStringNode.child(0); + const interpolationNode1 = templateStringNode.child(2); + const interpolationNode2 = templateStringNode.child(4); + const closeQuoteNode = templateStringNode.child(6); + + parser.setLanguage(HTML); + const htmlRanges = [ + { + startIndex: openQuoteNode.endIndex, + startPosition: openQuoteNode.endPosition, + endIndex: interpolationNode1.startIndex, + endPosition: interpolationNode1.startPosition, + }, + { + startIndex: interpolationNode1.endIndex, + startPosition: interpolationNode1.endPosition, + endIndex: interpolationNode2.startIndex, + endPosition: interpolationNode2.startPosition, + }, + { + startIndex: interpolationNode2.endIndex, + startPosition: interpolationNode2.endPosition, + endIndex: closeQuoteNode.startIndex, + endPosition: closeQuoteNode.startPosition, + }, + ]; + const htmlTree = parser.parse(sourceCode, null, {includedRanges: htmlRanges}); + + assert.equal( + htmlTree.rootNode.toString(), + '(document (element' + + ' (start_tag (tag_name))' + + ' (text)' + + ' (element (start_tag (tag_name)) (end_tag (tag_name)))' + + ' (text)' + + ' (end_tag (tag_name))))', + ); + assert.deepEqual(htmlTree.getIncludedRanges(), htmlRanges); + + const divElementNode = htmlTree.rootNode.child(0); + const helloTextNode = divElementNode.child(1); + const bElementNode = divElementNode.child(2); + const bStartTagNode = bElementNode.child(0); + const bEndTagNode = bElementNode.child(1); + + assert.equal(helloTextNode.type, 'text'); + assert.equal(helloTextNode.startIndex, sourceCode.indexOf('Hello')); + assert.equal(helloTextNode.endIndex, sourceCode.indexOf(' ')); + + assert.equal(bStartTagNode.type, 'start_tag'); + assert.equal(bStartTagNode.startIndex, sourceCode.indexOf('')); + assert.equal(bStartTagNode.endIndex, sourceCode.indexOf('${now()}')); + + assert.equal(bEndTagNode.type, 'end_tag'); + assert.equal(bEndTagNode.startIndex, sourceCode.indexOf('')); + assert.equal(bEndTagNode.endIndex, sourceCode.indexOf('.')); + }); + }); + + describe('an included range containing mismatched positions', () => { + it('parses the text within the range', () => { + const sourceCode = '
test
{_ignore_this_part_}'; + + parser.setLanguage(HTML); + + const endIndex = sourceCode.indexOf('{_ignore_this_part_'); + + const rangeToParse = { + startIndex: 0, + startPosition: {row: 10, column: 12}, + endIndex, + endPosition: {row: 10, column: 12 + endIndex}, + }; + + const htmlTree = parser.parse(sourceCode, null, {includedRanges: [rangeToParse]}); + + assert.deepEqual(htmlTree.getIncludedRanges()[0], rangeToParse); + + assert.equal( + htmlTree.rootNode.toString(), + '(document (element (start_tag (tag_name)) (text) (end_tag (tag_name))))', + ); + }); + }); + describe('.parse', () => { let tree; diff --git a/lib/binding_web/test/query-test.js b/lib/binding_web/test/query-test.js index 73c7d34f..fad6b3cf 100644 --- a/lib/binding_web/test/query-test.js +++ b/lib/binding_web/test/query-test.js @@ -71,8 +71,10 @@ describe('Query', () => { query = JavaScript.query('(identifier) @element'); const matches = query.matches( tree.rootNode, - {row: 1, column: 1}, - {row: 3, column: 1}, + { + startPosition: {row: 1, column: 1}, + endPosition: {row: 3, column: 1}, + }, ); assert.deepEqual(formatMatches(matches), [ {pattern: 0, captures: [{name: 'element', text: 'd'}]}, @@ -273,7 +275,7 @@ describe('Query', () => { (array (identifier) @pre (identifier) @post) `); - query.captures(tree.rootNode, null, null, {matchLimit: 32}); + query.captures(tree.rootNode, {matchLimit: 32}); assert.ok(query.didExceedMatchLimit()); }); @@ -295,7 +297,7 @@ describe('Query', () => { const expectCount = (tree, queryText, expectedCount) => { query = JavaScript.query(queryText); - captures = query.captures(tree.rootNode, null, null); + captures = query.captures(tree.rootNode); assert.equal(captures.length, expectedCount); }; @@ -406,6 +408,49 @@ describe('Query', () => { assert.deepEqual(query.predicatesForPattern(2), []); }); }); + + describe('.disableCapture', () => { + it('disables a capture', () => { + const query = JavaScript.query(` + (function_declaration + (identifier) @name1 @name2 @name3 + (statement_block) @body1 @body2) + `); + + const source = 'function foo() { return 1; }'; + const tree = parser.parse(source); + + let matches = query.matches(tree.rootNode); + assert.deepEqual(formatMatches(matches), [ + { + pattern: 0, + captures: [ + {name: 'name1', text: 'foo'}, + {name: 'name2', text: 'foo'}, + {name: 'name3', text: 'foo'}, + {name: 'body1', text: '{ return 1; }'}, + {name: 'body2', text: '{ return 1; }'}, + ], + }, + ]); + + // disabling captures still works when there are multiple captures on a + // single node. + query.disableCapture('name2'); + matches = query.matches(tree.rootNode); + assert.deepEqual(formatMatches(matches), [ + { + pattern: 0, + captures: [ + {name: 'name1', text: 'foo'}, + {name: 'name3', text: 'foo'}, + {name: 'body1', text: '{ return 1; }'}, + {name: 'body2', text: '{ return 1; }'}, + ], + }, + ]); + }); + }); }); function formatMatches(matches) { diff --git a/lib/binding_web/tree-sitter-web.d.ts b/lib/binding_web/tree-sitter-web.d.ts index 52aabeb5..6bf1af20 100644 --- a/lib/binding_web/tree-sitter-web.d.ts +++ b/lib/binding_web/tree-sitter-web.d.ts @@ -6,14 +6,15 @@ declare module 'web-tree-sitter' { */ static init(moduleOptions?: object): Promise; delete(): void; - parse(input: string | Parser.Input, previousTree?: Parser.Tree, options?: Parser.Options): Parser.Tree; + parse(input: string | Parser.Input, oldTree?: Parser.Tree, options?: Parser.Options): Parser.Tree; + getIncludedRanges(): Parser.Range[]; + getTimeoutMicros(): number; + setTimeoutMicros(timeout: number): void; reset(): void; getLanguage(): Parser.Language; - setLanguage(language?: Parser.Language | undefined | null): void; + setLanguage(language?: Parser.Language | null): void; getLogger(): Parser.Logger; - setLogger(logFunc?: Parser.Logger | undefined | null): void; - setTimeoutMicros(value: number): void; - getTimeoutMicros(): number; + setLogger(logFunc?: Parser.Logger | false | null): void; } namespace Parser { @@ -48,20 +49,20 @@ declare module 'web-tree-sitter' { type: "parse" | "lex" ) => void; - export type Input = ( - startIndex: number, - startPoint?: Point, - endIndex?: number, - ) => string | null; + export interface Input { + (index: number, position?: Point): string | null; + } export interface SyntaxNode { - grammarId: number; tree: Tree; - type: string; + id: number; typeId: number; + grammarId: number; + type: string; grammarType: string; isNamed: boolean; isMissing: boolean; + isExtra: boolean; hasChanges: boolean; hasError: boolean; isError: boolean; @@ -85,13 +86,19 @@ declare module 'web-tree-sitter' { nextNamedSibling: SyntaxNode | null; previousSibling: SyntaxNode | null; previousNamedSibling: SyntaxNode | null; + descendantCount: number; equals(other: SyntaxNode): boolean; toString(): string; child(index: number): SyntaxNode | null; namedChild(index: number): SyntaxNode | null; - childForFieldId(fieldId: number): SyntaxNode | null; childForFieldName(fieldName: string): SyntaxNode | null; + childForFieldId(fieldId: number): SyntaxNode | null; + fieldNameForChild(childIndex: number): string | null; + childrenForFieldName(fieldName: string, cursor: TreeCursor): Array; + childrenForFieldId(fieldId: number, cursor: TreeCursor): Array; + firstChildForIndex(index: number): SyntaxNode | null; + firstNamedChildForIndex(index: number): SyntaxNode | null; descendantForIndex(index: number): SyntaxNode; descendantForIndex(startIndex: number, endIndex: number): SyntaxNode; @@ -119,8 +126,10 @@ declare module 'web-tree-sitter' { startIndex: number; endIndex: number; readonly currentNode: SyntaxNode; - readonly currentFieldId: number; readonly currentFieldName: string; + readonly currentFieldId: number; + readonly currentDepth: number; + readonly currentDescendantIndex: number; reset(node: SyntaxNode): void; resetTo(cursor: TreeCursor): void; @@ -128,23 +137,76 @@ declare module 'web-tree-sitter' { gotoParent(): boolean; gotoFirstChild(): boolean; gotoLastChild(): boolean; - gotoFirstChildForIndex(index: number): boolean; + gotoFirstChildForIndex(goalIndex: number): boolean; + gotoFirstChildForPosition(goalPosition: Point): boolean; gotoNextSibling(): boolean; gotoPreviousSibling(): boolean; + gotoDescendant(goalDescendantIndex: number): void; } export interface Tree { readonly rootNode: SyntaxNode; + rootNodeWithOffset(offsetBytes: number, offsetExtent: Point): SyntaxNode; copy(): Tree; delete(): void; edit(delta: Edit): Tree; walk(): TreeCursor; getChangedRanges(other: Tree): Range[]; + getIncludedRanges(): Range[]; getEditedRange(other: Tree): Range; getLanguage(): Language; } + export interface QueryCapture { + name: string; + text?: string; + node: SyntaxNode; + setProperties?: { [prop: string]: string | null }; + assertedProperties?: { [prop: string]: string | null }; + refutedProperties?: { [prop: string]: string | null }; + } + + export interface QueryMatch { + pattern: number; + captures: QueryCapture[]; + } + + export type QueryOptions = { + startPosition?: Point; + endPosition?: Point; + startIndex?: number; + endIndex?: number; + matchLimit?: number; + maxStartDepth?: number; + }; + + export interface PredicateResult { + operator: string; + operands: { name: string; type: string }[]; + } + + export class Query { + captureNames: string[]; + readonly predicates: { [name: string]: Function }[]; + readonly setProperties: any[]; + readonly assertedProperties: any[]; + readonly refutedProperties: any[]; + readonly matchLimit: number; + + delete(): void; + captures(node: SyntaxNode, options?: QueryOptions): QueryCapture[]; + matches(node: SyntaxNode, options?: QueryOptions): QueryMatch[]; + predicatesForPattern(patternIndex: number): PredicateResult[]; + disableCapture(captureName: string): void; + disablePattern(patternIndex: number): void; + isPatternGuaranteedAtStep(byteOffset: number): boolean; + isPatternRooted(patternIndex: number): boolean; + isPatternNonLocal(patternIndex: number): boolean; + startIndexForPattern(patternIndex: number): number; + didExceedMatchLimit(): boolean; + } + class Language { static load(input: string | Uint8Array): Promise; @@ -164,40 +226,16 @@ declare module 'web-tree-sitter' { lookaheadIterator(stateId: number): LookaheadIterable | null; } - class LookaheadIterable { + export class LookaheadIterable { readonly language: Language; readonly currentTypeId: number; readonly currentType: string; delete(): void; - resetState(stateId: number): boolean; reset(language: Language, stateId: number): boolean; + resetState(stateId: number): boolean; [Symbol.iterator](): Iterator; } - - interface QueryCapture { - name: string; - node: SyntaxNode; - } - - interface QueryMatch { - pattern: number; - captures: QueryCapture[]; - } - - interface PredicateResult { - operator: string; - operands: { name: string; type: string }[]; - } - - class Query { - captureNames: string[]; - - delete(): void; - matches(node: SyntaxNode, startPosition?: Point, endPosition?: Point): QueryMatch[]; - captures(node: SyntaxNode, startPosition?: Point, endPosition?: Point): QueryCapture[]; - predicatesForPattern(patternIndex: number): PredicateResult[]; - } } export = Parser