From 3425b6e1c2b6b5e74ad765d87bd6062d747ce49d Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 23 Apr 2019 14:46:46 -0700 Subject: [PATCH] Add wasm binding for TreeCursor --- lib/web/binding.c | 103 +++++++++++++++++++- lib/web/binding.js | 107 +++++++++++++++++++++ lib/web/test/parser-test.js | 4 +- lib/web/test/tree-test.js | 183 +++++++++++++++++++++++++++++++++++- script/build-wasm | 34 ++++--- 5 files changed, 411 insertions(+), 20 deletions(-) diff --git a/lib/web/binding.c b/lib/web/binding.c index 748c91a6..29818e5e 100644 --- a/lib/web/binding.c +++ b/lib/web/binding.c @@ -28,7 +28,7 @@ static uint32_t byte_to_code_unit(uint32_t byte) { return byte >> 1; } -static void marshal_node(const void **buffer, TSNode node) { +static inline void marshal_node(const void **buffer, TSNode node) { buffer[0] = (const void *)node.id; buffer[1] = (const void *)node.context[0]; buffer[2] = (const void *)node.context[1]; @@ -36,7 +36,7 @@ static void marshal_node(const void **buffer, TSNode node) { buffer[4] = (const void *)node.context[3]; } -static TSNode unmarshal_node(const TSTree *tree) { +static inline TSNode unmarshal_node(const TSTree *tree) { TSNode node; node.id = TRANSFER_BUFFER[0]; node.context[0] = (uint32_t)TRANSFER_BUFFER[1]; @@ -47,6 +47,21 @@ static TSNode unmarshal_node(const TSTree *tree) { return node; } +static inline void marshal_cursor(const TSTreeCursor *cursor) { + TRANSFER_BUFFER[0] = (const void *)cursor->id; + TRANSFER_BUFFER[1] = (const void *)cursor->context[0]; + TRANSFER_BUFFER[2] = (const void *)cursor->context[1]; +} + +static inline TSTreeCursor unmarshal_cursor(const void **buffer, const TSTree *tree) { + TSTreeCursor cursor; + cursor.id = buffer[0]; + cursor.context[0] = (uint32_t)buffer[1]; + cursor.context[1] = (uint32_t)buffer[2]; + cursor.tree = tree; + return cursor; +} + static void marshal_point(TSPoint point) { TRANSFER_BUFFER[0] = (const void *)point.row; TRANSFER_BUFFER[1] = (const void *)byte_to_code_unit(point.column); @@ -163,6 +178,90 @@ void ts_tree_edit_wasm(TSTree *tree) { ts_tree_edit(tree, &edit); } +/************************/ +/* Section - TreeCursor */ +/************************/ + +void ts_tree_cursor_new_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + TSTreeCursor cursor = ts_tree_cursor_new(node); + marshal_cursor(&cursor); +} + +void ts_tree_cursor_delete_wasm(const TSTree *tree) { + TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); + ts_tree_cursor_delete(&cursor); +} + +void ts_tree_cursor_reset_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + TSTreeCursor cursor = unmarshal_cursor(&TRANSFER_BUFFER[5], tree); + ts_tree_cursor_reset(&cursor, node); + marshal_cursor(&cursor); +} + +bool ts_tree_cursor_goto_first_child_wasm(const TSTree *tree) { + TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); + bool result = ts_tree_cursor_goto_first_child(&cursor); + marshal_cursor(&cursor); + return result; +} + +bool ts_tree_cursor_goto_next_sibling_wasm(const TSTree *tree) { + TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); + bool result = ts_tree_cursor_goto_next_sibling(&cursor); + marshal_cursor(&cursor); + return result; +} + +bool ts_tree_cursor_goto_parent_wasm(const TSTree *tree) { + TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); + bool result = ts_tree_cursor_goto_parent(&cursor); + marshal_cursor(&cursor); + return result; +} + +uint16_t ts_tree_cursor_current_node_type_id_wasm(const TSTree *tree) { + TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); + TSNode node = ts_tree_cursor_current_node(&cursor); + return ts_node_symbol(node); +} + +bool ts_tree_cursor_current_node_is_named_wasm(const TSTree *tree) { + TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); + TSNode node = ts_tree_cursor_current_node(&cursor); + return ts_node_is_named(node); +} + +void ts_tree_cursor_start_position_wasm(const TSTree *tree) { + TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); + TSNode node = ts_tree_cursor_current_node(&cursor); + marshal_point(ts_node_start_point(node)); +} + +void ts_tree_cursor_end_position_wasm(const TSTree *tree) { + TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); + TSNode node = ts_tree_cursor_current_node(&cursor); + marshal_point(ts_node_end_point(node)); +} + +uint32_t ts_tree_cursor_start_index_wasm(const TSTree *tree) { + TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); + TSNode node = ts_tree_cursor_current_node(&cursor); + return byte_to_code_unit(ts_node_start_byte(node)); +} + +uint32_t ts_tree_cursor_end_index_wasm(const TSTree *tree) { + TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); + TSNode node = ts_tree_cursor_current_node(&cursor); + return byte_to_code_unit(ts_node_end_byte(node)); +} + +void ts_tree_cursor_current_node_wasm(const TSTree *tree) { + TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); + marshal_node(TRANSFER_BUFFER, ts_tree_cursor_current_node(&cursor)); +} + /******************/ /* Section - Node */ /******************/ diff --git a/lib/web/binding.js b/lib/web/binding.js index 85bf1cb6..318ccf1f 100644 --- a/lib/web/binding.js +++ b/lib/web/binding.js @@ -171,6 +171,10 @@ class Tree { getLanguage() { return this.language; } + + walk() { + return this.rootNode.walk(); + } } class Node { @@ -412,6 +416,12 @@ class Node { return unmarshalNode(this.tree); } + walk() { + marshalNode(this); + C._ts_tree_cursor_new_wasm(this.tree[0]); + return new TreeCursor(INTERNAL, this.tree); + } + toString() { marshalNode(this); const address = C._ts_node_to_string_wasm(this.tree[0]); @@ -421,6 +431,91 @@ class Node { } } +class TreeCursor { + constructor(internal, tree) { + if (internal !== INTERNAL) { + throw new Error('Illegal constructor') + } + this.tree = tree; + unmarshalTreeCursor(this); + } + + delete() { + marshalTreeCursor(this); + C._ts_tree_cursor_delete_wasm(this.tree[0]); + } + + reset(node) { + marshalNode(node); + marshalTreeCursor(this, TRANSFER_BUFFER + SIZE_OF_NODE); + C._ts_tree_cursor_reset_wasm(this.tree[0]); + unmarshalTreeCursor(this); + } + + get nodeType() { + return this.tree.language.types[this.nodeTypeId] || 'ERROR'; + } + + get nodeTypeId() { + marshalTreeCursor(this); + return C._ts_tree_cursor_current_node_type_id_wasm(this.tree[0]); + } + + get nodeIsNamed() { + marshalTreeCursor(this); + return C._ts_tree_cursor_current_node_is_named_wasm(this.tree[0]) === 1; + } + + get startPosition() { + marshalTreeCursor(this); + C._ts_tree_cursor_start_position_wasm(this.tree[0]); + return unmarshalPoint(TRANSFER_BUFFER); + } + + get endPosition() { + marshalTreeCursor(this); + C._ts_tree_cursor_end_position_wasm(this.tree[0]); + return unmarshalPoint(TRANSFER_BUFFER); + } + + get startIndex() { + marshalTreeCursor(this); + return C._ts_tree_cursor_start_index_wasm(this.tree[0]); + } + + get endIndex() { + marshalTreeCursor(this); + return C._ts_tree_cursor_end_index_wasm(this.tree[0]); + } + + currentNode() { + marshalTreeCursor(this); + C._ts_tree_cursor_current_node_wasm(this.tree[0]); + return unmarshalNode(this.tree); + } + + gotoFirstChild() { + marshalTreeCursor(this); + const result = C._ts_tree_cursor_goto_first_child_wasm(this.tree[0]); + unmarshalTreeCursor(this); + return result === 1; + } + + gotoNextSibling() { + marshalTreeCursor(this); + const result = C._ts_tree_cursor_goto_next_sibling_wasm(this.tree[0]); + unmarshalTreeCursor(this); + return result === 1; + } + + gotoParent() { + marshalTreeCursor(this); + const result = C._ts_tree_cursor_goto_parent_wasm(this.tree[0]); + unmarshalTreeCursor(this); + return result === 1; + } +} + class Language { constructor(internal, address) { if (internal !== INTERNAL) { @@ -499,6 +594,18 @@ function unmarshalNode(tree, address = TRANSFER_BUFFER) { return result; } +function marshalTreeCursor(cursor, address = TRANSFER_BUFFER) { + setValue(address + 0 * SIZE_OF_INT, cursor[0], 'i32'), + setValue(address + 1 * SIZE_OF_INT, cursor[1], 'i32'), + setValue(address + 2 * SIZE_OF_INT, cursor[2], 'i32') +} + +function unmarshalTreeCursor(cursor) { + cursor[0] = getValue(TRANSFER_BUFFER + 0 * SIZE_OF_INT, 'i32'), + cursor[1] = getValue(TRANSFER_BUFFER + 1 * SIZE_OF_INT, 'i32'), + cursor[2] = getValue(TRANSFER_BUFFER + 2 * SIZE_OF_INT, 'i32') +} + function marshalPoint(address, point) { setValue(address, point.row, 'i32') setValue(address + SIZE_OF_INT, point.column, 'i32') diff --git a/lib/web/test/parser-test.js b/lib/web/test/parser-test.js index 038ed59f..81847f37 100644 --- a/lib/web/test/parser-test.js +++ b/lib/web/test/parser-test.js @@ -123,9 +123,9 @@ describe("Parser", () => { tree = parser.parse(inputString); assert.equal(tree.rootNode.type, "program"); assert.equal(tree.rootNode.firstChild.firstChild.namedChildCount, repeatCount); - }); + }).timeout(5000); - it.only("can use languages with external scanners written in C++", () => { + it("can use languages with external scanners written in C++", () => { parser.setLanguage(Python); tree = parser.parse("def foo():\n bar()"); assert.equal( diff --git a/lib/web/test/tree-test.js b/lib/web/test/tree-test.js index ea7f085f..648829ca 100644 --- a/lib/web/test/tree-test.js +++ b/lib/web/test/tree-test.js @@ -21,8 +21,6 @@ describe("Tree", () => { let input, edit it('updates the positions of nodes', () => { - parser.setLanguage(JavaScript) - input = 'abc + cde'; tree = parser.parse(input); assert.equal( @@ -82,6 +80,170 @@ describe("Tree", () => { ); }); }); + + describe(".walk()", () => { + let cursor + + afterEach(() => { + cursor.delete(); + }) + + it('returns a cursor that can be used to walk the tree', () => { + tree = parser.parse('a * b + c / d'); + cursor = tree.walk(); + + assertCursorState(cursor, { + nodeType: 'program', + nodeIsNamed: true, + startPosition: {row: 0, column: 0}, + endPosition: {row: 0, column: 13}, + startIndex: 0, + endIndex: 13 + }); + + assert(cursor.gotoFirstChild()); + assertCursorState(cursor, { + nodeType: 'expression_statement', + nodeIsNamed: true, + startPosition: {row: 0, column: 0}, + endPosition: {row: 0, column: 13}, + startIndex: 0, + endIndex: 13 + }); + + assert(cursor.gotoFirstChild()); + assertCursorState(cursor, { + nodeType: 'binary_expression', + nodeIsNamed: true, + startPosition: {row: 0, column: 0}, + endPosition: {row: 0, column: 13}, + startIndex: 0, + endIndex: 13 + }); + + assert(cursor.gotoFirstChild()); + assertCursorState(cursor, { + nodeType: 'binary_expression', + nodeIsNamed: true, + startPosition: {row: 0, column: 0}, + endPosition: {row: 0, column: 5}, + startIndex: 0, + endIndex: 5 + }); + + assert(cursor.gotoFirstChild()); + assertCursorState(cursor, { + nodeType: 'identifier', + nodeIsNamed: true, + startPosition: {row: 0, column: 0}, + endPosition: {row: 0, column: 1}, + startIndex: 0, + endIndex: 1 + }); + + assert(!cursor.gotoFirstChild()) + assert(cursor.gotoNextSibling()); + assertCursorState(cursor, { + nodeType: '*', + nodeIsNamed: false, + startPosition: {row: 0, column: 2}, + endPosition: {row: 0, column: 3}, + startIndex: 2, + endIndex: 3 + }); + + assert(cursor.gotoNextSibling()); + assertCursorState(cursor, { + nodeType: 'identifier', + nodeIsNamed: true, + startPosition: {row: 0, column: 4}, + endPosition: {row: 0, column: 5}, + startIndex: 4, + endIndex: 5 + }); + + assert(!cursor.gotoNextSibling()); + assert(cursor.gotoParent()); + assertCursorState(cursor, { + nodeType: 'binary_expression', + nodeIsNamed: true, + startPosition: {row: 0, column: 0}, + endPosition: {row: 0, column: 5}, + startIndex: 0, + endIndex: 5 + }); + + assert(cursor.gotoNextSibling()); + assertCursorState(cursor, { + nodeType: '+', + nodeIsNamed: false, + startPosition: {row: 0, column: 6}, + endPosition: {row: 0, column: 7}, + startIndex: 6, + endIndex: 7 + }); + + assert(cursor.gotoNextSibling()); + assertCursorState(cursor, { + nodeType: 'binary_expression', + nodeIsNamed: true, + startPosition: {row: 0, column: 8}, + endPosition: {row: 0, column: 13}, + startIndex: 8, + endIndex: 13 + }); + + // const childIndex = cursor.gotoFirstChildForIndex(12); + // assertCursorState(cursor, { + // nodeType: 'identifier', + // nodeIsNamed: true, + // startPosition: {row: 0, column: 12}, + // endPosition: {row: 0, column: 13}, + // startIndex: 12, + // endIndex: 13 + // }); + // assert.equal(childIndex, 2); + // assert(!cursor.gotoNextSibling()); + // assert(cursor.gotoParent()); + + assert(cursor.gotoParent()); + assert.equal(cursor.nodeType, 'binary_expression') + assert(cursor.gotoParent()); + assert.equal(cursor.nodeType, 'expression_statement') + assert(cursor.gotoParent()); + assert.equal(cursor.nodeType, 'program') + assert(!cursor.gotoParent()); + }); + + it('returns a cursor that can be reset anywhere in the tree', () => { + tree = parser.parse('a * b + c / d'); + cursor = tree.walk(); + const root = tree.rootNode.firstChild; + + cursor.reset(root.firstChild.firstChild); + assertCursorState(cursor, { + nodeType: 'binary_expression', + nodeIsNamed: true, + startPosition: {row: 0, column: 0}, + endPosition: {row: 0, column: 5}, + startIndex: 0, + endIndex: 5 + }); + + cursor.gotoFirstChild() + assertCursorState(cursor, { + nodeType: 'identifier', + nodeIsNamed: true, + startPosition: {row: 0, column: 0}, + endPosition: {row: 0, column: 1}, + startIndex: 0, + endIndex: 1 + }); + + assert(cursor.gotoParent()); + assert(!cursor.gotoParent()); + }) + }); }); function spliceInput(input, startIndex, lengthRemoved, newText) { @@ -110,3 +272,20 @@ function getExtent(text) { } return {row, column: text.length - index}; } + +function assertCursorState(cursor, params) { + assert.equal(cursor.nodeType, params.nodeType); + assert.equal(cursor.nodeIsNamed, params.nodeIsNamed); + assert.deepEqual(cursor.startPosition, params.startPosition); + assert.deepEqual(cursor.endPosition, params.endPosition); + assert.deepEqual(cursor.startIndex, params.startIndex); + assert.deepEqual(cursor.endIndex, params.endIndex); + + const node = cursor.currentNode() + assert.equal(node.type, params.nodeType); + assert.equal(node.isNamed(), params.nodeIsNamed); + assert.deepEqual(node.startPosition, params.startPosition); + assert.deepEqual(node.endPosition, params.endPosition); + assert.deepEqual(node.startIndex, params.startIndex); + assert.deepEqual(node.endIndex, params.endIndex); +} diff --git a/script/build-wasm b/script/build-wasm index 9f507687..8238f24c 100755 --- a/script/build-wasm +++ b/script/build-wasm @@ -3,8 +3,10 @@ set -e args="-Os" +minify=1 if [[ "$1" == "--debug" ]]; then - args="-s ASSERTIONS=1 SAFE_HEAP=1 -O0" + minify=0 + args="-s ASSERTIONS=1 -s SAFE_HEAP=1 -O0" fi mkdir -p target/scratch target/release @@ -35,19 +37,23 @@ docker run \ lib/web/binding.c \ -o target/scratch/tree-sitter.js -if [ ! -d lib/web/node_modules/terser ]; then - ( - cd lib/web - npm install - ) + +if [[ "$minify" == "1" ]]; then + if [ ! -d lib/web/node_modules/terser ]; then + ( + cd lib/web + npm install + ) + fi + lib/web/node_modules/.bin/terser \ + --compress \ + --mangle \ + --keep-fnames \ + --keep-classnames \ + -- target/scratch/tree-sitter.js \ + > target/release/tree-sitter.js +else + cp target/scratch/tree-sitter.js target/release/tree-sitter.js fi -lib/web/node_modules/.bin/terser \ - --compress \ - --mangle \ - --keep-fnames \ - --keep-classnames \ - -- target/scratch/tree-sitter.js \ - > target/release/tree-sitter.js - mv target/scratch/tree-sitter.wasm target/release/tree-sitter.wasm