Reorganize language bindings

* Move rust binding: lib/binding -> lib/binding_rust
* Move wasm bindinig: lib/web -> lib/binding_web
* Add wasm readme
This commit is contained in:
Max Brunsfeld 2019-05-07 10:27:45 -07:00
parent a3ceb8f3a5
commit 3fc459a84b
23 changed files with 125 additions and 18 deletions

105
lib/binding_web/README.md Normal file
View file

@ -0,0 +1,105 @@
tree-sitter.wasm
================
[![Build Status](https://travis-ci.org/tree-sitter/tree-sitter.svg?branch=master)](https://travis-ci.org/tree-sitter/tree-sitter)
Wasm bindings to the [Tree-sitter](https://github.com/tree-sitter/tree-sitter) parsing library.
### Basic Usage
You can either load the library as a standalone script:
```html
<script src="/public/js/tree-sitter.js"/>
<script>
const Parser = window.TreeSitter;
Parser.init().then(() => { /* the library is ready */ });
</script>
```
or using a packaging system like Webpack:
```js
const Parser = require('tree-sitter');
Parser.init().then(() => { /* the library is ready */ });
```
Create a parser:
```js
const parser = new Parser;
```
Then assign a language to the parser. Tree-sitter languages are packaged as individual `.wasm` files:
```js
const JavaScript = await Parser.Language.load('/path/to/tree-sitter-javascript.wasm');
parser.setLanguage(JavaScript);
```
Now you can parse source code:
```js
const sourceCode = 'let x = 1; console.log(x);';
const tree = parser.parse(sourceCode);
```
and inspect the syntax tree.
```javascript
console.log(tree.rootNode.toString());
// (program
// (lexical_declaration
// (variable_declarator (identifier) (number)))
// (expression_statement
// (call_expression
// (member_expression (identifier) (property_identifier))
// (arguments (identifier)))))
const callExpression = tree.rootNode.child(1).firstChild;
console.log(callExpression);
// { type: 'call_expression',
// startPosition: {row: 0, column: 16},
// endPosition: {row: 0, column: 30},
// startIndex: 0,
// endIndex: 30 }
```
### Editing
If your source code *changes*, you can update the syntax tree. This will take less time than the first parse.
```javascript
// Replace 'let' with 'const'
const newSourceCode = 'const x = 1; console.log(x);';
tree.edit({
startIndex: 0,
oldEndIndex: 3,
newEndIndex: 5,
startPosition: {row: 0, column: 0},
oldEndPosition: {row: 0, column: 3},
newEndPosition: {row: 0, column: 5},
});
const newTree = parser.parse(newSourceCode, tree);
```
### Parsing Text From a Custom Data Structure
If your text is stored in a data structure other than a single string, you can parse it by supplying a callback to `parse` instead of a string:
```javascript
const sourceLines = [
'let x = 1;',
'console.log(x);'
];
const tree = parser.parse((index, position) => {
let line = sourceLines[position.row];
if (line) return line.slice(position.column);
});
```

454
lib/binding_web/binding.c Normal file
View file

@ -0,0 +1,454 @@
#include <emscripten.h>
#include <tree_sitter/api.h>
#include <stdio.h>
/*****************************/
/* Section - Data marshaling */
/*****************************/
static const uint32_t INPUT_BUFFER_SIZE = 10 * 1024;
const void *TRANSFER_BUFFER[12] = {
NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL,
};
void *ts_init() {
TRANSFER_BUFFER[0] = (const void *)TREE_SITTER_LANGUAGE_VERSION;
TRANSFER_BUFFER[1] = (const void *)TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION;
return TRANSFER_BUFFER;
}
static uint32_t code_unit_to_byte(uint32_t unit) {
return unit << 1;
}
static uint32_t byte_to_code_unit(uint32_t byte) {
return byte >> 1;
}
static inline void marshal_node(const void **buffer, TSNode node) {
buffer[0] = (const void *)node.id;
buffer[1] = (const void *)node.context[0];
buffer[2] = (const void *)node.context[1];
buffer[3] = (const void *)node.context[2];
buffer[4] = (const void *)node.context[3];
}
static inline TSNode unmarshal_node(const TSTree *tree) {
TSNode node;
node.id = TRANSFER_BUFFER[0];
node.context[0] = (uint32_t)TRANSFER_BUFFER[1];
node.context[1] = (uint32_t)TRANSFER_BUFFER[2];
node.context[2] = (uint32_t)TRANSFER_BUFFER[3];
node.context[3] = (uint32_t)TRANSFER_BUFFER[4];
node.tree = tree;
return node;
}
static inline void marshal_cursor(const TSTreeCursor *cursor) {
TRANSFER_BUFFER[0] = (const void *)cursor->id;
TRANSFER_BUFFER[1] = (const void *)cursor->context[0];
TRANSFER_BUFFER[2] = (const void *)cursor->context[1];
}
static inline TSTreeCursor unmarshal_cursor(const void **buffer, const TSTree *tree) {
TSTreeCursor cursor;
cursor.id = buffer[0];
cursor.context[0] = (uint32_t)buffer[1];
cursor.context[1] = (uint32_t)buffer[2];
cursor.tree = tree;
return cursor;
}
static void marshal_point(TSPoint point) {
TRANSFER_BUFFER[0] = (const void *)point.row;
TRANSFER_BUFFER[1] = (const void *)byte_to_code_unit(point.column);
}
static TSPoint unmarshal_point(const void **address) {
TSPoint point;
point.row = (uint32_t)address[0];
point.column = code_unit_to_byte((uint32_t)address[1]);
return point;
}
static TSInputEdit unmarshal_edit() {
TSInputEdit edit;
const void **address = TRANSFER_BUFFER;
edit.start_point = unmarshal_point(address); address += 2;
edit.old_end_point = unmarshal_point(address); address += 2;
edit.new_end_point = unmarshal_point(address); address += 2;
edit.start_byte = code_unit_to_byte((uint32_t)*address); address += 1;
edit.old_end_byte = code_unit_to_byte((uint32_t)*address); address += 1;
edit.new_end_byte = code_unit_to_byte((uint32_t)*address); address += 1;
return edit;
}
/********************/
/* Section - Parser */
/********************/
extern void tree_sitter_parse_callback(
char *input_buffer,
uint32_t index,
uint32_t row,
uint32_t column,
uint32_t *length_read
);
extern void tree_sitter_log_callback(
void *payload,
TSLogType log_type,
const char *message
);
void ts_parser_new_wasm() {
TSParser *parser = ts_parser_new();
char *input_buffer = calloc(INPUT_BUFFER_SIZE, sizeof(char));
TRANSFER_BUFFER[0] = parser;
TRANSFER_BUFFER[1] = input_buffer;
}
static const char *call_parse_callback(
void *payload,
uint32_t byte,
TSPoint position,
uint32_t *bytes_read
) {
char *buffer = (char *)payload;
tree_sitter_parse_callback(
buffer,
byte_to_code_unit(byte),
position.row,
byte_to_code_unit(position.column),
bytes_read
);
*bytes_read = code_unit_to_byte(*bytes_read);
if (*bytes_read >= INPUT_BUFFER_SIZE) {
*bytes_read = INPUT_BUFFER_SIZE - 2;
}
return buffer;
}
void ts_parser_enable_logger_wasm(TSParser *self, bool should_log) {
TSLogger logger = {self, should_log ? tree_sitter_log_callback : NULL};
ts_parser_set_logger(self, logger);
}
TSTree *ts_parser_parse_wasm(
TSParser *self,
char *input_buffer,
const TSTree *old_tree,
TSRange *ranges,
uint32_t range_count
) {
TSInput input = {
input_buffer,
call_parse_callback,
TSInputEncodingUTF16
};
if (range_count) {
for (unsigned i = 0; i < range_count; i++) {
TSRange *range = &ranges[i];
range->start_byte = code_unit_to_byte(range->start_byte);
range->end_byte = code_unit_to_byte(range->end_byte);
range->start_point.column = code_unit_to_byte(range->start_point.column);
range->end_point.column = code_unit_to_byte(range->end_point.column);
}
ts_parser_set_included_ranges(self, ranges, range_count);
free(ranges);
} else {
ts_parser_set_included_ranges(self, NULL, 0);
}
return ts_parser_parse(self, old_tree, input);
}
/******************/
/* Section - Tree */
/******************/
void ts_tree_root_node_wasm(const TSTree *tree) {
marshal_node(TRANSFER_BUFFER, ts_tree_root_node(tree));
}
void ts_tree_edit_wasm(TSTree *tree) {
TSInputEdit edit = unmarshal_edit();
ts_tree_edit(tree, &edit);
}
/************************/
/* Section - TreeCursor */
/************************/
void ts_tree_cursor_new_wasm(const TSTree *tree) {
TSNode node = unmarshal_node(tree);
TSTreeCursor cursor = ts_tree_cursor_new(node);
marshal_cursor(&cursor);
}
void ts_tree_cursor_delete_wasm(const TSTree *tree) {
TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree);
ts_tree_cursor_delete(&cursor);
}
void ts_tree_cursor_reset_wasm(const TSTree *tree) {
TSNode node = unmarshal_node(tree);
TSTreeCursor cursor = unmarshal_cursor(&TRANSFER_BUFFER[5], tree);
ts_tree_cursor_reset(&cursor, node);
marshal_cursor(&cursor);
}
bool ts_tree_cursor_goto_first_child_wasm(const TSTree *tree) {
TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree);
bool result = ts_tree_cursor_goto_first_child(&cursor);
marshal_cursor(&cursor);
return result;
}
bool ts_tree_cursor_goto_next_sibling_wasm(const TSTree *tree) {
TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree);
bool result = ts_tree_cursor_goto_next_sibling(&cursor);
marshal_cursor(&cursor);
return result;
}
bool ts_tree_cursor_goto_parent_wasm(const TSTree *tree) {
TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree);
bool result = ts_tree_cursor_goto_parent(&cursor);
marshal_cursor(&cursor);
return result;
}
uint16_t ts_tree_cursor_current_node_type_id_wasm(const TSTree *tree) {
TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree);
TSNode node = ts_tree_cursor_current_node(&cursor);
return ts_node_symbol(node);
}
bool ts_tree_cursor_current_node_is_named_wasm(const TSTree *tree) {
TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree);
TSNode node = ts_tree_cursor_current_node(&cursor);
return ts_node_is_named(node);
}
bool ts_tree_cursor_current_node_is_missing_wasm(const TSTree *tree) {
TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree);
TSNode node = ts_tree_cursor_current_node(&cursor);
return ts_node_is_missing(node);
}
const uint32_t ts_tree_cursor_current_node_id_wasm(const TSTree *tree) {
TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree);
TSNode node = ts_tree_cursor_current_node(&cursor);
return (uint32_t)node.id;
}
void ts_tree_cursor_start_position_wasm(const TSTree *tree) {
TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree);
TSNode node = ts_tree_cursor_current_node(&cursor);
marshal_point(ts_node_start_point(node));
}
void ts_tree_cursor_end_position_wasm(const TSTree *tree) {
TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree);
TSNode node = ts_tree_cursor_current_node(&cursor);
marshal_point(ts_node_end_point(node));
}
uint32_t ts_tree_cursor_start_index_wasm(const TSTree *tree) {
TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree);
TSNode node = ts_tree_cursor_current_node(&cursor);
return byte_to_code_unit(ts_node_start_byte(node));
}
uint32_t ts_tree_cursor_end_index_wasm(const TSTree *tree) {
TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree);
TSNode node = ts_tree_cursor_current_node(&cursor);
return byte_to_code_unit(ts_node_end_byte(node));
}
void ts_tree_cursor_current_node_wasm(const TSTree *tree) {
TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree);
marshal_node(TRANSFER_BUFFER, ts_tree_cursor_current_node(&cursor));
}
/******************/
/* Section - Node */
/******************/
static TSTreeCursor scratch_cursor = {0};
uint16_t ts_node_symbol_wasm(const TSTree *tree) {
TSNode node = unmarshal_node(tree);
return ts_node_symbol(node);
}
uint32_t ts_node_child_count_wasm(const TSTree *tree) {
TSNode node = unmarshal_node(tree);
return ts_node_child_count(node);
}
uint32_t ts_node_named_child_count_wasm(const TSTree *tree) {
TSNode node = unmarshal_node(tree);
return ts_node_named_child_count(node);
}
void ts_node_child_wasm(const TSTree *tree, uint32_t index) {
TSNode node = unmarshal_node(tree);
marshal_node(TRANSFER_BUFFER, ts_node_child(node, index));
}
void ts_node_named_child_wasm(const TSTree *tree, uint32_t index) {
TSNode node = unmarshal_node(tree);
marshal_node(TRANSFER_BUFFER, ts_node_named_child(node, index));
}
void ts_node_next_sibling_wasm(const TSTree *tree) {
TSNode node = unmarshal_node(tree);
marshal_node(TRANSFER_BUFFER, ts_node_next_sibling(node));
}
void ts_node_prev_sibling_wasm(const TSTree *tree) {
TSNode node = unmarshal_node(tree);
marshal_node(TRANSFER_BUFFER, ts_node_prev_sibling(node));
}
void ts_node_next_named_sibling_wasm(const TSTree *tree) {
TSNode node = unmarshal_node(tree);
marshal_node(TRANSFER_BUFFER, ts_node_next_named_sibling(node));
}
void ts_node_prev_named_sibling_wasm(const TSTree *tree) {
TSNode node = unmarshal_node(tree);
marshal_node(TRANSFER_BUFFER, ts_node_prev_named_sibling(node));
}
void ts_node_parent_wasm(const TSTree *tree) {
TSNode node = unmarshal_node(tree);
marshal_node(TRANSFER_BUFFER, ts_node_parent(node));
}
void ts_node_descendant_for_index_wasm(const TSTree *tree) {
TSNode node = unmarshal_node(tree);
const void **address = TRANSFER_BUFFER + 5;
uint32_t start = code_unit_to_byte((uint32_t)address[0]);
uint32_t end = code_unit_to_byte((uint32_t)address[1]);
marshal_node(TRANSFER_BUFFER, ts_node_descendant_for_byte_range(node, start, end));
}
void ts_node_named_descendant_for_index_wasm(const TSTree *tree) {
TSNode node = unmarshal_node(tree);
const void **address = TRANSFER_BUFFER + 5;
uint32_t start = code_unit_to_byte((uint32_t)address[0]);
uint32_t end = code_unit_to_byte((uint32_t)address[1]);
marshal_node(TRANSFER_BUFFER, ts_node_named_descendant_for_byte_range(node, start, end));
}
void ts_node_descendant_for_position_wasm(const TSTree *tree) {
TSNode node = unmarshal_node(tree);
const void **address = TRANSFER_BUFFER + 5;
TSPoint start = unmarshal_point(address); address += 2;
TSPoint end = unmarshal_point(address);
marshal_node(TRANSFER_BUFFER, ts_node_descendant_for_point_range(node, start, end));
}
void ts_node_named_descendant_for_position_wasm(const TSTree *tree) {
TSNode node = unmarshal_node(tree);
const void **address = TRANSFER_BUFFER + 5;
TSPoint start = unmarshal_point(address); address += 2;
TSPoint end = unmarshal_point(address);
marshal_node(TRANSFER_BUFFER, ts_node_named_descendant_for_point_range(node, start, end));
}
void ts_node_start_point_wasm(const TSTree *tree) {
TSNode node = unmarshal_node(tree);
marshal_point(ts_node_start_point(node));
}
void ts_node_end_point_wasm(const TSTree *tree) {
TSNode node = unmarshal_node(tree);
marshal_point(ts_node_end_point(node));
}
uint32_t ts_node_start_index_wasm(const TSTree *tree) {
TSNode node = unmarshal_node(tree);
return byte_to_code_unit(ts_node_start_byte(node));
}
uint32_t ts_node_end_index_wasm(const TSTree *tree) {
TSNode node = unmarshal_node(tree);
return byte_to_code_unit(ts_node_end_byte(node));
}
char *ts_node_to_string_wasm(const TSTree *tree) {
TSNode node = unmarshal_node(tree);
return ts_node_string(node);
}
void ts_node_children_wasm(const TSTree *tree) {
TSNode node = unmarshal_node(tree);
uint32_t count = ts_node_child_count(node);
const void **result = NULL;
if (count > 0) {
result = calloc(sizeof(void *), 5 * count);
const void **address = result;
ts_tree_cursor_reset(&scratch_cursor, node);
ts_tree_cursor_goto_first_child(&scratch_cursor);
marshal_node(address, ts_tree_cursor_current_node(&scratch_cursor));
for (uint32_t i = 1; i < count; i++) {
address += 5;
ts_tree_cursor_goto_next_sibling(&scratch_cursor);
TSNode child = ts_tree_cursor_current_node(&scratch_cursor);
marshal_node(address, child);
}
}
TRANSFER_BUFFER[0] = (const void *)count;
TRANSFER_BUFFER[1] = result;
}
void ts_node_named_children_wasm(const TSTree *tree) {
TSNode node = unmarshal_node(tree);
uint32_t count = ts_node_named_child_count(node);
const void **result = NULL;
if (count > 0) {
result = calloc(sizeof(void *), 5 * count);
const void **address = result;
ts_tree_cursor_reset(&scratch_cursor, node);
ts_tree_cursor_goto_first_child(&scratch_cursor);
uint32_t i = 0;
for (;;) {
TSNode child = ts_tree_cursor_current_node(&scratch_cursor);
if (ts_node_is_named(child)) {
marshal_node(address, child);
address += 5;
i++;
if (i == count) break;
}
if (!ts_tree_cursor_goto_next_sibling(&scratch_cursor)) break;
}
}
TRANSFER_BUFFER[0] = (const void *)count;
TRANSFER_BUFFER[1] = result;
}
int ts_node_is_named_wasm(const TSTree *tree) {
TSNode node = unmarshal_node(tree);
return ts_node_is_named(node);
}
int ts_node_has_changes_wasm(const TSTree *tree) {
TSNode node = unmarshal_node(tree);
return ts_node_has_changes(node);
}
int ts_node_has_error_wasm(const TSTree *tree) {
TSNode node = unmarshal_node(tree);
return ts_node_has_error(node);
}
int ts_node_is_missing_wasm(const TSTree *tree) {
TSNode node = unmarshal_node(tree);
return ts_node_is_missing(node);
}

661
lib/binding_web/binding.js Normal file
View file

@ -0,0 +1,661 @@
const C = Module;
const INTERNAL = {};
const SIZE_OF_INT = 4;
const SIZE_OF_NODE = 5 * SIZE_OF_INT;
const SIZE_OF_POINT = 2 * SIZE_OF_INT;
const SIZE_OF_RANGE = 2 * SIZE_OF_INT + 2 * SIZE_OF_POINT;
var VERSION;
var MIN_COMPATIBLE_VERSION;
var TRANSFER_BUFFER;
var currentParseCallback;
var currentLogCallback;
var initPromise;
class Parser {
static init() {
if (!initPromise) {
initPromise = new Promise(resolve => {
Module.onRuntimeInitialized = resolve
}).then(() => {
TRANSFER_BUFFER = C._ts_init();
VERSION = getValue(TRANSFER_BUFFER, 'i32');
MIN_COMPATIBLE_VERSION = getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32');
});
}
return initPromise;
}
constructor() {
C._ts_parser_new_wasm();
this[0] = getValue(TRANSFER_BUFFER, 'i32');
this[1] = getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32');
}
delete() {
C._ts_parser_delete(this[0]);
C._free(this[1]);
}
setLanguage(language) {
let address;
if (!language) {
address = 0;
language = null;
} else if (language.constructor === Language) {
address = language[0];
const version = C._ts_language_version(address);
if (version < MIN_COMPATIBLE_VERSION || VERSION < version) {
throw new Error(
`Incompatible language version ${version}. ` +
`Compatibility range ${MIN_COMPATIBLE_VERSION} through ${VERSION}.`
);
}
} else {
throw new Error('Argument must be a Language');
}
this.language = language;
C._ts_parser_set_language(this[0], address);
return this;
}
getLanguage() {
return this.language
}
parse(callback, oldTree, options) {
if (typeof callback === 'string') {
currentParseCallback = index => callback.slice(index);
} else if (typeof callback === 'function') {
currentParseCallback = callback;
} else {
throw new Error("Argument must be a string or a function");
}
if (this.logCallback) {
currentLogCallback = this.logCallback;
C._ts_parser_enable_logger_wasm(this[0], 1);
} else {
currentLogCallback = null;
C._ts_parser_enable_logger_wasm(this[0], 0);
}
let rangeCount = 0;
let rangeAddress = 0;
if (options && options.includedRanges) {
rangeCount = options.includedRanges.length;
rangeAddress = C._calloc(rangeCount, SIZE_OF_RANGE);
let address = rangeAddress;
for (let i = 0; i < rangeCount; i++) {
marshalRange(address, options.includedRanges[i]);
address += SIZE_OF_RANGE;
}
}
const treeAddress = C._ts_parser_parse_wasm(
this[0],
this[1],
oldTree ? oldTree[0] : 0,
rangeAddress,
rangeCount
);
if (!treeAddress) {
currentParseCallback = null;
currentLogCallback = null;
throw new Error('Parsing failed');
}
const result = new Tree(INTERNAL, treeAddress, this.language, currentParseCallback);
currentParseCallback = null;
currentLogCallback = null;
return result;
}
reset() {
C._ts_parser_parse_wasm(this[0]);
}
setTimeoutMicros(timeout) {
C._ts_parser_set_timeout_micros(this[0], timeout);
}
getTimeoutMicros(timeout) {
C._ts_parser_timeout_micros(this[0]);
}
setLogger(callback) {
if (!callback) {
callback = null;
} else if (typeof callback !== "function") {
throw new Error("Logger callback must be a function");
}
this.logCallback = callback;
return this;
}
getLogger() {
return this.logCallback;
}
}
class Tree {
constructor(internal, address, language, textCallback) {
if (internal !== INTERNAL) {
throw new Error('Illegal constructor')
}
this[0] = address;
this.language = language;
this.textCallback = textCallback;
}
copy() {
const address = C._ts_tree_copy(this[0]);
return new Tree(INTERNAL, address, this.language, this.textCallback);
}
delete() {
C._ts_tree_delete(this[0]);
}
edit(edit) {
marshalEdit(edit);
C._ts_tree_edit_wasm(this[0]);
}
get rootNode() {
C._ts_tree_root_node_wasm(this[0]);
return unmarshalNode(this);
}
getLanguage() {
return this.language;
}
walk() {
return this.rootNode.walk();
}
}
class Node {
constructor(internal, tree) {
if (internal !== INTERNAL) {
throw new Error('Illegal constructor')
}
this.tree = tree;
}
get id() {
return this[0];
}
get typeId() {
marshalNode(this);
return C._ts_node_symbol_wasm(this.tree);
}
get type() {
return this.tree.language.types[this.typeId] || 'ERROR';
}
get startPosition() {
marshalNode(this);
C._ts_node_start_point_wasm(this.tree[0]);
return unmarshalPoint(TRANSFER_BUFFER);
}
get endPosition() {
marshalNode(this);
C._ts_node_end_point_wasm(this.tree[0]);
return unmarshalPoint(TRANSFER_BUFFER);
}
get startIndex() {
marshalNode(this);
return C._ts_node_start_index_wasm(this.tree[0]);
}
get endIndex() {
marshalNode(this);
return C._ts_node_end_index_wasm(this.tree[0]);
}
get text() {
const startIndex = this.startIndex;
const length = this.endIndex - startIndex;
let result = this.tree.textCallback(startIndex);
while (result.length < length) {
result += this.tree.textCallback(startIndex + result.length);
}
return result.slice(0, length);
}
isNamed() {
marshalNode(this);
return C._ts_node_is_named_wasm(this.tree[0]) === 1;
}
hasError() {
marshalNode(this);
return C._ts_node_has_error_wasm(this.tree[0]) === 1;
}
hasChanges() {
marshalNode(this);
return C._ts_node_has_changes_wasm(this.tree[0]) === 1;
}
isMissing() {
marshalNode(this);
return C._ts_node_is_missing_wasm(this.tree[0]) === 1;
}
equals(other) {
if (this === other) return true;
for (let i = 0; i < 5; i++) {
if (this[i] !== other[i]) return false;
}
return true;
}
child(index) {
marshalNode(this);
C._ts_node_child_wasm(this.tree[0], index);
return unmarshalNode(this.tree);
}
namedChild(index) {
marshalNode(this);
C._ts_node_named_child_wasm(this.tree[0], index);
return unmarshalNode(this.tree);
}
get childCount() {
marshalNode(this);
return C._ts_node_child_count_wasm(this.tree[0]);
}
get namedChildCount() {
marshalNode(this);
return C._ts_node_named_child_count_wasm(this.tree[0]);
}
get firstChild() {
return this.child(0);
}
get firstNamedChild() {
return this.namedChild(0);
}
get lastChild() {
return this.child(this.childCount - 1);
}
get lastNamedChild() {
return this.namedChild(this.namedChildCount - 1);
}
get children() {
if (!this._children) {
marshalNode(this);
C._ts_node_children_wasm(this.tree[0]);
const count = getValue(TRANSFER_BUFFER, 'i32');
const buffer = getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32');
this._children = new Array(count);
if (count > 0) {
let address = buffer;
for (let i = 0; i < count; i++) {
this._children[i] = unmarshalNode(this.tree, address);
address += SIZE_OF_NODE;
}
C._free(buffer);
}
}
return this._children;
}
get namedChildren() {
if (!this._namedChildren) {
marshalNode(this);
C._ts_node_named_children_wasm(this.tree[0]);
const count = getValue(TRANSFER_BUFFER, 'i32');
const buffer = getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32');
this._namedChildren = new Array(count);
if (count > 0) {
let address = buffer;
for (let i = 0; i < count; i++) {
this._namedChildren[i] = unmarshalNode(this.tree, address);
address += SIZE_OF_NODE;
}
C._free(buffer);
}
}
return this._namedChildren;
}
get nextSibling() {
marshalNode(this);
C._ts_node_next_sibling_wasm(this.tree[0]);
return unmarshalNode(this.tree);
}
get previousSibling() {
marshalNode(this);
C._ts_node_prev_sibling_wasm(this.tree[0]);
return unmarshalNode(this.tree);
}
get nextNamedSibling() {
marshalNode(this);
C._ts_node_next_named_sibling_wasm(this.tree[0]);
return unmarshalNode(this.tree);
}
get previousNamedSibling() {
marshalNode(this);
C._ts_node_prev_named_sibling_wasm(this.tree[0]);
return unmarshalNode(this.tree);
}
get parent() {
marshalNode(this);
C._ts_node_parent_wasm(this.tree[0]);
return unmarshalNode(this.tree);
}
descendantForIndex(start, end = start) {
if (typeof start !== 'number' || typeof end !== 'number') {
throw new Error('Arguments must be numbers');
}
marshalNode(this);
let address = TRANSFER_BUFFER + SIZE_OF_NODE;
setValue(address, start, 'i32');
setValue(address + SIZE_OF_INT, end, 'i32');
C._ts_node_descendant_for_index_wasm(this.tree[0]);
return unmarshalNode(this.tree);
}
namedDescendantForIndex(start, end = start) {
if (typeof start !== 'number' || typeof end !== 'number') {
throw new Error('Arguments must be numbers');
}
marshalNode(this);
let address = TRANSFER_BUFFER + SIZE_OF_NODE;
setValue(address, start, 'i32');
setValue(address + SIZE_OF_INT, end, 'i32');
C._ts_node_named_descendant_for_index_wasm(this.tree[0]);
return unmarshalNode(this.tree);
}
descendantForPosition(start, end = start) {
if (!isPoint(start) || !isPoint(end)) {
throw new Error('Arguments must be {row, column} objects');
}
marshalNode(this);
let address = TRANSFER_BUFFER + SIZE_OF_NODE;
marshalPoint(address, start);
marshalPoint(address + SIZE_OF_POINT, end);
C._ts_node_descendant_for_position_wasm(this.tree[0]);
return unmarshalNode(this.tree);
}
namedDescendantForPosition(start, end = start) {
if (!isPoint(start) || !isPoint(end)) {
throw new Error('Arguments must be {row, column} objects');
}
marshalNode(this);
let address = TRANSFER_BUFFER + SIZE_OF_NODE;
marshalPoint(address, start);
marshalPoint(address + SIZE_OF_POINT, end);
C._ts_node_named_descendant_for_position_wasm(this.tree[0]);
return unmarshalNode(this.tree);
}
walk() {
marshalNode(this);
C._ts_tree_cursor_new_wasm(this.tree[0]);
return new TreeCursor(INTERNAL, this.tree);
}
toString() {
marshalNode(this);
const address = C._ts_node_to_string_wasm(this.tree[0]);
const result = AsciiToString(address);
C._free(address);
return result;
}
}
class TreeCursor {
constructor(internal, tree) {
if (internal !== INTERNAL) {
throw new Error('Illegal constructor')
}
this.tree = tree;
unmarshalTreeCursor(this);
}
delete() {
marshalTreeCursor(this);
C._ts_tree_cursor_delete_wasm(this.tree[0]);
}
reset(node) {
marshalNode(node);
marshalTreeCursor(this, TRANSFER_BUFFER + SIZE_OF_NODE);
C._ts_tree_cursor_reset_wasm(this.tree[0]);
unmarshalTreeCursor(this);
}
get nodeType() {
return this.tree.language.types[this.nodeTypeId] || 'ERROR';
}
get nodeTypeId() {
marshalTreeCursor(this);
return C._ts_tree_cursor_current_node_type_id_wasm(this.tree[0]);
}
get nodeId() {
marshalTreeCursor(this);
return C._ts_tree_cursor_current_node_id_wasm(this.tree[0]);
}
get nodeIsNamed() {
marshalTreeCursor(this);
return C._ts_tree_cursor_current_node_is_named_wasm(this.tree[0]) === 1;
}
get nodeIsMissing() {
marshalTreeCursor(this);
return C._ts_tree_cursor_current_node_is_missing_wasm(this.tree[0]) === 1;
}
get startPosition() {
marshalTreeCursor(this);
C._ts_tree_cursor_start_position_wasm(this.tree[0]);
return unmarshalPoint(TRANSFER_BUFFER);
}
get endPosition() {
marshalTreeCursor(this);
C._ts_tree_cursor_end_position_wasm(this.tree[0]);
return unmarshalPoint(TRANSFER_BUFFER);
}
get startIndex() {
marshalTreeCursor(this);
return C._ts_tree_cursor_start_index_wasm(this.tree[0]);
}
get endIndex() {
marshalTreeCursor(this);
return C._ts_tree_cursor_end_index_wasm(this.tree[0]);
}
currentNode() {
marshalTreeCursor(this);
C._ts_tree_cursor_current_node_wasm(this.tree[0]);
return unmarshalNode(this.tree);
}
gotoFirstChild() {
marshalTreeCursor(this);
const result = C._ts_tree_cursor_goto_first_child_wasm(this.tree[0]);
unmarshalTreeCursor(this);
return result === 1;
}
gotoNextSibling() {
marshalTreeCursor(this);
const result = C._ts_tree_cursor_goto_next_sibling_wasm(this.tree[0]);
unmarshalTreeCursor(this);
return result === 1;
}
gotoParent() {
marshalTreeCursor(this);
const result = C._ts_tree_cursor_goto_parent_wasm(this.tree[0]);
unmarshalTreeCursor(this);
return result === 1;
}
}
class Language {
constructor(internal, address) {
if (internal !== INTERNAL) {
throw new Error('Illegal constructor')
}
this[0] = address;
this.types = new Array(C._ts_language_symbol_count(this[0]));
for (let i = 0, n = this.types.length; i < n; i++) {
if (C._ts_language_symbol_type(this[0], i) < 2) {
this.types[i] = UTF8ToString(C._ts_language_symbol_name(this[0], i));
}
}
}
get version() {
return C._ts_language_version(this[0]);
}
static load(url) {
let bytes;
if (
typeof require === 'function' &&
require('url').parse(url).protocol == null
) {
const fs = require('fs');
bytes = Promise.resolve(fs.readFileSync(url));
} else {
bytes = fetch(url)
.then(response => response.arrayBuffer()
.then(buffer => {
if (response.ok) {
return new Uint8Array(buffer);
} else {
const body = new TextDecoder('utf-8').decode(buffer);
throw new Error(`Language.load failed with status ${response.status}.\n\n${body}`)
}
}));
}
return bytes
.then(bytes => loadWebAssemblyModule(bytes, {loadAsync: true}))
.then(mod => {
const functionName = Object.keys(mod).find(key => key.includes("tree_sitter_"));
const languageAddress = mod[functionName]();
return new Language(INTERNAL, languageAddress);
});
}
}
function isPoint(point) {
return (
point &&
typeof point.row === 'number' &&
typeof point.row === 'number'
);
}
function marshalNode(node) {
let address = TRANSFER_BUFFER;
for (let i = 0; i < 5; i++) {
setValue(address, node[i], 'i32');
address += SIZE_OF_INT;
}
}
function unmarshalNode(tree, address = TRANSFER_BUFFER) {
const id = getValue(address, 'i32');
if (id === 0) return null;
const result = new Node(INTERNAL, tree);
result[0] = id;
address += SIZE_OF_INT;
for (let i = 1; i < 5; i++) {
result[i] = getValue(address, 'i32');
address += SIZE_OF_INT;
}
return result;
}
function marshalTreeCursor(cursor, address = TRANSFER_BUFFER) {
setValue(address + 0 * SIZE_OF_INT, cursor[0], 'i32'),
setValue(address + 1 * SIZE_OF_INT, cursor[1], 'i32'),
setValue(address + 2 * SIZE_OF_INT, cursor[2], 'i32')
}
function unmarshalTreeCursor(cursor) {
cursor[0] = getValue(TRANSFER_BUFFER + 0 * SIZE_OF_INT, 'i32'),
cursor[1] = getValue(TRANSFER_BUFFER + 1 * SIZE_OF_INT, 'i32'),
cursor[2] = getValue(TRANSFER_BUFFER + 2 * SIZE_OF_INT, 'i32')
}
function marshalPoint(address, point) {
setValue(address, point.row, 'i32')
setValue(address + SIZE_OF_INT, point.column, 'i32')
}
function unmarshalPoint(address) {
return {
row: getValue(address, 'i32'),
column: getValue(address + SIZE_OF_INT, 'i32')
}
}
function marshalRange(address, range) {
marshalPoint(address, range.startPosition); address += SIZE_OF_POINT;
marshalPoint(address, range.endPosition); address += SIZE_OF_POINT;
setValue(address, range.startIndex, 'i32'); address += SIZE_OF_INT;
setValue(address, range.endIndex, 'i32'); address += SIZE_OF_INT;
}
function unmarshalRange(address) {
const result = {};
result.startPosition = unmarshalPoint(address); address += SIZE_OF_POINT;
result.endPosition = unmarshalPoint(address); address += SIZE_OF_POINT;
result.startIndex = getValue(address, 'i32'); address += SIZE_OF_INT;
result.endIndex = getValue(address, 'i32');
return result;
}
function marshalEdit(edit) {
let address = TRANSFER_BUFFER;
marshalPoint(address, edit.startPosition); address += SIZE_OF_POINT;
marshalPoint(address, edit.oldEndPosition); address += SIZE_OF_POINT;
marshalPoint(address, edit.newEndPosition); address += SIZE_OF_POINT;
setValue(address, edit.startIndex, 'i32'); address += SIZE_OF_INT;
setValue(address, edit.oldEndIndex, 'i32'); address += SIZE_OF_INT;
setValue(address, edit.newEndIndex, 'i32'); address += SIZE_OF_INT;
}
Parser.Language = Language;
return Parser;
}));

View file

@ -0,0 +1,85 @@
[
"_calloc",
"_free",
"_malloc",
"__ZNKSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE4copyEPcmm",
"__ZNKSt3__220__vector_base_commonILb1EE20__throw_length_errorEv",
"__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE6__initEPKcm",
"__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE7reserveEm",
"__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE9__grow_byEmmmmmm",
"__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE9push_backEc",
"__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEC1ERKS5_",
"__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEED2Ev",
"__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE9push_backEw",
"__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEED2Ev",
"__ZdlPv",
"__Znwm",
"___assert_fail",
"_abort",
"_iswalnum",
"_iswalpha",
"_iswdigit",
"_iswlower",
"_iswspace",
"_memchr",
"_memcmp",
"_memcpy",
"_strlen",
"_towupper",
"abort",
"_ts_init",
"_ts_language_symbol_count",
"_ts_language_symbol_name",
"_ts_language_symbol_type",
"_ts_language_version",
"_ts_node_child_count_wasm",
"_ts_node_child_wasm",
"_ts_node_children_wasm",
"_ts_node_descendant_for_index_wasm",
"_ts_node_descendant_for_position_wasm",
"_ts_node_end_index_wasm",
"_ts_node_end_point_wasm",
"_ts_node_has_changes_wasm",
"_ts_node_has_error_wasm",
"_ts_node_is_missing_wasm",
"_ts_node_is_named_wasm",
"_ts_node_named_child_count_wasm",
"_ts_node_named_child_wasm",
"_ts_node_named_children_wasm",
"_ts_node_named_descendant_for_index_wasm",
"_ts_node_named_descendant_for_position_wasm",
"_ts_node_next_named_sibling_wasm",
"_ts_node_next_sibling_wasm",
"_ts_node_parent_wasm",
"_ts_node_prev_named_sibling_wasm",
"_ts_node_prev_sibling_wasm",
"_ts_node_start_index_wasm",
"_ts_node_start_point_wasm",
"_ts_node_symbol_wasm",
"_ts_node_to_string_wasm",
"_ts_parser_delete",
"_ts_parser_enable_logger_wasm",
"_ts_parser_new_wasm",
"_ts_parser_parse_wasm",
"_ts_parser_set_language",
"_ts_tree_cursor_current_node_id_wasm",
"_ts_tree_cursor_current_node_is_missing_wasm",
"_ts_tree_cursor_current_node_is_named_wasm",
"_ts_tree_cursor_current_node_type_id_wasm",
"_ts_tree_cursor_current_node_wasm",
"_ts_tree_cursor_delete_wasm",
"_ts_tree_cursor_end_index_wasm",
"_ts_tree_cursor_end_position_wasm",
"_ts_tree_cursor_goto_first_child_wasm",
"_ts_tree_cursor_goto_next_sibling_wasm",
"_ts_tree_cursor_goto_parent_wasm",
"_ts_tree_cursor_new_wasm",
"_ts_tree_cursor_reset_wasm",
"_ts_tree_cursor_start_index_wasm",
"_ts_tree_cursor_start_position_wasm",
"_ts_tree_delete",
"_ts_tree_edit_wasm",
"_ts_tree_root_node_wasm"
]

View file

@ -0,0 +1,25 @@
mergeInto(LibraryManager.library, {
tree_sitter_parse_callback: function(
inputBufferAddress,
index,
row,
column,
lengthAddress
) {
var INPUT_BUFFER_SIZE = 10 * 1024;
var string = currentParseCallback(index, {row: row, column: column});
if (typeof string === 'string') {
setValue(lengthAddress, string.length, 'i32');
stringToUTF16(string, inputBufferAddress, INPUT_BUFFER_SIZE);
} else {
setValue(lengthAddress, 0, 'i32');
}
},
tree_sitter_log_callback: function(_payload, isLexMessage, messageAddress) {
if (currentLogCallback) {
const message = UTF8ToString(messageAddress);
currentLogCallback(message, isLexMessage !== 0);
}
}
});

View file

@ -0,0 +1,31 @@
{
"name": "tree-sitter.wasm",
"version": "0.0.1",
"description": "Tree-sitter bindings for the web",
"main": "index.js",
"directories": {
"test": "test"
},
"scripts": {
"test": "mocha"
},
"repository": {
"type": "git",
"url": "git+https://github.com/tree-sitter/tree-sitter.git"
},
"keywords": [
"incremental",
"parsing"
],
"author": "Max Brunsfeld",
"license": "MIT",
"bugs": {
"url": "https://github.com/tree-sitter/tree-sitter/issues"
},
"homepage": "https://github.com/tree-sitter/tree-sitter#readme",
"devDependencies": {
"chai": "^4.2.0",
"mocha": "^6.1.4",
"terser": "^3.17.0"
}
}

11
lib/binding_web/prefix.js Normal file
View file

@ -0,0 +1,11 @@
(function (root, factory) {
if (typeof define === 'function' && define.amd) {
define([], factory);
} else if (typeof exports === 'object') {
module.exports = factory();
// module.exports.init();
// delete module.exports.init;
} else {
window.TreeSitter = factory();
}
}(this, function () {

View file

@ -0,0 +1,9 @@
const release = '../../../target/release'
const Parser = require(`${release}/tree-sitter.js`);
const languageURL = name => require.resolve(`${release}/tree-sitter-${name}.wasm`);
module.exports = Parser.init().then(async () => ({
Parser,
languageURL,
JavaScript: await Parser.Language.load(languageURL('javascript')),
}));

View file

@ -0,0 +1,358 @@
const {assert} = require('chai');
let Parser, JavaScript;
describe("Node", () => {
let parser, tree;
before(async () =>
({Parser, JavaScript} = await require('./helper'))
);
beforeEach(() => {
tree = null;
parser = new Parser().setLanguage(JavaScript);
});
afterEach(() => {
parser.delete();
tree.delete();
});
describe(".children", () => {
it("returns an array of child nodes", () => {
tree = parser.parse("x10 + 1000");
assert.equal(1, tree.rootNode.children.length);
const sumNode = tree.rootNode.firstChild.firstChild;
assert.deepEqual(
sumNode.children.map(child => child.type),
["identifier", "+", "number"]
);
});
});
describe(".namedChildren", () => {
it("returns an array of named child nodes", () => {
tree = parser.parse("x10 + 1000");
const sumNode = tree.rootNode.firstChild.firstChild;
assert.equal(1, tree.rootNode.namedChildren.length);
assert.deepEqual(
["identifier", "number"],
sumNode.namedChildren.map(child => child.type)
);
});
});
describe(".startIndex and .endIndex", () => {
it("returns the character index where the node starts/ends in the text", () => {
tree = parser.parse("a👍👎1 / b👎c👎");
const quotientNode = tree.rootNode.firstChild.firstChild;
assert.equal(0, quotientNode.startIndex);
assert.equal(15, quotientNode.endIndex);
assert.deepEqual(
[0, 7, 9],
quotientNode.children.map(child => child.startIndex)
);
assert.deepEqual(
[6, 8, 15],
quotientNode.children.map(child => child.endIndex)
);
});
});
describe(".startPosition and .endPosition", () => {
it("returns the row and column where the node starts/ends in the text", () => {
tree = parser.parse("x10 + 1000");
const sumNode = tree.rootNode.firstChild.firstChild;
assert.equal("binary_expression", sumNode.type);
assert.deepEqual({ row: 0, column: 0 }, sumNode.startPosition);
assert.deepEqual({ row: 0, column: 10 }, sumNode.endPosition);
assert.deepEqual(
[{ row: 0, column: 0 }, { row: 0, column: 4 }, { row: 0, column: 6 }],
sumNode.children.map(child => child.startPosition)
);
assert.deepEqual(
[{ row: 0, column: 3 }, { row: 0, column: 5 }, { row: 0, column: 10 }],
sumNode.children.map(child => child.endPosition)
);
});
it("handles characters that occupy two UTF16 code units", () => {
tree = parser.parse("a👍👎1 /\n b👎c👎");
const sumNode = tree.rootNode.firstChild.firstChild;
assert.deepEqual(
[
[{ row: 0, column: 0 }, { row: 0, column: 6 }],
[{ row: 0, column: 7 }, { row: 0, column: 8 }],
[{ row: 1, column: 1 }, { row: 1, column: 7 }]
],
sumNode.children.map(child => [child.startPosition, child.endPosition])
);
});
});
describe(".parent", () => {
it("returns the node's parent", () => {
tree = parser.parse("x10 + 1000");
const sumNode = tree.rootNode.firstChild;
const variableNode = sumNode.firstChild;
assert.notEqual(sumNode.id, variableNode.id);
assert.equal(sumNode.id, variableNode.parent.id);
assert.equal(tree.rootNode.id, sumNode.parent.id);
});
});
describe('.child(), .firstChild, .lastChild', () => {
it('returns null when the node has no children', () => {
tree = parser.parse("x10 + 1000");
const sumNode = tree.rootNode.firstChild.firstChild;
const variableNode = sumNode.firstChild;
assert.equal(variableNode.firstChild, null);
assert.equal(variableNode.lastChild, null);
assert.equal(variableNode.firstNamedChild, null);
assert.equal(variableNode.lastNamedChild, null);
assert.equal(variableNode.child(1), null);
})
});
describe(".nextSibling and .previousSibling", () => {
it("returns the node's next and previous sibling", () => {
tree = parser.parse("x10 + 1000");
const sumNode = tree.rootNode.firstChild.firstChild;
assert.equal(sumNode.children[1].id, sumNode.children[0].nextSibling.id);
assert.equal(sumNode.children[2].id, sumNode.children[1].nextSibling.id);
assert.equal(
sumNode.children[0].id,
sumNode.children[1].previousSibling.id
);
assert.equal(
sumNode.children[1].id,
sumNode.children[2].previousSibling.id
);
});
});
describe(".nextNamedSibling and .previousNamedSibling", () => {
it("returns the node's next and previous named sibling", () => {
tree = parser.parse("x10 + 1000");
const sumNode = tree.rootNode.firstChild.firstChild;
assert.equal(
sumNode.namedChildren[1].id,
sumNode.namedChildren[0].nextNamedSibling.id
);
assert.equal(
sumNode.namedChildren[0].id,
sumNode.namedChildren[1].previousNamedSibling.id
);
});
});
describe(".descendantForIndex(min, max)", () => {
it("returns the smallest node that spans the given range", () => {
tree = parser.parse("x10 + 1000");
const sumNode = tree.rootNode.firstChild.firstChild;
assert.equal("identifier", sumNode.descendantForIndex(1, 2).type);
assert.equal("+", sumNode.descendantForIndex(4, 4).type);
assert.throws(() => {
sumNode.descendantForIndex(1, {});
}, "Arguments must be numbers");
assert.throws(() => {
sumNode.descendantForIndex();
}, "Arguments must be numbers");
});
});
describe(".namedDescendantForIndex", () => {
it("returns the smallest node that spans the given range", () => {
tree = parser.parse("x10 + 1000");
const sumNode = tree.rootNode.firstChild;
assert.equal("identifier", sumNode.descendantForIndex(1, 2).type);
assert.equal("+", sumNode.descendantForIndex(4, 4).type);
});
});
describe(".descendantForPosition(min, max)", () => {
it("returns the smallest node that spans the given range", () => {
tree = parser.parse("x10 + 1000");
const sumNode = tree.rootNode.firstChild;
assert.equal(
"identifier",
sumNode.descendantForPosition(
{ row: 0, column: 1 },
{ row: 0, column: 2 }
).type
);
assert.equal(
"+",
sumNode.descendantForPosition({ row: 0, column: 4 }).type
);
assert.throws(() => {
sumNode.descendantForPosition(1, {});
}, "Arguments must be {row, column} objects");
assert.throws(() => {
sumNode.descendantForPosition();
}, "Arguments must be {row, column} objects");
});
});
describe(".namedDescendantForPosition(min, max)", () => {
it("returns the smallest named node that spans the given range", () => {
tree = parser.parse("x10 + 1000");
const sumNode = tree.rootNode.firstChild;
assert.equal(
sumNode.namedDescendantForPosition(
{ row: 0, column: 1 },
{ row: 0, column: 2 }
).type,
"identifier",
);
assert.equal(
sumNode.namedDescendantForPosition({ row: 0, column: 4 }).type,
'binary_expression'
);
});
});
describe(".hasError()", () => {
it("returns true if the node contains an error", () => {
tree = parser.parse("1 + 2 * * 3");
const node = tree.rootNode;
assert.equal(
node.toString(),
'(program (expression_statement (binary_expression (number) (binary_expression (number) (ERROR) (number)))))'
);
const sum = node.firstChild.firstChild;
assert(sum.hasError());
assert(!sum.children[0].hasError());
assert(!sum.children[1].hasError());
assert(sum.children[2].hasError());
});
});
describe(".isMissing()", () => {
it("returns true if the node is missing from the source and was inserted via error recovery", () => {
tree = parser.parse("(2 ||)");
const node = tree.rootNode;
assert.equal(
node.toString(),
"(program (expression_statement (parenthesized_expression (binary_expression (number) (MISSING identifier)))))"
);
const sum = node.firstChild.firstChild.firstNamedChild;
assert.equal(sum.type, 'binary_expression')
assert(sum.hasError());
assert(!sum.children[0].isMissing());
assert(!sum.children[1].isMissing());
assert(sum.children[2].isMissing());
});
});
describe(".text", () => {
const text = "α0 / b👎c👎";
Object.entries({
'.parse(String)': text,
'.parse(Function)': offset => text.substr(offset, 4)
}).forEach(([method, parse]) =>
it(`returns the text of a node generated by ${method}`, async () => {
const [numeratorSrc, denominatorSrc] = text.split(/\s*\/\s+/)
tree = await parser.parse(text)
const quotientNode = tree.rootNode.firstChild.firstChild;
const [numerator, slash, denominator] = quotientNode.children;
assert.equal(text, tree.rootNode.text, 'root node text');
assert.equal(denominatorSrc, denominator.text, 'denominator text');
assert.equal(text, quotientNode.text, 'quotient text');
assert.equal(numeratorSrc, numerator.text, 'numerator text');
assert.equal('/', slash.text, '"/" text');
})
);
});
describe.skip('.descendantsOfType(type, min, max)', () => {
it('finds all of the descendants of the given type in the given range', () => {
tree = parser.parse("a + 1 * b * 2 + c + 3");
const outerSum = tree.rootNode.firstChild.firstChild;
let descendants = outerSum.descendantsOfType('number', {row: 0, column: 2}, {row: 0, column: 15})
assert.deepEqual(
descendants.map(node => node.startIndex),
[4, 12]
);
descendants = outerSum.descendantsOfType('identifier', {row: 0, column: 2}, {row: 0, column: 15})
assert.deepEqual(
descendants.map(node => node.startIndex),
[8]
);
descendants = outerSum.descendantsOfType('identifier', {row: 0, column: 0}, {row: 0, column: 30})
assert.deepEqual(
descendants.map(node => node.startIndex),
[0, 8, 16]
);
descendants = outerSum.descendantsOfType('number', {row: 0, column: 0}, {row: 0, column: 30})
assert.deepEqual(
descendants.map(node => node.startIndex),
[4, 12, 20]
);
descendants = outerSum.descendantsOfType(
['identifier', 'number'],
{row: 0, column: 0},
{row: 0, column: 30}
)
assert.deepEqual(
descendants.map(node => node.startIndex),
[0, 4, 8, 12, 16, 20]
);
descendants = outerSum.descendantsOfType('number')
assert.deepEqual(
descendants.map(node => node.startIndex),
[4, 12, 20]
);
descendants = outerSum.firstChild.descendantsOfType('number', {row: 0, column: 0}, {row: 0, column: 30})
assert.deepEqual(
descendants.map(node => node.startIndex),
[4, 12]
);
})
});
describe.skip('.closest(type)', () => {
it('returns the closest ancestor of the given type', () => {
tree = parser.parse("a(b + -d.e)");
const property = tree.rootNode.descendantForIndex("a(b + -d.".length);
assert.equal(property.type, 'property_identifier');
const unary = property.closest('unary_expression')
assert.equal(unary.type, 'unary_expression')
assert.equal(unary.startIndex, 'a(b + '.length)
assert.equal(unary.endIndex, 'a(b + -d.e'.length)
const sum = property.closest(['binary_expression', 'call_expression'])
assert.equal(sum.type, 'binary_expression')
assert.equal(sum.startIndex, 2)
assert.equal(sum.endIndex, 'a(b + -d.e'.length)
});
it('throws an exception when an invalid argument is given', () => {
tree = parser.parse("a + 1 * b * 2 + c + 3");
const number = tree.rootNode.descendantForIndex(4)
assert.throws(() => number.closest({a: 1}), /Argument must be a string or array of strings/)
});
});
});

View file

@ -0,0 +1,209 @@
const {assert} = require('chai');
let Parser, JavaScript, languageURL;
describe("Parser", () => {
let parser;
before(async () =>
({Parser, JavaScript, languageURL} = await require('./helper'))
);
beforeEach(() => {
parser = new Parser();
});
afterEach(() => {
parser.delete()
});
describe(".setLanguage", () => {
it("allows setting the language to null", () => {
assert.equal(parser.getLanguage(), null);
parser.setLanguage(JavaScript);
assert.equal(parser.getLanguage(), JavaScript);
parser.setLanguage(null);
assert.equal(parser.getLanguage(), null);
});
it("throws an exception when the given object is not a tree-sitter language", () => {
assert.throws(() => parser.setLanguage({}), /Argument must be a Language/);
assert.throws(() => parser.setLanguage(1), /Argument must be a Language/);
});
});
describe(".setLogger", () => {
beforeEach(() => {
parser.setLanguage(JavaScript)
});
it("calls the given callback for each parse event", () => {
const debugMessages = [];
parser.setLogger((message) => debugMessages.push(message));
parser.parse("a + b + c");
assert.includeMembers(debugMessages, [
"skip character:' '",
"consume character:'b'",
"reduce sym:program, child_count:1",
"accept"
]);
});
it("allows the callback to be retrieved later", () => {
const callback = () => {}
parser.setLogger(callback);
assert.equal(parser.getLogger(), callback);
parser.setLogger(false);
assert.equal(parser.getLogger(), null);
});
it("disables debugging when given a falsy value", () => {
const debugMessages = [];
parser.setLogger((message) => debugMessages.push(message));
parser.setLogger(false);
parser.parse("a + b * c");
assert.equal(debugMessages.length, 0);
});
it("throws an error when given a truthy value that isn't a function ", () => {
assert.throws(
() => parser.setLogger("5"),
"Logger callback must be a function"
);
});
it("rethrows errors thrown by the logging callback", () => {
const error = new Error("The error message");
parser.setLogger((msg, params) => { throw error; });
assert.throws(
() => parser.parse("ok;"),
"The error message"
);
});
});
describe(".parse", () => {
let tree;
beforeEach(() => {
tree = null;
parser.setLanguage(JavaScript)
});
afterEach(() => {
if (tree) tree.delete();
});
it("reads from the given input", () => {
const parts = ["first", "_", "second", "_", "third"];
tree = parser.parse(() => parts.shift());
assert.equal(tree.rootNode.toString(), "(program (expression_statement (identifier)))");
});
it("stops reading when the input callback return something that's not a string", () => {
const parts = ["abc", "def", "ghi", {}, {}, {}, "second-word", " "];
tree = parser.parse(() => parts.shift());
assert.equal(
tree.rootNode.toString(),
"(program (expression_statement (identifier)))"
);
assert.equal(tree.rootNode.endIndex, 9);
assert.equal(parts.length, 2);
});
it("throws an exception when the given input is not a function", () => {
assert.throws(() => parser.parse(null), "Argument must be a string or a function");
assert.throws(() => parser.parse(5), "Argument must be a string or a function");
assert.throws(() => parser.parse({}), "Argument must be a string or a function");
});
it("handles long input strings", () => {
const repeatCount = 10000;
const inputString = "[" + "0,".repeat(repeatCount) + "]";
tree = parser.parse(inputString);
assert.equal(tree.rootNode.type, "program");
assert.equal(tree.rootNode.firstChild.firstChild.namedChildCount, repeatCount);
}).timeout(5000);
it("can use the bash parser", async () => {
parser.setLanguage(await Parser.Language.load(languageURL('bash')));
tree = parser.parse("FOO=bar echo <<EOF 2> err.txt > hello.txt \nhello\nEOF");
assert.equal(
tree.rootNode.toString(),
'(program (redirected_statement (command ' +
'(variable_assignment (variable_name) (word)) ' +
'(command_name (word))) ' +
'(heredoc_redirect (heredoc_start)) ' +
'(file_redirect (file_descriptor) (word)) ' +
'(file_redirect (word))) ' +
'(heredoc_body))'
);
}).timeout(5000);
it("can use the c++ parser", async () => {
parser.setLanguage(await Parser.Language.load(languageURL('cpp')));
tree = parser.parse("const char *s = R\"EOF(HELLO WORLD)EOF\";");
assert.equal(
tree.rootNode.toString(),
'(translation_unit (declaration (type_qualifier) (primitive_type) (init_declarator (pointer_declarator (identifier)) (raw_string_literal))))'
);
}).timeout(5000);
it("can use the HTML parser", async () => {
parser.setLanguage(await Parser.Language.load(languageURL('html')));
tree = parser.parse("<div><span><custom></custom></span></div>");
assert.equal(
tree.rootNode.toString(),
'(fragment (element (start_tag (tag_name)) (element (start_tag (tag_name)) (element (start_tag (tag_name)) (end_tag (tag_name))) (end_tag (tag_name))) (end_tag (tag_name))))'
);
}).timeout(5000);
it("can use the python parser", async () => {
parser.setLanguage(await Parser.Language.load(languageURL('python')));
tree = parser.parse("class A:\n def b():\n c()");
assert.equal(
tree.rootNode.toString(),
'(module (class_definition (identifier) (function_definition (identifier) (parameters) (expression_statement (call (identifier) (argument_list))))))'
);
}).timeout(5000);
it("can use the rust parser", async () => {
parser.setLanguage(await Parser.Language.load(languageURL('rust')));
tree = parser.parse("const x: &'static str = r###\"hello\"###;");
assert.equal(
tree.rootNode.toString(),
'(source_file (const_item (identifier) (reference_type (lifetime (identifier)) (primitive_type)) (raw_string_literal)))'
);
}).timeout(5000);
it('parses only the text within the `includedRanges` if they are specified', () => {
const sourceCode = "<% foo() %> <% bar %>";
const start1 = sourceCode.indexOf('foo');
const end1 = start1 + 5
const start2 = sourceCode.indexOf('bar');
const end2 = start2 + 3
const tree = parser.parse(sourceCode, null, {
includedRanges: [
{
startIndex: start1,
endIndex: end1,
startPosition: {row: 0, column: start1},
endPosition: {row: 0, column: end1}
},
{
startIndex: start2,
endIndex: end2,
startPosition: {row: 0, column: start2},
endPosition: {row: 0, column: end2}
},
]
});
assert.equal(
tree.rootNode.toString(),
'(program (expression_statement (call_expression (identifier) (arguments))) (expression_statement (identifier)))'
);
})
});});

View file

@ -0,0 +1,291 @@
const {assert} = require('chai');
let Parser, JavaScript;
describe("Tree", () => {
let parser, tree;
before(async () =>
({Parser, JavaScript} = await require('./helper'))
);
beforeEach(() => {
parser = new Parser().setLanguage(JavaScript);
});
afterEach(() => {
parser.delete();
tree.delete();
});
describe('.edit', () => {
let input, edit
it('updates the positions of nodes', () => {
input = 'abc + cde';
tree = parser.parse(input);
assert.equal(
tree.rootNode.toString(),
"(program (expression_statement (binary_expression (identifier) (identifier))))"
);
let sumNode = tree.rootNode.firstChild.firstChild;
let variableNode1 = sumNode.firstChild;
let variableNode2 = sumNode.lastChild;
assert.equal(variableNode1.startIndex, 0);
assert.equal(variableNode1.endIndex, 3);
assert.equal(variableNode2.startIndex, 6);
assert.equal(variableNode2.endIndex, 9);
([input, edit] = spliceInput(input, input.indexOf('bc'), 0, ' * '));
assert.equal(input, 'a * bc + cde');
tree.edit(edit);
sumNode = tree.rootNode.firstChild.firstChild;
variableNode1 = sumNode.firstChild;
variableNode2 = sumNode.lastChild;
assert.equal(variableNode1.startIndex, 0);
assert.equal(variableNode1.endIndex, 6);
assert.equal(variableNode2.startIndex, 9);
assert.equal(variableNode2.endIndex, 12);
tree = parser.parse(input, tree);
assert.equal(
tree.rootNode.toString(),
"(program (expression_statement (binary_expression (binary_expression (identifier) (identifier)) (identifier))))"
);
});
it("handles non-ascii characters", () => {
input = 'αβδ + cde';
tree = parser.parse(input);
assert.equal(
tree.rootNode.toString(),
"(program (expression_statement (binary_expression (identifier) (identifier))))"
);
let variableNode = tree.rootNode.firstChild.firstChild.lastChild;
([input, edit] = spliceInput(input, input.indexOf('δ'), 0, '👍 * '));
assert.equal(input, 'αβ👍 * δ + cde');
tree.edit(edit);
variableNode = tree.rootNode.firstChild.firstChild.lastChild;
assert.equal(variableNode.startIndex, input.indexOf('cde'));
tree = parser.parse(input, tree);
assert.equal(
tree.rootNode.toString(),
"(program (expression_statement (binary_expression (binary_expression (identifier) (identifier)) (identifier))))"
);
});
});
describe(".walk()", () => {
let cursor
afterEach(() => {
cursor.delete();
})
it('returns a cursor that can be used to walk the tree', () => {
tree = parser.parse('a * b + c / d');
cursor = tree.walk();
assertCursorState(cursor, {
nodeType: 'program',
nodeIsNamed: true,
startPosition: {row: 0, column: 0},
endPosition: {row: 0, column: 13},
startIndex: 0,
endIndex: 13
});
assert(cursor.gotoFirstChild());
assertCursorState(cursor, {
nodeType: 'expression_statement',
nodeIsNamed: true,
startPosition: {row: 0, column: 0},
endPosition: {row: 0, column: 13},
startIndex: 0,
endIndex: 13
});
assert(cursor.gotoFirstChild());
assertCursorState(cursor, {
nodeType: 'binary_expression',
nodeIsNamed: true,
startPosition: {row: 0, column: 0},
endPosition: {row: 0, column: 13},
startIndex: 0,
endIndex: 13
});
assert(cursor.gotoFirstChild());
assertCursorState(cursor, {
nodeType: 'binary_expression',
nodeIsNamed: true,
startPosition: {row: 0, column: 0},
endPosition: {row: 0, column: 5},
startIndex: 0,
endIndex: 5
});
assert(cursor.gotoFirstChild());
assertCursorState(cursor, {
nodeType: 'identifier',
nodeIsNamed: true,
startPosition: {row: 0, column: 0},
endPosition: {row: 0, column: 1},
startIndex: 0,
endIndex: 1
});
assert(!cursor.gotoFirstChild())
assert(cursor.gotoNextSibling());
assertCursorState(cursor, {
nodeType: '*',
nodeIsNamed: false,
startPosition: {row: 0, column: 2},
endPosition: {row: 0, column: 3},
startIndex: 2,
endIndex: 3
});
assert(cursor.gotoNextSibling());
assertCursorState(cursor, {
nodeType: 'identifier',
nodeIsNamed: true,
startPosition: {row: 0, column: 4},
endPosition: {row: 0, column: 5},
startIndex: 4,
endIndex: 5
});
assert(!cursor.gotoNextSibling());
assert(cursor.gotoParent());
assertCursorState(cursor, {
nodeType: 'binary_expression',
nodeIsNamed: true,
startPosition: {row: 0, column: 0},
endPosition: {row: 0, column: 5},
startIndex: 0,
endIndex: 5
});
assert(cursor.gotoNextSibling());
assertCursorState(cursor, {
nodeType: '+',
nodeIsNamed: false,
startPosition: {row: 0, column: 6},
endPosition: {row: 0, column: 7},
startIndex: 6,
endIndex: 7
});
assert(cursor.gotoNextSibling());
assertCursorState(cursor, {
nodeType: 'binary_expression',
nodeIsNamed: true,
startPosition: {row: 0, column: 8},
endPosition: {row: 0, column: 13},
startIndex: 8,
endIndex: 13
});
// const childIndex = cursor.gotoFirstChildForIndex(12);
// assertCursorState(cursor, {
// nodeType: 'identifier',
// nodeIsNamed: true,
// startPosition: {row: 0, column: 12},
// endPosition: {row: 0, column: 13},
// startIndex: 12,
// endIndex: 13
// });
// assert.equal(childIndex, 2);
// assert(!cursor.gotoNextSibling());
// assert(cursor.gotoParent());
assert(cursor.gotoParent());
assert.equal(cursor.nodeType, 'binary_expression')
assert(cursor.gotoParent());
assert.equal(cursor.nodeType, 'expression_statement')
assert(cursor.gotoParent());
assert.equal(cursor.nodeType, 'program')
assert(!cursor.gotoParent());
});
it('returns a cursor that can be reset anywhere in the tree', () => {
tree = parser.parse('a * b + c / d');
cursor = tree.walk();
const root = tree.rootNode.firstChild;
cursor.reset(root.firstChild.firstChild);
assertCursorState(cursor, {
nodeType: 'binary_expression',
nodeIsNamed: true,
startPosition: {row: 0, column: 0},
endPosition: {row: 0, column: 5},
startIndex: 0,
endIndex: 5
});
cursor.gotoFirstChild()
assertCursorState(cursor, {
nodeType: 'identifier',
nodeIsNamed: true,
startPosition: {row: 0, column: 0},
endPosition: {row: 0, column: 1},
startIndex: 0,
endIndex: 1
});
assert(cursor.gotoParent());
assert(!cursor.gotoParent());
})
});
});
function spliceInput(input, startIndex, lengthRemoved, newText) {
const oldEndIndex = startIndex + lengthRemoved;
const newEndIndex = startIndex + newText.length;
const startPosition = getExtent(input.slice(0, startIndex));
const oldEndPosition = getExtent(input.slice(0, oldEndIndex));
input = input.slice(0, startIndex) + newText + input.slice(oldEndIndex);
const newEndPosition = getExtent(input.slice(0, newEndIndex));
return [
input,
{
startIndex, startPosition,
oldEndIndex, oldEndPosition,
newEndIndex, newEndPosition
}
];
}
function getExtent(text) {
let row = 0
let index;
for (index = 0; index != -1; index = text.indexOf('\n', index)) {
index++
row++;
}
return {row, column: text.length - index};
}
function assertCursorState(cursor, params) {
assert.equal(cursor.nodeType, params.nodeType);
assert.equal(cursor.nodeIsNamed, params.nodeIsNamed);
assert.deepEqual(cursor.startPosition, params.startPosition);
assert.deepEqual(cursor.endPosition, params.endPosition);
assert.deepEqual(cursor.startIndex, params.startIndex);
assert.deepEqual(cursor.endIndex, params.endIndex);
const node = cursor.currentNode()
assert.equal(node.type, params.nodeType);
assert.equal(node.isNamed(), params.nodeIsNamed);
assert.deepEqual(node.startPosition, params.startPosition);
assert.deepEqual(node.endPosition, params.endPosition);
assert.deepEqual(node.startIndex, params.startIndex);
assert.deepEqual(node.endIndex, params.endIndex);
}