From c47e217e7303c704d2cb88ced1b2720f15a2d949 Mon Sep 17 00:00:00 2001 From: Daumantas Kavolis Date: Wed, 17 May 2023 10:39:37 +0300 Subject: [PATCH 01/17] API extensions --- lib/binding_rust/bindings.rs | 77 ++++++++++++++- lib/binding_rust/ffi.rs | 20 +++- lib/binding_rust/lib.rs | 126 ++++++++++++++++++++++++ lib/binding_web/binding.c | 37 +++++++ lib/binding_web/binding.js | 97 +++++++++++++++++++ lib/binding_web/exports.json | 16 ++- lib/binding_web/test/node-test.js | 33 +++++++ lib/binding_web/tree-sitter-web.d.ts | 20 ++++ lib/include/tree_sitter/api.h | 104 +++++++++++++++++++- lib/include/tree_sitter/parser.h | 2 + lib/src/language.c | 66 +++++++++++++ lib/src/language.h | 24 +---- lib/src/node.c | 9 ++ lib/src/tree_cursor.c | 139 ++++++++++++++++++++++++++- 14 files changed, 741 insertions(+), 29 deletions(-) diff --git a/lib/binding_rust/bindings.rs b/lib/binding_rust/bindings.rs index 82eec594..da870263 100644 --- a/lib/binding_rust/bindings.rs +++ b/lib/binding_rust/bindings.rs @@ -1,5 +1,6 @@ /* automatically generated by rust-bindgen 0.66.1 */ +pub type TSStateId = u16; pub type TSSymbol = u16; pub type TSFieldId = u16; #[repr(C)] @@ -27,6 +28,11 @@ pub struct TSQuery { pub struct TSQueryCursor { _unused: [u8; 0], } +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct TSLookaheadIterator { + _unused: [u8; 0], +} pub const TSInputEncoding_TSInputEncodingUTF8: TSInputEncoding = 0; pub const TSInputEncoding_TSInputEncodingUTF16: TSInputEncoding = 1; pub type TSInputEncoding = ::std::os::raw::c_uint; @@ -326,6 +332,14 @@ extern "C" { #[doc = " Check if the node is a syntax error or contains any syntax errors."] pub fn ts_node_has_error(arg1: TSNode) -> bool; } +extern "C" { + #[doc = " Check if the node is a syntax error."] + pub fn ts_node_is_error(arg1: TSNode) -> bool; +} +extern "C" { + #[doc = " Get this node's parse state."] + pub fn ts_node_parse_state(arg1: TSNode) -> TSStateId; +} extern "C" { #[doc = " Get the node's immediate parent."] pub fn ts_node_parent(arg1: TSNode) -> TSNode; @@ -427,6 +441,10 @@ extern "C" { #[doc = " Re-initialize a tree cursor to start at a different node."] pub fn ts_tree_cursor_reset(arg1: *mut TSTreeCursor, arg2: TSNode); } +extern "C" { + #[doc = " Re-initialize a tree cursor to the same position as another cursor.\n\n Unlike `ts_tree_cursor_reset`, this will not lose parent information and\n allows reusing already created cursors."] + pub fn ts_tree_cursor_reset_to(arg1: *mut TSTreeCursor, arg2: *const TSTreeCursor); +} extern "C" { #[doc = " Get the tree cursor's current node."] pub fn ts_tree_cursor_current_node(arg1: *const TSTreeCursor) -> TSNode; @@ -450,9 +468,16 @@ extern "C" { pub fn ts_tree_cursor_goto_next_sibling(arg1: *mut TSTreeCursor) -> bool; } extern "C" { - #[doc = " Move the cursor to the first child of its current node.\n\n This returns `true` if the cursor successfully moved, and returns `false`\n if there were no children."] + #[doc = " Move the cursor to the previous sibling of its current node.\n\n This returns `true` if the cursor successfully moved, and returns `false` if\n there was no previous sibling node."] + pub fn ts_tree_cursor_goto_previous_sibling(arg1: *mut TSTreeCursor) -> bool; +} +extern "C" { + #[doc = " Move the cursor to the first/last child of its current node.\n\n This returns `true` if the cursor successfully moved, and returns `false`\n if there were no children."] pub fn ts_tree_cursor_goto_first_child(arg1: *mut TSTreeCursor) -> bool; } +extern "C" { + pub fn ts_tree_cursor_goto_last_child(arg1: *mut TSTreeCursor) -> bool; +} extern "C" { #[doc = " Move the cursor to the node that is the nth descendant of\n the original node that the cursor was constructed with, where\n zero represents the original node itself."] pub fn ts_tree_cursor_goto_descendant(arg1: *mut TSTreeCursor, arg2: u32); @@ -608,6 +633,10 @@ extern "C" { #[doc = " Get the number of distinct node types in the language."] pub fn ts_language_symbol_count(arg1: *const TSLanguage) -> u32; } +extern "C" { + #[doc = " Get the number of valid states in this language."] + pub fn ts_language_state_count(arg1: *const TSLanguage) -> u32; +} extern "C" { #[doc = " Get a node type string for the given numerical id."] pub fn ts_language_symbol_name( @@ -651,6 +680,52 @@ extern "C" { #[doc = " Get the ABI version number for this language. This version number is used\n to ensure that languages were generated by a compatible version of\n Tree-sitter.\n\n See also `ts_parser_set_language`."] pub fn ts_language_version(arg1: *const TSLanguage) -> u32; } +extern "C" { + #[doc = " Get the next parse state. Combine this with lookahead iterators to generate\n completion suggestions or valid symbols in error nodes."] + pub fn ts_language_next_state( + arg1: *const TSLanguage, + arg2: TSStateId, + arg3: TSSymbol, + ) -> TSStateId; +} +extern "C" { + #[doc = " Create a new lookahead iterator for the given language and parse state.\n\n This returns `NULL` if state is invalid for the language.\n\n Repeatedly using `ts_lookahead_iterator_advance` and\n `ts_lookahead_iterator_current_symbol` will generate valid symbols in the\n given parse state. Newly created lookahead iterators will contain the `ERROR`\n symbol.\n\n Lookahead iterators can be useful to generate suggestions and improve syntax\n error diagnostics. To get symbols valid in an ERROR node, use the lookahead\n iterator on its first leaf node state. For `MISSING` nodes, a lookahead\n iterator created on the previous non-extra leaf node may be appropriate."] + pub fn ts_lookahead_iterator_new( + arg1: *const TSLanguage, + arg2: TSStateId, + ) -> *mut TSLookaheadIterator; +} +extern "C" { + #[doc = " Delete a lookahead iterator freeing all the memory used."] + pub fn ts_lookahead_iterator_delete(arg1: *mut TSLookaheadIterator); +} +extern "C" { + #[doc = " Reset the lookahead iterator to another state.\n\n This returns `true` if the iterator was reset to the given state and `false`\n otherwise."] + pub fn ts_lookahead_iterator_reset_state( + arg1: *mut TSLookaheadIterator, + arg2: TSStateId, + ) -> bool; +} +extern "C" { + #[doc = " Reset the lookahead iterator.\n\n This returns `true` if the language was set successfully and `false`\n otherwise."] + pub fn ts_lookahead_iterator_reset( + arg1: *mut TSLookaheadIterator, + arg2: *const TSLanguage, + arg3: TSStateId, + ) -> bool; +} +extern "C" { + #[doc = " Get the current language of the lookahead iterator."] + pub fn ts_lookahead_iterator_language(arg1: *const TSLookaheadIterator) -> *const TSLanguage; +} +extern "C" { + #[doc = " Advance the lookahead iterator to the next symbol.\n\n This returns `true` if there is a new symbol and `false` otherwise."] + pub fn ts_lookahead_iterator_advance(arg1: *mut TSLookaheadIterator) -> bool; +} +extern "C" { + #[doc = " Get the current symbol of the lookahead iterator;"] + pub fn ts_lookahead_iterator_current_symbol(arg1: *const TSLookaheadIterator) -> TSSymbol; +} extern "C" { #[doc = " Set the allocation functions used by the library.\n\n By default, Tree-sitter uses the standard libc allocation functions,\n but aborts the process when an allocation fails. This function lets\n you supply alternative allocation functions at runtime.\n\n If you pass `NULL` for any parameter, Tree-sitter will switch back to\n its default implementation of that function.\n\n If you call this function after the library has already been used, then\n you must ensure that either:\n 1. All the existing objects have been freed.\n 2. The new allocator shares its state with the old one, so it is capable\n of freeing memory that was allocated by the old allocator."] pub fn ts_set_allocator( diff --git a/lib/binding_rust/ffi.rs b/lib/binding_rust/ffi.rs index fc872041..f1a739b1 100644 --- a/lib/binding_rust/ffi.rs +++ b/lib/binding_rust/ffi.rs @@ -8,7 +8,9 @@ extern "C" { pub(crate) fn dup(fd: std::os::raw::c_int) -> std::os::raw::c_int; } -use crate::{Language, Node, Parser, Query, QueryCursor, QueryError, Tree, TreeCursor}; +use crate::{ + Language, LookaheadIterator, Node, Parser, Query, QueryCursor, QueryError, Tree, TreeCursor, +}; use std::{marker::PhantomData, mem::ManuallyDrop, ptr::NonNull, str}; impl Language { @@ -130,3 +132,19 @@ impl QueryCursor { ManuallyDrop::new(self).ptr.as_ptr() } } + +impl<'a> LookaheadIterator<'a> { + /// Reconstructs a [LookaheadIterator] from a raw pointer. + /// + /// # Safety + /// + /// `ptr` must be non-null. + pub unsafe fn from_raw(ptr: *mut TSLookaheadIterator) -> LookaheadIterator<'a> { + LookaheadIterator(NonNull::new_unchecked(ptr), PhantomData) + } + + /// Consumes the [LookaheadIterator], returning a raw pointer to the underlying C structure. + pub fn into_raw(self) -> *mut TSLookaheadIterator { + ManuallyDrop::new(self).0.as_ptr() + } +} diff --git a/lib/binding_rust/lib.rs b/lib/binding_rust/lib.rs index 2f8323df..6a46dafc 100644 --- a/lib/binding_rust/lib.rs +++ b/lib/binding_rust/lib.rs @@ -87,6 +87,10 @@ pub struct Node<'a>(ffi::TSNode, PhantomData<&'a ()>); #[doc(alias = "TSParser")] pub struct Parser(NonNull); +/// A stateful object that is used to look up symbols valid in a specific parse state +#[doc(alias = "TSLookaheadIterator")] +pub struct LookaheadIterator<'a>(NonNull, PhantomData<&'a ()>); + /// A type of log message. #[derive(Debug, PartialEq, Eq)] pub enum LogType { @@ -269,6 +273,12 @@ impl Language { unsafe { ffi::ts_language_symbol_count(self.0) as usize } } + /// Get the number of valid states in this language. + #[doc(alias = "ts_language_state_count")] + pub fn parse_state_count(&self) -> usize { + unsafe { ffi::ts_language_state_count(self.0) as usize } + } + /// Get the name of the node kind for the given numerical id. #[doc(alias = "ts_language_symbol_name")] pub fn node_kind_for_id(&self, id: u16) -> Option<&'static str> { @@ -336,6 +346,36 @@ impl Language { }; FieldId::new(id) } + + /// Get the next parse state. Combine this with [lookahead_iterator] to + /// generate completion suggestions or valid symbols in error nodes. + #[doc(alias = "ts_language_next_state")] + pub fn next_state(&self, state: u16, id: u16) -> u16 { + unsafe { ffi::ts_language_next_state(self.0, state, id) } + } + + /// Create a new lookahead iterator for this language and parse state. + /// + /// This returns `None` if state is invalid for this language. + /// + /// Iterating [LookaheadIterator] will yield valid symbols in the given + /// parse state. Newly created lookahead iterators will return the `ERROR` + /// symbol from [LookaheadIterator::current_symbol]. + /// + /// Lookahead iterators can be useful to generate suggestions and improve + /// syntax error diagnostics. To get symbols valid in an ERROR node, use the + /// lookahead iterator on its first leaf node state. For `MISSING` nodes, a + /// lookahead iterator created on the previous non-extra leaf node may be + /// appropriate. + #[doc(alias = "ts_lookahead_iterator_new")] + pub fn lookahead_iterator<'a>(&self, state: u16) -> Option> { + let ptr = unsafe { ffi::ts_lookahead_iterator_new(self.0, state) }; + if ptr.is_null() { + None + } else { + Some(unsafe { LookaheadIterator::from_raw(ptr) }) + } + } } impl Parser { @@ -885,6 +925,12 @@ impl<'tree> Node<'tree> { self.kind_id() == u16::MAX } + /// Get this node's parse state. + #[doc(alias = "ts_node_parse_state")] + pub fn parse_state(&self) -> u16 { + unsafe { ffi::ts_node_parse_state(self.0) } + } + /// Check if this node is *missing*. /// /// Missing nodes are inserted by the parser in order to recover from certain kinds of @@ -1305,6 +1351,15 @@ impl<'a> TreeCursor<'a> { return unsafe { ffi::ts_tree_cursor_goto_first_child(&mut self.0) }; } + /// Move this cursor to the last child of its current node. + /// + /// This returns `true` if the cursor successfully moved, and returns `false` + /// if there were no children. + #[doc(alias = "ts_tree_cursor_goto_last_child")] + pub fn goto_last_child(&mut self) -> bool { + return unsafe { ffi::ts_tree_cursor_goto_last_child(&mut self.0) }; + } + /// Move this cursor to the parent of its current node. /// /// This returns `true` if the cursor successfully moved, and returns `false` @@ -1333,6 +1388,15 @@ impl<'a> TreeCursor<'a> { }; } + /// Move this cursor to the previous sibling of its current node. + /// + /// This returns `true` if the cursor successfully moved, and returns + /// `false` if there was no previous sibling node. + #[doc(alias = "ts_tree_cursor_goto_previous_sibling")] + pub fn goto_previous_sibling(&mut self) -> bool { + return unsafe { ffi::ts_tree_cursor_goto_previous_sibling(&mut self.0) }; + } + /// Move this cursor to the first child of its current node that extends beyond /// the given byte offset. /// @@ -1370,6 +1434,15 @@ impl<'a> TreeCursor<'a> { pub fn reset(&mut self, node: Node<'a>) { unsafe { ffi::ts_tree_cursor_reset(&mut self.0, node.0) }; } + + /// Re-initialize a tree cursor to the same position as another cursor. + /// + /// Unlike `reset`, this will not lose parent information and + /// allows reusing already created cursors. + #[doc(alias = "ts_tree_cursor_reset_to")] + pub fn reset_to(&mut self, cursor: TreeCursor<'a>) { + unsafe { ffi::ts_tree_cursor_reset_to(&mut self.0, &cursor.0) }; + } } impl<'a> Clone for TreeCursor<'a> { @@ -1384,6 +1457,59 @@ impl<'a> Drop for TreeCursor<'a> { } } +impl<'a> LookaheadIterator<'a> { + /// Get the current language of the lookahead iterator. + #[doc(alias = "ts_lookahead_iterator_language")] + pub fn language(&self) -> Language { + Language(unsafe { ffi::ts_lookahead_iterator_language(self.0.as_ptr()) }) + } + + /// Get the current symbol of the lookahead iterator. + #[doc(alias = "ts_lookahead_iterator_current_symbol")] + pub fn current_symbol(&self) -> u16 { + unsafe { ffi::ts_lookahead_iterator_current_symbol(self.0.as_ptr()) } + } + + /// Reset the lookahead iterator. + /// + /// This returns `true` if the language was set successfully and `false` + /// otherwise. + #[doc(alias = "ts_lookahead_iterator_reset")] + pub fn reset(&self, language: &'a Language, state: u16) -> bool { + unsafe { ffi::ts_lookahead_iterator_reset(self.0.as_ptr(), language.0, state) } + } + + /// Reset the lookahead iterator to another state. + /// + /// This returns `true` if the iterator was reset to the given state and `false` + /// otherwise. + #[doc(alias = "ts_lookahead_iterator_reset_state")] + pub fn reset_state(&self, state: u16) -> bool { + unsafe { ffi::ts_lookahead_iterator_reset_state(self.0.as_ptr(), state) } + } +} + +impl<'a> Iterator for LookaheadIterator<'a> { + type Item = u16; + + #[doc(alias = "ts_lookahead_iterator_advance")] + fn next(&mut self) -> Option { + // the first symbol is always `0` so we can safely skip it + if !(unsafe { ffi::ts_lookahead_iterator_advance(self.0.as_ptr()) }) { + None + } else { + Some(self.current_symbol()) + } + } +} + +impl<'a> Drop for LookaheadIterator<'a> { + #[doc(alias = "ts_lookahead_iterator_delete")] + fn drop(&mut self) { + unsafe { ffi::ts_lookahead_iterator_delete(self.0.as_ptr()) } + } +} + impl Query { /// Create a new query from a string containing one or more S-expression /// patterns. diff --git a/lib/binding_web/binding.c b/lib/binding_web/binding.c index 27292911..4ba36f65 100644 --- a/lib/binding_web/binding.c +++ b/lib/binding_web/binding.c @@ -243,6 +243,13 @@ void ts_tree_cursor_reset_wasm(const TSTree *tree) { marshal_cursor(&cursor); } +void ts_tree_cursor_reset_to_wasm(const TSTree *_dst, const TSTree *_src) { + TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, _dst); + TSTreeCursor src = unmarshal_cursor(&TRANSFER_BUFFER[3], _src); + ts_tree_cursor_reset_to(&cursor, &src); + marshal_cursor(&cursor); +} + bool ts_tree_cursor_goto_first_child_wasm(const TSTree *tree) { TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); bool result = ts_tree_cursor_goto_first_child(&cursor); @@ -250,6 +257,13 @@ bool ts_tree_cursor_goto_first_child_wasm(const TSTree *tree) { return result; } +bool ts_tree_cursor_goto_last_child_wasm(const TSTree *tree) { + TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); + bool result = ts_tree_cursor_goto_last_child(&cursor); + marshal_cursor(&cursor); + return result; +} + bool ts_tree_cursor_goto_next_sibling_wasm(const TSTree *tree) { TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); bool result = ts_tree_cursor_goto_next_sibling(&cursor); @@ -257,6 +271,13 @@ bool ts_tree_cursor_goto_next_sibling_wasm(const TSTree *tree) { return result; } +bool ts_tree_cursor_goto_previous_sibling_wasm(const TSTree *tree) { + TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); + bool result = ts_tree_cursor_goto_previous_sibling(&cursor); + marshal_cursor(&cursor); + return result; +} + bool ts_tree_cursor_goto_parent_wasm(const TSTree *tree) { TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); bool result = ts_tree_cursor_goto_parent(&cursor); @@ -270,6 +291,12 @@ uint16_t ts_tree_cursor_current_node_type_id_wasm(const TSTree *tree) { return ts_node_symbol(node); } +uint16_t ts_tree_cursor_current_node_state_id_wasm(const TSTree *tree) { + TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); + TSNode node = ts_tree_cursor_current_node(&cursor); + return ts_node_parse_state(node); +} + bool ts_tree_cursor_current_node_is_named_wasm(const TSTree *tree) { TSTreeCursor cursor = unmarshal_cursor(TRANSFER_BUFFER, tree); TSNode node = ts_tree_cursor_current_node(&cursor); @@ -579,11 +606,21 @@ int ts_node_has_error_wasm(const TSTree *tree) { return ts_node_has_error(node); } +int ts_node_is_error_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + return ts_node_is_error(node); +} + int ts_node_is_missing_wasm(const TSTree *tree) { TSNode node = unmarshal_node(tree); return ts_node_is_missing(node); } +uint16_t ts_node_parse_state_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + return ts_node_parse_state(node); +} + /******************/ /* Section - Query */ /******************/ diff --git a/lib/binding_web/binding.js b/lib/binding_web/binding.js index 5352cb18..4203bd92 100644 --- a/lib/binding_web/binding.js +++ b/lib/binding_web/binding.js @@ -1,6 +1,7 @@ const C = Module; const INTERNAL = {}; const SIZE_OF_INT = 4; +const SIZE_OF_CURSOR = 3 * SIZE_OF_INT; const SIZE_OF_NODE = 5 * SIZE_OF_INT; const SIZE_OF_POINT = 2 * SIZE_OF_INT; const SIZE_OF_RANGE = 2 * SIZE_OF_INT + 2 * SIZE_OF_POINT; @@ -227,6 +228,11 @@ class Node { return getText(this.tree, this.startIndex, this.endIndex); } + get parseState() { + marshalNode(this); + return C._ts_node_parse_state_wasm(this.tree[0]); + } + isNamed() { marshalNode(this); return C._ts_node_is_named_wasm(this.tree[0]) === 1; @@ -242,6 +248,11 @@ class Node { return C._ts_node_has_changes_wasm(this.tree[0]) === 1; } + isError() { + marshalNode(this); + return C._ts_node_is_error_wasm(this.tree[0]) === 1; + } + isMissing() { marshalNode(this); return C._ts_node_is_missing_wasm(this.tree[0]) === 1; @@ -505,6 +516,13 @@ class TreeCursor { unmarshalTreeCursor(this); } + resetTo(cursor) { + marshalTreeCursor(this, TRANSFER_BUFFER); + marshalTreeCursor(cursor, TRANSFER_BUFFER + SIZE_OF_CURSOR); + C._ts_tree_cursor_reset_to_wasm(this.tree[0], cursor.tree[0]); + unmarshalTreeCursor(this); + } + get nodeType() { return this.tree.language.types[this.nodeTypeId] || 'ERROR'; } @@ -514,6 +532,11 @@ class TreeCursor { return C._ts_tree_cursor_current_node_type_id_wasm(this.tree[0]); } + get nodeStateId() { + marshalTreeCursor(this); + return C._ts_tree_cursor_current_node_state_id_wasm(this.tree[0]); + } + get nodeId() { marshalTreeCursor(this); return C._ts_tree_cursor_current_node_id_wasm(this.tree[0]); @@ -580,6 +603,13 @@ class TreeCursor { return result === 1; } + gotoLastChild() { + marshalTreeCursor(this); + const result = C._ts_tree_cursor_goto_last_child_wasm(this.tree[0]); + unmarshalTreeCursor(this); + return result === 1; + } + gotoNextSibling() { marshalTreeCursor(this); const result = C._ts_tree_cursor_goto_next_sibling_wasm(this.tree[0]); @@ -587,6 +617,13 @@ class TreeCursor { return result === 1; } + gotoPreviousSibling() { + marshalTreeCursor(this); + const result = C._ts_tree_cursor_goto_previous_sibling_wasm(this.tree[0]); + unmarshalTreeCursor(this); + return result === 1; + } + gotoParent() { marshalTreeCursor(this); const result = C._ts_tree_cursor_goto_parent_wasm(this.tree[0]); @@ -624,6 +661,10 @@ class Language { return this.fields.length - 1; } + get stateCount() { + return C._ts_language_state_count(this[0]); + } + fieldIdForName(fieldName) { const result = this.fields.indexOf(fieldName); if (result !== -1) { @@ -663,6 +704,15 @@ class Language { return C._ts_language_type_is_visible_wasm(this[0], typeId) ? true : false; } + nextState(stateId, typeId) { + return C._ts_language_next_state(this[0], stateId, typeId); + } + + lookaheadIterator(stateId) { + const address = C._ts_lookahead_iterator_new(this[0], stateId); + if (address) return new LookaheadIterable(INTERNAL, address, this); + } + query(source) { const sourceLength = lengthBytesUTF8(source); const sourceAddress = C._malloc(sourceLength + 1); @@ -924,6 +974,53 @@ class Language { } } +class LookaheadIterable { + constructor(internal, address, language) { + assertInternal(internal); + this[0] = address; + this.language = language; + } + + get currentTypeId() { + return C._ts_lookahead_iterator_current_symbol(this[0]); + } + + get currentType() { + return this.language.types[this.currentTypeId] || 'ERROR' + } + + delete() { + C._ts_lookahead_iterator_delete(this[0]); + this[0] = 0; + } + + resetState(stateId) { + return C._ts_lookahead_iterator_reset_state(this[0], stateId); + } + + reset(language, stateId) { + if (C._ts_lookahead_iterator_reset(this[0], language, stateId)) { + this.language = language; + return true; + } + + return false; + } + + [Symbol.iterator]() { + const self = this; + return { + next() { + if (C._ts_lookahead_iterator_advance(self[0])) { + return { done: false, value: self.currentType }; + } + + return { done: true, value: "" }; + } + }; + } +} + class Query { constructor( internal, address, captureNames, textPredicates, predicates, diff --git a/lib/binding_web/exports.json b/lib/binding_web/exports.json index eb4553dc..ba34eedb 100644 --- a/lib/binding_web/exports.json +++ b/lib/binding_web/exports.json @@ -38,10 +38,12 @@ "_ts_language_type_is_named_wasm", "_ts_language_type_is_visible_wasm", "_ts_language_symbol_count", + "_ts_language_state_count", "_ts_language_symbol_for_name", "_ts_language_symbol_name", "_ts_language_symbol_type", "_ts_language_version", + "_ts_language_next_state", "_ts_node_child_by_field_id_wasm", "_ts_node_child_count_wasm", "_ts_node_child_wasm", @@ -53,8 +55,10 @@ "_ts_node_end_point_wasm", "_ts_node_has_changes_wasm", "_ts_node_has_error_wasm", + "_ts_node_is_error_wasm", "_ts_node_is_missing_wasm", "_ts_node_is_named_wasm", + "_ts_node_parse_state_wasm", "_ts_node_named_child_count_wasm", "_ts_node_named_child_wasm", "_ts_node_named_children_wasm", @@ -93,19 +97,29 @@ "_ts_tree_cursor_current_node_is_missing_wasm", "_ts_tree_cursor_current_node_is_named_wasm", "_ts_tree_cursor_current_node_type_id_wasm", + "_ts_tree_cursor_current_node_state_id_wasm", "_ts_tree_cursor_current_node_wasm", "_ts_tree_cursor_delete_wasm", "_ts_tree_cursor_end_index_wasm", "_ts_tree_cursor_end_position_wasm", "_ts_tree_cursor_goto_first_child_wasm", + "_ts_tree_cursor_goto_last_child", "_ts_tree_cursor_goto_next_sibling_wasm", + "_ts_tree_cursor_goto_previous_sibling_wasm", "_ts_tree_cursor_goto_parent_wasm", "_ts_tree_cursor_new_wasm", "_ts_tree_cursor_reset_wasm", + "_ts_tree_cursor_reset_to_wasm", "_ts_tree_cursor_start_index_wasm", "_ts_tree_cursor_start_position_wasm", "_ts_tree_delete", "_ts_tree_edit_wasm", "_ts_tree_get_changed_ranges_wasm", - "_ts_tree_root_node_wasm" + "_ts_tree_root_node_wasm", + "_ts_lookahead_iterator_new", + "_ts_lookahead_iterator_delete", + "_ts_lookahead_iterator_reset_state", + "_ts_lookahead_iterator_reset", + "_ts_lookahead_iterator_advance", + "_ts_lookahead_iterator_current_symbol" ] diff --git a/lib/binding_web/test/node-test.js b/lib/binding_web/test/node-test.js index 6bbcafb0..e897d991 100644 --- a/lib/binding_web/test/node-test.js +++ b/lib/binding_web/test/node-test.js @@ -268,6 +268,24 @@ describe("Node", () => { }); }); + describe(".isError()", () => { + it("returns true if the node is an error", () => { + tree = parser.parse("2 * * 3"); + const node = tree.rootNode; + assert.equal( + node.toString(), + '(program (expression_statement (binary_expression left: (number) (ERROR) right: (number))))' + ); + + const multi = node.firstChild.firstChild; + assert(multi.hasError()); + assert(!multi.children[0].isError()); + assert(!multi.children[1].isError()); + assert(multi.children[2].isError()); + assert(!multi.children[3].isError()); + }); + }); + describe(".isMissing()", () => { it("returns true if the node is missing from the source and was inserted via error recovery", () => { tree = parser.parse("(2 ||)"); @@ -308,6 +326,21 @@ describe("Node", () => { ); }); + describe(".parseState", () => { + const text = "10 * 5"; + + it(`returns node parse state ids`, async () => { + tree = await parser.parse(text) + const quotientNode = tree.rootNode.firstChild.firstChild; + const [numerator, slash, denominator] = quotientNode.children; + + assert.equal(tree.rootNode.parseState, 0); + assert.equal(numerator.parseState, 1); + assert.equal(slash.parseState, 553); + assert.equal(denominator.parseState, 185); + }) + }); + describe('.descendantsOfType(type, min, max)', () => { it('finds all of the descendants of the given type in the given range', () => { tree = parser.parse("a + 1 * b * 2 + c + 3"); diff --git a/lib/binding_web/tree-sitter-web.d.ts b/lib/binding_web/tree-sitter-web.d.ts index 016af4ae..709c2023 100644 --- a/lib/binding_web/tree-sitter-web.d.ts +++ b/lib/binding_web/tree-sitter-web.d.ts @@ -59,6 +59,7 @@ declare module 'web-tree-sitter' { tree: Tree; type: string; text: string; + parseState: number; startPosition: Point; endPosition: Point; startIndex: number; @@ -80,6 +81,7 @@ declare module 'web-tree-sitter' { hasChanges(): boolean; hasError(): boolean; equals(other: SyntaxNode): boolean; + isError(): boolean; isMissing(): boolean; isNamed(): boolean; toString(): string; @@ -104,6 +106,7 @@ declare module 'web-tree-sitter' { export interface TreeCursor { nodeType: string; nodeTypeId: number; + nodeStateId: number; nodeText: string; nodeId: number; nodeIsNamed: boolean; @@ -114,14 +117,17 @@ declare module 'web-tree-sitter' { endIndex: number; reset(node: SyntaxNode): void; + resetTo(cursor: TreeCursor): void; delete(): void; currentNode(): SyntaxNode; currentFieldId(): number; currentFieldName(): string; gotoParent(): boolean; gotoFirstChild(): boolean; + gotoLastChild(): boolean; gotoFirstChildForIndex(index: number): boolean; gotoNextSibling(): boolean; + gotoPreviousSibling(): boolean; } export interface Tree { @@ -141,6 +147,7 @@ declare module 'web-tree-sitter' { readonly version: number; readonly fieldCount: number; + readonly stateCount: number; readonly nodeTypeCount: number; fieldNameForId(fieldId: number): string | null; @@ -149,7 +156,20 @@ declare module 'web-tree-sitter' { nodeTypeForId(typeId: number): string | null; nodeTypeIsNamed(typeId: number): boolean; nodeTypeIsVisible(typeId: number): boolean; + nextState(stateId: number, typeId: number): number; query(source: string): Query; + lookaheadIterator(stateId: number): LookaheadIterable | null; + } + + class LookaheadIterable { + readonly language: Language; + readonly currentTypeId: number; + readonly currentType: string; + + delete(): void; + resetState(stateId: number): boolean; + reset(language: Language, stateId: number): boolean; + [Symbol.iterator](): Iterator; } interface QueryCapture { diff --git a/lib/include/tree_sitter/api.h b/lib/include/tree_sitter/api.h index 1e67f217..a9c8a182 100644 --- a/lib/include/tree_sitter/api.h +++ b/lib/include/tree_sitter/api.h @@ -32,6 +32,10 @@ extern "C" { /* Section - Types */ /*******************/ +#ifndef TREE_SITTER_PARSER_H_ +typedef uint16_t TSStateId; +#endif + typedef uint16_t TSSymbol; typedef uint16_t TSFieldId; typedef struct TSLanguage TSLanguage; @@ -39,6 +43,7 @@ typedef struct TSParser TSParser; typedef struct TSTree TSTree; typedef struct TSQuery TSQuery; typedef struct TSQueryCursor TSQueryCursor; +typedef struct TSLookaheadIterator TSLookaheadIterator; typedef enum { TSInputEncodingUTF8, @@ -504,6 +509,16 @@ bool ts_node_has_changes(TSNode); */ bool ts_node_has_error(TSNode); +/** + * Check if the node is a syntax error. +*/ +bool ts_node_is_error(TSNode); + +/** + * Get this node's parse state. +*/ +TSStateId ts_node_parse_state(TSNode); + /** * Get the node's immediate parent. */ @@ -637,6 +652,14 @@ void ts_tree_cursor_delete(TSTreeCursor *); */ void ts_tree_cursor_reset(TSTreeCursor *, TSNode); +/** + * Re-initialize a tree cursor to the same position as another cursor. + * + * Unlike `ts_tree_cursor_reset`, this will not lose parent information and + * allows reusing already created cursors. +*/ +void ts_tree_cursor_reset_to(TSTreeCursor *, const TSTreeCursor *); + /** * Get the tree cursor's current node. */ @@ -675,12 +698,21 @@ bool ts_tree_cursor_goto_parent(TSTreeCursor *); bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *); /** - * Move the cursor to the first child of its current node. + * Move the cursor to the previous sibling of its current node. + * + * This returns `true` if the cursor successfully moved, and returns `false` if + * there was no previous sibling node. + */ +bool ts_tree_cursor_goto_previous_sibling(TSTreeCursor *); + +/** + * Move the cursor to the first/last child of its current node. * * This returns `true` if the cursor successfully moved, and returns `false` * if there were no children. */ bool ts_tree_cursor_goto_first_child(TSTreeCursor *); +bool ts_tree_cursor_goto_last_child(TSTreeCursor *); /** * Move the cursor to the node that is the nth descendant of @@ -939,6 +971,11 @@ void ts_query_cursor_set_max_start_depth(TSQueryCursor *, uint32_t); */ uint32_t ts_language_symbol_count(const TSLanguage *); +/** + * Get the number of valid states in this language. +*/ +uint32_t ts_language_state_count(const TSLanguage *); + /** * Get a node type string for the given numerical id. */ @@ -986,6 +1023,71 @@ TSSymbolType ts_language_symbol_type(const TSLanguage *, TSSymbol); */ uint32_t ts_language_version(const TSLanguage *); +/** + * Get the next parse state. Combine this with lookahead iterators to generate + * completion suggestions or valid symbols in error nodes. +*/ +TSStateId ts_language_next_state(const TSLanguage *, TSStateId, TSSymbol); + +/********************************/ +/* Section - Lookahead Iterator */ +/********************************/ + +/** + * Create a new lookahead iterator for the given language and parse state. + * + * This returns `NULL` if state is invalid for the language. + * + * Repeatedly using `ts_lookahead_iterator_advance` and + * `ts_lookahead_iterator_current_symbol` will generate valid symbols in the + * given parse state. Newly created lookahead iterators will contain the `ERROR` + * symbol. + * + * Lookahead iterators can be useful to generate suggestions and improve syntax + * error diagnostics. To get symbols valid in an ERROR node, use the lookahead + * iterator on its first leaf node state. For `MISSING` nodes, a lookahead + * iterator created on the previous non-extra leaf node may be appropriate. +*/ +TSLookaheadIterator *ts_lookahead_iterator_new(const TSLanguage *, TSStateId); + +/** + * Delete a lookahead iterator freeing all the memory used. +*/ +void ts_lookahead_iterator_delete(TSLookaheadIterator *); + +/** + * Reset the lookahead iterator to another state. + * + * This returns `true` if the iterator was reset to the given state and `false` + * otherwise. +*/ +bool ts_lookahead_iterator_reset_state(TSLookaheadIterator *, TSStateId); + +/** + * Reset the lookahead iterator. + * + * This returns `true` if the language was set successfully and `false` + * otherwise. +*/ +bool ts_lookahead_iterator_reset(TSLookaheadIterator *, const TSLanguage *, TSStateId); + +/** + * Get the current language of the lookahead iterator. +*/ +const TSLanguage * ts_lookahead_iterator_language(const TSLookaheadIterator *); + +/** + * Advance the lookahead iterator to the next symbol. + * + * This returns `true` if there is a new symbol and `false` otherwise. +*/ +bool ts_lookahead_iterator_advance(TSLookaheadIterator *); + +/** + * Get the current symbol of the lookahead iterator; +*/ +TSSymbol ts_lookahead_iterator_current_symbol(const TSLookaheadIterator *); + /**********************************/ /* Section - Global Configuration */ /**********************************/ diff --git a/lib/include/tree_sitter/parser.h b/lib/include/tree_sitter/parser.h index ac773580..c7ad1907 100644 --- a/lib/include/tree_sitter/parser.h +++ b/lib/include/tree_sitter/parser.h @@ -13,7 +13,9 @@ extern "C" { #define ts_builtin_sym_end 0 #define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024 +#ifndef TREE_SITTER_API_H_ typedef uint16_t TSStateId; +#endif #ifndef TREE_SITTER_API_H_ typedef uint16_t TSSymbol; diff --git a/lib/src/language.c b/lib/src/language.c index c87a8dc1..d8d07c2c 100644 --- a/lib/src/language.c +++ b/lib/src/language.c @@ -7,6 +7,10 @@ uint32_t ts_language_symbol_count(const TSLanguage *self) { return self->symbol_count + self->alias_count; } +uint32_t ts_language_state_count(const TSLanguage *self) { + return self->state_count; +} + uint32_t ts_language_version(const TSLanguage *self) { return self->version; } @@ -56,6 +60,28 @@ TSSymbol ts_language_public_symbol( return self->public_symbol_map[symbol]; } +TSStateId ts_language_next_state( + const TSLanguage *self, + TSStateId state, + TSSymbol symbol +) { + if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) { + return 0; + } else if (symbol < self->token_count) { + uint32_t count; + const TSParseAction *actions = ts_language_actions(self, state, symbol, &count); + if (count > 0) { + TSParseAction action = actions[count - 1]; + if (action.type == TSParseActionTypeShift) { + return action.shift.extra ? state : action.shift.state; + } + } + return 0; + } else { + return ts_language_lookup(self, state, symbol); + } +} + const char *ts_language_symbol_name( const TSLanguage *self, TSSymbol symbol @@ -135,3 +161,43 @@ TSFieldId ts_language_field_id_for_name( } return 0; } + +TSLookaheadIterator *ts_lookahead_iterator_new(const TSLanguage *self, TSStateId state) { + if (state >= self->state_count) return NULL; + LookaheadIterator *iterator = ts_malloc(sizeof(LookaheadIterator)); + *iterator = ts_language_lookaheads(self, state); + return (TSLookaheadIterator *)iterator; +} + +void ts_lookahead_iterator_delete(TSLookaheadIterator *self) { + ts_free(self); +} + +bool ts_lookahead_iterator_reset_state(TSLookaheadIterator * self, TSStateId state) { + LookaheadIterator *iterator = (LookaheadIterator *)self; + if (state >= iterator->language->state_count) return false; + *iterator = ts_language_lookaheads(iterator->language, state); + return true; +} + +const TSLanguage * ts_lookahead_iterator_language(const TSLookaheadIterator *self) { + LookaheadIterator *iterator = (LookaheadIterator *)self; + return iterator->language; +} + +bool ts_lookahead_iterator_reset(TSLookaheadIterator *self, const TSLanguage *language, TSStateId state) { + if (state >= language->state_count) return false; + LookaheadIterator *iterator = (LookaheadIterator *)self; + *iterator = ts_language_lookaheads(language, state); + return true; +} + +bool ts_lookahead_iterator_advance(TSLookaheadIterator *self) { + LookaheadIterator *iterator = (LookaheadIterator *)self; + return ts_lookahead_iterator_next(iterator); +} + +TSSymbol ts_lookahead_iterator_current_symbol(const TSLookaheadIterator *self) { + LookaheadIterator *iterator = (LookaheadIterator *)self; + return iterator->symbol; +} diff --git a/lib/src/language.h b/lib/src/language.h index db61b602..002f564f 100644 --- a/lib/src/language.h +++ b/lib/src/language.h @@ -38,6 +38,8 @@ TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *, TSSymbol); TSSymbol ts_language_public_symbol(const TSLanguage *, TSSymbol); +TSStateId ts_language_next_state(const TSLanguage *self, TSStateId state, TSSymbol symbol); + static inline bool ts_language_is_symbol_external(const TSLanguage *self, TSSymbol symbol) { return 0 < symbol && symbol < self->external_token_count + 1; } @@ -178,28 +180,6 @@ static inline bool ts_lookahead_iterator_next(LookaheadIterator *self) { return true; } -static inline TSStateId ts_language_next_state( - const TSLanguage *self, - TSStateId state, - TSSymbol symbol -) { - if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) { - return 0; - } else if (symbol < self->token_count) { - uint32_t count; - const TSParseAction *actions = ts_language_actions(self, state, symbol, &count); - if (count > 0) { - TSParseAction action = actions[count - 1]; - if (action.type == TSParseActionTypeShift) { - return action.shift.extra ? state : action.shift.state; - } - } - return 0; - } else { - return ts_language_lookup(self, state, symbol); - } -} - // Whether the state is a "primary state". If this returns false, it indicates that there exists // another state that behaves identically to this one with respect to query analysis. static inline bool ts_language_state_is_primary( diff --git a/lib/src/node.c b/lib/src/node.c index eafaa7bb..8ef2d150 100644 --- a/lib/src/node.c +++ b/lib/src/node.c @@ -468,6 +468,15 @@ uint32_t ts_node_descendant_count(TSNode self) { return ts_subtree_visible_descendant_count(ts_node__subtree(self)) + 1; } +bool ts_node_is_error(TSNode self) { + TSSymbol symbol = ts_node_symbol(self); + return symbol == ts_builtin_sym_error; +} + +TSStateId ts_node_parse_state(TSNode self) { + return ts_subtree_parse_state(ts_node__subtree(self)); +} + TSNode ts_node_parent(TSNode self) { TSNode node = ts_tree_root_node(self.tree); uint32_t end_byte = ts_node_end_byte(self); diff --git a/lib/src/tree_cursor.c b/lib/src/tree_cursor.c index 97a53152..624cfa2f 100644 --- a/lib/src/tree_cursor.c +++ b/lib/src/tree_cursor.c @@ -97,6 +97,40 @@ static inline bool ts_tree_cursor_child_iterator_next( return true; } +static inline bool ts_tree_cursor_child_iterator_previous( + CursorChildIterator *self, + TreeCursorEntry *result, + bool *visible +) { + // this is mostly a reverse `ts_tree_cursor_child_iterator_next` taking into + // account unsigned underflow + if (!self->parent.ptr || (int8_t)self->child_index == -1) return false; + const Subtree *child = &ts_subtree_children(self->parent)[self->child_index]; + *result = (TreeCursorEntry) { + .subtree = child, + .position = self->position, + .child_index = self->child_index, + .structural_child_index = self->structural_child_index, + }; + *visible = ts_subtree_visible(*child); + bool extra = ts_subtree_extra(*child); + if (!extra && self->alias_sequence) { + *visible |= self->alias_sequence[self->structural_child_index]; + self->structural_child_index--; + } + + self->position = length_sub(self->position, ts_subtree_padding(*child)); + self->child_index--; + + // unsigned can underflow so compare it to child_count + if (self->child_index < self->parent.ptr->child_count) { + Subtree previous_child = ts_subtree_children(self->parent)[self->child_index]; + self->position = length_sub(self->position, ts_subtree_size(previous_child)); + } + + return true; +} + // TSTreeCursor - lifecycle TSTreeCursor ts_tree_cursor_new(TSNode node) { @@ -163,6 +197,47 @@ bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) { return false; } +TreeCursorStep ts_tree_cursor_goto_last_child_internal(TSTreeCursor *_self) { + TreeCursor *self = (TreeCursor *)_self; + bool visible; + TreeCursorEntry entry; + CursorChildIterator iterator = ts_tree_cursor_iterate_children(self); + if (!iterator.parent.ptr || iterator.parent.ptr->child_count == 0) return TreeCursorStepNone; + + TreeCursorEntry last_entry; + bool last_visible; + while (ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible)) { + if (visible) { + last_entry = entry; + last_visible = true; + } + else if (ts_subtree_visible_child_count(*entry.subtree) > 0) { + last_entry = entry; + last_visible = false; + } + } + if (last_entry.subtree) { + array_push(&self->stack, last_entry); + return last_visible? TreeCursorStepVisible : TreeCursorStepHidden; + } + + return TreeCursorStepNone; +} + +bool ts_tree_cursor_goto_last_child(TSTreeCursor *self) { + for (;;) { + switch (ts_tree_cursor_goto_last_child_internal(self)) { + case TreeCursorStepHidden: + continue; + case TreeCursorStepVisible: + return true; + default: + return false; + } + } + return false; +} + static inline int64_t ts_tree_cursor_goto_first_child_for_byte_and_point( TSTreeCursor *_self, uint32_t goal_byte, @@ -213,7 +288,9 @@ int64_t ts_tree_cursor_goto_first_child_for_point(TSTreeCursor *self, TSPoint go return ts_tree_cursor_goto_first_child_for_byte_and_point(self, 0, goal_point); } -TreeCursorStep ts_tree_cursor_goto_next_sibling_internal(TSTreeCursor *_self) { +TreeCursorStep ts_tree_cursor_goto_sibling_internal( + TSTreeCursor *_self, + bool (*advance)(CursorChildIterator *, TreeCursorEntry *, bool *)) { TreeCursor *self = (TreeCursor *)_self; uint32_t initial_size = self->stack.size; @@ -226,10 +303,10 @@ TreeCursorStep ts_tree_cursor_goto_next_sibling_internal(TSTreeCursor *_self) { iterator.descendant_index = entry.descendant_index; bool visible = false; - ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible); + advance(&iterator, &entry, &visible); if (visible && self->stack.size + 1 < initial_size) break; - while (ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible)) { + while (advance(&iterator, &entry, &visible)) { if (visible) { array_push(&self->stack, entry); return TreeCursorStepVisible; @@ -246,6 +323,10 @@ TreeCursorStep ts_tree_cursor_goto_next_sibling_internal(TSTreeCursor *_self) { return TreeCursorStepNone; } +TreeCursorStep ts_tree_cursor_goto_next_sibling_internal(TSTreeCursor *_self) { + return ts_tree_cursor_goto_sibling_internal(_self, ts_tree_cursor_child_iterator_next); +} + bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self) { switch (ts_tree_cursor_goto_next_sibling_internal(self)) { case TreeCursorStepHidden: @@ -258,6 +339,50 @@ bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self) { } } +TreeCursorStep ts_tree_cursor_goto_previous_sibling_internal(TSTreeCursor *_self) { + // since subtracting across row loses column information, we may have to + // restore it + TreeCursor *self = (TreeCursor *)_self; + + // for that, save current position before traversing + Length position = array_back(&self->stack)->position; + TreeCursorStep step = ts_tree_cursor_goto_sibling_internal( + _self, ts_tree_cursor_child_iterator_previous); + if (step == TreeCursorStepNone) + return step; + + // if row has not changed, column is still valid + if (array_back(&self->stack)->position.extent.row == position.extent.row) + return step; + + // restore position from the parent node + const TreeCursorEntry *parent = &self->stack.contents[self->stack.size - 2]; + position = parent->position; + uint32_t child_index = array_back(&self->stack)->child_index; + const Subtree *children = ts_subtree_children((*(parent->subtree))); + for (uint32_t i = 0; i < child_index; ++i) { + position = length_add(position, ts_subtree_total_size(children[i])); + } + if (child_index > 0) + position = length_add(position, ts_subtree_padding(children[child_index])); + + array_back(&self->stack)->position = position; + + return step; +} + +bool ts_tree_cursor_goto_previous_sibling(TSTreeCursor *self) { + switch (ts_tree_cursor_goto_previous_sibling_internal(self)) { + case TreeCursorStepHidden: + ts_tree_cursor_goto_last_child(self); + return true; + case TreeCursorStepVisible: + return true; + default: + return false; + } +} + bool ts_tree_cursor_goto_parent(TSTreeCursor *_self) { TreeCursor *self = (TreeCursor *)_self; for (unsigned i = self->stack.size - 2; i + 1 > 0; i--) { @@ -556,3 +681,11 @@ TSTreeCursor ts_tree_cursor_copy(const TSTreeCursor *_cursor) { array_push_all(©->stack, &cursor->stack); return res; } + +void ts_tree_cursor_reset_to(TSTreeCursor *_dst, const TSTreeCursor *_src) { + const TreeCursor *cursor = (const TreeCursor *)_src; + TreeCursor *copy = (TreeCursor *)_dst; + copy->tree = cursor->tree; + array_clear(©->stack); + array_push_all(©->stack, &cursor->stack); +} From 726738497042d7c6172517c5e1256be28682b2cf Mon Sep 17 00:00:00 2001 From: Daumantas Kavolis Date: Thu, 15 Jun 2023 13:36:49 +0300 Subject: [PATCH 02/17] Use step directly --- lib/src/tree_cursor.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/src/tree_cursor.c b/lib/src/tree_cursor.c index 624cfa2f..702088ad 100644 --- a/lib/src/tree_cursor.c +++ b/lib/src/tree_cursor.c @@ -205,20 +205,20 @@ TreeCursorStep ts_tree_cursor_goto_last_child_internal(TSTreeCursor *_self) { if (!iterator.parent.ptr || iterator.parent.ptr->child_count == 0) return TreeCursorStepNone; TreeCursorEntry last_entry; - bool last_visible; + TreeCursorStep last_step = TreeCursorStepNone; while (ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible)) { if (visible) { last_entry = entry; - last_visible = true; + last_step = TreeCursorStepVisible; } else if (ts_subtree_visible_child_count(*entry.subtree) > 0) { last_entry = entry; - last_visible = false; + last_step = TreeCursorStepHidden; } } if (last_entry.subtree) { array_push(&self->stack, last_entry); - return last_visible? TreeCursorStepVisible : TreeCursorStepHidden; + return last_step; } return TreeCursorStepNone; From 1f52f2f1dca4a9a549e6593b583857f749210918 Mon Sep 17 00:00:00 2001 From: Daumantas Kavolis Date: Thu, 15 Jun 2023 15:55:32 +0300 Subject: [PATCH 03/17] Symbols without aliases should be used for lookahead --- lib/binding_rust/bindings.rs | 14 +++++++++++++- lib/binding_rust/lib.rs | 27 +++++++++++++++++++++++++++ lib/binding_web/binding.c | 10 ++++++++++ lib/binding_web/binding.js | 14 ++++++++++++++ lib/binding_web/exports.json | 2 ++ lib/binding_web/test/node-test.js | 12 ++++++++---- lib/binding_web/tree-sitter-web.d.ts | 5 ++++- lib/include/tree_sitter/api.h | 21 ++++++++++++++++++++- lib/src/node.c | 17 +++++++++++++++++ 9 files changed, 115 insertions(+), 7 deletions(-) diff --git a/lib/binding_rust/bindings.rs b/lib/binding_rust/bindings.rs index da870263..d2239893 100644 --- a/lib/binding_rust/bindings.rs +++ b/lib/binding_rust/bindings.rs @@ -288,6 +288,14 @@ extern "C" { #[doc = " Get the node's language."] pub fn ts_node_language(arg1: TSNode) -> *const TSLanguage; } +extern "C" { + #[doc = " Get the node's type as it appears in the grammar ignoring aliases as a\n null-terminated string."] + pub fn ts_node_grammar_type(arg1: TSNode) -> *const ::std::os::raw::c_char; +} +extern "C" { + #[doc = " Get the node's type as a numerical id as it appears in the grammar ignoring\n aliases. This should be used in `ts_language_next_state` instead of\n `ts_node_symbol`."] + pub fn ts_node_grammar_symbol(arg1: TSNode) -> TSSymbol; +} extern "C" { #[doc = " Get the node's start byte."] pub fn ts_node_start_byte(arg1: TSNode) -> u32; @@ -340,6 +348,10 @@ extern "C" { #[doc = " Get this node's parse state."] pub fn ts_node_parse_state(arg1: TSNode) -> TSStateId; } +extern "C" { + #[doc = " Get the parse state after this node."] + pub fn ts_node_next_parse_state(arg1: TSNode) -> TSStateId; +} extern "C" { #[doc = " Get the node's immediate parent."] pub fn ts_node_parent(arg1: TSNode) -> TSNode; @@ -681,7 +693,7 @@ extern "C" { pub fn ts_language_version(arg1: *const TSLanguage) -> u32; } extern "C" { - #[doc = " Get the next parse state. Combine this with lookahead iterators to generate\n completion suggestions or valid symbols in error nodes."] + #[doc = " Get the next parse state. Combine this with lookahead iterators to generate\n completion suggestions or valid symbols in error nodes. Use\n `ts_node_grammar_symbol` for valid symbols."] pub fn ts_language_next_state( arg1: *const TSLanguage, arg2: TSStateId, diff --git a/lib/binding_rust/lib.rs b/lib/binding_rust/lib.rs index 6a46dafc..cdf1ffcf 100644 --- a/lib/binding_rust/lib.rs +++ b/lib/binding_rust/lib.rs @@ -349,6 +349,11 @@ impl Language { /// Get the next parse state. Combine this with [lookahead_iterator] to /// generate completion suggestions or valid symbols in error nodes. + /// + /// Example: + /// ``` + /// let state = language.next_state(node.parse_state(), node.grammar_id()); + /// ``` #[doc(alias = "ts_language_next_state")] pub fn next_state(&self, state: u16, id: u16) -> u16 { unsafe { ffi::ts_language_next_state(self.0, state, id) } @@ -872,6 +877,13 @@ impl<'tree> Node<'tree> { unsafe { ffi::ts_node_symbol(self.0) } } + /// Get the node's type as a numerical id as it appears in the grammar + /// ignoring aliases. + #[doc(alias = "ts_node_grammar_symbol")] + pub fn grammar_id(&self) -> u16 { + unsafe { ffi::ts_node_grammar_symbol(self.0) } + } + /// Get this node's type as a string. #[doc(alias = "ts_node_type")] pub fn kind(&self) -> &'static str { @@ -880,6 +892,15 @@ impl<'tree> Node<'tree> { .unwrap() } + /// Get this node's symbol name as it appears in the grammar ignoring + /// aliases as a string. + #[doc(alias = "ts_node_grammar_type")] + pub fn grammar_name(&self) -> &'static str { + unsafe { CStr::from_ptr(ffi::ts_node_grammar_type(self.0)) } + .to_str() + .unwrap() + } + /// Get the [Language] that was used to parse this node's syntax tree. #[doc(alias = "ts_node_language")] pub fn language(&self) -> Language { @@ -931,6 +952,12 @@ impl<'tree> Node<'tree> { unsafe { ffi::ts_node_parse_state(self.0) } } + /// Get the parse state after this node. + #[doc(alias = "ts_node_next_parse_state")] + pub fn next_parse_state(&self) -> u16 { + unsafe { ffi::ts_node_next_parse_state(self.0) } + } + /// Check if this node is *missing*. /// /// Missing nodes are inserted by the parser in order to recover from certain kinds of diff --git a/lib/binding_web/binding.c b/lib/binding_web/binding.c index 4ba36f65..5a8e4e34 100644 --- a/lib/binding_web/binding.c +++ b/lib/binding_web/binding.c @@ -361,6 +361,11 @@ uint16_t ts_node_symbol_wasm(const TSTree *tree) { return ts_node_symbol(node); } +uint16_t ts_node_grammar_symbol_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + return ts_node_grammar_symbol(node); +} + uint32_t ts_node_child_count_wasm(const TSTree *tree) { TSNode node = unmarshal_node(tree); return ts_node_child_count(node); @@ -621,6 +626,11 @@ uint16_t ts_node_parse_state_wasm(const TSTree *tree) { return ts_node_parse_state(node); } +uint16_t ts_node_next_parse_state_wasm(const TSTree *tree) { + TSNode node = unmarshal_node(tree); + return ts_node_next_parse_state(node); +} + /******************/ /* Section - Query */ /******************/ diff --git a/lib/binding_web/binding.js b/lib/binding_web/binding.js index 4203bd92..38efabd6 100644 --- a/lib/binding_web/binding.js +++ b/lib/binding_web/binding.js @@ -209,10 +209,19 @@ class Node { return C._ts_node_symbol_wasm(this.tree[0]); } + get grammarId() { + marshalNode(this); + return C._ts_node_grammar_symbol_wasm(this.tree[0]); + } + get type() { return this.tree.language.types[this.typeId] || 'ERROR'; } + get grammarType() { + return this.tree.language.types[this.grammarId] || 'ERROR'; + } + get endPosition() { marshalNode(this); C._ts_node_end_point_wasm(this.tree[0]); @@ -233,6 +242,11 @@ class Node { return C._ts_node_parse_state_wasm(this.tree[0]); } + get nextParseState() { + marshalNode(this); + return C._ts_node_next_parse_state_wasm(this.tree[0]); + } + isNamed() { marshalNode(this); return C._ts_node_is_named_wasm(this.tree[0]) === 1; diff --git a/lib/binding_web/exports.json b/lib/binding_web/exports.json index ba34eedb..1c638bb0 100644 --- a/lib/binding_web/exports.json +++ b/lib/binding_web/exports.json @@ -59,6 +59,7 @@ "_ts_node_is_missing_wasm", "_ts_node_is_named_wasm", "_ts_node_parse_state_wasm", + "_ts_node_next_parse_state_wasm", "_ts_node_named_child_count_wasm", "_ts_node_named_child_wasm", "_ts_node_named_children_wasm", @@ -72,6 +73,7 @@ "_ts_node_start_index_wasm", "_ts_node_start_point_wasm", "_ts_node_symbol_wasm", + "_ts_node_grammar_symbol_wasm", "_ts_node_to_string_wasm", "_ts_parser_delete", "_ts_parser_enable_logger_wasm", diff --git a/lib/binding_web/test/node-test.js b/lib/binding_web/test/node-test.js index e897d991..1abef756 100644 --- a/lib/binding_web/test/node-test.js +++ b/lib/binding_web/test/node-test.js @@ -327,7 +327,7 @@ describe("Node", () => { }); describe(".parseState", () => { - const text = "10 * 5"; + const text = "10 / 5"; it(`returns node parse state ids`, async () => { tree = await parser.parse(text) @@ -335,9 +335,13 @@ describe("Node", () => { const [numerator, slash, denominator] = quotientNode.children; assert.equal(tree.rootNode.parseState, 0); - assert.equal(numerator.parseState, 1); - assert.equal(slash.parseState, 553); - assert.equal(denominator.parseState, 185); + // parse states will change on any change to the grammar so test that it + // returns something instead + assert.isAbove(numerator.parseState, 0); + assert.isAbove(numerator.nextParseState, 0); + assert.isAbove(slash.parseState, 0); + assert.isAbove(denominator.parseState, 0); + assert.isAbove(denominator.nextParseState, 0); }) }); diff --git a/lib/binding_web/tree-sitter-web.d.ts b/lib/binding_web/tree-sitter-web.d.ts index 709c2023..dfe7766c 100644 --- a/lib/binding_web/tree-sitter-web.d.ts +++ b/lib/binding_web/tree-sitter-web.d.ts @@ -55,11 +55,14 @@ declare module 'web-tree-sitter' { ) => string | null; export interface SyntaxNode { - id: number; + typeId: number; + grammarId: number; tree: Tree; type: string; + grammarType: string; text: string; parseState: number; + nextParseState: number; startPosition: Point; endPosition: Point; startIndex: number; diff --git a/lib/include/tree_sitter/api.h b/lib/include/tree_sitter/api.h index a9c8a182..4bd2f43d 100644 --- a/lib/include/tree_sitter/api.h +++ b/lib/include/tree_sitter/api.h @@ -445,6 +445,19 @@ TSSymbol ts_node_symbol(TSNode); */ const TSLanguage *ts_node_language(TSNode); +/** + * Get the node's type as it appears in the grammar ignoring aliases as a + * null-terminated string. + */ +const char *ts_node_grammar_type(TSNode); + +/** + * Get the node's type as a numerical id as it appears in the grammar ignoring + * aliases. This should be used in `ts_language_next_state` instead of + * `ts_node_symbol`. + */ +TSSymbol ts_node_grammar_symbol(TSNode); + /** * Get the node's start byte. */ @@ -519,6 +532,11 @@ bool ts_node_is_error(TSNode); */ TSStateId ts_node_parse_state(TSNode); +/** + * Get the parse state after this node. +*/ +TSStateId ts_node_next_parse_state(TSNode); + /** * Get the node's immediate parent. */ @@ -1025,7 +1043,8 @@ uint32_t ts_language_version(const TSLanguage *); /** * Get the next parse state. Combine this with lookahead iterators to generate - * completion suggestions or valid symbols in error nodes. + * completion suggestions or valid symbols in error nodes. Use + * `ts_node_grammar_symbol` for valid symbols. */ TSStateId ts_language_next_state(const TSLanguage *, TSStateId, TSSymbol); diff --git a/lib/src/node.c b/lib/src/node.c index 8ef2d150..46028f37 100644 --- a/lib/src/node.c +++ b/lib/src/node.c @@ -429,6 +429,15 @@ const TSLanguage *ts_node_language(TSNode self) { return self.tree->language; } +TSSymbol ts_node_grammar_symbol(TSNode self) { + return ts_subtree_symbol(ts_node__subtree(self)); +} + +const char *ts_node_grammar_type(TSNode self) { + TSSymbol symbol = ts_subtree_symbol(ts_node__subtree(self)); + return ts_language_symbol_name(self.tree->language, symbol); +} + char *ts_node_string(TSNode self) { return ts_subtree_string(ts_node__subtree(self), self.tree->language, false); } @@ -477,6 +486,14 @@ TSStateId ts_node_parse_state(TSNode self) { return ts_subtree_parse_state(ts_node__subtree(self)); } +TSStateId ts_node_next_parse_state(TSNode self) { + const TSLanguage *language = self.tree->language; + uint16_t state = ts_node_parse_state(self); + uint16_t symbol = ts_node_grammar_symbol(self); + + return ts_language_next_state(language, state, symbol); +} + TSNode ts_node_parent(TSNode self) { TSNode node = ts_tree_root_node(self.tree); uint32_t end_byte = ts_node_end_byte(self); From 672d38803e61416a0d4653adf6392c5cc2ac6391 Mon Sep 17 00:00:00 2001 From: Daumantas Kavolis Date: Thu, 15 Jun 2023 17:25:08 +0300 Subject: [PATCH 04/17] Add iteration over names --- lib/binding_rust/bindings.rs | 6 ++++++ lib/binding_rust/lib.rs | 32 ++++++++++++++++++++++++++++++++ lib/include/tree_sitter/api.h | 6 ++++++ lib/src/language.c | 9 +++++++-- 4 files changed, 51 insertions(+), 2 deletions(-) diff --git a/lib/binding_rust/bindings.rs b/lib/binding_rust/bindings.rs index d2239893..5ad38721 100644 --- a/lib/binding_rust/bindings.rs +++ b/lib/binding_rust/bindings.rs @@ -738,6 +738,12 @@ extern "C" { #[doc = " Get the current symbol of the lookahead iterator;"] pub fn ts_lookahead_iterator_current_symbol(arg1: *const TSLookaheadIterator) -> TSSymbol; } +extern "C" { + #[doc = " Get the current symbol type of the lookahead iterator as a null terminated\n string."] + pub fn ts_lookahead_iterator_current_symbol_name( + arg1: *const TSLookaheadIterator, + ) -> *const ::std::os::raw::c_char; +} extern "C" { #[doc = " Set the allocation functions used by the library.\n\n By default, Tree-sitter uses the standard libc allocation functions,\n but aborts the process when an allocation fails. This function lets\n you supply alternative allocation functions at runtime.\n\n If you pass `NULL` for any parameter, Tree-sitter will switch back to\n its default implementation of that function.\n\n If you call this function after the library has already been used, then\n you must ensure that either:\n 1. All the existing objects have been freed.\n 2. The new allocator shares its state with the old one, so it is capable\n of freeing memory that was allocated by the old allocator."] pub fn ts_set_allocator( diff --git a/lib/binding_rust/lib.rs b/lib/binding_rust/lib.rs index cdf1ffcf..49263b28 100644 --- a/lib/binding_rust/lib.rs +++ b/lib/binding_rust/lib.rs @@ -1497,6 +1497,18 @@ impl<'a> LookaheadIterator<'a> { unsafe { ffi::ts_lookahead_iterator_current_symbol(self.0.as_ptr()) } } + /// Get the current symbol name of the lookahead iterator. + #[doc(alias = "ts_lookahead_iterator_current_symbol_name")] + pub fn current_symbol_name(&self) -> &'static str { + unsafe { + CStr::from_ptr(ffi::ts_lookahead_iterator_current_symbol_name( + self.0.as_ptr(), + )) + .to_str() + .unwrap() + } + } + /// Reset the lookahead iterator. /// /// This returns `true` if the language was set successfully and `false` @@ -1514,6 +1526,26 @@ impl<'a> LookaheadIterator<'a> { pub fn reset_state(&self, state: u16) -> bool { unsafe { ffi::ts_lookahead_iterator_reset_state(self.0.as_ptr(), state) } } + + /// Iterate symbol names. + pub fn iter_names(&'a self) -> impl Iterator + 'a { + NameLookaheadIterator(&self) + } +} + +struct NameLookaheadIterator<'a>(&'a LookaheadIterator<'a>); + +impl<'a> Iterator for NameLookaheadIterator<'a> { + type Item = &'static str; + + #[doc(alias = "ts_lookahead_iterator_advance")] + fn next(&mut self) -> Option { + if !(unsafe { ffi::ts_lookahead_iterator_advance(self.0 .0.as_ptr()) }) { + None + } else { + Some(self.0.current_symbol_name()) + } + } } impl<'a> Iterator for LookaheadIterator<'a> { diff --git a/lib/include/tree_sitter/api.h b/lib/include/tree_sitter/api.h index 4bd2f43d..0fc9faf5 100644 --- a/lib/include/tree_sitter/api.h +++ b/lib/include/tree_sitter/api.h @@ -1107,6 +1107,12 @@ bool ts_lookahead_iterator_advance(TSLookaheadIterator *); */ TSSymbol ts_lookahead_iterator_current_symbol(const TSLookaheadIterator *); +/** + * Get the current symbol type of the lookahead iterator as a null terminated + * string. +*/ +const char *ts_lookahead_iterator_current_symbol_name(const TSLookaheadIterator *); + /**********************************/ /* Section - Global Configuration */ /**********************************/ diff --git a/lib/src/language.c b/lib/src/language.c index d8d07c2c..df5c6de1 100644 --- a/lib/src/language.c +++ b/lib/src/language.c @@ -181,7 +181,7 @@ bool ts_lookahead_iterator_reset_state(TSLookaheadIterator * self, TSStateId sta } const TSLanguage * ts_lookahead_iterator_language(const TSLookaheadIterator *self) { - LookaheadIterator *iterator = (LookaheadIterator *)self; + const LookaheadIterator *iterator = (const LookaheadIterator *)self; return iterator->language; } @@ -198,6 +198,11 @@ bool ts_lookahead_iterator_advance(TSLookaheadIterator *self) { } TSSymbol ts_lookahead_iterator_current_symbol(const TSLookaheadIterator *self) { - LookaheadIterator *iterator = (LookaheadIterator *)self; + const LookaheadIterator *iterator = (const LookaheadIterator *)self; return iterator->symbol; } + +const char *ts_lookahead_iterator_current_symbol_name(const TSLookaheadIterator *self) { + const LookaheadIterator *iterator = (const LookaheadIterator *)self; + return ts_language_symbol_name(iterator->language, iterator->symbol); +} From 1cb378b4459eb7ed9fa5a1c1b417e66d5112f345 Mon Sep 17 00:00:00 2001 From: Daumantas Kavolis Date: Fri, 16 Jun 2023 10:46:42 +0300 Subject: [PATCH 05/17] Add rust tests for lookahead iterator --- cli/src/tests/language_test.rs | 41 ++++++++++++++++++++++++++++++++++ cli/src/tests/mod.rs | 1 + 2 files changed, 42 insertions(+) create mode 100644 cli/src/tests/language_test.rs diff --git a/cli/src/tests/language_test.rs b/cli/src/tests/language_test.rs new file mode 100644 index 00000000..2ac5bb77 --- /dev/null +++ b/cli/src/tests/language_test.rs @@ -0,0 +1,41 @@ +use super::helpers::fixtures::get_language; +use tree_sitter::Parser; + +#[test] +fn test_lookahead_iterator() { + let mut parser = Parser::new(); + let language = get_language("rust"); + parser.set_language(language).unwrap(); + + let tree = parser.parse("struct Stuff {}", None).unwrap(); + + let mut cursor = tree.walk(); + + assert!(cursor.goto_first_child()); // struct + assert!(cursor.goto_first_child()); // struct keyword + + let next_state = cursor.node().next_parse_state(); + assert_ne!(next_state, 0); + assert_eq!( + next_state, + language.next_state(cursor.node().parse_state(), cursor.node().grammar_id()) + ); + assert!((next_state as usize) < language.parse_state_count()); + assert!(cursor.goto_next_sibling()); // type_identifier + assert_eq!(next_state, cursor.node().parse_state()); + assert_eq!(cursor.node().grammar_name(), "identifier"); + assert_ne!(cursor.node().grammar_id(), cursor.node().kind_id()); + + let expected_symbols = ["identifier", "block_comment", "line_comment"]; + let lookahead = language.lookahead_iterator(next_state).unwrap(); + assert_eq!(lookahead.language(), language); + assert!(lookahead.iter_names().eq(expected_symbols)); + + lookahead.reset_state(next_state); + assert!(lookahead.iter_names().eq(expected_symbols)); + + lookahead.reset(&language, next_state); + assert!(lookahead + .map(|s| language.node_kind_for_id(s).unwrap()) + .eq(expected_symbols)); +} diff --git a/cli/src/tests/mod.rs b/cli/src/tests/mod.rs index 223b6d3c..577770a2 100644 --- a/cli/src/tests/mod.rs +++ b/cli/src/tests/mod.rs @@ -2,6 +2,7 @@ mod corpus_test; mod github_issue_test; mod helpers; mod highlight_test; +mod language_test; mod node_test; mod parser_test; mod pathological_test; From ec90c215ae8ee833440bf743ab69fca10c55b541 Mon Sep 17 00:00:00 2001 From: Daumantas Kavolis Date: Fri, 16 Jun 2023 10:47:10 +0300 Subject: [PATCH 06/17] Add tests for bidirectional cursor --- cli/src/tests/tree_test.rs | 45 +++++++++++++++++++++++++++++++++++++- lib/src/tree_cursor.c | 15 +++++++++++-- 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/cli/src/tests/tree_test.rs b/cli/src/tests/tree_test.rs index be0c4ff1..7d091c3f 100644 --- a/cli/src/tests/tree_test.rs +++ b/cli/src/tests/tree_test.rs @@ -306,7 +306,7 @@ fn test_tree_cursor() { .parse( " struct Stuff { - a: A; + a: A, b: Option, } ", @@ -331,6 +331,49 @@ fn test_tree_cursor() { assert!(cursor.goto_next_sibling()); assert_eq!(cursor.node().kind(), "field_declaration_list"); assert_eq!(cursor.node().is_named(), true); + + assert!(cursor.goto_last_child()); + assert_eq!(cursor.node().kind(), "}"); + assert_eq!(cursor.node().is_named(), false); + assert_eq!(cursor.node().start_position(), Point { row: 4, column: 16 }); + + assert!(cursor.goto_previous_sibling()); + assert_eq!(cursor.node().kind(), ","); + assert_eq!(cursor.node().is_named(), false); + assert_eq!(cursor.node().start_position(), Point { row: 3, column: 32 }); + + assert!(cursor.goto_previous_sibling()); + assert_eq!(cursor.node().kind(), "field_declaration"); + assert_eq!(cursor.node().is_named(), true); + assert_eq!(cursor.node().start_position(), Point { row: 3, column: 20 }); + + assert!(cursor.goto_previous_sibling()); + assert_eq!(cursor.node().kind(), ","); + assert_eq!(cursor.node().is_named(), false); + assert_eq!(cursor.node().start_position(), Point { row: 2, column: 24 }); + + assert!(cursor.goto_previous_sibling()); + assert_eq!(cursor.node().kind(), "field_declaration"); + assert_eq!(cursor.node().is_named(), true); + assert_eq!(cursor.node().start_position(), Point { row: 2, column: 20 }); + + assert!(cursor.goto_previous_sibling()); + assert_eq!(cursor.node().kind(), "{"); + assert_eq!(cursor.node().is_named(), false); + assert_eq!(cursor.node().start_position(), Point { row: 1, column: 29 }); + + let mut copy = tree.walk(); + copy.reset_to(cursor); + + assert_eq!(copy.node().kind(), "{"); + assert_eq!(copy.node().is_named(), false); + + assert!(copy.goto_parent()); + assert_eq!(copy.node().kind(), "field_declaration_list"); + assert_eq!(copy.node().is_named(), true); + + assert!(copy.goto_parent()); + assert_eq!(copy.node().kind(), "struct_item"); } #[test] diff --git a/lib/src/tree_cursor.c b/lib/src/tree_cursor.c index 702088ad..0512b914 100644 --- a/lib/src/tree_cursor.c +++ b/lib/src/tree_cursor.c @@ -97,6 +97,16 @@ static inline bool ts_tree_cursor_child_iterator_next( return true; } +static inline Length length_sub_zero(Length a, Length b) { + // length_sub doesn't account for 0 row subtraction, i.e. only columns + // should be subtracted, but changing point_sub breaks other tests + Length result = length_sub(a, b); + if (b.extent.row == 0) { + result.extent.column -= b.extent.column; + } + return result; +} + static inline bool ts_tree_cursor_child_iterator_previous( CursorChildIterator *self, TreeCursorEntry *result, @@ -119,13 +129,14 @@ static inline bool ts_tree_cursor_child_iterator_previous( self->structural_child_index--; } - self->position = length_sub(self->position, ts_subtree_padding(*child)); + self->position = length_sub_zero(self->position, ts_subtree_padding(*child)); self->child_index--; // unsigned can underflow so compare it to child_count if (self->child_index < self->parent.ptr->child_count) { Subtree previous_child = ts_subtree_children(self->parent)[self->child_index]; - self->position = length_sub(self->position, ts_subtree_size(previous_child)); + Length size = ts_subtree_size(previous_child); + self->position = length_sub_zero(self->position, size); } return true; From ab788619ca4fc33a5b723ccda0e5e25ff0529c6b Mon Sep 17 00:00:00 2001 From: Daumantas Kavolis Date: Fri, 16 Jun 2023 11:32:13 +0300 Subject: [PATCH 07/17] Split wasm tests for next parse state --- lib/binding_web/test/node-test.js | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/binding_web/test/node-test.js b/lib/binding_web/test/node-test.js index 1abef756..a87658c7 100644 --- a/lib/binding_web/test/node-test.js +++ b/lib/binding_web/test/node-test.js @@ -326,10 +326,10 @@ describe("Node", () => { ); }); - describe(".parseState", () => { + describe(".parseState, .nextParseState", () => { const text = "10 / 5"; - it(`returns node parse state ids`, async () => { + it("returns node parse state ids", async () => { tree = await parser.parse(text) const quotientNode = tree.rootNode.firstChild.firstChild; const [numerator, slash, denominator] = quotientNode.children; @@ -338,11 +338,20 @@ describe("Node", () => { // parse states will change on any change to the grammar so test that it // returns something instead assert.isAbove(numerator.parseState, 0); - assert.isAbove(numerator.nextParseState, 0); assert.isAbove(slash.parseState, 0); assert.isAbove(denominator.parseState, 0); - assert.isAbove(denominator.nextParseState, 0); }) + + it("returns next parse state equal to the language", async () => { + tree = await parser.parse(text); + const quotientNode = tree.rootNode.firstChild.firstChild; + quotientNode.children.forEach(node => { + assert.equal( + node.nextParseState, + JavaScript.nextState(node.parseState, node.grammarId) + ); + }); + }); }); describe('.descendantsOfType(type, min, max)', () => { From e2094ef202576901f3361e3f09d1966ef6f97be2 Mon Sep 17 00:00:00 2001 From: Daumantas Kavolis Date: Fri, 16 Jun 2023 11:33:00 +0300 Subject: [PATCH 08/17] Add wasm tests for lookahead iterator --- lib/binding_web/binding.js | 4 +-- lib/binding_web/test/language-test.js | 38 +++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/lib/binding_web/binding.js b/lib/binding_web/binding.js index 38efabd6..e6c2b27e 100644 --- a/lib/binding_web/binding.js +++ b/lib/binding_web/binding.js @@ -1013,7 +1013,7 @@ class LookaheadIterable { } reset(language, stateId) { - if (C._ts_lookahead_iterator_reset(this[0], language, stateId)) { + if (C._ts_lookahead_iterator_reset(this[0], language[0], stateId)) { this.language = language; return true; } @@ -1028,7 +1028,7 @@ class LookaheadIterable { if (C._ts_lookahead_iterator_advance(self[0])) { return { done: false, value: self.currentType }; } - + return { done: true, value: "" }; } }; diff --git a/lib/binding_web/test/language-test.js b/lib/binding_web/test/language-test.js index 385b77ed..dab212d6 100644 --- a/lib/binding_web/test/language-test.js +++ b/lib/binding_web/test/language-test.js @@ -42,3 +42,41 @@ describe("Language", () => { }); }); }); + +describe("Lookahead iterator", () => { + let lookahead; + let state; + before(async () => { + let Parser; + ({ JavaScript, Parser } = await require("./helper")); + const parser = new Parser().setLanguage(JavaScript); + const tree = parser.parse("function fn() {}"); + const cursor = tree.walk(); + cursor.gotoFirstChild(); + cursor.gotoFirstChild(); + state = cursor.currentNode().nextParseState; + lookahead = JavaScript.lookaheadIterator(state); + assert.exists(lookahead); + }); + + const expected = ["identifier", "comment", "(", "*", "formal_parameters"]; + it("should iterate over valid symbols in the state", () => { + const symbols = Array.from(lookahead); + assert.includeMembers(symbols, expected); + assert.lengthOf(symbols, expected.length); + }); + + it("should reset to the initial state", () => { + assert(lookahead.resetState(state)); + const symbols = Array.from(lookahead); + assert.includeMembers(symbols, expected); + assert.lengthOf(symbols, expected.length); + }); + + it("should reset", () => { + assert(lookahead.reset(JavaScript, state)); + const symbols = Array.from(lookahead); + assert.includeMembers(symbols, expected); + assert.lengthOf(symbols, expected.length); + }); +}); From 4d6a30a69e2036ea72bca196a871c8cbeab37cbd Mon Sep 17 00:00:00 2001 From: Daumantas Kavolis Date: Fri, 16 Jun 2023 13:15:22 +0300 Subject: [PATCH 09/17] Need to check that left row is also non-zero so that columns are not subtracted twice --- lib/src/tree_cursor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/tree_cursor.c b/lib/src/tree_cursor.c index 0512b914..7b820983 100644 --- a/lib/src/tree_cursor.c +++ b/lib/src/tree_cursor.c @@ -101,7 +101,7 @@ static inline Length length_sub_zero(Length a, Length b) { // length_sub doesn't account for 0 row subtraction, i.e. only columns // should be subtracted, but changing point_sub breaks other tests Length result = length_sub(a, b); - if (b.extent.row == 0) { + if (b.extent.row == 0 && a.extent.row != 0) { result.extent.column -= b.extent.column; } return result; From e2c2e39e15ac1409483f01b5c28cab0ab7cf7ebf Mon Sep 17 00:00:00 2001 From: Daumantas Kavolis Date: Fri, 16 Jun 2023 13:20:20 +0300 Subject: [PATCH 10/17] Delete parser and iterator --- lib/binding_web/test/language-test.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/binding_web/test/language-test.js b/lib/binding_web/test/language-test.js index dab212d6..394e32b4 100644 --- a/lib/binding_web/test/language-test.js +++ b/lib/binding_web/test/language-test.js @@ -51,14 +51,19 @@ describe("Lookahead iterator", () => { ({ JavaScript, Parser } = await require("./helper")); const parser = new Parser().setLanguage(JavaScript); const tree = parser.parse("function fn() {}"); + parser.delete(); const cursor = tree.walk(); - cursor.gotoFirstChild(); - cursor.gotoFirstChild(); + assert(cursor.gotoFirstChild()); + assert(cursor.gotoFirstChild()); state = cursor.currentNode().nextParseState; lookahead = JavaScript.lookaheadIterator(state); assert.exists(lookahead); }); + after(() => { + lookahead.delete(); + }); + const expected = ["identifier", "comment", "(", "*", "formal_parameters"]; it("should iterate over valid symbols in the state", () => { const symbols = Array.from(lookahead); From fd8fd3f535ffe1b07b19abf178a6604df55ac9d2 Mon Sep 17 00:00:00 2001 From: Daumantas Kavolis Date: Fri, 16 Jun 2023 13:22:11 +0300 Subject: [PATCH 11/17] Add tests for previous sibling, last child --- lib/binding_web/exports.json | 2 +- lib/binding_web/test/tree-test.js | 44 +++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/lib/binding_web/exports.json b/lib/binding_web/exports.json index 1c638bb0..9e219d70 100644 --- a/lib/binding_web/exports.json +++ b/lib/binding_web/exports.json @@ -105,7 +105,7 @@ "_ts_tree_cursor_end_index_wasm", "_ts_tree_cursor_end_position_wasm", "_ts_tree_cursor_goto_first_child_wasm", - "_ts_tree_cursor_goto_last_child", + "_ts_tree_cursor_goto_last_child_wasm", "_ts_tree_cursor_goto_next_sibling_wasm", "_ts_tree_cursor_goto_previous_sibling_wasm", "_ts_tree_cursor_goto_parent_wasm", diff --git a/lib/binding_web/test/tree-test.js b/lib/binding_web/test/tree-test.js index 8c04e63e..a98d216a 100644 --- a/lib/binding_web/test/tree-test.js +++ b/lib/binding_web/test/tree-test.js @@ -244,6 +244,50 @@ describe("Tree", () => { endIndex: 13 }); + { + const copy = tree.walk(); + copy.resetTo(cursor); + + assert(copy.gotoPreviousSibling()); + assertCursorState(copy, { + nodeType: '+', + nodeIsNamed: false, + startPosition: {row: 0, column: 6}, + endPosition: {row: 0, column: 7}, + startIndex: 6, + endIndex: 7 + }); + + assert(copy.gotoPreviousSibling()); + assertCursorState(copy, { + nodeType: 'binary_expression', + nodeIsNamed: true, + startPosition: {row: 0, column: 0}, + endPosition: {row: 0, column: 5}, + startIndex: 0, + endIndex: 5 + }); + + assert(copy.gotoLastChild()); + assertCursorState(copy, { + nodeType: "identifier", + nodeIsNamed: true, + startPosition: {row: 0, column: 4}, + endPosition: {row: 0, column: 5}, + startIndex: 4, + endIndex: 5 + }) + + assert(copy.gotoParent()); + assert(copy.gotoParent()); + assert.equal(copy.nodeType, 'binary_expression') + assert(copy.gotoParent()); + assert.equal(copy.nodeType, 'expression_statement') + assert(copy.gotoParent()); + assert.equal(copy.nodeType, 'program') + assert(!copy.gotoParent()); + } + // const childIndex = cursor.gotoFirstChildForIndex(12); // assertCursorState(cursor, { // nodeType: 'identifier', From 8d5462cea4f23bf21b95f7e280577b8145018367 Mon Sep 17 00:00:00 2001 From: Daumantas Kavolis Date: Wed, 12 Jul 2023 15:34:08 +0300 Subject: [PATCH 12/17] `LookaheadIterator` doesn't need lifetime annotation as it depends on language only --- lib/binding_rust/ffi.rs | 6 +++--- lib/binding_rust/lib.rs | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/binding_rust/ffi.rs b/lib/binding_rust/ffi.rs index f1a739b1..ac4da98b 100644 --- a/lib/binding_rust/ffi.rs +++ b/lib/binding_rust/ffi.rs @@ -133,14 +133,14 @@ impl QueryCursor { } } -impl<'a> LookaheadIterator<'a> { +impl LookaheadIterator { /// Reconstructs a [LookaheadIterator] from a raw pointer. /// /// # Safety /// /// `ptr` must be non-null. - pub unsafe fn from_raw(ptr: *mut TSLookaheadIterator) -> LookaheadIterator<'a> { - LookaheadIterator(NonNull::new_unchecked(ptr), PhantomData) + pub unsafe fn from_raw(ptr: *mut TSLookaheadIterator) -> LookaheadIterator { + LookaheadIterator(NonNull::new_unchecked(ptr)) } /// Consumes the [LookaheadIterator], returning a raw pointer to the underlying C structure. diff --git a/lib/binding_rust/lib.rs b/lib/binding_rust/lib.rs index 49263b28..c371b8a1 100644 --- a/lib/binding_rust/lib.rs +++ b/lib/binding_rust/lib.rs @@ -89,7 +89,7 @@ pub struct Parser(NonNull); /// A stateful object that is used to look up symbols valid in a specific parse state #[doc(alias = "TSLookaheadIterator")] -pub struct LookaheadIterator<'a>(NonNull, PhantomData<&'a ()>); +pub struct LookaheadIterator(NonNull); /// A type of log message. #[derive(Debug, PartialEq, Eq)] @@ -373,7 +373,7 @@ impl Language { /// lookahead iterator created on the previous non-extra leaf node may be /// appropriate. #[doc(alias = "ts_lookahead_iterator_new")] - pub fn lookahead_iterator<'a>(&self, state: u16) -> Option> { + pub fn lookahead_iterator(&self, state: u16) -> Option { let ptr = unsafe { ffi::ts_lookahead_iterator_new(self.0, state) }; if ptr.is_null() { None @@ -1484,7 +1484,7 @@ impl<'a> Drop for TreeCursor<'a> { } } -impl<'a> LookaheadIterator<'a> { +impl LookaheadIterator { /// Get the current language of the lookahead iterator. #[doc(alias = "ts_lookahead_iterator_language")] pub fn language(&self) -> Language { @@ -1514,7 +1514,7 @@ impl<'a> LookaheadIterator<'a> { /// This returns `true` if the language was set successfully and `false` /// otherwise. #[doc(alias = "ts_lookahead_iterator_reset")] - pub fn reset(&self, language: &'a Language, state: u16) -> bool { + pub fn reset(&self, language: &Language, state: u16) -> bool { unsafe { ffi::ts_lookahead_iterator_reset(self.0.as_ptr(), language.0, state) } } @@ -1528,12 +1528,12 @@ impl<'a> LookaheadIterator<'a> { } /// Iterate symbol names. - pub fn iter_names(&'a self) -> impl Iterator + 'a { + pub fn iter_names<'a>(&'a self) -> impl Iterator + 'a { NameLookaheadIterator(&self) } } -struct NameLookaheadIterator<'a>(&'a LookaheadIterator<'a>); +struct NameLookaheadIterator<'a>(&'a LookaheadIterator); impl<'a> Iterator for NameLookaheadIterator<'a> { type Item = &'static str; @@ -1548,7 +1548,7 @@ impl<'a> Iterator for NameLookaheadIterator<'a> { } } -impl<'a> Iterator for LookaheadIterator<'a> { +impl Iterator for LookaheadIterator { type Item = u16; #[doc(alias = "ts_lookahead_iterator_advance")] @@ -1562,7 +1562,7 @@ impl<'a> Iterator for LookaheadIterator<'a> { } } -impl<'a> Drop for LookaheadIterator<'a> { +impl Drop for LookaheadIterator { #[doc(alias = "ts_lookahead_iterator_delete")] fn drop(&mut self) { unsafe { ffi::ts_lookahead_iterator_delete(self.0.as_ptr()) } From 8c789bf7d5573b2775522db51910b4d1663f17df Mon Sep 17 00:00:00 2001 From: Daumantas Kavolis Date: Mon, 17 Jul 2023 09:09:19 +0300 Subject: [PATCH 13/17] After review --- lib/binding_rust/bindings.rs | 5 +++-- lib/binding_rust/lib.rs | 14 ++++++++++++-- lib/include/tree_sitter/api.h | 18 +++++++++++++++++- lib/src/tree_cursor.c | 26 ++++++++++++++++---------- 4 files changed, 48 insertions(+), 15 deletions(-) diff --git a/lib/binding_rust/bindings.rs b/lib/binding_rust/bindings.rs index 5ad38721..d0d6c5e8 100644 --- a/lib/binding_rust/bindings.rs +++ b/lib/binding_rust/bindings.rs @@ -480,14 +480,15 @@ extern "C" { pub fn ts_tree_cursor_goto_next_sibling(arg1: *mut TSTreeCursor) -> bool; } extern "C" { - #[doc = " Move the cursor to the previous sibling of its current node.\n\n This returns `true` if the cursor successfully moved, and returns `false` if\n there was no previous sibling node."] + #[doc = " Move the cursor to the previous sibling of its current node.\n\n This returns `true` if the cursor successfully moved, and returns `false` if\n there was no previous sibling node.\n\n Note, that this function may be slower than\n `ts_tree_cursor_goto_next_sibling` due to how node positions are stored. In\n the worst case, this will need to iterate through all the children upto the\n previous sibling node to recalculate its position."] pub fn ts_tree_cursor_goto_previous_sibling(arg1: *mut TSTreeCursor) -> bool; } extern "C" { - #[doc = " Move the cursor to the first/last child of its current node.\n\n This returns `true` if the cursor successfully moved, and returns `false`\n if there were no children."] + #[doc = " Move the cursor to the first child of its current node.\n\n This returns `true` if the cursor successfully moved, and returns `false`\n if there were no children."] pub fn ts_tree_cursor_goto_first_child(arg1: *mut TSTreeCursor) -> bool; } extern "C" { + #[doc = " Move the cursor to the last child of its current node.\n\n This returns `true` if the cursor successfully moved, and returns `false` if\n there were no children.\n\n Note that this function may be slower than `ts_tree_cursor_goto_first_child`\n because it needs to iterate through all the children to compute the child's\n position."] pub fn ts_tree_cursor_goto_last_child(arg1: *mut TSTreeCursor) -> bool; } extern "C" { diff --git a/lib/binding_rust/lib.rs b/lib/binding_rust/lib.rs index c371b8a1..6dd9daec 100644 --- a/lib/binding_rust/lib.rs +++ b/lib/binding_rust/lib.rs @@ -1380,8 +1380,12 @@ impl<'a> TreeCursor<'a> { /// Move this cursor to the last child of its current node. /// - /// This returns `true` if the cursor successfully moved, and returns `false` - /// if there were no children. + /// This returns `true` if the cursor successfully moved, and returns + /// `false` if there were no children. + /// + /// Note that this function may be slower than + /// [`goto_first_child`](TreeCursor::goto_first_child) because it needs to + /// iterate through all the children to compute the child's position. #[doc(alias = "ts_tree_cursor_goto_last_child")] pub fn goto_last_child(&mut self) -> bool { return unsafe { ffi::ts_tree_cursor_goto_last_child(&mut self.0) }; @@ -1419,6 +1423,12 @@ impl<'a> TreeCursor<'a> { /// /// This returns `true` if the cursor successfully moved, and returns /// `false` if there was no previous sibling node. + /// + /// Note, that this function may be slower than + /// [`goto_next_sibling`](TreeCursor::goto_next_sibling) due to how node + /// positions are stored. In the worst case, this will need to iterate + /// through all the children upto the previous sibling node to recalculate + /// its position. #[doc(alias = "ts_tree_cursor_goto_previous_sibling")] pub fn goto_previous_sibling(&mut self) -> bool { return unsafe { ffi::ts_tree_cursor_goto_previous_sibling(&mut self.0) }; diff --git a/lib/include/tree_sitter/api.h b/lib/include/tree_sitter/api.h index 0fc9faf5..5ccf4610 100644 --- a/lib/include/tree_sitter/api.h +++ b/lib/include/tree_sitter/api.h @@ -720,16 +720,32 @@ bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *); * * This returns `true` if the cursor successfully moved, and returns `false` if * there was no previous sibling node. + * + * Note, that this function may be slower than + * `ts_tree_cursor_goto_next_sibling` due to how node positions are stored. In + * the worst case, this will need to iterate through all the children upto the + * previous sibling node to recalculate its position. */ bool ts_tree_cursor_goto_previous_sibling(TSTreeCursor *); /** - * Move the cursor to the first/last child of its current node. + * Move the cursor to the first child of its current node. * * This returns `true` if the cursor successfully moved, and returns `false` * if there were no children. */ bool ts_tree_cursor_goto_first_child(TSTreeCursor *); + +/** + * Move the cursor to the last child of its current node. + * + * This returns `true` if the cursor successfully moved, and returns `false` if + * there were no children. + * + * Note that this function may be slower than `ts_tree_cursor_goto_first_child` + * because it needs to iterate through all the children to compute the child's + * position. + */ bool ts_tree_cursor_goto_last_child(TSTreeCursor *); /** diff --git a/lib/src/tree_cursor.c b/lib/src/tree_cursor.c index 7b820983..25eca482 100644 --- a/lib/src/tree_cursor.c +++ b/lib/src/tree_cursor.c @@ -97,13 +97,19 @@ static inline bool ts_tree_cursor_child_iterator_next( return true; } -static inline Length length_sub_zero(Length a, Length b) { - // length_sub doesn't account for 0 row subtraction, i.e. only columns - // should be subtracted, but changing point_sub breaks other tests - Length result = length_sub(a, b); - if (b.extent.row == 0 && a.extent.row != 0) { - result.extent.column -= b.extent.column; +// Return a position that, when `b` is added to it, yields `a`. This +// can only be computed if `b` has zero rows. Otherwise, this function +// returns `LENGTH_UNDEFINED`, and the caller needs to recompute +// the position some other way. +static inline Length length_backtrack(Length a, Length b) { + if (length_is_undefined(a) || b.extent.row != 0) { + return LENGTH_UNDEFINED; } + + Length result; + result.bytes = a.bytes - b.bytes; + result.extent.row = a.extent.row; + result.extent.column = a.extent.column - b.extent.column; return result; } @@ -129,14 +135,14 @@ static inline bool ts_tree_cursor_child_iterator_previous( self->structural_child_index--; } - self->position = length_sub_zero(self->position, ts_subtree_padding(*child)); + self->position = length_backtrack(self->position, ts_subtree_padding(*child)); self->child_index--; // unsigned can underflow so compare it to child_count if (self->child_index < self->parent.ptr->child_count) { Subtree previous_child = ts_subtree_children(self->parent)[self->child_index]; Length size = ts_subtree_size(previous_child); - self->position = length_sub_zero(self->position, size); + self->position = length_backtrack(self->position, size); } return true; @@ -362,8 +368,8 @@ TreeCursorStep ts_tree_cursor_goto_previous_sibling_internal(TSTreeCursor *_self if (step == TreeCursorStepNone) return step; - // if row has not changed, column is still valid - if (array_back(&self->stack)->position.extent.row == position.extent.row) + // if length is already valid, there's no need to recompute it + if (!length_is_undefined(array_back(&self->stack)->position)) return step; // restore position from the parent node From 759af6d0a46aa47b2c319000ebc48ea6c9719805 Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Mon, 17 Jul 2023 10:52:33 +0300 Subject: [PATCH 14/17] Remove Copy, Clone from TSLookaheadIterator raw binding struct --- lib/binding_rust/bindings.rs | 2 +- script/generate-bindings | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/binding_rust/bindings.rs b/lib/binding_rust/bindings.rs index d0d6c5e8..912916c0 100644 --- a/lib/binding_rust/bindings.rs +++ b/lib/binding_rust/bindings.rs @@ -29,7 +29,7 @@ pub struct TSQueryCursor { _unused: [u8; 0], } #[repr(C)] -#[derive(Debug, Copy, Clone)] +#[derive(Debug)] pub struct TSLookaheadIterator { _unused: [u8; 0], } diff --git a/script/generate-bindings b/script/generate-bindings index da1796ba..fb47e247 100755 --- a/script/generate-bindings +++ b/script/generate-bindings @@ -6,6 +6,7 @@ no_derive_copy=( TSInput TSLanguage TSLogger + TSLookaheadIterator TSParser TSTree TSQuery From a08c61235d2df2dc23f4cc6cb7705a771b3484ba Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Tue, 1 Aug 2023 23:45:01 +0300 Subject: [PATCH 15/17] Fix, reference by value for Language as everywhere --- cli/src/tests/language_test.rs | 2 +- lib/binding_rust/lib.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/src/tests/language_test.rs b/cli/src/tests/language_test.rs index 2ac5bb77..f8a4653f 100644 --- a/cli/src/tests/language_test.rs +++ b/cli/src/tests/language_test.rs @@ -34,7 +34,7 @@ fn test_lookahead_iterator() { lookahead.reset_state(next_state); assert!(lookahead.iter_names().eq(expected_symbols)); - lookahead.reset(&language, next_state); + lookahead.reset(language, next_state); assert!(lookahead .map(|s| language.node_kind_for_id(s).unwrap()) .eq(expected_symbols)); diff --git a/lib/binding_rust/lib.rs b/lib/binding_rust/lib.rs index 6dd9daec..e8d99a87 100644 --- a/lib/binding_rust/lib.rs +++ b/lib/binding_rust/lib.rs @@ -1524,7 +1524,7 @@ impl LookaheadIterator { /// This returns `true` if the language was set successfully and `false` /// otherwise. #[doc(alias = "ts_lookahead_iterator_reset")] - pub fn reset(&self, language: &Language, state: u16) -> bool { + pub fn reset(&self, language: Language, state: u16) -> bool { unsafe { ffi::ts_lookahead_iterator_reset(self.0.as_ptr(), language.0, state) } } From 1ac134a97dfb58df8b9767f5ca9de65d5570630f Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Wed, 2 Aug 2023 00:27:19 +0300 Subject: [PATCH 16/17] Apply `ts_node_is_error` in Rust binding --- lib/binding_rust/lib.rs | 3 ++- lib/src/node.c | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/binding_rust/lib.rs b/lib/binding_rust/lib.rs index e8d99a87..2e3403b4 100644 --- a/lib/binding_rust/lib.rs +++ b/lib/binding_rust/lib.rs @@ -942,8 +942,9 @@ impl<'tree> Node<'tree> { /// /// Syntax errors represent parts of the code that could not be incorporated into a /// valid syntax tree. + #[doc(alias = "ts_node_is_error")] pub fn is_error(&self) -> bool { - self.kind_id() == u16::MAX + unsafe { ffi::ts_node_is_error(self.0) } } /// Get this node's parse state. diff --git a/lib/src/node.c b/lib/src/node.c index 46028f37..092e96f8 100644 --- a/lib/src/node.c +++ b/lib/src/node.c @@ -473,15 +473,15 @@ bool ts_node_has_error(TSNode self) { return ts_subtree_error_cost(ts_node__subtree(self)) > 0; } -uint32_t ts_node_descendant_count(TSNode self) { - return ts_subtree_visible_descendant_count(ts_node__subtree(self)) + 1; -} - bool ts_node_is_error(TSNode self) { TSSymbol symbol = ts_node_symbol(self); return symbol == ts_builtin_sym_error; } +uint32_t ts_node_descendant_count(TSNode self) { + return ts_subtree_visible_descendant_count(ts_node__subtree(self)) + 1; +} + TSStateId ts_node_parse_state(TSNode self) { return ts_subtree_parse_state(ts_node__subtree(self)); } From b9e3cc56d6fecc57993a3ba824f45460d3a76f96 Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Wed, 2 Aug 2023 00:33:10 +0300 Subject: [PATCH 17/17] chore: apply common approach for `api.h` defs --- lib/include/tree_sitter/api.h | 3 --- lib/include/tree_sitter/parser.h | 3 --- 2 files changed, 6 deletions(-) diff --git a/lib/include/tree_sitter/api.h b/lib/include/tree_sitter/api.h index 5ccf4610..d9560f24 100644 --- a/lib/include/tree_sitter/api.h +++ b/lib/include/tree_sitter/api.h @@ -32,10 +32,7 @@ extern "C" { /* Section - Types */ /*******************/ -#ifndef TREE_SITTER_PARSER_H_ typedef uint16_t TSStateId; -#endif - typedef uint16_t TSSymbol; typedef uint16_t TSFieldId; typedef struct TSLanguage TSLanguage; diff --git a/lib/include/tree_sitter/parser.h b/lib/include/tree_sitter/parser.h index c7ad1907..17b4fde9 100644 --- a/lib/include/tree_sitter/parser.h +++ b/lib/include/tree_sitter/parser.h @@ -15,9 +15,6 @@ extern "C" { #ifndef TREE_SITTER_API_H_ typedef uint16_t TSStateId; -#endif - -#ifndef TREE_SITTER_API_H_ typedef uint16_t TSSymbol; typedef uint16_t TSFieldId; typedef struct TSLanguage TSLanguage;