From 78010722a49ed6224c773c22b0d25a8c9fbde584 Mon Sep 17 00:00:00 2001 From: Douglas Creager Date: Thu, 13 May 2021 15:38:11 -0400 Subject: [PATCH 1/6] query: Allow unlimited pending matches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Well, not completely unlimited — we're still using a 16-bit counter to keep track of them. But we longer have a static maximum of 32 pending matches when executing a query. --- cli/src/tests/query_test.rs | 4 +++ lib/src/query.c | 72 +++++++++++++++---------------------- 2 files changed, 33 insertions(+), 43 deletions(-) diff --git a/cli/src/tests/query_test.rs b/cli/src/tests/query_test.rs index d6153dd4..b47dd69a 100644 --- a/cli/src/tests/query_test.rs +++ b/cli/src/tests/query_test.rs @@ -1625,6 +1625,7 @@ fn test_query_matches_with_different_tokens_with_the_same_string_value() { }); } +/* #[test] fn test_query_matches_with_too_many_permutations_to_track() { allocations::record(|| { @@ -1695,6 +1696,7 @@ fn test_query_matches_with_alternatives_and_too_many_permutations_to_track() { assert_eq!(cursor.did_exceed_match_limit(), true); }); } +*/ #[test] fn test_query_matches_with_anonymous_tokens() { @@ -2702,6 +2704,7 @@ fn test_query_captures_with_many_nested_results_with_fields() { }); } +/* #[test] fn test_query_captures_with_too_many_nested_results() { allocations::record(|| { @@ -2792,6 +2795,7 @@ fn test_query_captures_with_too_many_nested_results() { ); }); } +*/ #[test] fn test_query_captures_with_definite_pattern_containing_many_nested_matches() { diff --git a/lib/src/query.c b/lib/src/query.c index 65dbe1fe..5043554f 100644 --- a/lib/src/query.c +++ b/lib/src/query.c @@ -12,7 +12,6 @@ // #define LOG(...) fprintf(stderr, __VA_ARGS__) #define LOG(...) -#define MAX_CAPTURE_LIST_COUNT 32 #define MAX_STEP_CAPTURE_COUNT 3 #define MAX_STATE_PREDECESSOR_COUNT 100 #define MAX_ANALYSIS_STATE_DEPTH 12 @@ -165,6 +164,7 @@ typedef struct { } QueryState; typedef Array(TSQueryCapture) CaptureList; +typedef Array(CaptureList) CaptureListPoolEntry; /* * CaptureListPool - A collection of *lists* of captures. Each query state needs @@ -173,9 +173,8 @@ typedef Array(TSQueryCapture) CaptureList; * currently in use by a query state. */ typedef struct { - CaptureList list[MAX_CAPTURE_LIST_COUNT]; + CaptureListPoolEntry list; CaptureList empty_list; - uint32_t usage_map; } CaptureListPool; /* @@ -357,54 +356,55 @@ static uint32_t stream_offset(Stream *self) { static CaptureListPool capture_list_pool_new(void) { return (CaptureListPool) { + .list = array_new(), .empty_list = array_new(), - .usage_map = UINT32_MAX, }; } static void capture_list_pool_reset(CaptureListPool *self) { - self->usage_map = UINT32_MAX; - for (unsigned i = 0; i < MAX_CAPTURE_LIST_COUNT; i++) { - array_clear(&self->list[i]); + for (uint16_t i = 0; i < self->list.size; i++) { + // This invalid size means that the list is not in use. + self->list.contents[i].size = UINT32_MAX; } } static void capture_list_pool_delete(CaptureListPool *self) { - for (unsigned i = 0; i < MAX_CAPTURE_LIST_COUNT; i++) { - array_delete(&self->list[i]); + for (uint16_t i = 0; i < self->list.size; i++) { + array_delete(&self->list.contents[i]); } + array_delete(&self->list); } static const CaptureList *capture_list_pool_get(const CaptureListPool *self, uint16_t id) { - if (id >= MAX_CAPTURE_LIST_COUNT) return &self->empty_list; - return &self->list[id]; + if (id >= self->list.size) return &self->empty_list; + return &self->list.contents[id]; } static CaptureList *capture_list_pool_get_mut(CaptureListPool *self, uint16_t id) { - assert(id < MAX_CAPTURE_LIST_COUNT); - return &self->list[id]; -} - -static bool capture_list_pool_is_empty(const CaptureListPool *self) { - return self->usage_map == 0; + assert(id < self->list.size); + return &self->list.contents[id]; } static uint16_t capture_list_pool_acquire(CaptureListPool *self) { - // In the usage_map bitmask, ones represent free lists, and zeros represent - // lists that are in use. A free list id can quickly be found by counting - // the leading zeros in the usage map. An id of zero corresponds to the - // highest-order bit in the bitmask. - uint16_t id = count_leading_zeros(self->usage_map); - if (id >= MAX_CAPTURE_LIST_COUNT) return NONE; - self->usage_map &= ~bitmask_for_index(id); - array_clear(&self->list[id]); - return id; + // First see if any already allocated capture lists are currently unused. + for (uint16_t i = 0; i < self->list.size; i++) { + if (self->list.contents[i].size == UINT32_MAX) { + array_clear(&self->list.contents[i]); + return i; + } + } + + // Otherwise allocate and initialize a new capture list. + uint16_t i = self->list.size; + CaptureList list; + array_init(&list); + array_push(&self->list, list); + return i; } static void capture_list_pool_release(CaptureListPool *self, uint16_t id) { - if (id >= MAX_CAPTURE_LIST_COUNT) return; - array_clear(&self->list[id]); - self->usage_map |= bitmask_for_index(id); + if (id >= self->list.size) return; + self->list.contents[id].size = UINT32_MAX; } /************** @@ -3186,20 +3186,6 @@ bool ts_query_cursor_next_capture( return true; } - if (capture_list_pool_is_empty(&self->capture_list_pool)) { - LOG( - " abandon state. index:%u, pattern:%u, offset:%u.\n", - first_unfinished_state_index, - first_unfinished_pattern_index, - first_unfinished_capture_byte - ); - capture_list_pool_release( - &self->capture_list_pool, - self->states.contents[first_unfinished_state_index].capture_list_id - ); - array_erase(&self->states, first_unfinished_state_index); - } - // If there are no finished matches that are ready to be returned, then // continue finding more matches. if ( From cd96552448a6e0d4eb27fc54b27cb5130c4b6f76 Mon Sep 17 00:00:00 2001 From: Douglas Creager Date: Wed, 2 Jun 2021 11:24:58 -0400 Subject: [PATCH 2/6] query: Allow configurable match limit The default is now a whopping 64K matches, which "should be enough for everyone". You can use the new `ts_query_cursor_set_match_limit` function to set this to a lower limit, such as the previous default of 32. --- cli/src/tests/query_test.rs | 7 ++-- lib/binding_rust/bindings.rs | 24 +++++++++---- lib/binding_rust/lib.rs | 13 +++++++ lib/include/tree_sitter/api.h | 20 +++++++---- lib/src/query.c | 65 +++++++++++++++++++++++++++++++---- 5 files changed, 106 insertions(+), 23 deletions(-) diff --git a/cli/src/tests/query_test.rs b/cli/src/tests/query_test.rs index b47dd69a..5a19eead 100644 --- a/cli/src/tests/query_test.rs +++ b/cli/src/tests/query_test.rs @@ -1625,7 +1625,6 @@ fn test_query_matches_with_different_tokens_with_the_same_string_value() { }); } -/* #[test] fn test_query_matches_with_too_many_permutations_to_track() { allocations::record(|| { @@ -1646,6 +1645,7 @@ fn test_query_matches_with_too_many_permutations_to_track() { parser.set_language(language).unwrap(); let tree = parser.parse(&source, None).unwrap(); let mut cursor = QueryCursor::new(); + cursor.set_match_limit(32); let matches = cursor.matches(&query, tree.root_node(), to_callback(&source)); // For this pathological query, some match permutations will be dropped. @@ -1687,6 +1687,7 @@ fn test_query_matches_with_alternatives_and_too_many_permutations_to_track() { parser.set_language(language).unwrap(); let tree = parser.parse(&source, None).unwrap(); let mut cursor = QueryCursor::new(); + cursor.set_match_limit(32); let matches = cursor.matches(&query, tree.root_node(), to_callback(&source)); assert_eq!( @@ -1696,7 +1697,6 @@ fn test_query_matches_with_alternatives_and_too_many_permutations_to_track() { assert_eq!(cursor.did_exceed_match_limit(), true); }); } -*/ #[test] fn test_query_matches_with_anonymous_tokens() { @@ -2704,7 +2704,6 @@ fn test_query_captures_with_many_nested_results_with_fields() { }); } -/* #[test] fn test_query_captures_with_too_many_nested_results() { allocations::record(|| { @@ -2768,6 +2767,7 @@ fn test_query_captures_with_too_many_nested_results() { parser.set_language(language).unwrap(); let tree = parser.parse(&source, None).unwrap(); let mut cursor = QueryCursor::new(); + cursor.set_match_limit(32); let captures = cursor.captures(&query, tree.root_node(), to_callback(&source)); let captures = collect_captures(captures, &query, &source); @@ -2795,7 +2795,6 @@ fn test_query_captures_with_too_many_nested_results() { ); }); } -*/ #[test] fn test_query_captures_with_definite_pattern_containing_many_nested_matches() { diff --git a/lib/binding_rust/bindings.rs b/lib/binding_rust/bindings.rs index 50da12fc..9b8c0f65 100644 --- a/lib/binding_rust/bindings.rs +++ b/lib/binding_rust/bindings.rs @@ -726,15 +726,27 @@ extern "C" { pub fn ts_query_cursor_exec(arg1: *mut TSQueryCursor, arg2: *const TSQuery, arg3: TSNode); } extern "C" { - #[doc = " Check if this cursor has exceeded its maximum number of in-progress"] - #[doc = " matches."] + #[doc = " Manage the maximum number of in-progress matches allowed by this query"] + #[doc = " cursor."] #[doc = ""] - #[doc = " Currently, query cursors have a fixed capacity for storing lists"] - #[doc = " of in-progress captures. If this capacity is exceeded, then the"] - #[doc = " earliest-starting match will silently be dropped to make room for"] - #[doc = " further matches."] + #[doc = " Query cursors have a maximum capacity for storing lists of in-progress"] + #[doc = " captures. If this capacity is exceeded, then the earliest-starting match will"] + #[doc = " silently be dropped to make room for further matches."] + #[doc = ""] + #[doc = " By default, this limit is 65,536 pending matches, which is effectively"] + #[doc = " unlimited for most queries and syntax trees. You can optionally set this to a"] + #[doc = " lower number if you want to have (and check) a tighter bound on query"] + #[doc = " complexity."] + #[doc = ""] + #[doc = " If you update the match limit, it must be > 0 and <= 65536."] pub fn ts_query_cursor_did_exceed_match_limit(arg1: *const TSQueryCursor) -> bool; } +extern "C" { + pub fn ts_query_cursor_match_limit(arg1: *const TSQueryCursor) -> u32; +} +extern "C" { + pub fn ts_query_cursor_set_match_limit(arg1: *mut TSQueryCursor, arg2: u32); +} extern "C" { #[doc = " Set the range of bytes or (row, column) positions in which the query"] #[doc = " will be executed."] diff --git a/lib/binding_rust/lib.rs b/lib/binding_rust/lib.rs index 801f773f..bfdc843c 100644 --- a/lib/binding_rust/lib.rs +++ b/lib/binding_rust/lib.rs @@ -1598,6 +1598,19 @@ impl<'a> QueryCursor { QueryCursor(unsafe { NonNull::new_unchecked(ffi::ts_query_cursor_new()) }) } + /// Return the maximum number of in-progress matches for this cursor. + pub fn match_limit(&self) -> u32 { + unsafe { ffi::ts_query_cursor_match_limit(self.0.as_ptr()) } + } + + /// Set the maximum number of in-progress matches for this cursor. The limit must be > 0 and + /// <= 65536. + pub fn set_match_limit(&mut self, limit: u32) { + unsafe { + ffi::ts_query_cursor_set_match_limit(self.0.as_ptr(), limit); + } + } + /// Check if, on its last execution, this cursor exceeded its maximum number of /// in-progress matches. pub fn did_exceed_match_limit(&self) -> bool { diff --git a/lib/include/tree_sitter/api.h b/lib/include/tree_sitter/api.h index 43315415..fad7a589 100644 --- a/lib/include/tree_sitter/api.h +++ b/lib/include/tree_sitter/api.h @@ -798,15 +798,23 @@ void ts_query_cursor_delete(TSQueryCursor *); void ts_query_cursor_exec(TSQueryCursor *, const TSQuery *, TSNode); /** - * Check if this cursor has exceeded its maximum number of in-progress - * matches. + * Manage the maximum number of in-progress matches allowed by this query + * cursor. * - * Currently, query cursors have a fixed capacity for storing lists - * of in-progress captures. If this capacity is exceeded, then the - * earliest-starting match will silently be dropped to make room for - * further matches. + * Query cursors have a maximum capacity for storing lists of in-progress + * captures. If this capacity is exceeded, then the earliest-starting match will + * silently be dropped to make room for further matches. + * + * By default, this limit is 65,536 pending matches, which is effectively + * unlimited for most queries and syntax trees. You can optionally set this to a + * lower number if you want to have (and check) a tighter bound on query + * complexity. + * + * If you update the match limit, it must be > 0 and <= 65536. */ bool ts_query_cursor_did_exceed_match_limit(const TSQueryCursor *); +uint32_t ts_query_cursor_match_limit(const TSQueryCursor *); +void ts_query_cursor_set_match_limit(TSQueryCursor *, uint32_t); /** * Set the range of bytes or (row, column) positions in which the query diff --git a/lib/src/query.c b/lib/src/query.c index 5043554f..2c8e7193 100644 --- a/lib/src/query.c +++ b/lib/src/query.c @@ -175,6 +175,15 @@ typedef Array(CaptureList) CaptureListPoolEntry; typedef struct { CaptureListPoolEntry list; CaptureList empty_list; + // The maximum number of capture lists that we are allowed to allocate. We + // never allow `list` to allocate more entries than this, dropping pending + // matches if needed to stay under the limit. + uint32_t max_capture_list_count; + // The number of capture lists allocated in `list` that are not currently in + // use. We reuse those existing-but-unused capture lists before trying to + // allocate any new ones. We use an invalid value (UINT32_MAX) for a capture + // list's length to indicate that it's not in use. + uint16_t free_capture_list_count; } CaptureListPool; /* @@ -358,6 +367,10 @@ static CaptureListPool capture_list_pool_new(void) { return (CaptureListPool) { .list = array_new(), .empty_list = array_new(), + // The maximum maxmimum is 64K, since we use `uint16_t` as our capture list + // index type. + .max_capture_list_count = 65536, + .free_capture_list_count = 0, }; } @@ -366,6 +379,7 @@ static void capture_list_pool_reset(CaptureListPool *self) { // This invalid size means that the list is not in use. self->list.contents[i].size = UINT32_MAX; } + self->free_capture_list_count = self->list.size; } static void capture_list_pool_delete(CaptureListPool *self) { @@ -385,17 +399,30 @@ static CaptureList *capture_list_pool_get_mut(CaptureListPool *self, uint16_t id return &self->list.contents[id]; } +static bool capture_list_pool_is_empty(const CaptureListPool *self) { + // The capture list pool is empty if all allocated lists are in use, and we + // have reached the maximum allowed number of allocated lists. + return self->free_capture_list_count == 0 && self->list.size >= self->max_capture_list_count; +} + static uint16_t capture_list_pool_acquire(CaptureListPool *self) { - // First see if any already allocated capture lists are currently unused. - for (uint16_t i = 0; i < self->list.size; i++) { - if (self->list.contents[i].size == UINT32_MAX) { - array_clear(&self->list.contents[i]); - return i; + // First see if any already allocated capture list is currently unused. + if (self->free_capture_list_count > 0) { + for (uint16_t i = 0; i < self->list.size; i++) { + if (self->list.contents[i].size == UINT32_MAX) { + array_clear(&self->list.contents[i]); + self->free_capture_list_count--; + return i; + } } } - // Otherwise allocate and initialize a new capture list. - uint16_t i = self->list.size; + // Otherwise allocate and initialize a new capture list, as long as that + // doesn't put us over the requested maximum. + uint32_t i = self->list.size; + if (i >= self->max_capture_list_count) { + return NONE; + } CaptureList list; array_init(&list); array_push(&self->list, list); @@ -405,6 +432,7 @@ static uint16_t capture_list_pool_acquire(CaptureListPool *self) { static void capture_list_pool_release(CaptureListPool *self, uint16_t id) { if (id >= self->list.size) return; self->list.contents[id].size = UINT32_MAX; + self->free_capture_list_count++; } /************** @@ -2285,6 +2313,15 @@ bool ts_query_cursor_did_exceed_match_limit(const TSQueryCursor *self) { return self->did_exceed_match_limit; } +uint32_t ts_query_cursor_match_limit(const TSQueryCursor *self) { + return self->capture_list_pool.max_capture_list_count; +} + +void ts_query_cursor_set_match_limit(TSQueryCursor *self, uint32_t limit) { + assert(limit > 0 && limit <= 65536); + self->capture_list_pool.max_capture_list_count = limit; +} + void ts_query_cursor_exec( TSQueryCursor *self, const TSQuery *query, @@ -3186,6 +3223,20 @@ bool ts_query_cursor_next_capture( return true; } + if (capture_list_pool_is_empty(&self->capture_list_pool)) { + LOG( + " abandon state. index:%u, pattern:%u, offset:%u.\n", + first_unfinished_state_index, + first_unfinished_pattern_index, + first_unfinished_capture_byte + ); + capture_list_pool_release( + &self->capture_list_pool, + self->states.contents[first_unfinished_state_index].capture_list_id + ); + array_erase(&self->states, first_unfinished_state_index); + } + // If there are no finished matches that are ready to be returned, then // continue finding more matches. if ( From 1f6eac555cb74a003405fc680e139c5d8faeb6b2 Mon Sep 17 00:00:00 2001 From: Douglas Creager Date: Wed, 2 Jun 2021 13:19:52 -0400 Subject: [PATCH 3/6] query: Use uint32_t for capture list IDs --- lib/include/tree_sitter/api.h | 16 ++++++---------- lib/src/query.c | 10 ++++------ 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/lib/include/tree_sitter/api.h b/lib/include/tree_sitter/api.h index fad7a589..6f826604 100644 --- a/lib/include/tree_sitter/api.h +++ b/lib/include/tree_sitter/api.h @@ -801,16 +801,12 @@ void ts_query_cursor_exec(TSQueryCursor *, const TSQuery *, TSNode); * Manage the maximum number of in-progress matches allowed by this query * cursor. * - * Query cursors have a maximum capacity for storing lists of in-progress - * captures. If this capacity is exceeded, then the earliest-starting match will - * silently be dropped to make room for further matches. - * - * By default, this limit is 65,536 pending matches, which is effectively - * unlimited for most queries and syntax trees. You can optionally set this to a - * lower number if you want to have (and check) a tighter bound on query - * complexity. - * - * If you update the match limit, it must be > 0 and <= 65536. + * Query cursors have an optional maximum capacity for storing lists of + * in-progress captures. If this capacity is exceeded, then the + * earliest-starting match will silently be dropped to make room for further + * matches. This maximum capacity is optional — by default, query cursors allow + * any number of pending matches, dynamically allocating new space for them as + * needed as the query is executed. */ bool ts_query_cursor_did_exceed_match_limit(const TSQueryCursor *); uint32_t ts_query_cursor_match_limit(const TSQueryCursor *); diff --git a/lib/src/query.c b/lib/src/query.c index 2c8e7193..b082234f 100644 --- a/lib/src/query.c +++ b/lib/src/query.c @@ -152,10 +152,10 @@ typedef struct { */ typedef struct { uint32_t id; + uint32_t capture_list_id; uint16_t start_depth; uint16_t step_index; uint16_t pattern_index; - uint16_t capture_list_id; uint16_t consumed_capture_count: 12; bool seeking_immediate_match: 1; bool has_in_progress_alternatives: 1; @@ -183,7 +183,7 @@ typedef struct { // use. We reuse those existing-but-unused capture lists before trying to // allocate any new ones. We use an invalid value (UINT32_MAX) for a capture // list's length to indicate that it's not in use. - uint16_t free_capture_list_count; + uint32_t free_capture_list_count; } CaptureListPool; /* @@ -367,9 +367,7 @@ static CaptureListPool capture_list_pool_new(void) { return (CaptureListPool) { .list = array_new(), .empty_list = array_new(), - // The maximum maxmimum is 64K, since we use `uint16_t` as our capture list - // index type. - .max_capture_list_count = 65536, + .max_capture_list_count = UINT32_MAX, .free_capture_list_count = 0, }; } @@ -2318,7 +2316,7 @@ uint32_t ts_query_cursor_match_limit(const TSQueryCursor *self) { } void ts_query_cursor_set_match_limit(TSQueryCursor *self, uint32_t limit) { - assert(limit > 0 && limit <= 65536); + assert(limit > 0); self->capture_list_pool.max_capture_list_count = limit; } From ad3907c2a6f9aee249677493f9444de868ed06bd Mon Sep 17 00:00:00 2001 From: Douglas Creager Date: Wed, 2 Jun 2021 13:51:00 -0400 Subject: [PATCH 4/6] wasm: Add matchLimit option to query methods This exposes the new configurable match limits for query cursors. --- lib/binding_web/binding.c | 16 ++++++++++++++-- lib/binding_web/binding.js | 26 ++++++++++++++++++++++---- lib/binding_web/test/query-test.js | 2 +- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/lib/binding_web/binding.c b/lib/binding_web/binding.c index 8adaec75..27292911 100644 --- a/lib/binding_web/binding.c +++ b/lib/binding_web/binding.c @@ -594,9 +594,15 @@ void ts_query_matches_wasm( uint32_t start_row, uint32_t start_column, uint32_t end_row, - uint32_t end_column + uint32_t end_column, + uint32_t match_limit ) { if (!scratch_query_cursor) scratch_query_cursor = ts_query_cursor_new(); + if (match_limit == 0) { + ts_query_cursor_set_match_limit(scratch_query_cursor, UINT32_MAX); + } else { + ts_query_cursor_set_match_limit(scratch_query_cursor, match_limit); + } TSNode node = unmarshal_node(tree); TSPoint start_point = {start_row, code_unit_to_byte(start_column)}; @@ -635,9 +641,15 @@ void ts_query_captures_wasm( uint32_t start_row, uint32_t start_column, uint32_t end_row, - uint32_t end_column + uint32_t end_column, + uint32_t match_limit ) { if (!scratch_query_cursor) scratch_query_cursor = ts_query_cursor_new(); + if (match_limit == 0) { + ts_query_cursor_set_match_limit(scratch_query_cursor, UINT32_MAX); + } else { + ts_query_cursor_set_match_limit(scratch_query_cursor, match_limit); + } TSNode node = unmarshal_node(tree); TSPoint start_point = {start_row, code_unit_to_byte(start_column)}; diff --git a/lib/binding_web/binding.js b/lib/binding_web/binding.js index 1f9ef412..bf0a91ce 100644 --- a/lib/binding_web/binding.js +++ b/lib/binding_web/binding.js @@ -953,9 +953,17 @@ class Query { this[0] = 0; } - matches(node, startPosition, endPosition) { + matches(node, startPosition, endPosition, options) { if (!startPosition) startPosition = ZERO_POINT; if (!endPosition) endPosition = ZERO_POINT; + if (!options) options = {}; + + let matchLimit = options.matchLimit; + if (typeof matchLimit === 'undefined') { + matchLimit = 0; + } else if (typeof matchLimit !== 'number') { + throw new Error('Arguments must be numbers'); + } marshalNode(node); @@ -965,7 +973,8 @@ class Query { startPosition.row, startPosition.column, endPosition.row, - endPosition.column + endPosition.column, + matchLimit ); const rawCount = getValue(TRANSFER_BUFFER, 'i32'); @@ -1000,9 +1009,17 @@ class Query { return result; } - captures(node, startPosition, endPosition) { + captures(node, startPosition, endPosition, options) { if (!startPosition) startPosition = ZERO_POINT; if (!endPosition) endPosition = ZERO_POINT; + if (!options) options = {}; + + let matchLimit = options.matchLimit; + if (typeof matchLimit === 'undefined') { + matchLimit = 0; + } else if (typeof matchLimit !== 'number') { + throw new Error('Arguments must be numbers'); + } marshalNode(node); @@ -1012,7 +1029,8 @@ class Query { startPosition.row, startPosition.column, endPosition.row, - endPosition.column + endPosition.column, + matchLimit ); const count = getValue(TRANSFER_BUFFER, 'i32'); diff --git a/lib/binding_web/test/query-test.js b/lib/binding_web/test/query-test.js index b7b2e053..2b2aebe0 100644 --- a/lib/binding_web/test/query-test.js +++ b/lib/binding_web/test/query-test.js @@ -256,7 +256,7 @@ describe("Query", () => { (array (identifier) @pre (identifier) @post) `); - const captures = query.captures(tree.rootNode); + const captures = query.captures(tree.rootNode, null, null, {matchLimit: 32}); assert.ok(query.didExceedMatchLimit()); }); }); From 47f1af818a1866a84fbcc679ca286ec8571bc815 Mon Sep 17 00:00:00 2001 From: Douglas Creager Date: Wed, 2 Jun 2021 14:14:57 -0400 Subject: [PATCH 5/6] query: Remove bits.h --- lib/src/bits.h | 42 ------------------------------------------ lib/src/query.c | 1 - 2 files changed, 43 deletions(-) delete mode 100644 lib/src/bits.h diff --git a/lib/src/bits.h b/lib/src/bits.h deleted file mode 100644 index ca8caf30..00000000 --- a/lib/src/bits.h +++ /dev/null @@ -1,42 +0,0 @@ -#ifndef TREE_SITTER_BITS_H_ -#define TREE_SITTER_BITS_H_ - -#include - -static inline uint32_t bitmask_for_index(uint16_t id) { - return (1u << (31 - id)); -} - -#ifdef __TINYC__ - -// Algorithm taken from the Hacker's Delight book -// See also https://graphics.stanford.edu/~seander/bithacks.html -static inline uint32_t count_leading_zeros(uint32_t x) { - int count = 0; - if (x == 0) return 32; - x = x - ((x >> 1) & 0x55555555); - x = (x & 0x33333333) + ((x >> 2) & 0x33333333); - count = (((x + (x >> 4)) & 0x0f0f0f0f) * 0x01010101) >> 24; - return count; -} - -#elif defined _WIN32 && !defined __GNUC__ - -#include - -static inline uint32_t count_leading_zeros(uint32_t x) { - if (x == 0) return 32; - uint32_t result; - _BitScanReverse(&result, x); - return 31 - result; -} - -#else - -static inline uint32_t count_leading_zeros(uint32_t x) { - if (x == 0) return 32; - return __builtin_clz(x); -} - -#endif -#endif // TREE_SITTER_BITS_H_ diff --git a/lib/src/query.c b/lib/src/query.c index b082234f..cf5b6267 100644 --- a/lib/src/query.c +++ b/lib/src/query.c @@ -1,7 +1,6 @@ #include "tree_sitter/api.h" #include "./alloc.h" #include "./array.h" -#include "./bits.h" #include "./language.h" #include "./point.h" #include "./tree_cursor.h" From cc20708a33676e80352543d395c4d37ebd895669 Mon Sep 17 00:00:00 2001 From: Douglas Creager Date: Wed, 2 Jun 2021 14:16:04 -0400 Subject: [PATCH 6/6] query: Minor cleanups --- lib/src/query.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/src/query.c b/lib/src/query.c index cf5b6267..13149eb8 100644 --- a/lib/src/query.c +++ b/lib/src/query.c @@ -163,7 +163,6 @@ typedef struct { } QueryState; typedef Array(TSQueryCapture) CaptureList; -typedef Array(CaptureList) CaptureListPoolEntry; /* * CaptureListPool - A collection of *lists* of captures. Each query state needs @@ -172,7 +171,7 @@ typedef Array(CaptureList) CaptureListPoolEntry; * currently in use by a query state. */ typedef struct { - CaptureListPoolEntry list; + Array(CaptureList) list; CaptureList empty_list; // The maximum number of capture lists that we are allowed to allocate. We // never allow `list` to allocate more entries than this, dropping pending @@ -2315,7 +2314,6 @@ uint32_t ts_query_cursor_match_limit(const TSQueryCursor *self) { } void ts_query_cursor_set_match_limit(TSQueryCursor *self, uint32_t limit) { - assert(limit > 0); self->capture_list_pool.max_capture_list_count = limit; }