From 78010722a49ed6224c773c22b0d25a8c9fbde584 Mon Sep 17 00:00:00 2001 From: Douglas Creager Date: Thu, 13 May 2021 15:38:11 -0400 Subject: [PATCH] query: Allow unlimited pending matches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Well, not completely unlimited — we're still using a 16-bit counter to keep track of them. But we longer have a static maximum of 32 pending matches when executing a query. --- cli/src/tests/query_test.rs | 4 +++ lib/src/query.c | 72 +++++++++++++++---------------------- 2 files changed, 33 insertions(+), 43 deletions(-) diff --git a/cli/src/tests/query_test.rs b/cli/src/tests/query_test.rs index d6153dd4..b47dd69a 100644 --- a/cli/src/tests/query_test.rs +++ b/cli/src/tests/query_test.rs @@ -1625,6 +1625,7 @@ fn test_query_matches_with_different_tokens_with_the_same_string_value() { }); } +/* #[test] fn test_query_matches_with_too_many_permutations_to_track() { allocations::record(|| { @@ -1695,6 +1696,7 @@ fn test_query_matches_with_alternatives_and_too_many_permutations_to_track() { assert_eq!(cursor.did_exceed_match_limit(), true); }); } +*/ #[test] fn test_query_matches_with_anonymous_tokens() { @@ -2702,6 +2704,7 @@ fn test_query_captures_with_many_nested_results_with_fields() { }); } +/* #[test] fn test_query_captures_with_too_many_nested_results() { allocations::record(|| { @@ -2792,6 +2795,7 @@ fn test_query_captures_with_too_many_nested_results() { ); }); } +*/ #[test] fn test_query_captures_with_definite_pattern_containing_many_nested_matches() { diff --git a/lib/src/query.c b/lib/src/query.c index 65dbe1fe..5043554f 100644 --- a/lib/src/query.c +++ b/lib/src/query.c @@ -12,7 +12,6 @@ // #define LOG(...) fprintf(stderr, __VA_ARGS__) #define LOG(...) -#define MAX_CAPTURE_LIST_COUNT 32 #define MAX_STEP_CAPTURE_COUNT 3 #define MAX_STATE_PREDECESSOR_COUNT 100 #define MAX_ANALYSIS_STATE_DEPTH 12 @@ -165,6 +164,7 @@ typedef struct { } QueryState; typedef Array(TSQueryCapture) CaptureList; +typedef Array(CaptureList) CaptureListPoolEntry; /* * CaptureListPool - A collection of *lists* of captures. Each query state needs @@ -173,9 +173,8 @@ typedef Array(TSQueryCapture) CaptureList; * currently in use by a query state. */ typedef struct { - CaptureList list[MAX_CAPTURE_LIST_COUNT]; + CaptureListPoolEntry list; CaptureList empty_list; - uint32_t usage_map; } CaptureListPool; /* @@ -357,54 +356,55 @@ static uint32_t stream_offset(Stream *self) { static CaptureListPool capture_list_pool_new(void) { return (CaptureListPool) { + .list = array_new(), .empty_list = array_new(), - .usage_map = UINT32_MAX, }; } static void capture_list_pool_reset(CaptureListPool *self) { - self->usage_map = UINT32_MAX; - for (unsigned i = 0; i < MAX_CAPTURE_LIST_COUNT; i++) { - array_clear(&self->list[i]); + for (uint16_t i = 0; i < self->list.size; i++) { + // This invalid size means that the list is not in use. + self->list.contents[i].size = UINT32_MAX; } } static void capture_list_pool_delete(CaptureListPool *self) { - for (unsigned i = 0; i < MAX_CAPTURE_LIST_COUNT; i++) { - array_delete(&self->list[i]); + for (uint16_t i = 0; i < self->list.size; i++) { + array_delete(&self->list.contents[i]); } + array_delete(&self->list); } static const CaptureList *capture_list_pool_get(const CaptureListPool *self, uint16_t id) { - if (id >= MAX_CAPTURE_LIST_COUNT) return &self->empty_list; - return &self->list[id]; + if (id >= self->list.size) return &self->empty_list; + return &self->list.contents[id]; } static CaptureList *capture_list_pool_get_mut(CaptureListPool *self, uint16_t id) { - assert(id < MAX_CAPTURE_LIST_COUNT); - return &self->list[id]; -} - -static bool capture_list_pool_is_empty(const CaptureListPool *self) { - return self->usage_map == 0; + assert(id < self->list.size); + return &self->list.contents[id]; } static uint16_t capture_list_pool_acquire(CaptureListPool *self) { - // In the usage_map bitmask, ones represent free lists, and zeros represent - // lists that are in use. A free list id can quickly be found by counting - // the leading zeros in the usage map. An id of zero corresponds to the - // highest-order bit in the bitmask. - uint16_t id = count_leading_zeros(self->usage_map); - if (id >= MAX_CAPTURE_LIST_COUNT) return NONE; - self->usage_map &= ~bitmask_for_index(id); - array_clear(&self->list[id]); - return id; + // First see if any already allocated capture lists are currently unused. + for (uint16_t i = 0; i < self->list.size; i++) { + if (self->list.contents[i].size == UINT32_MAX) { + array_clear(&self->list.contents[i]); + return i; + } + } + + // Otherwise allocate and initialize a new capture list. + uint16_t i = self->list.size; + CaptureList list; + array_init(&list); + array_push(&self->list, list); + return i; } static void capture_list_pool_release(CaptureListPool *self, uint16_t id) { - if (id >= MAX_CAPTURE_LIST_COUNT) return; - array_clear(&self->list[id]); - self->usage_map |= bitmask_for_index(id); + if (id >= self->list.size) return; + self->list.contents[id].size = UINT32_MAX; } /************** @@ -3186,20 +3186,6 @@ bool ts_query_cursor_next_capture( return true; } - if (capture_list_pool_is_empty(&self->capture_list_pool)) { - LOG( - " abandon state. index:%u, pattern:%u, offset:%u.\n", - first_unfinished_state_index, - first_unfinished_pattern_index, - first_unfinished_capture_byte - ); - capture_list_pool_release( - &self->capture_list_pool, - self->states.contents[first_unfinished_state_index].capture_list_id - ); - array_erase(&self->states, first_unfinished_state_index); - } - // If there are no finished matches that are ready to be returned, then // continue finding more matches. if (