From e742186c25f55349313ce6c50ae91a8744b5ee56 Mon Sep 17 00:00:00 2001 From: Mingkai Dong Date: Fri, 17 Dec 2021 19:53:02 +0800 Subject: [PATCH 1/6] Allow to change the allocator dynamically --- lib/src/alloc.c | 72 +++++++++++++++++++++++++++++++++++++++++++++++++ lib/src/alloc.h | 70 ++++++++--------------------------------------- lib/src/lib.c | 1 + 3 files changed, 84 insertions(+), 59 deletions(-) create mode 100644 lib/src/alloc.c diff --git a/lib/src/alloc.c b/lib/src/alloc.c new file mode 100644 index 00000000..5ffc050c --- /dev/null +++ b/lib/src/alloc.c @@ -0,0 +1,72 @@ +#include "alloc.h" + +#if defined(TREE_SITTER_ALLOCATION_TRACKING) + +void *ts_record_malloc(size_t); +void *ts_record_calloc(size_t, size_t); +void *ts_record_realloc(void *, size_t); +void ts_record_free(void *); +bool ts_toggle_allocation_recording(bool); + +static struct allocator ts_tracking_allocator = { + .malloc = ts_record_malloc, + .calloc = ts_record_calloc, + .realloc = ts_record_realloc, + .free = ts_record_free, +}; + +struct allocator *ts_allocator = &ts_tracking_allocator; + +#else + +#include + +static inline bool ts_toggle_allocation_recording(bool value) { + (void)value; + return false; +} + + +static inline void *ts_malloc_default(size_t size) { + void *result = malloc(size); + if (size > 0 && !result) { + fprintf(stderr, "tree-sitter failed to allocate %zu bytes", size); + exit(1); + } + return result; +} + +static inline void *ts_calloc_default(size_t count, size_t size) { + void *result = calloc(count, size); + if (count > 0 && !result) { + fprintf(stderr, "tree-sitter failed to allocate %zu bytes", count * size); + exit(1); + } + return result; +} + +static inline void *ts_realloc_default(void *buffer, size_t size) { + void *result = realloc(buffer, size); + if (size > 0 && !result) { + fprintf(stderr, "tree-sitter failed to reallocate %zu bytes", size); + exit(1); + } + return result; +} + +static inline void ts_free_default(void *buffer) { + free(buffer); +} + +static struct allocator ts_default_allocator = { + .malloc = ts_malloc_default, + .calloc = ts_calloc_default, + .realloc = ts_realloc_default, + .free = ts_free_default, +}; + +// Allow clients to override allocation functions dynamically +struct allocator *ts_allocator = &ts_default_allocator; + + +#endif diff --git a/lib/src/alloc.h b/lib/src/alloc.h index dd487ca2..131a8b49 100644 --- a/lib/src/alloc.h +++ b/lib/src/alloc.h @@ -9,75 +9,27 @@ extern "C" { #include #include -#if defined(TREE_SITTER_ALLOCATION_TRACKING) +struct allocator { + void *(*malloc)(size_t); + void *(*calloc)(size_t, size_t); + void *(*realloc)(void *, size_t); + void (*free)(void *); +}; -void *ts_record_malloc(size_t); -void *ts_record_calloc(size_t, size_t); -void *ts_record_realloc(void *, size_t); -void ts_record_free(void *); -bool ts_toggle_allocation_recording(bool); - -#define ts_malloc ts_record_malloc -#define ts_calloc ts_record_calloc -#define ts_realloc ts_record_realloc -#define ts_free ts_record_free - -#else +extern struct allocator *ts_allocator; // Allow clients to override allocation functions - #ifndef ts_malloc -#define ts_malloc ts_malloc_default +#define ts_malloc ts_allocator->malloc #endif #ifndef ts_calloc -#define ts_calloc ts_calloc_default +#define ts_calloc ts_allocator->calloc #endif #ifndef ts_realloc -#define ts_realloc ts_realloc_default +#define ts_realloc ts_allocator->realloc #endif #ifndef ts_free -#define ts_free ts_free_default -#endif - -#include - -static inline bool ts_toggle_allocation_recording(bool value) { - (void)value; - return false; -} - - -static inline void *ts_malloc_default(size_t size) { - void *result = malloc(size); - if (size > 0 && !result) { - fprintf(stderr, "tree-sitter failed to allocate %zu bytes", size); - exit(1); - } - return result; -} - -static inline void *ts_calloc_default(size_t count, size_t size) { - void *result = calloc(count, size); - if (count > 0 && !result) { - fprintf(stderr, "tree-sitter failed to allocate %zu bytes", count * size); - exit(1); - } - return result; -} - -static inline void *ts_realloc_default(void *buffer, size_t size) { - void *result = realloc(buffer, size); - if (size > 0 && !result) { - fprintf(stderr, "tree-sitter failed to reallocate %zu bytes", size); - exit(1); - } - return result; -} - -static inline void ts_free_default(void *buffer) { - free(buffer); -} - +#define ts_free ts_allocator->free #endif #ifdef __cplusplus diff --git a/lib/src/lib.c b/lib/src/lib.c index 289d32f4..5aab20d5 100644 --- a/lib/src/lib.c +++ b/lib/src/lib.c @@ -5,6 +5,7 @@ #define _POSIX_C_SOURCE 200112L +#include "./alloc.c" #include "./get_changed_ranges.c" #include "./language.c" #include "./lexer.c" From b516f96f375ac975bdc7910543a163e5cc952fb0 Mon Sep 17 00:00:00 2001 From: Mingkai Dong Date: Sat, 18 Dec 2021 00:33:49 +0800 Subject: [PATCH 2/6] Fix declaration of ts_toggle_allocation_recording --- lib/src/alloc.c | 2 +- lib/src/alloc.h | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/src/alloc.c b/lib/src/alloc.c index 5ffc050c..6f62a777 100644 --- a/lib/src/alloc.c +++ b/lib/src/alloc.c @@ -21,7 +21,7 @@ struct allocator *ts_allocator = &ts_tracking_allocator; #include -static inline bool ts_toggle_allocation_recording(bool value) { +bool ts_toggle_allocation_recording(bool value) { (void)value; return false; } diff --git a/lib/src/alloc.h b/lib/src/alloc.h index 131a8b49..35ba77d2 100644 --- a/lib/src/alloc.h +++ b/lib/src/alloc.h @@ -32,6 +32,8 @@ extern struct allocator *ts_allocator; #define ts_free ts_allocator->free #endif +bool ts_toggle_allocation_recording(bool); + #ifdef __cplusplus } #endif From 578bf74bf359a29b1d02b2b7dfb6e0808be90b4b Mon Sep 17 00:00:00 2001 From: Mingkai Dong Date: Sat, 18 Dec 2021 09:53:58 +0800 Subject: [PATCH 3/6] Add TSAllocator and ts_set_allocator in api.h --- lib/include/tree_sitter/api.h | 21 +++++++++++++++++++++ lib/src/alloc.c | 17 ++++++++++++----- lib/src/alloc.h | 11 +++-------- 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/lib/include/tree_sitter/api.h b/lib/include/tree_sitter/api.h index 66ceaea1..f1192e34 100644 --- a/lib/include/tree_sitter/api.h +++ b/lib/include/tree_sitter/api.h @@ -134,10 +134,31 @@ typedef enum { TSQueryErrorLanguage, } TSQueryError; +typedef struct { + void *(*malloc)(size_t); + void *(*calloc)(size_t, size_t); + void *(*realloc)(void *, size_t); + void (*free)(void *); +} TSAllocator; + /********************/ /* Section - Parser */ /********************/ + +/** + * The allocator. + */ +extern TSAllocator *ts_allocator; + +/** + * Switch to a new allocator. + * + * Returns the old allocator. The caller needs to take care of any allocated + * memory managed by the old allocator. + */ +TSAllocator *ts_set_allocator(TSAllocator *new_alloc); + /** * Create a new parser. */ diff --git a/lib/src/alloc.c b/lib/src/alloc.c index 6f62a777..391430f7 100644 --- a/lib/src/alloc.c +++ b/lib/src/alloc.c @@ -8,14 +8,14 @@ void *ts_record_realloc(void *, size_t); void ts_record_free(void *); bool ts_toggle_allocation_recording(bool); -static struct allocator ts_tracking_allocator = { +static TSAllocator ts_tracking_allocator = { .malloc = ts_record_malloc, .calloc = ts_record_calloc, .realloc = ts_record_realloc, .free = ts_record_free, }; -struct allocator *ts_allocator = &ts_tracking_allocator; +TSAllocator *ts_allocator = &ts_tracking_allocator; #else @@ -58,7 +58,7 @@ static inline void ts_free_default(void *buffer) { free(buffer); } -static struct allocator ts_default_allocator = { +static TSAllocator ts_default_allocator = { .malloc = ts_malloc_default, .calloc = ts_calloc_default, .realloc = ts_realloc_default, @@ -66,7 +66,14 @@ static struct allocator ts_default_allocator = { }; // Allow clients to override allocation functions dynamically -struct allocator *ts_allocator = &ts_default_allocator; +TSAllocator *ts_allocator = &ts_default_allocator; +#endif // defined(TREE_SITTER_ALLOCATION_TRACKING) + +TSAllocator *ts_set_allocator(TSAllocator *new_alloc) +{ + TSAllocator *old = ts_allocator; + ts_allocator = new_alloc; + return old; +} -#endif diff --git a/lib/src/alloc.h b/lib/src/alloc.h index 35ba77d2..6c04ae18 100644 --- a/lib/src/alloc.h +++ b/lib/src/alloc.h @@ -1,6 +1,8 @@ #ifndef TREE_SITTER_ALLOC_H_ #define TREE_SITTER_ALLOC_H_ +#include "tree_sitter/api.h" + #ifdef __cplusplus extern "C" { #endif @@ -9,14 +11,7 @@ extern "C" { #include #include -struct allocator { - void *(*malloc)(size_t); - void *(*calloc)(size_t, size_t); - void *(*realloc)(void *, size_t); - void (*free)(void *); -}; - -extern struct allocator *ts_allocator; +extern TSAllocator *ts_allocator; // Allow clients to override allocation functions #ifndef ts_malloc From b9b051e933712428d8d0b0a9999a9a2fdc0d71be Mon Sep 17 00:00:00 2001 From: Mingkai Dong Date: Sat, 18 Dec 2021 11:26:18 +0800 Subject: [PATCH 4/6] Remove ts_allocator from api.h Since we have exposed the function to set allocator, we don't need to expose the global variable in `api.h`. --- lib/include/tree_sitter/api.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/include/tree_sitter/api.h b/lib/include/tree_sitter/api.h index f1192e34..c7de7662 100644 --- a/lib/include/tree_sitter/api.h +++ b/lib/include/tree_sitter/api.h @@ -145,12 +145,6 @@ typedef struct { /* Section - Parser */ /********************/ - -/** - * The allocator. - */ -extern TSAllocator *ts_allocator; - /** * Switch to a new allocator. * From 486ea2569dc0bdfab3ff15874a86da815e375572 Mon Sep 17 00:00:00 2001 From: Mingkai Dong Date: Sat, 18 Dec 2021 16:45:18 +0800 Subject: [PATCH 5/6] Avoid allocator from being switched more than once --- lib/include/tree_sitter/api.h | 11 ++++++++--- lib/src/alloc.c | 13 +++++++++---- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/lib/include/tree_sitter/api.h b/lib/include/tree_sitter/api.h index c7de7662..0b64e60e 100644 --- a/lib/include/tree_sitter/api.h +++ b/lib/include/tree_sitter/api.h @@ -148,10 +148,15 @@ typedef struct { /** * Switch to a new allocator. * - * Returns the old allocator. The caller needs to take care of any allocated - * memory managed by the old allocator. + * Returns true iff the switch successes. + * + * This function can only be invoked *once*, before tree-sitter has allocated + * any memory via malloc/calloc/realloc. Otherwise, memory bugs could occur + * since some memory is allocated by the old allocator, but freed by the new + * one. + * */ -TSAllocator *ts_set_allocator(TSAllocator *new_alloc); +bool ts_set_allocator(TSAllocator *new_alloc); /** * Create a new parser. diff --git a/lib/src/alloc.c b/lib/src/alloc.c index 391430f7..65c72f11 100644 --- a/lib/src/alloc.c +++ b/lib/src/alloc.c @@ -70,10 +70,15 @@ TSAllocator *ts_allocator = &ts_default_allocator; #endif // defined(TREE_SITTER_ALLOCATION_TRACKING) -TSAllocator *ts_set_allocator(TSAllocator *new_alloc) +bool ts_set_allocator(TSAllocator *new_alloc) { - TSAllocator *old = ts_allocator; - ts_allocator = new_alloc; - return old; + static bool using_default_allocator = true; + if (!using_default_allocator) { + fprintf(stderr, "tree-sitter's allocator can only be set once!"); + return false; + } + using_default_allocator = false; + ts_allocator = new_alloc; + return true; } From 8e4d4ef8b9257d1293c42c4dbb07151d1df6d614 Mon Sep 17 00:00:00 2001 From: Mingkai Dong Date: Fri, 24 Dec 2021 09:28:23 +0800 Subject: [PATCH 6/6] Replace allocator struct with function pointers --- lib/include/tree_sitter/api.h | 24 +++++++++------------ lib/src/alloc.c | 40 ++++++++++++++--------------------- lib/src/alloc.h | 13 +++++++----- 3 files changed, 34 insertions(+), 43 deletions(-) diff --git a/lib/include/tree_sitter/api.h b/lib/include/tree_sitter/api.h index 0b64e60e..4d558a5c 100644 --- a/lib/include/tree_sitter/api.h +++ b/lib/include/tree_sitter/api.h @@ -134,13 +134,6 @@ typedef enum { TSQueryErrorLanguage, } TSQueryError; -typedef struct { - void *(*malloc)(size_t); - void *(*calloc)(size_t, size_t); - void *(*realloc)(void *, size_t); - void (*free)(void *); -} TSAllocator; - /********************/ /* Section - Parser */ /********************/ @@ -148,15 +141,18 @@ typedef struct { /** * Switch to a new allocator. * - * Returns true iff the switch successes. - * - * This function can only be invoked *once*, before tree-sitter has allocated - * any memory via malloc/calloc/realloc. Otherwise, memory bugs could occur - * since some memory is allocated by the old allocator, but freed by the new - * one. + * This function can be invoked more than once. However, the application needs + * to pay attention to the memory allocated by the old allocator but might be + * freed by the new one. * + * Specifically, the application either, + * 1. ensures all parsers and trees are freed before calling it; + * 2. provides an allocator that shares its state with the old allocator. */ -bool ts_set_allocator(TSAllocator *new_alloc); +void ts_set_allocator(void *(*new_malloc)(size_t), + void *(*new_calloc)(size_t, size_t), + void *(*new_realloc)(void *, size_t), + void (*new_free)(void *)); /** * Create a new parser. diff --git a/lib/src/alloc.c b/lib/src/alloc.c index 65c72f11..9c0458ff 100644 --- a/lib/src/alloc.c +++ b/lib/src/alloc.c @@ -8,14 +8,10 @@ void *ts_record_realloc(void *, size_t); void ts_record_free(void *); bool ts_toggle_allocation_recording(bool); -static TSAllocator ts_tracking_allocator = { - .malloc = ts_record_malloc, - .calloc = ts_record_calloc, - .realloc = ts_record_realloc, - .free = ts_record_free, -}; - -TSAllocator *ts_allocator = &ts_tracking_allocator; +void *(*ts_current_malloc)(size_t) = ts_record_malloc; +void *(*ts_current_calloc)(size_t, size_t) = ts_record_calloc; +void *(*ts_current_realloc)(void *, size_t) = ts_record_realloc; +void (*ts_current_free)(void *) = ts_record_free; #else @@ -58,27 +54,23 @@ static inline void ts_free_default(void *buffer) { free(buffer); } -static TSAllocator ts_default_allocator = { - .malloc = ts_malloc_default, - .calloc = ts_calloc_default, - .realloc = ts_realloc_default, - .free = ts_free_default, -}; // Allow clients to override allocation functions dynamically -TSAllocator *ts_allocator = &ts_default_allocator; +void *(*ts_current_malloc)(size_t) = ts_malloc_default; +void *(*ts_current_calloc)(size_t, size_t) = ts_calloc_default; +void *(*ts_current_realloc)(void *, size_t) = ts_realloc_default; +void (*ts_current_free)(void *) = ts_free_default; #endif // defined(TREE_SITTER_ALLOCATION_TRACKING) -bool ts_set_allocator(TSAllocator *new_alloc) +void ts_set_allocator(void *(*new_malloc)(size_t), + void *(*new_calloc)(size_t, size_t), + void *(*new_realloc)(void *, size_t), + void (*new_free)(void *)) { - static bool using_default_allocator = true; - if (!using_default_allocator) { - fprintf(stderr, "tree-sitter's allocator can only be set once!"); - return false; - } - using_default_allocator = false; - ts_allocator = new_alloc; - return true; + ts_current_malloc = new_malloc; + ts_current_calloc = new_calloc; + ts_current_realloc = new_realloc; + ts_current_free = new_free; } diff --git a/lib/src/alloc.h b/lib/src/alloc.h index 6c04ae18..faf7c4ef 100644 --- a/lib/src/alloc.h +++ b/lib/src/alloc.h @@ -11,20 +11,23 @@ extern "C" { #include #include -extern TSAllocator *ts_allocator; +extern void *(*ts_current_malloc)(size_t); +extern void *(*ts_current_calloc)(size_t, size_t); +extern void *(*ts_current_realloc)(void *, size_t); +extern void (*ts_current_free)(void *); // Allow clients to override allocation functions #ifndef ts_malloc -#define ts_malloc ts_allocator->malloc +#define ts_malloc ts_current_malloc #endif #ifndef ts_calloc -#define ts_calloc ts_allocator->calloc +#define ts_calloc ts_current_calloc #endif #ifndef ts_realloc -#define ts_realloc ts_allocator->realloc +#define ts_realloc ts_current_realloc #endif #ifndef ts_free -#define ts_free ts_allocator->free +#define ts_free ts_current_free #endif bool ts_toggle_allocation_recording(bool);