From 555277a102e15c8b4366346fa97b2421165f3068 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 23 Jan 2023 11:45:10 -0800 Subject: [PATCH] Allow testing highlight and tag queries when testing wasm build Replace non-mutating `ts_parser_wasm_store` function with `ts_parser_take_wasm_store`, which removes and returns the wasm store, in order to facilitate single ownership. --- cli/src/main.rs | 17 +++++++++++++++-- cli/src/test_highlight.rs | 9 ++++++--- cli/src/test_tags.rs | 5 ++--- lib/binding_rust/bindings.rs | 18 +++++++++++++++++- lib/binding_rust/wasm_language.rs | 13 ++++++++++--- lib/include/tree_sitter/api.h | 27 ++++++++++++++++++++++++++- lib/src/parser.c | 8 ++++++++ 7 files changed, 84 insertions(+), 13 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 2bb290eb..f6bb88a1 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -9,7 +9,9 @@ use tree_sitter_cli::{ util, wasm, }; use tree_sitter_config::Config; +use tree_sitter_highlight::Highlighter; use tree_sitter_loader as loader; +use tree_sitter_tags::TagsContext; const BUILD_VERSION: &'static str = env!("CARGO_PKG_VERSION"); const BUILD_SHA: Option<&'static str> = option_env!("BUILD_SHA"); @@ -350,18 +352,29 @@ fn run() -> Result<()> { )?; } + let mut store = parser.take_wasm_store(); + // Check that all of the queries are valid. test::check_queries_at_path(*language, ¤t_dir.join("queries"))?; // Run the syntax highlighting tests. let test_highlight_dir = test_dir.join("highlight"); if test_highlight_dir.is_dir() { - test_highlight::test_highlights(&loader, &test_highlight_dir)?; + let mut highlighter = Highlighter::new(); + if let Some(store) = store.take() { + highlighter.parser().set_wasm_store(store).unwrap(); + } + test_highlight::test_highlights(&loader, &mut highlighter, &test_highlight_dir)?; + store = highlighter.parser().take_wasm_store(); } let test_tag_dir = test_dir.join("tags"); if test_tag_dir.is_dir() { - test_tags::test_tags(&loader, &test_tag_dir)?; + let mut tags_context = TagsContext::new(); + if let Some(store) = store.take() { + tags_context.parser().set_wasm_store(store).unwrap(); + } + test_tags::test_tags(&loader, &mut tags_context, &test_tag_dir)?; } } diff --git a/cli/src/test_highlight.rs b/cli/src/test_highlight.rs index 2d9d536a..5411fcb1 100644 --- a/cli/src/test_highlight.rs +++ b/cli/src/test_highlight.rs @@ -38,9 +38,12 @@ impl std::fmt::Display for Failure { } } -pub fn test_highlights(loader: &Loader, directory: &Path) -> Result<()> { +pub fn test_highlights( + loader: &Loader, + highlighter: &mut Highlighter, + directory: &Path, +) -> Result<()> { let mut failed = false; - let mut highlighter = Highlighter::new(); println!("syntax highlighting:"); for highlight_test_file in fs::read_dir(directory)? { @@ -55,7 +58,7 @@ pub fn test_highlights(loader: &Loader, directory: &Path) -> Result<()> { .ok_or_else(|| anyhow!("No highlighting config found for {:?}", test_file_path))?; match test_highlight( &loader, - &mut highlighter, + highlighter, highlight_config, fs::read(&test_file_path)?.as_slice(), ) { diff --git a/cli/src/test_tags.rs b/cli/src/test_tags.rs index 024d094c..28694db9 100644 --- a/cli/src/test_tags.rs +++ b/cli/src/test_tags.rs @@ -38,9 +38,8 @@ impl std::fmt::Display for Failure { } } -pub fn test_tags(loader: &Loader, directory: &Path) -> Result<()> { +pub fn test_tags(loader: &Loader, tags_context: &mut TagsContext, directory: &Path) -> Result<()> { let mut failed = false; - let mut tags_context = TagsContext::new(); println!("tags:"); for tag_test_file in fs::read_dir(directory)? { @@ -54,7 +53,7 @@ pub fn test_tags(loader: &Loader, directory: &Path) -> Result<()> { .tags_config(language)? .ok_or_else(|| anyhow!("No tags config found for {:?}", test_file_path))?; match test_tag( - &mut tags_context, + tags_context, tags_config, fs::read(&test_file_path)?.as_slice(), ) { diff --git a/lib/binding_rust/bindings.rs b/lib/binding_rust/bindings.rs index 464aa8d7..be155be1 100644 --- a/lib/binding_rust/bindings.rs +++ b/lib/binding_rust/bindings.rs @@ -346,6 +346,9 @@ extern "C" { pub fn ts_tree_language(arg1: *const TSTree) -> *const TSLanguage; } extern "C" { + #[doc = " Get the array of included ranges that was used to parse the syntax tree."] + #[doc = ""] + #[doc = " The returned pointer must be freed by the caller."] pub fn ts_tree_included_ranges(arg1: *const TSTree, length: *mut u32) -> *mut TSRange; } extern "C" { @@ -865,12 +868,19 @@ pub struct TSWasmStore { _unused: [u8; 0], } extern "C" { + #[doc = " Create a Wasm store."] pub fn ts_wasm_store_new(engine: *mut TSWasmEngine) -> *mut TSWasmStore; } extern "C" { + #[doc = " Free the memory associated with the given Wasm store."] pub fn ts_wasm_store_delete(arg1: *mut TSWasmStore); } extern "C" { + #[doc = " Create a language from a buffer of Wasm. The resulting language behaves"] + #[doc = " like any other Tree-sitter language, except that in order to use it with"] + #[doc = " a parser, that parser must have a Wasm store. Note that the language"] + #[doc = " can be used with any Wasm store, it doesn't need to be the same store that"] + #[doc = " was used to originally load it."] pub fn ts_wasm_store_load_language( arg1: *mut TSWasmStore, name: *const ::std::os::raw::c_char, @@ -879,13 +889,19 @@ extern "C" { ) -> *const TSLanguage; } extern "C" { + #[doc = " Check if the language came from a Wasm module. If so, then in order to use"] + #[doc = " this langauge with a Parser, that parser must have a Wasm store assigned."] pub fn ts_language_is_wasm(arg1: *const TSLanguage) -> bool; } extern "C" { + #[doc = " Assign the given Wasm store to the parser. A parser must have a Wasm store"] + #[doc = " in order to use Wasm languages."] pub fn ts_parser_set_wasm_store(arg1: *mut TSParser, arg2: *mut TSWasmStore); } extern "C" { - pub fn ts_parser_wasm_store(arg1: *mut TSParser) -> *mut TSWasmStore; + #[doc = " Remove the parser's current Wasm store and return it. This returns NULL if"] + #[doc = " the parser doesn't have a Wasm store."] + pub fn ts_parser_take_wasm_store(arg1: *mut TSParser) -> *mut TSWasmStore; } extern "C" { #[doc = " Set the allocation functions used by the library."] diff --git a/lib/binding_rust/wasm_language.rs b/lib/binding_rust/wasm_language.rs index 1e7f21d7..653cd8cc 100644 --- a/lib/binding_rust/wasm_language.rs +++ b/lib/binding_rust/wasm_language.rs @@ -48,12 +48,19 @@ impl Language { impl Parser { pub fn set_wasm_store(&mut self, store: WasmStore) -> Result<(), LanguageError> { - unsafe { - ffi::ts_parser_set_wasm_store(self.0.as_ptr(), store.0); - } + unsafe { ffi::ts_parser_set_wasm_store(self.0.as_ptr(), store.0) }; mem::forget(store); Ok(()) } + + pub fn take_wasm_store(&mut self) -> Option { + let ptr = unsafe { ffi::ts_parser_take_wasm_store(self.0.as_ptr()) }; + if ptr.is_null() { + None + } else { + Some(WasmStore(ptr)) + } + } } impl Drop for WasmStore { diff --git a/lib/include/tree_sitter/api.h b/lib/include/tree_sitter/api.h index 6824f032..0944adc9 100644 --- a/lib/include/tree_sitter/api.h +++ b/lib/include/tree_sitter/api.h @@ -944,10 +944,23 @@ uint32_t ts_language_version(const TSLanguage *); typedef struct wasm_engine_t TSWasmEngine; typedef struct TSWasmStore TSWasmStore; +/** + * Create a Wasm store. + */ TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine); +/** + * Free the memory associated with the given Wasm store. + */ void ts_wasm_store_delete(TSWasmStore *); +/** + * Create a language from a buffer of Wasm. The resulting language behaves + * like any other Tree-sitter language, except that in order to use it with + * a parser, that parser must have a Wasm store. Note that the language + * can be used with any Wasm store, it doesn't need to be the same store that + * was used to originally load it. + */ const TSLanguage *ts_wasm_store_load_language( TSWasmStore *, const char *name, @@ -955,11 +968,23 @@ const TSLanguage *ts_wasm_store_load_language( uint32_t wasm_len ); +/** + * Check if the language came from a Wasm module. If so, then in order to use + * this langauge with a Parser, that parser must have a Wasm store assigned. + */ bool ts_language_is_wasm(const TSLanguage *); +/** + * Assign the given Wasm store to the parser. A parser must have a Wasm store + * in order to use Wasm languages. + */ void ts_parser_set_wasm_store(TSParser *, TSWasmStore *); -TSWasmStore *ts_parser_wasm_store(TSParser *); +/** + * Remove the parser's current Wasm store and return it. This returns NULL if + * the parser doesn't have a Wasm store. + */ +TSWasmStore *ts_parser_take_wasm_store(TSParser *); /**********************************/ /* Section - Global Configuration */ diff --git a/lib/src/parser.c b/lib/src/parser.c index 7e0448ee..19ebd94d 100644 --- a/lib/src/parser.c +++ b/lib/src/parser.c @@ -1850,6 +1850,7 @@ void ts_parser_delete(TSParser *self) { ts_subtree_release(&self->tree_pool, self->old_tree); self->old_tree = NULL_SUBTREE; } + ts_wasm_store_delete(self->wasm_store); ts_lexer_delete(&self->lexer); ts_parser__set_cached_token(self, 0, NULL_SUBTREE, NULL_SUBTREE); ts_subtree_pool_delete(&self->tree_pool); @@ -2092,7 +2093,14 @@ TSTree *ts_parser_parse_string_encoding( } void ts_parser_set_wasm_store(TSParser *self, TSWasmStore *store) { + ts_wasm_store_delete(self->wasm_store); self->wasm_store = store; } +TSWasmStore *ts_parser_take_wasm_store(TSParser *self) { + TSWasmStore *result = self->wasm_store; + self->wasm_store = NULL; + return result; +} + #undef LOG