From 1a571ae20877c7bfac1fa59f0cc38027fe669685 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Tue, 4 Aug 2020 17:53:47 -0400 Subject: [PATCH 1/8] Add errors_present field to tagging context. --- tags/src/c_lib.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tags/src/c_lib.rs b/tags/src/c_lib.rs index 07e1e19a..b93c69a2 100644 --- a/tags/src/c_lib.rs +++ b/tags/src/c_lib.rs @@ -52,6 +52,7 @@ pub struct TSTagsBuffer { context: TagsContext, tags: Vec, docs: Vec, + errors_present: bool, } #[no_mangle] @@ -184,6 +185,7 @@ pub extern "C" fn ts_tags_buffer_new() -> *mut TSTagsBuffer { context: TagsContext::new(), tags: Vec::with_capacity(64), docs: Vec::with_capacity(64), + errors_present: false, })) } @@ -216,6 +218,12 @@ pub extern "C" fn ts_tags_buffer_docs_len(this: *const TSTagsBuffer) -> u32 { buffer.docs.len() as u32 } +#[no_mangle] +pub extern "C" fn ts_tagger_errors_present(this: *const TSTagsBuffer) -> bool { + let buffer = unwrap_ptr(this); + buffer.errors_present +} + #[no_mangle] pub extern "C" fn ts_tagger_syntax_kinds_for_scope_name( this: *mut TSTagger, From 5a52dc2cd700170196753481db1e8aa261e50d50 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Wed, 5 Aug 2020 11:18:59 -0400 Subject: [PATCH 2/8] Return an iterator-bool tuple instead of just an iterator. --- cli/src/tags.rs | 3 ++- tags/src/c_lib.rs | 5 ++++- tags/src/lib.rs | 6 +++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/cli/src/tags.rs b/cli/src/tags.rs index 5ea00f39..5e999693 100644 --- a/cli/src/tags.rs +++ b/cli/src/tags.rs @@ -53,7 +53,8 @@ pub fn generate_tags( let source = fs::read(path)?; let t0 = Instant::now(); - for tag in context.generate_tags(tags_config, &source, Some(&cancellation_flag))? { + let (tagged, _) = context.generate_tags(tags_config, &source, Some(&cancellation_flag))?; + for tag in tagged { let tag = tag?; if !quiet { write!( diff --git a/tags/src/c_lib.rs b/tags/src/c_lib.rs index b93c69a2..84f8c97b 100644 --- a/tags/src/c_lib.rs +++ b/tags/src/c_lib.rs @@ -126,7 +126,10 @@ pub extern "C" fn ts_tagger_tag( .context .generate_tags(config, source_code, cancellation_flag) { - Ok(tags) => tags, + Ok((tags, found_error)) => { + buffer.errors_present = found_error; + tags + } Err(e) => { return match e { Error::InvalidLanguage => TSTagsError::InvalidLanguage, diff --git a/tags/src/lib.rs b/tags/src/lib.rs index c247c13e..dd55d4be 100644 --- a/tags/src/lib.rs +++ b/tags/src/lib.rs @@ -255,7 +255,7 @@ impl TagsContext { config: &'a TagsConfiguration, source: &'a [u8], cancellation_flag: Option<&'a AtomicUsize>, - ) -> Result> + 'a, Error> { + ) -> Result<(impl Iterator> + 'a, bool), Error> { self.parser .set_language(config.language) .map_err(|_| Error::InvalidLanguage)?; @@ -271,7 +271,7 @@ impl TagsContext { .matches(&config.query, tree_ref.root_node(), move |node| { &source[node.byte_range()] }); - Ok(TagsIter { + Ok((TagsIter { _tree: tree, matches, source, @@ -285,7 +285,7 @@ impl TagsContext { inherits: false, local_defs: Vec::new(), }], - }) + }, tree_ref.root_node().has_error())) } } From f4108056b0b5be57441493a279cb22fc3fd95829 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Wed, 5 Aug 2020 11:33:04 -0400 Subject: [PATCH 3/8] Remove otiose pattern match. --- cli/src/tags.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cli/src/tags.rs b/cli/src/tags.rs index 5e999693..122b58d2 100644 --- a/cli/src/tags.rs +++ b/cli/src/tags.rs @@ -53,8 +53,7 @@ pub fn generate_tags( let source = fs::read(path)?; let t0 = Instant::now(); - let (tagged, _) = context.generate_tags(tags_config, &source, Some(&cancellation_flag))?; - for tag in tagged { + for tag in context.generate_tags(tags_config, &source, Some(&cancellation_flag))?.0 { let tag = tag?; if !quiet { write!( From 5c86a9c654b7f2be39f55039ad114f277aa64a64 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Wed, 5 Aug 2020 11:52:07 -0400 Subject: [PATCH 4/8] Fix the tests --- cli/src/tests/tags_test.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cli/src/tests/tags_test.rs b/cli/src/tests/tags_test.rs index 3ff1c92b..88e57ec1 100644 --- a/cli/src/tests/tags_test.rs +++ b/cli/src/tests/tags_test.rs @@ -102,6 +102,7 @@ fn test_tags_python() { let tags = tag_context .generate_tags(&tags_config, source, None) .unwrap() + .0 .collect::, _>>() .unwrap(); @@ -153,6 +154,7 @@ fn test_tags_javascript() { let tags = tag_context .generate_tags(&tags_config, source, None) .unwrap() + .0 .collect::, _>>() .unwrap(); @@ -189,6 +191,7 @@ fn test_tags_columns_measured_in_utf16_code_units() { let tag = tag_context .generate_tags(&tags_config, source, None) .unwrap() + .0 .next() .unwrap() .unwrap(); @@ -229,6 +232,7 @@ fn test_tags_ruby() { let tags = tag_context .generate_tags(&tags_config, source.as_bytes(), None) .unwrap() + .0 .collect::, _>>() .unwrap(); @@ -271,7 +275,7 @@ fn test_tags_cancellation() { .generate_tags(&tags_config, source.as_bytes(), Some(&cancellation_flag)) .unwrap(); - for (i, tag) in tags.enumerate() { + for (i, tag) in tags.0.enumerate() { if i == 150 { cancellation_flag.store(1, Ordering::SeqCst); } From 94ab884ee4d0b965c8c16212979e15927976f068 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Wed, 5 Aug 2020 12:16:09 -0400 Subject: [PATCH 5/8] Add a test. --- cli/src/tests/tags_test.rs | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/cli/src/tests/tags_test.rs b/cli/src/tests/tags_test.rs index 88e57ec1..2b058c0b 100644 --- a/cli/src/tests/tags_test.rs +++ b/cli/src/tests/tags_test.rs @@ -297,6 +297,39 @@ fn test_invalid_capture() { assert_eq!(e, Error::InvalidCapture("method".to_string())); } +#[test] +fn test_tags_with_parse_error() { + let language = get_language("python"); + let tags_config = TagsConfiguration::new(language, PYTHON_TAG_QUERY, "").unwrap(); + let mut tag_context = TagsContext::new(); + + let source = br#" + class Fine: pass + class Bad + "#; + + let (tags, failed) = tag_context + .generate_tags(&tags_config, source, None) + .unwrap(); + + let newtags = tags.collect::, _>>().unwrap(); + + assert!(failed, "syntax error should have been detected"); + + assert_eq!( + newtags.iter() + .map(|t| ( + substr(source, &t.name_range), + tags_config.syntax_type_name(t.syntax_type_id) + )) + .collect::>(), + &[ + ("Fine", "class"), + ] + ); +} + + #[test] fn test_tags_via_c_api() { allocations::record(|| { From 7576b0b4485343902f54ab1dbe0464dd7ef4f920 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Wed, 5 Aug 2020 12:21:42 -0400 Subject: [PATCH 6/8] Add accessor to the C header. --- tags/include/tree_sitter/tags.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tags/include/tree_sitter/tags.h b/tags/include/tree_sitter/tags.h index f2b17075..42109bee 100644 --- a/tags/include/tree_sitter/tags.h +++ b/tags/include/tree_sitter/tags.h @@ -88,6 +88,9 @@ uint32_t ts_tags_buffer_docs_len(const TSTagsBuffer *); // Get the syntax kinds for a scope. const char **ts_tagger_syntax_kinds_for_scope_name(const TSTagger *, const char *scope_name, uint32_t *len); +// Determine whether a parse error was encountered while tagging. +bool ts_tagger_errors_present(); + #ifdef __cplusplus } #endif From ec6af791af5761130238134e935ad6236aeb151c Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Wed, 5 Aug 2020 12:24:39 -0400 Subject: [PATCH 7/8] Bikeshed this name a little bit. --- tags/include/tree_sitter/tags.h | 2 +- tags/src/c_lib.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tags/include/tree_sitter/tags.h b/tags/include/tree_sitter/tags.h index 42109bee..773113d7 100644 --- a/tags/include/tree_sitter/tags.h +++ b/tags/include/tree_sitter/tags.h @@ -89,7 +89,7 @@ uint32_t ts_tags_buffer_docs_len(const TSTagsBuffer *); const char **ts_tagger_syntax_kinds_for_scope_name(const TSTagger *, const char *scope_name, uint32_t *len); // Determine whether a parse error was encountered while tagging. -bool ts_tagger_errors_present(); +bool ts_tags_buffer_found_parse_error(); #ifdef __cplusplus } diff --git a/tags/src/c_lib.rs b/tags/src/c_lib.rs index 84f8c97b..8cb5abb4 100644 --- a/tags/src/c_lib.rs +++ b/tags/src/c_lib.rs @@ -222,7 +222,7 @@ pub extern "C" fn ts_tags_buffer_docs_len(this: *const TSTagsBuffer) -> u32 { } #[no_mangle] -pub extern "C" fn ts_tagger_errors_present(this: *const TSTagsBuffer) -> bool { +pub extern "C" fn ts_tags_buffer_found_parse_error(this: *const TSTagsBuffer) -> bool { let buffer = unwrap_ptr(this); buffer.errors_present } From 8d58a0d33a070af73dd6548d8000e0e7ddd04331 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Wed, 5 Aug 2020 13:10:02 -0400 Subject: [PATCH 8/8] Add parameter in the header. --- tags/include/tree_sitter/tags.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tags/include/tree_sitter/tags.h b/tags/include/tree_sitter/tags.h index 773113d7..4784abbb 100644 --- a/tags/include/tree_sitter/tags.h +++ b/tags/include/tree_sitter/tags.h @@ -89,7 +89,7 @@ uint32_t ts_tags_buffer_docs_len(const TSTagsBuffer *); const char **ts_tagger_syntax_kinds_for_scope_name(const TSTagger *, const char *scope_name, uint32_t *len); // Determine whether a parse error was encountered while tagging. -bool ts_tags_buffer_found_parse_error(); +bool ts_tags_buffer_found_parse_error(const TSTagsBuffer*); #ifdef __cplusplus }