Merge pull request #708 from tree-sitter/add-tagger-error-detection

Add ts_tags_buffer_found_parse_error capabilities for error detection during tagging.

cli/src/tags.rs

@@ -53,7 +53,7 @@ pub fn generate_tags(
 
             let source = fs::read(path)?;
             let t0 = Instant::now();
-            for tag in context.generate_tags(tags_config, &source, Some(&cancellation_flag))? {
+            for tag in context.generate_tags(tags_config, &source, Some(&cancellation_flag))?.0 {
                 let tag = tag?;
                 if !quiet {
                     write!(

cli/src/tests/tags_test.rs

@@ -102,6 +102,7 @@ fn test_tags_python() {
     let tags = tag_context
         .generate_tags(&tags_config, source, None)
         .unwrap()
+        .0
         .collect::<Result<Vec<_>, _>>()
         .unwrap();
 
@@ -153,6 +154,7 @@ fn test_tags_javascript() {
     let tags = tag_context
         .generate_tags(&tags_config, source, None)
         .unwrap()
+        .0
         .collect::<Result<Vec<_>, _>>()
         .unwrap();
 
@@ -189,6 +191,7 @@ fn test_tags_columns_measured_in_utf16_code_units() {
     let tag = tag_context
         .generate_tags(&tags_config, source, None)
         .unwrap()
+        .0
         .next()
         .unwrap()
         .unwrap();
@@ -229,6 +232,7 @@ fn test_tags_ruby() {
     let tags = tag_context
         .generate_tags(&tags_config, source.as_bytes(), None)
         .unwrap()
+        .0
         .collect::<Result<Vec<_>, _>>()
         .unwrap();
 
@@ -271,7 +275,7 @@ fn test_tags_cancellation() {
             .generate_tags(&tags_config, source.as_bytes(), Some(&cancellation_flag))
             .unwrap();
 
-        for (i, tag) in tags.enumerate() {
+        for (i, tag) in tags.0.enumerate() {
             if i == 150 {
                 cancellation_flag.store(1, Ordering::SeqCst);
             }
@@ -293,6 +297,39 @@ fn test_invalid_capture() {
     assert_eq!(e, Error::InvalidCapture("method".to_string()));
 }
 
+#[test]
+fn test_tags_with_parse_error() {
+    let language = get_language("python");
+    let tags_config = TagsConfiguration::new(language, PYTHON_TAG_QUERY, "").unwrap();
+    let mut tag_context = TagsContext::new();
+
+    let source = br#"
+    class Fine: pass
+    class Bad
+    "#;
+
+    let (tags, failed) = tag_context
+        .generate_tags(&tags_config, source, None)
+        .unwrap();
+
+    let newtags = tags.collect::<Result<Vec<_>, _>>().unwrap();
+
+    assert!(failed, "syntax error should have been detected");
+
+    assert_eq!(
+        newtags.iter()
+            .map(|t| (
+                substr(source, &t.name_range),
+                tags_config.syntax_type_name(t.syntax_type_id)
+            ))
+            .collect::<Vec<_>>(),
+        &[
+            ("Fine", "class"),
+        ]
+    );
+}
+
+
 #[test]
 fn test_tags_via_c_api() {
     allocations::record(|| {

tags/include/tree_sitter/tags.h

@@ -88,6 +88,9 @@ uint32_t ts_tags_buffer_docs_len(const TSTagsBuffer *);
 // Get the syntax kinds for a scope.
 const char **ts_tagger_syntax_kinds_for_scope_name(const TSTagger *, const char *scope_name, uint32_t *len);
 
+// Determine whether a parse error was encountered while tagging.
+bool ts_tags_buffer_found_parse_error(const TSTagsBuffer*);
+
 #ifdef __cplusplus
 }
 #endif

tags/src/c_lib.rs

@@ -55,6 +55,7 @@ pub struct TSTagsBuffer {
     context: TagsContext,
     tags: Vec<TSTag>,
     docs: Vec<u8>,
+    errors_present: bool,
 }
 
 #[no_mangle]
@@ -129,7 +130,10 @@ pub extern "C" fn ts_tagger_tag(
             .context
             .generate_tags(config, source_code, cancellation_flag)
         {
-            Ok(tags) => tags,
+            Ok((tags, found_error)) => {
+                buffer.errors_present = found_error;
+                tags
+            }
             Err(e) => {
                 return match e {
                     Error::InvalidLanguage => TSTagsError::InvalidLanguage,
@@ -188,6 +192,7 @@ pub extern "C" fn ts_tags_buffer_new() -> *mut TSTagsBuffer {
         context: TagsContext::new(),
         tags: Vec::with_capacity(BUFFER_TAGS_RESERVE_CAPACITY),
         docs: Vec::with_capacity(BUFFER_DOCS_RESERVE_CAPACITY),
+        errors_present: false,
     }))
 }
 
@@ -220,6 +225,12 @@ pub extern "C" fn ts_tags_buffer_docs_len(this: *const TSTagsBuffer) -> u32 {
     buffer.docs.len() as u32
 }
 
+#[no_mangle]
+pub extern "C" fn ts_tags_buffer_found_parse_error(this: *const TSTagsBuffer) -> bool {
+    let buffer = unwrap_ptr(this);
+    buffer.errors_present
+}
+
 #[no_mangle]
 pub extern "C" fn ts_tagger_syntax_kinds_for_scope_name(
     this: *mut TSTagger,

tags/src/lib.rs

@@ -255,7 +255,7 @@ impl TagsContext {
         config: &'a TagsConfiguration,
         source: &'a [u8],
         cancellation_flag: Option<&'a AtomicUsize>,
-    ) -> Result<impl Iterator<Item = Result<Tag, Error>> + 'a, Error> {
+    ) -> Result<(impl Iterator<Item = Result<Tag, Error>> + 'a, bool), Error> {
         self.parser
             .set_language(config.language)
             .map_err(|_| Error::InvalidLanguage)?;
@@ -271,7 +271,7 @@ impl TagsContext {
             .matches(&config.query, tree_ref.root_node(), move |node| {
                 &source[node.byte_range()]
             });
-        Ok(TagsIter {
+        Ok((TagsIter {
             _tree: tree,
             matches,
             source,
@@ -285,7 +285,7 @@ impl TagsContext {
                 inherits: false,
                 local_defs: Vec::new(),
             }],
-        })
+        }, tree_ref.root_node().has_error()))
     }
 }