highlight: add built-in support for carriage-return highlight

cli/src/tests/highlight_test.rs

@@ -4,7 +4,7 @@ use std::ffi::CString;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::{fs, ptr, slice, str};
 use tree_sitter_highlight::{
-    c, Error, HighlightConfiguration, HighlightEvent, Highlighter, HtmlRenderer,
+    c, Error, Highlight, HighlightConfiguration, HighlightEvent, Highlighter, HtmlRenderer,
 };
 
 lazy_static! {
@@ -23,6 +23,7 @@ lazy_static! {
         get_highlight_config("rust", Some("injections.scm"), &HIGHLIGHT_NAMES);
     static ref HIGHLIGHT_NAMES: Vec<String> = [
         "attribute",
+        "carriage-return",
         "comment",
         "constant",
         "constructor",
@@ -322,6 +323,19 @@ fn test_highlighting_empty_lines() {
     );
 }
 
+#[test]
+fn test_highlighting_carriage_returns() {
+    let source = "a = \"a\rb\"\r\nb\r";
+
+    assert_eq!(
+        &to_html(&source, &JS_HIGHLIGHT).unwrap(),
+        &[
+            "<span class=variable>a</span> <span class=operator>=</span> <span class=string>&quot;a<span class=carriage-return></span>b&quot;</span>\n",
+            "<span class=variable>b</span>\n",
+        ],
+    );
+}
+
 #[test]
 fn test_highlighting_ejs_with_html_and_javascript() {
     let source = vec!["<div><% foo() %></div><script> bar() </script>"].join("\n");
@@ -617,6 +631,12 @@ fn to_html<'a>(
         &test_language_for_injection_string,
     )?;
 
+    renderer.set_carriage_return_highlight(
+        HIGHLIGHT_NAMES
+            .iter()
+            .position(|s| s == "carriage-return")
+            .map(Highlight),
+    );
     renderer
         .render(events, src, &|highlight| HTML_ATTRS[highlight.0].as_bytes())
         .unwrap();

highlight/src/c_lib.rs

@@ -1,4 +1,4 @@
-use super::{Error, HighlightConfiguration, Highlighter, HtmlRenderer};
+use super::{Error, Highlight, HighlightConfiguration, Highlighter, HtmlRenderer};
 use regex::Regex;
 use std::collections::HashMap;
 use std::ffi::CStr;
@@ -12,6 +12,7 @@ pub struct TSHighlighter {
     languages: HashMap<String, (Option<Regex>, HighlightConfiguration)>,
     attribute_strings: Vec<&'static [u8]>,
     highlight_names: Vec<String>,
+    carriage_return_index: Option<usize>,
 }
 
 pub struct TSHighlightBuffer {
@@ -43,15 +44,17 @@ pub extern "C" fn ts_highlighter_new(
     let highlight_names = highlight_names
         .into_iter()
         .map(|s| unsafe { CStr::from_ptr(*s).to_string_lossy().to_string() })
-        .collect();
+        .collect::<Vec<_>>();
     let attribute_strings = attribute_strings
         .into_iter()
         .map(|s| unsafe { CStr::from_ptr(*s).to_bytes() })
         .collect();
+    let carriage_return_index = highlight_names.iter().position(|s| s == "carriage-return");
     Box::into_raw(Box::new(TSHighlighter {
         languages: HashMap::new(),
         attribute_strings,
         highlight_names,
+        carriage_return_index,
     }))
 }
 
@@ -215,6 +218,9 @@ impl TSHighlighter {
 
         if let Ok(highlights) = highlights {
             output.renderer.reset();
+            output
+                .renderer
+                .set_carriage_return_highlight(self.carriage_return_index.map(Highlight));
             let result = output
                 .renderer
                 .render(highlights, source_code, &|s| self.attribute_strings[s.0]);

highlight/src/lib.rs

@@ -64,6 +64,7 @@ pub struct Highlighter {
 pub struct HtmlRenderer {
     pub html: Vec<u8>,
     pub line_offsets: Vec<u32>,
+    carriage_return_highlight: Option<Highlight>,
 }
 
 #[derive(Debug)]
@@ -899,9 +900,14 @@ impl HtmlRenderer {
         HtmlRenderer {
             html: Vec::new(),
             line_offsets: vec![0],
+            carriage_return_highlight: None,
         }
     }
 
+    pub fn set_carriage_return_highlight(&mut self, highlight: Option<Highlight>) {
+        self.carriage_return_highlight = highlight;
+    }
+
     pub fn reset(&mut self) {
         self.html.clear();
         self.line_offsets.clear();
@@ -958,6 +964,20 @@ impl HtmlRenderer {
             })
     }
 
+    fn add_carriage_return<'a, F>(&mut self, attribute_callback: &F)
+    where
+        F: Fn(Highlight) -> &'a [u8],
+    {
+        if let Some(highlight) = self.carriage_return_highlight {
+            let attribute_string = (attribute_callback)(highlight);
+            if !attribute_string.is_empty() {
+                self.html.extend(b"<span ");
+                self.html.extend(attribute_string);
+                self.html.extend(b"></span>");
+            }
+        }
+    }
+
     fn start_highlight<'a, F>(&mut self, h: Highlight, attribute_callback: &F)
     where
         F: Fn(Highlight) -> &'a [u8],
@@ -979,11 +999,23 @@ impl HtmlRenderer {
     where
         F: Fn(Highlight) -> &'a [u8],
     {
+        let mut last_char_was_cr = false;
         for c in util::LossyUtf8::new(src).flat_map(|p| p.bytes()) {
-            if c == b'\n' {
-                if self.html.ends_with(b"\r") {
-                    self.html.pop();
+            // Don't render carriage return characters, but allow lone carriage returns (not
+            // followed by line feeds) to be styled via the attribute callback.
+            if c == b'\r' {
+                last_char_was_cr = true;
+                continue;
+            }
+            if last_char_was_cr {
+                if c != b'\n' {
+                    self.add_carriage_return(attribute_callback);
                 }
+                last_char_was_cr = false;
+            }
+
+            // At line boundaries, close and re-open all of the open tags.
+            if c == b'\n' {
                 highlights.iter().for_each(|_| self.end_highlight());
                 self.html.push(c);
                 self.line_offsets.push(self.html.len() as u32);