diff --git a/cli/src/tests/highlight_test.rs b/cli/src/tests/highlight_test.rs index 652ace98..1f7106dd 100644 --- a/cli/src/tests/highlight_test.rs +++ b/cli/src/tests/highlight_test.rs @@ -4,7 +4,7 @@ use std::ffi::CString; use std::sync::atomic::{AtomicUsize, Ordering}; use std::{fs, ptr, slice, str}; use tree_sitter_highlight::{ - c, Error, HighlightConfiguration, HighlightEvent, Highlighter, HtmlRenderer, + c, Error, Highlight, HighlightConfiguration, HighlightEvent, Highlighter, HtmlRenderer, }; lazy_static! { @@ -23,6 +23,7 @@ lazy_static! { get_highlight_config("rust", Some("injections.scm"), &HIGHLIGHT_NAMES); static ref HIGHLIGHT_NAMES: Vec = [ "attribute", + "carriage-return", "comment", "constant", "constructor", @@ -322,6 +323,19 @@ fn test_highlighting_empty_lines() { ); } +#[test] +fn test_highlighting_carriage_returns() { + let source = "a = \"a\rb\"\r\nb\r"; + + assert_eq!( + &to_html(&source, &JS_HIGHLIGHT).unwrap(), + &[ + "a = "ab"\n", + "b\n", + ], + ); +} + #[test] fn test_highlighting_ejs_with_html_and_javascript() { let source = vec!["
<% foo() %>
"].join("\n"); @@ -617,6 +631,12 @@ fn to_html<'a>( &test_language_for_injection_string, )?; + renderer.set_carriage_return_highlight( + HIGHLIGHT_NAMES + .iter() + .position(|s| s == "carriage-return") + .map(Highlight), + ); renderer .render(events, src, &|highlight| HTML_ATTRS[highlight.0].as_bytes()) .unwrap(); diff --git a/highlight/src/c_lib.rs b/highlight/src/c_lib.rs index d96e246f..334b8db0 100644 --- a/highlight/src/c_lib.rs +++ b/highlight/src/c_lib.rs @@ -1,4 +1,4 @@ -use super::{Error, HighlightConfiguration, Highlighter, HtmlRenderer}; +use super::{Error, Highlight, HighlightConfiguration, Highlighter, HtmlRenderer}; use regex::Regex; use std::collections::HashMap; use std::ffi::CStr; @@ -12,6 +12,7 @@ pub struct TSHighlighter { languages: HashMap, HighlightConfiguration)>, attribute_strings: Vec<&'static [u8]>, highlight_names: Vec, + carriage_return_index: Option, } pub struct TSHighlightBuffer { @@ -43,15 +44,17 @@ pub extern "C" fn ts_highlighter_new( let highlight_names = highlight_names .into_iter() .map(|s| unsafe { CStr::from_ptr(*s).to_string_lossy().to_string() }) - .collect(); + .collect::>(); let attribute_strings = attribute_strings .into_iter() .map(|s| unsafe { CStr::from_ptr(*s).to_bytes() }) .collect(); + let carriage_return_index = highlight_names.iter().position(|s| s == "carriage-return"); Box::into_raw(Box::new(TSHighlighter { languages: HashMap::new(), attribute_strings, highlight_names, + carriage_return_index, })) } @@ -215,6 +218,9 @@ impl TSHighlighter { if let Ok(highlights) = highlights { output.renderer.reset(); + output + .renderer + .set_carriage_return_highlight(self.carriage_return_index.map(Highlight)); let result = output .renderer .render(highlights, source_code, &|s| self.attribute_strings[s.0]); diff --git a/highlight/src/lib.rs b/highlight/src/lib.rs index 31bca7e0..6f1b7bbd 100644 --- a/highlight/src/lib.rs +++ b/highlight/src/lib.rs @@ -64,6 +64,7 @@ pub struct Highlighter { pub struct HtmlRenderer { pub html: Vec, pub line_offsets: Vec, + carriage_return_highlight: Option, } #[derive(Debug)] @@ -899,9 +900,14 @@ impl HtmlRenderer { HtmlRenderer { html: Vec::new(), line_offsets: vec![0], + carriage_return_highlight: None, } } + pub fn set_carriage_return_highlight(&mut self, highlight: Option) { + self.carriage_return_highlight = highlight; + } + pub fn reset(&mut self) { self.html.clear(); self.line_offsets.clear(); @@ -958,6 +964,20 @@ impl HtmlRenderer { }) } + fn add_carriage_return<'a, F>(&mut self, attribute_callback: &F) + where + F: Fn(Highlight) -> &'a [u8], + { + if let Some(highlight) = self.carriage_return_highlight { + let attribute_string = (attribute_callback)(highlight); + if !attribute_string.is_empty() { + self.html.extend(b""); + } + } + } + fn start_highlight<'a, F>(&mut self, h: Highlight, attribute_callback: &F) where F: Fn(Highlight) -> &'a [u8], @@ -979,11 +999,23 @@ impl HtmlRenderer { where F: Fn(Highlight) -> &'a [u8], { + let mut last_char_was_cr = false; for c in util::LossyUtf8::new(src).flat_map(|p| p.bytes()) { - if c == b'\n' { - if self.html.ends_with(b"\r") { - self.html.pop(); + // Don't render carriage return characters, but allow lone carriage returns (not + // followed by line feeds) to be styled via the attribute callback. + if c == b'\r' { + last_char_was_cr = true; + continue; + } + if last_char_was_cr { + if c != b'\n' { + self.add_carriage_return(attribute_callback); } + last_char_was_cr = false; + } + + // At line boundaries, close and re-open all of the open tags. + if c == b'\n' { highlights.iter().for_each(|_| self.end_highlight()); self.html.push(c); self.line_offsets.push(self.html.len() as u32);