From 31c2086273f02b303f7757ba834dfc5bc88f178d Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 9 May 2019 09:40:15 -0700 Subject: [PATCH 1/7] :art: Refactor TreePropertyCursor --- lib/binding_rust/lib.rs | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/lib/binding_rust/lib.rs b/lib/binding_rust/lib.rs index b553a28d..4c0f3043 100644 --- a/lib/binding_rust/lib.rs +++ b/lib/binding_rust/lib.rs @@ -708,9 +708,7 @@ impl<'a, P> TreePropertyCursor<'a, P> { property_sheet, source, }; - let kind_id = result.cursor.node().kind_id(); - let field_id = result.cursor.field_id(); - let state = result.next_state(&result.current_state(), kind_id, field_id, 0); + let state = result.next_state(0); result.state_stack.push(state); result } @@ -725,15 +723,9 @@ impl<'a, P> TreePropertyCursor<'a, P> { pub fn goto_first_child(&mut self) -> bool { if self.cursor.goto_first_child() { - let child_index = 0; - let next_state_id = { - let state = &self.current_state(); - let kind_id = self.cursor.node().kind_id(); - let field_id = self.cursor.field_id(); - self.next_state(state, kind_id, field_id, child_index) - }; + let next_state_id = self.next_state(0); self.state_stack.push(next_state_id); - self.child_index_stack.push(child_index); + self.child_index_stack.push(0); true } else { false @@ -744,12 +736,7 @@ impl<'a, P> TreePropertyCursor<'a, P> { if self.cursor.goto_next_sibling() { let child_index = self.child_index_stack.pop().unwrap() + 1; self.state_stack.pop(); - let next_state_id = { - let state = &self.current_state(); - let kind_id = self.cursor.node().kind_id(); - let field_id = self.cursor.field_id(); - self.next_state(state, kind_id, field_id, child_index) - }; + let next_state_id = self.next_state(child_index); self.state_stack.push(next_state_id); self.child_index_stack.push(child_index); true @@ -768,13 +755,10 @@ impl<'a, P> TreePropertyCursor<'a, P> { } } - fn next_state( - &self, - state: &PropertyState, - node_kind_id: u16, - node_field_id: Option, - node_child_index: usize, - ) -> usize { + fn next_state(&self, node_child_index: usize) -> usize { + let state = self.current_state(); + let node_field_id = self.cursor.field_id(); + let node_kind_id = self.cursor.node().kind_id(); let transitions = node_field_id .and_then(|field_id| state.field_transitions.get(&field_id)) .or_else(|| state.kind_transitions.get(&node_kind_id)); From d78ac581f32783215ef4baf81816dec986b86dea Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 9 May 2019 09:42:40 -0700 Subject: [PATCH 2/7] Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes. --- cli/src/highlight.rs | 58 ++++++------ cli/src/tests/highlight_test.rs | 131 +++++++++++++++----------- highlight/src/c_lib.rs | 34 +++---- highlight/src/lib.rs | 162 ++++++++++++++++---------------- 4 files changed, 207 insertions(+), 178 deletions(-) diff --git a/cli/src/highlight.rs b/cli/src/highlight.rs index e7bb8818..17abb6e5 100644 --- a/cli/src/highlight.rs +++ b/cli/src/highlight.rs @@ -9,7 +9,7 @@ use std::collections::HashMap; use std::time::Instant; use std::{fmt, fs, io, path}; use tree_sitter::{Language, PropertySheet}; -use tree_sitter_highlight::{highlight, highlight_html, HighlightEvent, Properties, Scope}; +use tree_sitter_highlight::{highlight, highlight_html, Highlight, HighlightEvent, Properties}; lazy_static! { static ref CSS_STYLES_BY_COLOR_ID: Vec = @@ -27,12 +27,14 @@ impl Theme { Ok(serde_json::from_str(&json).unwrap_or_default()) } - fn ansi_style(&self, scope: Scope) -> Option<&Style> { - self.ansi_styles[scope as usize].as_ref() + fn ansi_style(&self, highlight: Highlight) -> Option<&Style> { + self.ansi_styles[highlight as usize].as_ref() } - fn css_style(&self, scope: Scope) -> Option<&str> { - self.css_styles[scope as usize].as_ref().map(|s| s.as_str()) + fn css_style(&self, highlight: Highlight) -> Option<&str> { + self.css_styles[highlight as usize] + .as_ref() + .map(|s| s.as_str()) } } @@ -41,15 +43,15 @@ impl<'de> Deserialize<'de> for Theme { where D: Deserializer<'de>, { - let scope_count = Scope::Unknown as usize + 1; - let mut ansi_styles = vec![None; scope_count]; - let mut css_styles = vec![None; scope_count]; - if let Ok(colors) = HashMap::::deserialize(deserializer) { - for (scope, style_value) in colors { + let highlight_count = Highlight::Unknown as usize + 1; + let mut ansi_styles = vec![None; highlight_count]; + let mut css_styles = vec![None; highlight_count]; + if let Ok(colors) = HashMap::::deserialize(deserializer) { + for (highlight, style_value) in colors { let mut style = Style::default(); parse_style(&mut style, style_value); - ansi_styles[scope as usize] = Some(style); - css_styles[scope as usize] = Some(style_to_css(style)); + ansi_styles[highlight as usize] = Some(style); + css_styles[highlight as usize] = Some(style_to_css(style)); } } Ok(Self { @@ -67,8 +69,8 @@ impl Serialize for Theme { let entry_count = self.ansi_styles.iter().filter(|i| i.is_some()).count(); let mut map = serializer.serialize_map(Some(entry_count))?; for (i, style) in self.ansi_styles.iter().enumerate() { - let scope = Scope::from_usize(i).unwrap(); - if scope == Scope::Unknown { + let highlight = Highlight::from_usize(i).unwrap(); + if highlight == Highlight::Unknown { break; } if let Some(style) = style { @@ -98,14 +100,14 @@ impl Serialize for Theme { if style.is_underline { entry.insert("underline", Value::Bool(true)); } - map.serialize_entry(&scope, &entry)?; + map.serialize_entry(&highlight, &entry)?; } else if let Some(color) = color { - map.serialize_entry(&scope, &color)?; + map.serialize_entry(&highlight, &color)?; } else { - map.serialize_entry(&scope, &Value::Null)?; + map.serialize_entry(&highlight, &Value::Null)?; } } else { - map.serialize_entry(&scope, &Value::Null)?; + map.serialize_entry(&highlight, &Value::Null)?; } } map.end() @@ -150,11 +152,11 @@ impl fmt::Debug for Theme { let mut first = true; for (i, style) in self.ansi_styles.iter().enumerate() { if let Some(style) = style { - let scope = Scope::from_usize(i).unwrap(); + let highlight = Highlight::from_usize(i).unwrap(); if !first { write!(f, ", ")?; } - write!(f, "{:?}: {:?}", scope, style)?; + write!(f, "{:?}: {:?}", highlight, style)?; first = false; } } @@ -262,23 +264,23 @@ pub fn ansi( let mut stdout = stdout.lock(); let time = Instant::now(); - let mut scope_stack = Vec::new(); + let mut highlight_stack = Vec::new(); for event in highlight(source, language, property_sheet, |s| { language_for_injection_string(loader, s) })? { match event { HighlightEvent::Source(s) => { - if let Some(style) = scope_stack.last().and_then(|s| theme.ansi_style(*s)) { + if let Some(style) = highlight_stack.last().and_then(|s| theme.ansi_style(*s)) { write!(&mut stdout, "{}", style.paint(s))?; } else { write!(&mut stdout, "{}", s)?; } } - HighlightEvent::ScopeStart(s) => { - scope_stack.push(s); + HighlightEvent::HighlightStart(h) => { + highlight_stack.push(h); } - HighlightEvent::ScopeEnd => { - scope_stack.pop(); + HighlightEvent::HighlightEnd => { + highlight_stack.pop(); } } } @@ -334,8 +336,8 @@ pub fn html( language, property_sheet, |s| language_for_injection_string(loader, s), - |scope| { - if let Some(css_style) = theme.css_style(scope) { + |highlight| { + if let Some(css_style) = theme.css_style(highlight) { css_style } else { "" diff --git a/cli/src/tests/highlight_test.rs b/cli/src/tests/highlight_test.rs index 8cc7f7e6..4a384716 100644 --- a/cli/src/tests/highlight_test.rs +++ b/cli/src/tests/highlight_test.rs @@ -3,7 +3,7 @@ use lazy_static::lazy_static; use std::ffi::CString; use std::{ptr, slice, str}; use tree_sitter::{Language, PropertySheet}; -use tree_sitter_highlight::{c, highlight, highlight_html, HighlightEvent, Properties, Scope}; +use tree_sitter_highlight::{c, highlight, highlight_html, Highlight, HighlightEvent, Properties}; lazy_static! { static ref JS_SHEET: PropertySheet = @@ -15,8 +15,8 @@ lazy_static! { static ref SCOPE_CLASS_STRINGS: Vec = { let mut result = Vec::new(); let mut i = 0; - while let Some(scope) = Scope::from_usize(i) { - result.push(format!("class={:?}", scope)); + while let Some(highlight) = Highlight::from_usize(i) { + result.push(format!("class={:?}", highlight)); i += 1; } result @@ -30,34 +30,51 @@ fn test_highlighting_injected_html_in_javascript() { assert_eq!( &to_token_vector(&source, get_language("javascript"), &JS_SHEET).unwrap(), &[vec![ - ("const", vec![Scope::Keyword]), + ("const", vec![Highlight::Keyword]), (" ", vec![]), - ("s", vec![Scope::Variable]), + ("s", vec![Highlight::Variable]), (" ", vec![]), - ("=", vec![Scope::Operator]), + ("=", vec![Highlight::Operator]), (" ", vec![]), - ("html", vec![Scope::Function]), + ("html", vec![Highlight::Function]), (" ", vec![]), - ("`<", vec![Scope::String]), - ("div", vec![Scope::String, Scope::Tag]), - (">", vec![Scope::String]), + ("`<", vec![Highlight::String]), + ("div", vec![Highlight::String, Highlight::Tag]), + (">", vec![Highlight::String]), ( "${", - vec![Scope::String, Scope::Embedded, Scope::PunctuationSpecial] + vec![ + Highlight::String, + Highlight::Embedded, + Highlight::PunctuationSpecial + ] + ), + ( + "a", + vec![Highlight::String, Highlight::Embedded, Highlight::Variable] + ), + (" ", vec![Highlight::String, Highlight::Embedded]), + ( + "<", + vec![Highlight::String, Highlight::Embedded, Highlight::Operator] + ), + (" ", vec![Highlight::String, Highlight::Embedded]), + ( + "b", + vec![Highlight::String, Highlight::Embedded, Highlight::Variable] ), - ("a", vec![Scope::String, Scope::Embedded, Scope::Variable]), - (" ", vec![Scope::String, Scope::Embedded]), - ("<", vec![Scope::String, Scope::Embedded, Scope::Operator]), - (" ", vec![Scope::String, Scope::Embedded]), - ("b", vec![Scope::String, Scope::Embedded, Scope::Variable]), ( "}", - vec![Scope::String, Scope::Embedded, Scope::PunctuationSpecial] + vec![ + Highlight::String, + Highlight::Embedded, + Highlight::PunctuationSpecial + ] ), - ("`", vec![Scope::String]), - (";", vec![Scope::PunctuationDelimiter]), + ("`", vec![Highlight::String]), + (";", vec![Highlight::PunctuationDelimiter]), ]] ); } @@ -76,35 +93,43 @@ fn test_highlighting_injected_javascript_in_html() { assert_eq!( &to_token_vector(&source, get_language("html"), &HTML_SHEET).unwrap(), &[ - vec![("<", vec![]), ("body", vec![Scope::Tag]), (">", vec![]),], - vec![(" <", vec![]), ("script", vec![Scope::Tag]), (">", vec![]),], + vec![("<", vec![]), ("body", vec![Highlight::Tag]), (">", vec![]),], + vec![ + (" <", vec![]), + ("script", vec![Highlight::Tag]), + (">", vec![]), + ], vec![ (" ", vec![]), - ("const", vec![Scope::Keyword]), + ("const", vec![Highlight::Keyword]), (" ", vec![]), - ("x", vec![Scope::Variable]), + ("x", vec![Highlight::Variable]), (" ", vec![]), - ("=", vec![Scope::Operator]), + ("=", vec![Highlight::Operator]), (" ", vec![]), - ("new", vec![Scope::Keyword]), + ("new", vec![Highlight::Keyword]), (" ", vec![]), - ("Thing", vec![Scope::Constructor]), - ("(", vec![Scope::PunctuationBracket]), - (")", vec![Scope::PunctuationBracket]), - (";", vec![Scope::PunctuationDelimiter]), + ("Thing", vec![Highlight::Constructor]), + ("(", vec![Highlight::PunctuationBracket]), + (")", vec![Highlight::PunctuationBracket]), + (";", vec![Highlight::PunctuationDelimiter]), ], vec![ (" ", vec![]), + ], + vec![ + ("", vec![]), ], - vec![("", vec![]),], ] ); } #[test] -fn test_highlighting_multiline_scopes_to_html() { +fn test_highlighting_multiline_nodes_to_html() { let source = vec![ "const SOMETHING = `", " one ${", @@ -166,17 +191,17 @@ fn test_highlighting_ejs() { &to_token_vector(&source, get_language("embedded-template"), &EJS_SHEET).unwrap(), &[[ ("<", vec![]), - ("div", vec![Scope::Tag]), + ("div", vec![Highlight::Tag]), (">", vec![]), - ("<%", vec![Scope::Keyword]), + ("<%", vec![Highlight::Keyword]), (" ", vec![]), - ("foo", vec![Scope::Function]), - ("(", vec![Scope::PunctuationBracket]), - (")", vec![Scope::PunctuationBracket]), + ("foo", vec![Highlight::Function]), + ("(", vec![Highlight::PunctuationBracket]), + (")", vec![Highlight::PunctuationBracket]), (" ", vec![]), - ("%>", vec![Scope::Keyword]), + ("%>", vec![Highlight::Keyword]), ("", vec![]) ]], ); @@ -201,11 +226,11 @@ fn test_highlighting_via_c_api() { let injection_regex = c_string("^(javascript|js)$"); let source_code = c_string(""); - let attribute_strings = &mut [ptr::null(); Scope::Unknown as usize + 1]; - attribute_strings[Scope::Tag as usize] = class_tag.as_ptr(); - attribute_strings[Scope::String as usize] = class_string.as_ptr(); - attribute_strings[Scope::Keyword as usize] = class_keyword.as_ptr(); - attribute_strings[Scope::Function as usize] = class_function.as_ptr(); + let attribute_strings = &mut [ptr::null(); Highlight::Unknown as usize + 1]; + attribute_strings[Highlight::Tag as usize] = class_tag.as_ptr(); + attribute_strings[Highlight::String as usize] = class_string.as_ptr(); + attribute_strings[Highlight::Keyword as usize] = class_keyword.as_ptr(); + attribute_strings[Highlight::Function as usize] = class_function.as_ptr(); let highlighter = c::ts_highlighter_new(attribute_strings.as_ptr()); let buffer = c::ts_highlight_buffer_new(); @@ -290,7 +315,7 @@ fn to_html<'a>( language, property_sheet, &test_language_for_injection_string, - &|scope| SCOPE_CLASS_STRINGS[scope as usize].as_str(), + &|highlight| SCOPE_CLASS_STRINGS[highlight as usize].as_str(), ) } @@ -298,9 +323,9 @@ fn to_token_vector<'a>( src: &'a str, language: Language, property_sheet: &'a PropertySheet, -) -> Result)>>, String> { +) -> Result)>>, String> { let mut lines = Vec::new(); - let mut scopes = Vec::new(); + let mut highlights = Vec::new(); let mut line = Vec::new(); for event in highlight( src.as_bytes(), @@ -309,9 +334,9 @@ fn to_token_vector<'a>( &test_language_for_injection_string, )? { match event { - HighlightEvent::ScopeStart(s) => scopes.push(s), - HighlightEvent::ScopeEnd => { - scopes.pop(); + HighlightEvent::HighlightStart(s) => highlights.push(s), + HighlightEvent::HighlightEnd => { + highlights.pop(); } HighlightEvent::Source(s) => { for (i, l) in s.lines().enumerate() { @@ -320,7 +345,7 @@ fn to_token_vector<'a>( line = Vec::new(); } if l.len() > 0 { - line.push((l, scopes.clone())); + line.push((l, highlights.clone())); } } } diff --git a/highlight/src/c_lib.rs b/highlight/src/c_lib.rs index b8098980..73ebda7d 100644 --- a/highlight/src/c_lib.rs +++ b/highlight/src/c_lib.rs @@ -1,4 +1,4 @@ -use super::{escape, load_property_sheet, HighlightEvent, Highlighter, Properties, Scope}; +use super::{escape, load_property_sheet, Highlight, HighlightEvent, Highlighter, Properties}; use regex::Regex; use std::collections::HashMap; use std::ffi::CStr; @@ -37,7 +37,7 @@ pub extern "C" fn ts_highlighter_new( attribute_strings: *const *const c_char, ) -> *mut TSHighlighter { let attribute_strings = - unsafe { slice::from_raw_parts(attribute_strings, Scope::Unknown as usize + 1) }; + unsafe { slice::from_raw_parts(attribute_strings, Highlight::Unknown as usize + 1) }; let attribute_strings = attribute_strings .into_iter() .map(|s| { @@ -185,19 +185,19 @@ impl TSHighlighter { output.html.clear(); output.line_offsets.clear(); output.line_offsets.push(0); - let mut scopes = Vec::new(); + let mut highlights = Vec::new(); for event in highlighter { match event { - HighlightEvent::ScopeStart(s) => { - scopes.push(s); - output.start_scope(s, &self.attribute_strings); + HighlightEvent::HighlightStart(s) => { + highlights.push(s); + output.start_highlight(s, &self.attribute_strings); } - HighlightEvent::ScopeEnd => { - scopes.pop(); - output.end_scope(); + HighlightEvent::HighlightEnd => { + highlights.pop(); + output.end_highlight(); } HighlightEvent::Source(src) => { - output.add_text(src, &scopes, &self.attribute_strings); + output.add_text(src, &highlights, &self.attribute_strings); } }; } @@ -209,8 +209,8 @@ impl TSHighlighter { } impl TSHighlightBuffer { - fn start_scope(&mut self, s: Scope, attribute_strings: &[&[u8]]) { - let attribute_string = attribute_strings[s as usize]; + fn start_highlight(&mut self, h: Highlight, attribute_strings: &[&[u8]]) { + let attribute_string = attribute_strings[h as usize]; self.html.extend(b""); } - fn end_scope(&mut self) { + fn end_highlight(&mut self) { self.html.extend(b""); } @@ -227,16 +227,16 @@ impl TSHighlightBuffer { self.line_offsets.push(self.html.len() as u32); } - fn add_text(&mut self, src: &str, scopes: &Vec, attribute_strings: &[&[u8]]) { + fn add_text(&mut self, src: &str, highlights: &Vec, attribute_strings: &[&[u8]]) { let mut multiline = false; for line in src.split('\n') { let line = line.trim_end_matches('\r'); if multiline { - scopes.iter().for_each(|_| self.end_scope()); + highlights.iter().for_each(|_| self.end_highlight()); self.finish_line(); - scopes + highlights .iter() - .for_each(|scope| self.start_scope(*scope, attribute_strings)); + .for_each(|scope| self.start_highlight(*scope, attribute_strings)); } write!(&mut self.html, "{}", escape::Escape(line)).unwrap(); multiline = true; diff --git a/highlight/src/lib.rs b/highlight/src/lib.rs index 79837cf2..f05179dc 100644 --- a/highlight/src/lib.rs +++ b/highlight/src/lib.rs @@ -40,13 +40,13 @@ struct Injection { #[derive(Debug)] pub struct Properties { - scope: Option, + highlight: Option, injections: Vec, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] #[repr(u16)] -pub enum Scope { +pub enum Highlight { Attribute, Comment, Constant, @@ -101,8 +101,8 @@ where #[derive(Copy, Clone, Debug)] pub enum HighlightEvent<'a> { Source(&'a str), - ScopeStart(Scope), - ScopeEnd, + HighlightStart(Highlight), + HighlightEnd, } #[derive(Debug, Deserialize)] @@ -143,7 +143,7 @@ enum InjectionContentJSON { #[derive(Debug, Deserialize)] struct PropertiesJSON { - scope: Option, + highlight: Option, #[serde(rename = "injection-language")] injection_language: Option, #[serde(rename = "injection-content")] @@ -193,9 +193,9 @@ pub fn load_property_sheet( Ok(sheet) } -impl Scope { +impl Highlight { pub fn from_usize(i: usize) -> Option { - if i <= (Scope::Unknown as usize) { + if i <= (Highlight::Unknown as usize) { Some(unsafe { transmute(i as u16) }) } else { None @@ -281,7 +281,7 @@ impl Properties { }?; Ok(Self { - scope: json.scope, + highlight: json.highlight, injections, }) } @@ -657,19 +657,19 @@ where // Determine if any scopes start or end at the current position. let scope_event; - if let Some(scope) = properties.scope { + if let Some(highlight) = properties.highlight { let next_offset = cmp::min(self.source.len(), self.layers[0].offset()); - // Before returning any scope boundaries, return any remaining slice of - // the source code the precedes that scope boundary. + // Before returning any highlight boundaries, return any remaining slice of + // the source code the precedes that highlight boundary. if self.source_offset < next_offset { return self.emit_source(next_offset); } scope_event = if self.layers[0].at_node_end { - Some(HighlightEvent::ScopeEnd) + Some(HighlightEvent::HighlightEnd) } else { - Some(HighlightEvent::ScopeStart(scope)) + Some(HighlightEvent::HighlightStart(highlight)) }; } else { scope_event = None; @@ -753,8 +753,8 @@ impl<'a> Layer<'a> { fn cmp(&self, other: &Layer) -> cmp::Ordering { // Events are ordered primarily by their position in the document. But if - // one scope starts at a given position and another scope ends at that - // same position, return the scope end event before the scope start event. + // one highlight starts at a given position and another highlight ends at that + // same position, return the highlight end event before the highlight start event. self.offset() .cmp(&other.offset()) .then_with(|| other.at_node_end.cmp(&self.at_node_end)) @@ -783,82 +783,82 @@ impl<'a> Layer<'a> { } } -impl<'de> Deserialize<'de> for Scope { +impl<'de> Deserialize<'de> for Highlight { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { let s = String::deserialize(deserializer)?; match s.as_str() { - "attribute" => Ok(Scope::Attribute), - "comment" => Ok(Scope::Comment), - "constant" => Ok(Scope::Constant), - "constant.builtin" => Ok(Scope::ConstantBuiltin), - "constructor" => Ok(Scope::Constructor), - "constructor.builtin" => Ok(Scope::ConstructorBuiltin), - "embedded" => Ok(Scope::Embedded), - "escape" => Ok(Scope::Escape), - "function" => Ok(Scope::Function), - "function.builtin" => Ok(Scope::FunctionBuiltin), - "keyword" => Ok(Scope::Keyword), - "number" => Ok(Scope::Number), - "operator" => Ok(Scope::Operator), - "property" => Ok(Scope::Property), - "property.builtin" => Ok(Scope::PropertyBuiltin), - "punctuation" => Ok(Scope::Punctuation), - "punctuation.bracket" => Ok(Scope::PunctuationBracket), - "punctuation.delimiter" => Ok(Scope::PunctuationDelimiter), - "punctuation.special" => Ok(Scope::PunctuationSpecial), - "string" => Ok(Scope::String), - "string.special" => Ok(Scope::StringSpecial), - "type" => Ok(Scope::Type), - "type.builtin" => Ok(Scope::TypeBuiltin), - "variable" => Ok(Scope::Variable), - "variable.builtin" => Ok(Scope::VariableBuiltin), - "tag" => Ok(Scope::Tag), - _ => Ok(Scope::Unknown), + "attribute" => Ok(Highlight::Attribute), + "comment" => Ok(Highlight::Comment), + "constant" => Ok(Highlight::Constant), + "constant.builtin" => Ok(Highlight::ConstantBuiltin), + "constructor" => Ok(Highlight::Constructor), + "constructor.builtin" => Ok(Highlight::ConstructorBuiltin), + "embedded" => Ok(Highlight::Embedded), + "escape" => Ok(Highlight::Escape), + "function" => Ok(Highlight::Function), + "function.builtin" => Ok(Highlight::FunctionBuiltin), + "keyword" => Ok(Highlight::Keyword), + "number" => Ok(Highlight::Number), + "operator" => Ok(Highlight::Operator), + "property" => Ok(Highlight::Property), + "property.builtin" => Ok(Highlight::PropertyBuiltin), + "punctuation" => Ok(Highlight::Punctuation), + "punctuation.bracket" => Ok(Highlight::PunctuationBracket), + "punctuation.delimiter" => Ok(Highlight::PunctuationDelimiter), + "punctuation.special" => Ok(Highlight::PunctuationSpecial), + "string" => Ok(Highlight::String), + "string.special" => Ok(Highlight::StringSpecial), + "type" => Ok(Highlight::Type), + "type.builtin" => Ok(Highlight::TypeBuiltin), + "variable" => Ok(Highlight::Variable), + "variable.builtin" => Ok(Highlight::VariableBuiltin), + "tag" => Ok(Highlight::Tag), + _ => Ok(Highlight::Unknown), } } } -impl Serialize for Scope { +impl Serialize for Highlight { fn serialize(&self, serializer: S) -> Result where S: Serializer, { match self { - Scope::Attribute => serializer.serialize_str("attribute"), - Scope::Comment => serializer.serialize_str("comment"), - Scope::Constant => serializer.serialize_str("constant"), - Scope::ConstantBuiltin => serializer.serialize_str("constant.builtin"), - Scope::Constructor => serializer.serialize_str("constructor"), - Scope::ConstructorBuiltin => serializer.serialize_str("constructor.builtin"), - Scope::Embedded => serializer.serialize_str("embedded"), - Scope::Escape => serializer.serialize_str("escape"), - Scope::Function => serializer.serialize_str("function"), - Scope::FunctionBuiltin => serializer.serialize_str("function.builtin"), - Scope::Keyword => serializer.serialize_str("keyword"), - Scope::Number => serializer.serialize_str("number"), - Scope::Operator => serializer.serialize_str("operator"), - Scope::Property => serializer.serialize_str("property"), - Scope::PropertyBuiltin => serializer.serialize_str("property.builtin"), - Scope::Punctuation => serializer.serialize_str("punctuation"), - Scope::PunctuationBracket => serializer.serialize_str("punctuation.bracket"), - Scope::PunctuationDelimiter => serializer.serialize_str("punctuation.delimiter"), - Scope::PunctuationSpecial => serializer.serialize_str("punctuation.special"), - Scope::String => serializer.serialize_str("string"), - Scope::StringSpecial => serializer.serialize_str("string.special"), - Scope::Type => serializer.serialize_str("type"), - Scope::TypeBuiltin => serializer.serialize_str("type.builtin"), - Scope::Variable => serializer.serialize_str("variable"), - Scope::VariableBuiltin => serializer.serialize_str("variable.builtin"), - Scope::Tag => serializer.serialize_str("tag"), - Scope::Unknown => serializer.serialize_str(""), + Highlight::Attribute => serializer.serialize_str("attribute"), + Highlight::Comment => serializer.serialize_str("comment"), + Highlight::Constant => serializer.serialize_str("constant"), + Highlight::ConstantBuiltin => serializer.serialize_str("constant.builtin"), + Highlight::Constructor => serializer.serialize_str("constructor"), + Highlight::ConstructorBuiltin => serializer.serialize_str("constructor.builtin"), + Highlight::Embedded => serializer.serialize_str("embedded"), + Highlight::Escape => serializer.serialize_str("escape"), + Highlight::Function => serializer.serialize_str("function"), + Highlight::FunctionBuiltin => serializer.serialize_str("function.builtin"), + Highlight::Keyword => serializer.serialize_str("keyword"), + Highlight::Number => serializer.serialize_str("number"), + Highlight::Operator => serializer.serialize_str("operator"), + Highlight::Property => serializer.serialize_str("property"), + Highlight::PropertyBuiltin => serializer.serialize_str("property.builtin"), + Highlight::Punctuation => serializer.serialize_str("punctuation"), + Highlight::PunctuationBracket => serializer.serialize_str("punctuation.bracket"), + Highlight::PunctuationDelimiter => serializer.serialize_str("punctuation.delimiter"), + Highlight::PunctuationSpecial => serializer.serialize_str("punctuation.special"), + Highlight::String => serializer.serialize_str("string"), + Highlight::StringSpecial => serializer.serialize_str("string.special"), + Highlight::Type => serializer.serialize_str("type"), + Highlight::TypeBuiltin => serializer.serialize_str("type.builtin"), + Highlight::Variable => serializer.serialize_str("variable"), + Highlight::VariableBuiltin => serializer.serialize_str("variable.builtin"), + Highlight::Tag => serializer.serialize_str("tag"), + Highlight::Unknown => serializer.serialize_str(""), } } } -pub trait HTMLAttributeCallback<'a>: Fn(Scope) -> &'a str {} +pub trait HTMLAttributeCallback<'a>: Fn(Highlight) -> &'a str {} pub fn highlight<'a, F>( source: &'a [u8], @@ -881,18 +881,18 @@ pub fn highlight_html<'a, F1, F2>( ) -> Result, String> where F1: Fn(&str) -> Option<(Language, &'a PropertySheet)>, - F2: Fn(Scope) -> &'a str, + F2: Fn(Highlight) -> &'a str, { let highlighter = Highlighter::new(source, language, property_sheet, injection_callback, None)?; let mut renderer = HtmlRenderer::new(attribute_callback); let mut scopes = Vec::new(); for event in highlighter { match event { - HighlightEvent::ScopeStart(s) => { + HighlightEvent::HighlightStart(s) => { scopes.push(s); renderer.start_scope(s); } - HighlightEvent::ScopeEnd => { + HighlightEvent::HighlightEnd => { scopes.pop(); renderer.end_scope(); } @@ -907,7 +907,7 @@ where Ok(renderer.result) } -struct HtmlRenderer<'a, F: Fn(Scope) -> &'a str> { +struct HtmlRenderer<'a, F: Fn(Highlight) -> &'a str> { result: Vec, current_line: String, attribute_callback: F, @@ -915,7 +915,7 @@ struct HtmlRenderer<'a, F: Fn(Scope) -> &'a str> { impl<'a, F> HtmlRenderer<'a, F> where - F: Fn(Scope) -> &'a str, + F: Fn(Highlight) -> &'a str, { fn new(attribute_callback: F) -> Self { HtmlRenderer { @@ -925,7 +925,7 @@ where } } - fn start_scope(&mut self, s: Scope) { + fn start_scope(&mut self, s: Highlight) { write!( &mut self.current_line, "", @@ -944,14 +944,16 @@ where self.current_line.clear(); } - fn add_text(&mut self, src: &str, scopes: &Vec) { + fn add_text(&mut self, src: &str, scopes: &Vec) { let mut multiline = false; for line in src.split('\n') { let line = line.trim_end_matches('\r'); if multiline { scopes.iter().for_each(|_| self.end_scope()); self.finish_line(); - scopes.iter().for_each(|scope| self.start_scope(*scope)); + scopes + .iter() + .for_each(|highlight| self.start_scope(*highlight)); } write!(&mut self.current_line, "{}", escape::Escape(line)).unwrap(); multiline = true; From a7d02e72764c56fc49276834f49877a93f0a8fdc Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 9 May 2019 12:02:18 -0700 Subject: [PATCH 3/7] Add support for highlight properties that track local variables --- cli/src/highlight.rs | 3 +- cli/src/properties.rs | 5 +- highlight/src/lib.rs | 110 +++++++++++++++++++++++++++++++++++++++- lib/binding_rust/lib.rs | 4 ++ 4 files changed, 117 insertions(+), 5 deletions(-) diff --git a/cli/src/highlight.rs b/cli/src/highlight.rs index 17abb6e5..91e6b5b3 100644 --- a/cli/src/highlight.rs +++ b/cli/src/highlight.rs @@ -138,7 +138,8 @@ impl Default for Theme { "tag": 18, "type": 23, "type.builtin": {"color": 23, "bold": true}, - "variable.builtin": {"bold": true} + "variable.builtin": {"bold": true}, + "variable.parameter": {"underline": true} } "#, ) diff --git a/cli/src/properties.rs b/cli/src/properties.rs index ad9e2552..44a444ab 100644 --- a/cli/src/properties.rs +++ b/cli/src/properties.rs @@ -16,6 +16,7 @@ use tree_sitter::{self, PropertyStateJSON, PropertyTransitionJSON}; #[serde(untagged)] enum PropertyValue { Number(isize), + Boolean(bool), String(String), Object(PropertySet), Array(Vec), @@ -737,8 +738,8 @@ fn parse_sass_value(value: &Value) -> Result { } Value::Color(_, Some(name)) => Ok(PropertyValue::String(name.clone())), Value::Numeric(n, _) => Ok(PropertyValue::Number(n.to_integer())), - Value::True => Ok(PropertyValue::String("true".to_string())), - Value::False => Ok(PropertyValue::String("false".to_string())), + Value::True => Ok(PropertyValue::Boolean(true)), + Value::False => Ok(PropertyValue::Boolean(false)), _ => Err(Error(format!( "Property values must be strings or function calls. Got {:?}", value diff --git a/highlight/src/lib.rs b/highlight/src/lib.rs index f05179dc..cfc6d3a5 100644 --- a/highlight/src/lib.rs +++ b/highlight/src/lib.rs @@ -41,7 +41,11 @@ struct Injection { #[derive(Debug)] pub struct Properties { highlight: Option, + highlight_nonlocal: Option, injections: Vec, + local_scope: Option, + local_definition: bool, + local_reference: bool, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] @@ -73,15 +77,24 @@ pub enum Highlight { TypeBuiltin, Variable, VariableBuiltin, + VariableParameter, Unknown, } +#[derive(Debug)] +struct Scope<'a> { + inherits: bool, + local_defs: Vec<(&'a str, Highlight)>, +} + struct Layer<'a> { _tree: Tree, cursor: TreePropertyCursor<'a, Properties>, ranges: Vec, at_node_end: bool, depth: usize, + scope_stack: Vec>, + local_highlight: Option, } struct Highlighter<'a, T> @@ -144,10 +157,22 @@ enum InjectionContentJSON { #[derive(Debug, Deserialize)] struct PropertiesJSON { highlight: Option, + #[serde(rename = "highlight-nonlocal")] + highlight_nonlocal: Option, + #[serde(rename = "injection-language")] injection_language: Option, #[serde(rename = "injection-content")] injection_content: Option, + + #[serde(default, rename = "local-scope")] + local_scope: bool, + #[serde(default, rename = "local-scope-inherit")] + local_scope_inherit: bool, + #[serde(default, rename = "local-definition")] + local_definition: bool, + #[serde(default, rename = "local-reference")] + local_reference: bool, } #[derive(Debug)] @@ -282,6 +307,14 @@ impl Properties { Ok(Self { highlight: json.highlight, + highlight_nonlocal: json.highlight_nonlocal, + local_scope: if json.local_scope { + Some(json.local_scope_inherit) + } else { + None + }, + local_definition: json.local_definition, + local_reference: json.local_reference, injections, }) } @@ -629,6 +662,7 @@ where while !self.layers.is_empty() { let first_layer = &self.layers[0]; + let local_highlight = first_layer.local_highlight; let properties = &first_layer.cursor.node_properties(); // Add any injections for the current node. @@ -657,7 +691,10 @@ where // Determine if any scopes start or end at the current position. let scope_event; - if let Some(highlight) = properties.highlight { + if let Some(highlight) = local_highlight + .or(properties.highlight_nonlocal) + .or(properties.highlight) + { let next_offset = cmp::min(self.source.len(), self.layers[0].offset()); // Before returning any highlight boundaries, return any remaining slice of @@ -748,6 +785,11 @@ impl<'a> Layer<'a> { ranges, depth, at_node_end: false, + scope_stack: vec![Scope { + inherits: false, + local_defs: Vec::new(), + }], + local_highlight: None, } } @@ -770,17 +812,79 @@ impl<'a> Layer<'a> { } fn advance(&mut self) -> bool { + // Clear the current local highlighting class, which may be re-populated + // if we enter a node that represents a local definition or local reference. + self.local_highlight = None; + + // Step through the tree in a depth-first traversal, stopping at both + // the start and end position of every node. if self.at_node_end { + self.leave_node(); if self.cursor.goto_next_sibling() { + self.enter_node(); self.at_node_end = false; } else if !self.cursor.goto_parent() { return false; } - } else if !self.cursor.goto_first_child() { + } else if self.cursor.goto_first_child() { + self.enter_node(); + } else { self.at_node_end = true; } true } + + fn enter_node(&mut self) { + let node = self.cursor.node(); + let props = self.cursor.node_properties(); + let node_text = if props.local_definition || props.local_reference { + node.utf8_text(self.cursor.source()).ok() + } else { + None + }; + + // If this node represents a local definition, then record its highlighting class + // and store the highlighting class in the current local scope. + if props.local_definition { + if let (Some(text), Some(inner_scope), Some(highlight)) = + (node_text, self.scope_stack.last_mut(), props.highlight) + { + self.local_highlight = props.highlight; + if let Err(i) = inner_scope.local_defs.binary_search_by_key(&text, |e| e.0) { + inner_scope.local_defs.insert(i, (text, highlight)); + } + } + } + // If this node represents a local reference, then look it up in the current scope + // stack. If a local definition is found, record its highlighting class. + else if props.local_reference { + if let Some(text) = node_text { + for scope in self.scope_stack.iter().rev() { + if let Ok(i) = scope.local_defs.binary_search_by_key(&text, |e| e.0) { + self.local_highlight = Some(scope.local_defs[i].1); + break; + } + if !scope.inherits { + break; + } + } + } + } + // If this node represents a new local scope, then push it onto the scope stack. + if let Some(inherits) = props.local_scope { + self.scope_stack.push(Scope { + inherits, + local_defs: Vec::new(), + }); + } + } + + fn leave_node(&mut self) { + let props = self.cursor.node_properties(); + if props.local_scope.is_some() { + self.scope_stack.pop(); + } + } } impl<'de> Deserialize<'de> for Highlight { @@ -815,6 +919,7 @@ impl<'de> Deserialize<'de> for Highlight { "type.builtin" => Ok(Highlight::TypeBuiltin), "variable" => Ok(Highlight::Variable), "variable.builtin" => Ok(Highlight::VariableBuiltin), + "variable.parameter" => Ok(Highlight::VariableParameter), "tag" => Ok(Highlight::Tag), _ => Ok(Highlight::Unknown), } @@ -852,6 +957,7 @@ impl Serialize for Highlight { Highlight::TypeBuiltin => serializer.serialize_str("type.builtin"), Highlight::Variable => serializer.serialize_str("variable"), Highlight::VariableBuiltin => serializer.serialize_str("variable.builtin"), + Highlight::VariableParameter => serializer.serialize_str("variable.parameter"), Highlight::Tag => serializer.serialize_str("tag"), Highlight::Unknown => serializer.serialize_str(""), } diff --git a/lib/binding_rust/lib.rs b/lib/binding_rust/lib.rs index 4c0f3043..74b275a0 100644 --- a/lib/binding_rust/lib.rs +++ b/lib/binding_rust/lib.rs @@ -755,6 +755,10 @@ impl<'a, P> TreePropertyCursor<'a, P> { } } + pub fn source(&self) -> &'a [u8] { + &self.source + } + fn next_state(&self, node_child_index: usize) -> usize { let state = self.current_state(); let node_field_id = self.cursor.field_id(); From b25af0f05f73da6b17bb1d52e95247e468b19277 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 9 May 2019 17:45:48 -0700 Subject: [PATCH 4/7] Fix property state merging bug --- cli/src/properties.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cli/src/properties.rs b/cli/src/properties.rs index 44a444ab..33489f4d 100644 --- a/cli/src/properties.rs +++ b/cli/src/properties.rs @@ -315,6 +315,9 @@ impl Builder { transition.state_id = *replacement; } } + if let Some(replacement) = state_replacements.get(&state.default_next_state_id) { + state.default_next_state_id = *replacement; + } state.transitions.dedup(); } } @@ -338,6 +341,7 @@ impl Builder { for transition in state.transitions.iter_mut() { transition.state_id = final_state_replacements[transition.state_id]; } + state.default_next_state_id = final_state_replacements[state.default_next_state_id] } let mut i = 0; From b069e75149e64d06e8733f806705ca0f18f27b3b Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 10 May 2019 09:18:39 -0700 Subject: [PATCH 5/7] Use branches of fixture repos with the new highlight API --- script/fetch-fixtures | 10 +++++----- script/fetch-fixtures.cmd | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/script/fetch-fixtures b/script/fetch-fixtures index 94f9eddd..473138a2 100755 --- a/script/fetch-fixtures +++ b/script/fetch-fixtures @@ -24,12 +24,12 @@ fetch_grammar() { fetch_grammar bash master fetch_grammar c master fetch_grammar cpp master -fetch_grammar embedded-template master +fetch_grammar embedded-template highlight-locals fetch_grammar go master -fetch_grammar html master -fetch_grammar javascript master +fetch_grammar html highlight-locals +fetch_grammar javascript highlight-locals fetch_grammar json master -fetch_grammar python master -fetch_grammar ruby master +fetch_grammar python highlight-locals +fetch_grammar ruby highlight-locals fetch_grammar rust master fetch_grammar typescript master diff --git a/script/fetch-fixtures.cmd b/script/fetch-fixtures.cmd index 011d73ff..e1dd4b50 100644 --- a/script/fetch-fixtures.cmd +++ b/script/fetch-fixtures.cmd @@ -3,13 +3,13 @@ call:fetch_grammar bash master call:fetch_grammar c master call:fetch_grammar cpp master -call:fetch_grammar embedded-template master +call:fetch_grammar embedded-template highlight-locals call:fetch_grammar go master -call:fetch_grammar html master -call:fetch_grammar javascript master +call:fetch_grammar html highlight-locals +call:fetch_grammar javascript highlight-locals call:fetch_grammar json master -call:fetch_grammar python master -call:fetch_grammar ruby master +call:fetch_grammar python highlight-locals +call:fetch_grammar ruby highlight-locals call:fetch_grammar rust master call:fetch_grammar typescript master exit /B 0 From e648dc8616d7331dd61de2ff95dc988b31dfdc6e Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 10 May 2019 10:06:05 -0700 Subject: [PATCH 6/7] Rename TSHighlightScope -> TSHighlightValue --- highlight/include/tree_sitter/highlight.h | 58 +++++++++++------------ 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/highlight/include/tree_sitter/highlight.h b/highlight/include/tree_sitter/highlight.h index 347999d2..34bccbc6 100644 --- a/highlight/include/tree_sitter/highlight.h +++ b/highlight/include/tree_sitter/highlight.h @@ -16,41 +16,41 @@ typedef enum { // The list of scopes which can be styled for syntax highlighting. // When constructing a `TSHighlighter`, you need to construct an // `attribute_strings` array whose elements correspond to these values. -enum TSHighlightScope { - TSHighlightScopeAttribute, - TSHighlightScopeComment, - TSHighlightScopeConstant, - TSHighlightScopeConstantBuiltin, - TSHighlightScopeConstructor, - TSHighlightScopeConstructorBuiltin, - TSHighlightScopeEmbedded, - TSHighlightScopeEscape, - TSHighlightScopeFunction, - TSHighlightScopeFunctionBuiltin, - TSHighlightScopeKeyword, - TSHighlightScopeNumber, - TSHighlightScopeOperator, - TSHighlightScopeProperty, - TSHighlightScopePropertyBuiltin, - TSHighlightScopePunctuation, - TSHighlightScopePunctuationBracket, - TSHighlightScopePunctuationDelimiter, - TSHighlightScopePunctuationSpecial, - TSHighlightScopeString, - TSHighlightScopeStringSpecial, - TSHighlightScopeTag, - TSHighlightScopeType, - TSHighlightScopeTypeBuiltin, - TSHighlightScopeVariable, - TSHighlightScopeVariableBuiltin, - TSHighlightScopeUnknown, +enum TSHighlightValue { + TSHighlightValueAttribute, + TSHighlightValueComment, + TSHighlightValueConstant, + TSHighlightValueConstantBuiltin, + TSHighlightValueConstructor, + TSHighlightValueConstructorBuiltin, + TSHighlightValueEmbedded, + TSHighlightValueEscape, + TSHighlightValueFunction, + TSHighlightValueFunctionBuiltin, + TSHighlightValueKeyword, + TSHighlightValueNumber, + TSHighlightValueOperator, + TSHighlightValueProperty, + TSHighlightValuePropertyBuiltin, + TSHighlightValuePunctuation, + TSHighlightValuePunctuationBracket, + TSHighlightValuePunctuationDelimiter, + TSHighlightValuePunctuationSpecial, + TSHighlightValueString, + TSHighlightValueStringSpecial, + TSHighlightValueTag, + TSHighlightValueType, + TSHighlightValueTypeBuiltin, + TSHighlightValueVariable, + TSHighlightValueVariableBuiltin, + TSHighlightValueUnknown, }; typedef struct TSHighlighter TSHighlighter; typedef struct TSHighlightBuffer TSHighlightBuffer; // Construct a `TSHighlighter` by providing a list of strings containing -// the HTML attributes that should be applied for each highlight scope. +// the HTML attributes that should be applied for each highlight value. TSHighlighter *ts_highlighter_new( const char **attribute_strings ); From 958ab27efbfba203966d47eedcaf3f5bc2fd136d Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 10 May 2019 10:32:31 -0700 Subject: [PATCH 7/7] Update JS highlighting unit test --- cli/src/tests/highlight_test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/tests/highlight_test.rs b/cli/src/tests/highlight_test.rs index 4a384716..c2f5924c 100644 --- a/cli/src/tests/highlight_test.rs +++ b/cli/src/tests/highlight_test.rs @@ -172,7 +172,7 @@ fn test_highlighting_empty_lines() { &[ "class A {\n".to_string(), "\n".to_string(), - " b(c) {\n".to_string(), + " b(c) {\n".to_string(), "\n".to_string(), " d(e)\n".to_string(), "\n".to_string(),