use super::helpers::fixtures::{get_language, get_property_sheet, get_property_sheet_json}; use lazy_static::lazy_static; use std::ffi::CString; use std::{ptr, slice, str}; use tree_sitter::{Language, PropertySheet}; use tree_sitter_highlight::{c, highlight, highlight_html, HighlightEvent, Properties, Scope}; lazy_static! { static ref JS_SHEET: PropertySheet = get_property_sheet("javascript", "highlights.json"); static ref HTML_SHEET: PropertySheet = get_property_sheet("html", "highlights.json"); static ref EJS_SHEET: PropertySheet = get_property_sheet("embedded-template", "highlights-ejs.json"); static ref SCOPE_CLASS_STRINGS: Vec = { let mut result = Vec::new(); let mut i = 0; while let Some(scope) = Scope::from_usize(i) { result.push(format!("class={:?}", scope)); i += 1; } result }; } #[test] fn test_highlighting_injected_html_in_javascript() { let source = vec!["const s = html `

${a < b}

`;"].join("\n"); assert_eq!( &to_token_vector(&source, get_language("javascript"), &JS_SHEET).unwrap(), &[vec![ ("const", vec![Scope::Keyword]), (" ", vec![]), ("s", vec![Scope::Variable]), (" ", vec![]), ("=", vec![Scope::Operator]), (" ", vec![]), ("html", vec![Scope::Function]), (" ", vec![]), ("`<", vec![Scope::String]), ("div", vec![Scope::String, Scope::Tag]), (">", vec![Scope::String]), ( "${", vec![Scope::String, Scope::Embedded, Scope::PunctuationSpecial] ), ("a", vec![Scope::String, Scope::Embedded, Scope::Variable]), (" ", vec![Scope::String, Scope::Embedded]), ("<", vec![Scope::String, Scope::Embedded, Scope::Operator]), (" ", vec![Scope::String, Scope::Embedded]), ("b", vec![Scope::String, Scope::Embedded, Scope::Variable]), ( "}", vec![Scope::String, Scope::Embedded, Scope::PunctuationSpecial] ), ("`", vec![Scope::String]), (";", vec![Scope::PunctuationDelimiter]), ]] ); } #[test] fn test_highlighting_injected_javascript_in_html() { let source = vec![ "", " ", "", ] .join("\n"); assert_eq!( &to_token_vector(&source, get_language("html"), &HTML_SHEET).unwrap(), &[ vec![("<", vec![]), ("body", vec![Scope::Tag]), (">", vec![]),], vec![(" <", vec![]), ("script", vec![Scope::Tag]), (">", vec![]),], vec![ (" ", vec![]), ("const", vec![Scope::Keyword]), (" ", vec![]), ("x", vec![Scope::Variable]), (" ", vec![]), ("=", vec![Scope::Operator]), (" ", vec![]), ("new", vec![Scope::Keyword]), (" ", vec![]), ("Thing", vec![Scope::Constructor]), ("(", vec![Scope::PunctuationBracket]), (")", vec![Scope::PunctuationBracket]), (";", vec![Scope::PunctuationDelimiter]), ], vec![ (" ", vec![]), ], vec![("", vec![]),], ] ); } #[test] fn test_highlighting_multiline_scopes_to_html() { let source = vec![ "const SOMETHING = `", " one ${", " two()", " } three", "`", ] .join("\n"); assert_eq!( &to_html(&source, get_language("javascript"), &JS_SHEET,).unwrap(), &[ "const SOMETHING = `\n".to_string(), " one ${\n".to_string(), " two()\n".to_string(), " } three\n".to_string(), "`\n".to_string(), ] ); } #[test] fn test_highlighting_empty_lines() { let source = vec![ "class A {", "", " b(c) {", "", " d(e)", "", " }", "", "}", ] .join("\n"); assert_eq!( &to_html(&source, get_language("javascript"), &JS_SHEET,).unwrap(), &[ "class A {\n".to_string(), "\n".to_string(), " b(c) {\n".to_string(), "\n".to_string(), " d(e)\n".to_string(), "\n".to_string(), " }\n".to_string(), "\n".to_string(), "}\n".to_string(), ] ); } #[test] fn test_highlighting_ejs() { let source = vec!["

<% foo() %>

"].join("\n"); assert_eq!( &to_token_vector(&source, get_language("embedded-template"), &EJS_SHEET).unwrap(), &[[ ("<", vec![]), ("div", vec![Scope::Tag]), (">", vec![]), ("<%", vec![Scope::Keyword]), (" ", vec![]), ("foo", vec![Scope::Function]), ("(", vec![Scope::PunctuationBracket]), (")", vec![Scope::PunctuationBracket]), (" ", vec![]), ("%>", vec![Scope::Keyword]), ("", vec![]) ]], ); } #[test] fn test_highlighting_via_c_api() { let js_lang = get_language("javascript"); let html_lang = get_language("html"); let js_sheet = get_property_sheet_json("javascript", "highlights.json"); let js_sheet = c_string(&js_sheet); let html_sheet = get_property_sheet_json("html", "highlights.json"); let html_sheet = c_string(&html_sheet); let class_tag = c_string("class=tag"); let class_function = c_string("class=function"); let class_string = c_string("class=string"); let class_keyword = c_string("class=keyword"); let js_scope_name = c_string("source.js"); let html_scope_name = c_string("text.html.basic"); let injection_regex = c_string("^(javascript|js)$"); let source_code = c_string(""); let attribute_strings = &mut [ptr::null(); Scope::Unknown as usize + 1]; attribute_strings[Scope::Tag as usize] = class_tag.as_ptr(); attribute_strings[Scope::String as usize] = class_string.as_ptr(); attribute_strings[Scope::Keyword as usize] = class_keyword.as_ptr(); attribute_strings[Scope::Function as usize] = class_function.as_ptr(); let highlighter = c::ts_highlighter_new(attribute_strings.as_ptr()); let buffer = c::ts_highlight_buffer_new(); c::ts_highlighter_add_language( highlighter, html_scope_name.as_ptr(), html_lang, html_sheet.as_ptr(), ptr::null_mut(), ); c::ts_highlighter_add_language( highlighter, js_scope_name.as_ptr(), js_lang, js_sheet.as_ptr(), injection_regex.as_ptr(), ); c::ts_highlighter_highlight( highlighter, html_scope_name.as_ptr(), source_code.as_ptr(), source_code.as_bytes().len() as u32, buffer, ptr::null_mut(), ); let output_bytes = c::ts_highlight_buffer_content(buffer); let output_line_offsets = c::ts_highlight_buffer_line_offsets(buffer); let output_len = c::ts_highlight_buffer_len(buffer); let output_line_count = c::ts_highlight_buffer_line_count(buffer); let output_bytes = unsafe { slice::from_raw_parts(output_bytes, output_len as usize) }; let output_line_offsets = unsafe { slice::from_raw_parts(output_line_offsets, output_line_count as usize) }; let mut lines = Vec::new(); for i in 0..(output_line_count as usize) { let line_start = output_line_offsets[i] as usize; let line_end = output_line_offsets .get(i + 1) .map(|x| *x as usize) .unwrap_or(output_bytes.len()); lines.push(str::from_utf8(&output_bytes[line_start..line_end]).unwrap()); } assert_eq!( lines, vec![ "<script>", "const a = b('c');", "c.d();", "</script>", ] ); c::ts_highlighter_delete(highlighter); c::ts_highlight_buffer_delete(buffer); } fn c_string(s: &str) -> CString { CString::new(s.as_bytes().to_vec()).unwrap() } fn test_language_for_injection_string<'a>( string: &str, ) -> Option<(Language, &'a PropertySheet)> { match string { "javascript" => Some((get_language("javascript"), &JS_SHEET)), "html" => Some((get_language("html"), &HTML_SHEET)), _ => None, } } fn to_html<'a>( src: &'a str, language: Language, property_sheet: &'a PropertySheet, ) -> Result, String> { highlight_html( src.as_bytes(), language, property_sheet, &test_language_for_injection_string, &|scope| SCOPE_CLASS_STRINGS[scope as usize].as_str(), ) } fn to_token_vector<'a>( src: &'a str, language: Language, property_sheet: &'a PropertySheet, ) -> Result)>>, String> { let mut lines = Vec::new(); let mut scopes = Vec::new(); let mut line = Vec::new(); for event in highlight( src.as_bytes(), language, property_sheet, &test_language_for_injection_string, )? { match event { HighlightEvent::ScopeStart(s) => scopes.push(s), HighlightEvent::ScopeEnd => { scopes.pop(); } HighlightEvent::Source(s) => { for (i, l) in s.lines().enumerate() { if i > 0 { lines.push(line); line = Vec::new(); } if l.len() > 0 { line.push((l, scopes.clone())); } } } } } lines.push(line); Ok(lines) }