traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
tree-sitter/cli/src/tests/highlight_test.rs

418 lines
14 KiB
Rust
Raw Normal View History

Start work on a C API for syntax highlighting
2019-03-08 13:13:02 -08:00
use super::helpers::fixtures::{get_language, get_property_sheet, get_property_sheet_json};
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
use lazy_static::lazy_static;
Start work on a C API for syntax highlighting
2019-03-08 13:13:02 -08:00
use std::ffi::CString;
use std::{ptr, slice, str};
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
use tree_sitter::{Language, PropertySheet};
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
use tree_sitter_highlight::{c, highlight, highlight_html, Highlight, HighlightEvent, Properties};
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
lazy_static! {
static ref JS_SHEET: PropertySheet<Properties> =
get_property_sheet("javascript", "highlights.json");
static ref HTML_SHEET: PropertySheet<Properties> =
get_property_sheet("html", "highlights.json");
Highlight: fix handling of root node injections like in ERB, EJS
2019-03-13 15:28:08 -07:00
static ref EJS_SHEET: PropertySheet<Properties> =
get_property_sheet("embedded-template", "highlights-ejs.json");
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
static ref SCOPE_CLASS_STRINGS: Vec<String> = {
let mut result = Vec::new();
let mut i = 0;
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
while let Some(highlight) = Highlight::from_usize(i) {
result.push(format!("class={:?}", highlight));
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
i += 1;
}
result
};
}
#[test]
fn test_highlighting_injected_html_in_javascript() {
let source = vec!["const s = html `<div>${a < b}</div>`;"].join("\n");
assert_eq!(
&to_token_vector(&source, get_language("javascript"), &JS_SHEET).unwrap(),
&[vec![
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
("const", vec![Highlight::Keyword]),
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
(" ", vec![]),
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
("s", vec![Highlight::Variable]),
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
(" ", vec![]),
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
("=", vec![Highlight::Operator]),
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
(" ", vec![]),
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
("html", vec![Highlight::Function]),
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
(" ", vec![]),
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
("`<", vec![Highlight::String]),
("div", vec![Highlight::String, Highlight::Tag]),
(">", vec![Highlight::String]),
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
(
"${",
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
vec![
Highlight::String,
Highlight::Embedded,
Highlight::PunctuationSpecial
]
),
(
"a",
vec![Highlight::String, Highlight::Embedded, Highlight::Variable]
),
(" ", vec![Highlight::String, Highlight::Embedded]),
(
"<",
vec![Highlight::String, Highlight::Embedded, Highlight::Operator]
),
(" ", vec![Highlight::String, Highlight::Embedded]),
(
"b",
vec![Highlight::String, Highlight::Embedded, Highlight::Variable]
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
),
(
"}",
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
vec![
Highlight::String,
Highlight::Embedded,
Highlight::PunctuationSpecial
]
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
),
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
("</", vec![Highlight::String]),
("div", vec![Highlight::String, Highlight::Tag]),
(">`", vec![Highlight::String]),
(";", vec![Highlight::PunctuationDelimiter]),
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
]]
);
}
#[test]
fn test_highlighting_injected_javascript_in_html() {
let source = vec![
"<body>",
" <script>",
" const x = new Thing();",
" </script>",
"</body>",
]
.join("\n");
assert_eq!(
&to_token_vector(&source, get_language("html"), &HTML_SHEET).unwrap(),
&[
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
vec![("<", vec![]), ("body", vec![Highlight::Tag]), (">", vec![]),],
vec![
(" <", vec![]),
("script", vec![Highlight::Tag]),
(">", vec![]),
],
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
vec![
(" ", vec![]),
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
("const", vec![Highlight::Keyword]),
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
(" ", vec![]),
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
("x", vec![Highlight::Variable]),
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
(" ", vec![]),
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
("=", vec![Highlight::Operator]),
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
(" ", vec![]),
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
("new", vec![Highlight::Keyword]),
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
(" ", vec![]),
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
("Thing", vec![Highlight::Constructor]),
("(", vec![Highlight::PunctuationBracket]),
(")", vec![Highlight::PunctuationBracket]),
(";", vec![Highlight::PunctuationDelimiter]),
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
],
vec![
(" </", vec![]),
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
("script", vec![Highlight::Tag]),
(">", vec![]),
],
vec![
("</", vec![]),
("body", vec![Highlight::Tag]),
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
(">", vec![]),
],
]
);
}
#[test]
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
fn test_highlighting_multiline_nodes_to_html() {
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
let source = vec![
"const SOMETHING = `",
" one ${",
" two()",
" } three",
"`",
Fix highlighting in the presence of node fields
2019-05-07 11:17:34 -07:00
"",
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
]
.join("\n");
assert_eq!(
&to_html(&source, get_language("javascript"), &JS_SHEET,).unwrap(),
&[
"<span class=Keyword>const</span> <span class=Constant>SOMETHING</span> <span class=Operator>=</span> <span class=String>`</span>\n".to_string(),
"<span class=String> one <span class=Embedded><span class=PunctuationSpecial>${</span></span></span>\n".to_string(),
"<span class=String><span class=Embedded> <span class=Function>two</span><span class=PunctuationBracket>(</span><span class=PunctuationBracket>)</span></span></span>\n".to_string(),
"<span class=String><span class=Embedded> <span class=PunctuationSpecial>}</span></span> three</span>\n".to_string(),
"<span class=String>`</span>\n".to_string(),
]
);
}
Add a unit test for local variable tracking in syntax highlighting
2019-05-10 14:38:43 -07:00
#[test]
fn test_highlighting_with_local_variable_tracking() {
let source = vec![
"module.exports = function a(b) {",
" const module = c;",
" console.log(module, b);",
"}",
"",
]
.join("\n");
assert_eq!(
&to_token_vector(&source, get_language("javascript"), &JS_SHEET).unwrap(),
&[
vec![
("module", vec![Highlight::VariableBuiltin]),
(".", vec![Highlight::PunctuationDelimiter]),
("exports", vec![Highlight::Property]),
(" ", vec![]),
("=", vec![Highlight::Operator]),
(" ", vec![]),
("function", vec![Highlight::Keyword]),
(" ", vec![]),
("a", vec![Highlight::Function]),
("(", vec![Highlight::PunctuationBracket]),
("b", vec![Highlight::VariableParameter]),
(")", vec![Highlight::PunctuationBracket]),
(" ", vec![]),
("{", vec![Highlight::PunctuationBracket])
],
vec![
(" ", vec![]),
("const", vec![Highlight::Keyword]),
(" ", vec![]),
("module", vec![Highlight::Variable]),
(" ", vec![]),
("=", vec![Highlight::Operator]),
(" ", vec![]),
("c", vec![Highlight::Variable]),
(";", vec![Highlight::PunctuationDelimiter])
],
vec![
(" ", vec![]),
("console", vec![Highlight::VariableBuiltin]),
(".", vec![Highlight::PunctuationDelimiter]),
("log", vec![Highlight::Function]),
("(", vec![Highlight::PunctuationBracket]),
// Not a builtin, because `module` was defined as a variable above.
("module", vec![Highlight::Variable]),
(",", vec![Highlight::PunctuationDelimiter]),
(" ", vec![]),
// A parameter, because `b` was defined as a parameter above.
("b", vec![Highlight::VariableParameter]),
(")", vec![Highlight::PunctuationBracket]),
(";", vec![Highlight::PunctuationDelimiter]),
("}", vec![Highlight::PunctuationBracket])
]
],
);
}
highlight: Fix HTML rendering of empty lines
2019-02-22 11:48:29 -08:00
#[test]
fn test_highlighting_empty_lines() {
let source = vec![
"class A {",
"",
" b(c) {",
"",
" d(e)",
"",
" }",
"",
"}",
]
.join("\n");
assert_eq!(
&to_html(&source, get_language("javascript"), &JS_SHEET,).unwrap(),
&[
"<span class=Keyword>class</span> <span class=Constructor>A</span> <span class=PunctuationBracket>{</span>\n".to_string(),
"\n".to_string(),
Update JS highlighting unit test
2019-05-10 10:32:31 -07:00
" <span class=Function>b</span><span class=PunctuationBracket>(</span><span class=VariableParameter>c</span><span class=PunctuationBracket>)</span> <span class=PunctuationBracket>{</span>\n".to_string(),
highlight: Fix HTML rendering of empty lines
2019-02-22 11:48:29 -08:00
"\n".to_string(),
" <span class=Function>d</span><span class=PunctuationBracket>(</span><span class=Variable>e</span><span class=PunctuationBracket>)</span>\n".to_string(),
"\n".to_string(),
" <span class=PunctuationBracket>}</span>\n".to_string(),
"\n".to_string(),
"<span class=PunctuationBracket>}</span>\n".to_string(),
]
);
}
Highlight: fix handling of root node injections like in ERB, EJS
2019-03-13 15:28:08 -07:00
#[test]
fn test_highlighting_ejs() {
let source = vec!["<div><% foo() %></div>"].join("\n");
assert_eq!(
&to_token_vector(&source, get_language("embedded-template"), &EJS_SHEET).unwrap(),
&[[
("<", vec![]),
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
("div", vec![Highlight::Tag]),
Highlight: fix handling of root node injections like in ERB, EJS
2019-03-13 15:28:08 -07:00
(">", vec![]),
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
("<%", vec![Highlight::Keyword]),
Highlight: fix handling of root node injections like in ERB, EJS
2019-03-13 15:28:08 -07:00
(" ", vec![]),
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
("foo", vec![Highlight::Function]),
("(", vec![Highlight::PunctuationBracket]),
(")", vec![Highlight::PunctuationBracket]),
Highlight: fix handling of root node injections like in ERB, EJS
2019-03-13 15:28:08 -07:00
(" ", vec![]),
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
("%>", vec![Highlight::Keyword]),
Highlight: fix handling of root node injections like in ERB, EJS
2019-03-13 15:28:08 -07:00
("</", vec![]),
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
("div", vec![Highlight::Tag]),
Highlight: fix handling of root node injections like in ERB, EJS
2019-03-13 15:28:08 -07:00
(">", vec![])
]],
);
}
Start work on a C API for syntax highlighting
2019-03-08 13:13:02 -08:00
#[test]
fn test_highlighting_via_c_api() {
let js_lang = get_language("javascript");
let html_lang = get_language("html");
let js_sheet = get_property_sheet_json("javascript", "highlights.json");
let js_sheet = c_string(&js_sheet);
let html_sheet = get_property_sheet_json("html", "highlights.json");
let html_sheet = c_string(&html_sheet);
let class_tag = c_string("class=tag");
let class_function = c_string("class=function");
let class_string = c_string("class=string");
let class_keyword = c_string("class=keyword");
let js_scope_name = c_string("source.js");
let html_scope_name = c_string("text.html.basic");
let injection_regex = c_string("^(javascript|js)$");
let source_code = c_string("<script>\nconst a = b('c');\nc.d();\n</script>");
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
let attribute_strings = &mut [ptr::null(); Highlight::Unknown as usize + 1];
attribute_strings[Highlight::Tag as usize] = class_tag.as_ptr();
attribute_strings[Highlight::String as usize] = class_string.as_ptr();
attribute_strings[Highlight::Keyword as usize] = class_keyword.as_ptr();
attribute_strings[Highlight::Function as usize] = class_function.as_ptr();
Start work on a C API for syntax highlighting
2019-03-08 13:13:02 -08:00
let highlighter = c::ts_highlighter_new(attribute_strings.as_ptr());
let buffer = c::ts_highlight_buffer_new();
c::ts_highlighter_add_language(
highlighter,
html_scope_name.as_ptr(),
html_lang,
html_sheet.as_ptr(),
ptr::null_mut(),
);
c::ts_highlighter_add_language(
highlighter,
js_scope_name.as_ptr(),
js_lang,
js_sheet.as_ptr(),
injection_regex.as_ptr(),
);
c::ts_highlighter_highlight(
highlighter,
html_scope_name.as_ptr(),
source_code.as_ptr(),
source_code.as_bytes().len() as u32,
buffer,
Add cancellation flag parameter to highlight API
2019-03-18 09:52:02 -07:00
ptr::null_mut(),
Start work on a C API for syntax highlighting
2019-03-08 13:13:02 -08:00
);
let output_bytes = c::ts_highlight_buffer_content(buffer);
let output_line_offsets = c::ts_highlight_buffer_line_offsets(buffer);
let output_len = c::ts_highlight_buffer_len(buffer);
let output_line_count = c::ts_highlight_buffer_line_count(buffer);
let output_bytes = unsafe { slice::from_raw_parts(output_bytes, output_len as usize) };
let output_line_offsets =
unsafe { slice::from_raw_parts(output_line_offsets, output_line_count as usize) };
let mut lines = Vec::new();
for i in 0..(output_line_count as usize) {
let line_start = output_line_offsets[i] as usize;
let line_end = output_line_offsets
.get(i + 1)
.map(|x| *x as usize)
.unwrap_or(output_bytes.len());
lines.push(str::from_utf8(&output_bytes[line_start..line_end]).unwrap());
}
assert_eq!(
lines,
vec![
"&lt;<span class=tag>script</span>&gt;",
"<span class=keyword>const</span> <span>a</span> <span>=</span> <span class=function>b</span><span>(</span><span class=string>&#39;c&#39;</span><span>)</span><span>;</span>",
"<span>c</span><span>.</span><span class=function>d</span><span>(</span><span>)</span><span>;</span>",
"&lt;/<span class=tag>script</span>&gt;",
]
);
c::ts_highlighter_delete(highlighter);
c::ts_highlight_buffer_delete(buffer);
}
fn c_string(s: &str) -> CString {
CString::new(s.as_bytes().to_vec()).unwrap()
}
Replace LanguageRegistry trait with a simple callback
2019-02-19 17:07:12 -08:00
fn test_language_for_injection_string<'a>(
string: &str,
) -> Option<(Language, &'a PropertySheet<Properties>)> {
match string {
"javascript" => Some((get_language("javascript"), &JS_SHEET)),
"html" => Some((get_language("html"), &HTML_SHEET)),
_ => None,
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
}
}
fn to_html<'a>(
src: &'a str,
language: Language,
property_sheet: &'a PropertySheet<Properties>,
) -> Result<Vec<String>, String> {
highlight_html(
src.as_bytes(),
language,
property_sheet,
Replace LanguageRegistry trait with a simple callback
2019-02-19 17:07:12 -08:00
&test_language_for_injection_string,
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
&|highlight| SCOPE_CLASS_STRINGS[highlight as usize].as_str(),
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
)
}
fn to_token_vector<'a>(
src: &'a str,
language: Language,
property_sheet: &'a PropertySheet<Properties>,
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
) -> Result<Vec<Vec<(&'a str, Vec<Highlight>)>>, String> {
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
let mut lines = Vec::new();
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
let mut highlights = Vec::new();
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
let mut line = Vec::new();
for event in highlight(
src.as_bytes(),
language,
property_sheet,
Replace LanguageRegistry trait with a simple callback
2019-02-19 17:07:12 -08:00
&test_language_for_injection_string,
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
)? {
match event {
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
HighlightEvent::HighlightStart(s) => highlights.push(s),
HighlightEvent::HighlightEnd => {
highlights.pop();
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
}
HighlightEvent::Source(s) => {
for (i, l) in s.lines().enumerate() {
if i > 0 {
lines.push(line);
line = Vec::new();
}
if l.len() > 0 {
Rename `scope` -> `highlight` in highlighting property sheets We need to use the word `scope` for a different purpose: tracking local scopes.
2019-05-09 09:42:40 -07:00
line.push((l, highlights.clone()));
Add a `highlight` subcommand
2019-02-19 11:24:50 -08:00
}
}
}
}
}
lines.push(line);
Ok(lines)
}
Reference in a new issue Copy permalink