traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Start reimplementing highlight crate with tree queries

Browse source
This commit is contained in:
Max Brunsfeld 2019-09-18 17:35:47 -07:00
parent ee9a3c0ebb
commit f4903578f8
9 changed files with 1259 additions and 1487 deletions
Download patch file Download diff file Expand all files Collapse all files

24
cli/src/error.rs
View file

@ -1,6 +1,6 @@
use std::fmt::Write;
use std::io;
use tree_sitter_highlight::PropertySheetError;
use tree_sitter::QueryError;
#[derive(Debug)]
pub struct Error(pub Vec<String>);
@ -50,6 +50,18 @@ impl Error {
}
}
impl<'a> From<QueryError> for Error {
fn from(error: QueryError) -> Self {
Error::new(format!("{:?}", error))
}
}
impl<'a> From<tree_sitter_highlight::Error> for Error {
fn from(error: tree_sitter_highlight::Error) -> Self {
Error::new(format!("{:?}", error))
}
}
impl From<serde_json::Error> for Error {
fn from(error: serde_json::Error) -> Self {
Error::new(error.to_string())
@ -79,13 +91,3 @@ impl From<String> for Error {
Error::new(error)
}
}
impl From<PropertySheetError> for Error {
fn from(error: PropertySheetError) -> Self {
match error {
PropertySheetError::InvalidFormat(e) => Self::from(e),
PropertySheetError::InvalidRegex(e) => Self::regex(&e.to_string()),
PropertySheetError::InvalidJSON(e) => Self::from(e),
}
}
}

317
cli/src/highlight.rs
View file

@ -1,6 +1,6 @@
use crate::error::Result;
use crate::loader::Loader;
use ansi_term::{Color, Style};
use ansi_term::Color;
use lazy_static::lazy_static;
use serde::ser::SerializeMap;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
@ -9,18 +9,52 @@ use std::collections::HashMap;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::Instant;
use std::{fmt, fs, io, path, thread};
use tree_sitter::{Language, PropertySheet};
use tree_sitter_highlight::{highlight, highlight_html, Highlight, HighlightEvent, Properties};
use std::{fs, io, path, str, thread, usize};
use tree_sitter_highlight::{
HighlightConfiguration, HighlightContext, HighlightEvent, Highlighter, HtmlRenderer,
};
pub const HTML_HEADER: &'static str = "
<!doctype HTML>
<head>
<title>Tree-sitter Highlighting</title>
<style>
body {
font-family: monospace
}
.line-number {
user-select: none;
text-align: right;
color: rgba(27,31,35,.3);
padding: 0 10px;
}
.line {
white-space: pre;
}
</style>
</head>
<body>
";
pub const HTML_FOOTER: &'static str = "
</body>
";
lazy_static! {
static ref CSS_STYLES_BY_COLOR_ID: Vec<String> =
serde_json::from_str(include_str!("../vendor/xterm-colors.json")).unwrap();
}
#[derive(Debug, Default)]
pub struct Style {
pub ansi: ansi_term::Style,
pub css: Option<String>,
}
#[derive(Debug)]
pub struct Theme {
ansi_styles: Vec<Option<Style>>,
css_styles: Vec<Option<String>>,
pub highlighter: Highlighter,
styles: Vec<Style>,
}
impl Theme {
@ -29,14 +63,8 @@ impl Theme {
Ok(serde_json::from_str(&json).unwrap_or_default())
}
fn ansi_style(&self, highlight: Highlight) -> Option<&Style> {
self.ansi_styles[highlight as usize].as_ref()
}
fn css_style(&self, highlight: Highlight) -> Option<&str> {
self.css_styles[highlight as usize]
.as_ref()
.map(|s| s.as_str())
pub fn default_style(&self) -> Style {
Style::default()
}
}
@ -45,20 +73,21 @@ impl<'de> Deserialize<'de> for Theme {
where
D: Deserializer<'de>,
{
let highlight_count = Highlight::Unknown as usize + 1;
let mut ansi_styles = vec![None; highlight_count];
let mut css_styles = vec![None; highlight_count];
if let Ok(colors) = HashMap::<Highlight, Value>::deserialize(deserializer) {
for (highlight, style_value) in colors {
let mut names = Vec::new();
let mut styles = Vec::new();
if let Ok(colors) = HashMap::<String, Value>::deserialize(deserializer) {
names.reserve(colors.len());
styles.reserve(colors.len());
for (name, style_value) in colors {
let mut style = Style::default();
parse_style(&mut style, style_value);
ansi_styles[highlight as usize] = Some(style);
css_styles[highlight as usize] = Some(style_to_css(style));
names.push(name);
styles.push(style);
}
}
Ok(Self {
ansi_styles,
css_styles,
highlighter: Highlighter::new(names),
styles,
})
}
}
@ -68,48 +97,40 @@ impl Serialize for Theme {
where
S: Serializer,
{
let entry_count = self.ansi_styles.iter().filter(|i| i.is_some()).count();
let mut map = serializer.serialize_map(Some(entry_count))?;
for (i, style) in self.ansi_styles.iter().enumerate() {
let highlight = Highlight::from_usize(i).unwrap();
if highlight == Highlight::Unknown {
break;
}
if let Some(style) = style {
let color = style.foreground.map(|color| match color {
Color::Black => json!("black"),
Color::Blue => json!("blue"),
Color::Cyan => json!("cyan"),
Color::Green => json!("green"),
Color::Purple => json!("purple"),
Color::Red => json!("red"),
Color::White => json!("white"),
Color::Yellow => json!("yellow"),
Color::RGB(r, g, b) => json!(format!("#{:x?}{:x?}{:x?}", r, g, b)),
Color::Fixed(n) => json!(n),
});
if style.is_bold || style.is_italic || style.is_underline {
let mut entry = HashMap::new();
if let Some(color) = color {
entry.insert("color", color);
}
if style.is_bold {
entry.insert("bold", Value::Bool(true));
}
if style.is_italic {
entry.insert("italic", Value::Bool(true));
}
if style.is_underline {
entry.insert("underline", Value::Bool(true));
}
map.serialize_entry(&highlight, &entry)?;
} else if let Some(color) = color {
map.serialize_entry(&highlight, &color)?;
} else {
map.serialize_entry(&highlight, &Value::Null)?;
let mut map = serializer.serialize_map(Some(self.styles.len()))?;
for (name, style) in self.highlighter.highlight_names.iter().zip(&self.styles) {
let style = &style.ansi;
let color = style.foreground.map(|color| match color {
Color::Black => json!("black"),
Color::Blue => json!("blue"),
Color::Cyan => json!("cyan"),
Color::Green => json!("green"),
Color::Purple => json!("purple"),
Color::Red => json!("red"),
Color::White => json!("white"),
Color::Yellow => json!("yellow"),
Color::RGB(r, g, b) => json!(format!("#{:x?}{:x?}{:x?}", r, g, b)),
Color::Fixed(n) => json!(n),
});
if style.is_bold || style.is_italic || style.is_underline {
let mut style_json = HashMap::new();
if let Some(color) = color {
style_json.insert("color", color);
}
if style.is_bold {
style_json.insert("bold", Value::Bool(true));
}
if style.is_italic {
style_json.insert("italic", Value::Bool(true));
}
if style.is_underline {
style_json.insert("underline", Value::Bool(true));
}
map.serialize_entry(&name, &style_json)?;
} else if let Some(color) = color {
map.serialize_entry(&name, &color)?;
} else {
map.serialize_entry(&highlight, &Value::Null)?;
map.serialize_entry(&name, &Value::Null)?;
}
}
map.end()
@ -149,42 +170,27 @@ impl Default for Theme {
}
}
impl fmt::Debug for Theme {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{{")?;
let mut first = true;
for (i, style) in self.ansi_styles.iter().enumerate() {
if let Some(style) = style {
let highlight = Highlight::from_usize(i).unwrap();
if !first {
write!(f, ", ")?;
}
write!(f, "{:?}: {:?}", highlight, style)?;
first = false;
}
}
write!(f, "}}")?;
Ok(())
}
}
fn parse_style(style: &mut Style, json: Value) {
if let Value::Object(entries) = json {
for (property_name, value) in entries {
match property_name.as_str() {
"bold" => *style = style.bold(),
"italic" => *style = style.italic(),
"underline" => *style = style.underline(),
"bold" => style.ansi = style.ansi.bold(),
"italic" => style.ansi = style.ansi.italic(),
"underline" => style.ansi = style.ansi.underline(),
"color" => {
if let Some(color) = parse_color(value) {
*style = style.fg(color);
style.ansi = style.ansi.fg(color);
}
}
_ => {}
}
}
style.css = Some(style_to_css(style.ansi));
} else if let Some(color) = parse_color(json) {
*style = style.fg(color);
style.ansi = style.ansi.fg(color);
style.css = Some(style_to_css(style.ansi));
} else {
style.css = None;
}
}
@ -223,7 +229,7 @@ fn parse_color(json: Value) -> Option<Color> {
}
}
fn style_to_css(style: Style) -> String {
fn style_to_css(style: ansi_term::Style) -> String {
use std::fmt::Write;
let mut result = "style='".to_string();
if style.is_bold {
@ -271,108 +277,82 @@ pub fn ansi(
loader: &Loader,
theme: &Theme,
source: &[u8],
language: Language,
property_sheet: &PropertySheet<Properties>,
config: &HighlightConfiguration,
print_time: bool,
) -> Result<()> {
use std::io::Write;
let stdout = io::stdout();
let mut stdout = stdout.lock();
let cancellation_flag = cancel_on_stdin();
let time = Instant::now();
let mut highlight_stack = Vec::new();
for event in highlight(
let cancellation_flag = cancel_on_stdin();
let mut context = HighlightContext::new();
let events = theme.highlighter.highlight(
&mut context,
config,
source,
language,
property_sheet,
Some(cancellation_flag.as_ref()),
|s| language_for_injection_string(loader, s),
)
.map_err(|e| e.to_string())?
{
let event = event.map_err(|e| e.to_string())?;
match event {
HighlightEvent::Source { start, end } => {
if let Some(style) = highlight_stack.last().and_then(|s| theme.ansi_style(*s)) {
style.paint(&source[start..end]).write_to(&mut stdout)?;
} else {
stdout.write_all(&source[start..end])?;
}
}
HighlightEvent::HighlightStart(h) => {
highlight_stack.push(h);
Some(&cancellation_flag),
|string| language_for_injection_string(loader, theme, string),
)?;
let mut style_stack = vec![theme.default_style().ansi];
for event in events {
match event? {
HighlightEvent::HighlightStart(highlight) => {
style_stack.push(theme.styles[highlight.0].ansi);
}
HighlightEvent::HighlightEnd => {
highlight_stack.pop();
style_stack.pop();
}
HighlightEvent::Source { start, end } => {
style_stack
.last()
.unwrap()
.paint(&source[start..end])
.write_to(&mut stdout)?;
}
}
}
if print_time {
let duration = time.elapsed();
let duration_ms = duration.as_secs() * 1000 + duration.subsec_nanos() as u64 / 1000000;
eprintln!("{} ms", duration_ms);
eprintln!("Time: {}ms", time.elapsed().as_millis());
}
Ok(())
}
pub const HTML_HEADER: &'static str = "
<!doctype HTML>
<head>
<title>Tree-sitter Highlighting</title>
<style>
body {
font-family: monospace
}
.line-number {
user-select: none;
text-align: right;
color: rgba(27,31,35,.3);
padding: 0 10px;
}
.line {
white-space: pre;
}
</style>
</head>
<body>
";
pub const HTML_FOOTER: &'static str = "
</body>
";
pub fn html(
loader: &Loader,
theme: &Theme,
source: &[u8],
language: Language,
property_sheet: &PropertySheet<Properties>,
config: &HighlightConfiguration,
print_time: bool,
) -> Result<()> {
use std::io::Write;
let stdout = io::stdout();
let mut stdout = stdout.lock();
write!(&mut stdout, "<table>\n")?;
let time = Instant::now();
let cancellation_flag = cancel_on_stdin();
let lines = highlight_html(
let mut context = HighlightContext::new();
let events = theme.highlighter.highlight(
&mut context,
config,
source,
language,
property_sheet,
Some(cancellation_flag.as_ref()),
|s| language_for_injection_string(loader, s),
|highlight| {
if let Some(css_style) = theme.css_style(highlight) {
css_style
} else {
""
}
},
)
.map_err(|e| e.to_string())?;
for (i, line) in lines.into_iter().enumerate() {
Some(&cancellation_flag),
|string| language_for_injection_string(loader, theme, string),
)?;
let mut renderer = HtmlRenderer::new();
renderer.render(events, source, &move |highlight| {
if let Some(css_style) = &theme.styles[highlight.0].css {
css_style.as_bytes()
} else {
"".as_bytes()
}
})?;
for (i, line) in renderer.lines().enumerate() {
write!(
&mut stdout,
"<tr><td class=line-number>{}</td><td class=line>{}</td></tr>\n",
@ -380,14 +360,21 @@ pub fn html(
line
)?;
}
write!(&mut stdout, "</table>\n")?;
if print_time {
eprintln!("Time: {}ms", time.elapsed().as_millis());
}
Ok(())
}
fn language_for_injection_string<'a>(
loader: &'a Loader,
theme: &Theme,
string: &str,
) -> Option<(Language, &'a PropertySheet<Properties>)> {
) -> Option<&'a HighlightConfiguration> {
match loader.language_configuration_for_injection_string(string) {
Err(e) => {
eprintln!(
@ -399,7 +386,7 @@ fn language_for_injection_string<'a>(
}
Ok(None) => None,
Ok(Some((language, configuration))) => {
match configuration.highlight_property_sheet(language) {
match configuration.highlight_config(&theme.highlighter, language) {
Err(e) => {
eprintln!(
"Failed to load property sheet for injection string '{}': {}",
@ -409,7 +396,7 @@ fn language_for_injection_string<'a>(
None
}
Ok(None) => None,
Ok(Some(sheet)) => Some((language, sheet)),
Ok(Some(config)) => Some(config),
}
}
}

71
cli/src/loader.rs
View file

@ -9,8 +9,8 @@ use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::SystemTime;
use std::{fs, mem};
use tree_sitter::{Language, PropertySheet};
use tree_sitter_highlight::{load_property_sheet, Properties};
use tree_sitter::Language;
use tree_sitter_highlight::{HighlightConfiguration, Highlighter};
#[cfg(unix)]
const DYLIB_EXTENSION: &'static str = "so";
@ -27,9 +27,9 @@ pub struct LanguageConfiguration {
pub _first_line_regex: Option<Regex>,
pub injection_regex: Option<Regex>,
pub file_types: Vec<String>,
pub highlight_property_sheet_path: Option<PathBuf>,
pub root_path: PathBuf,
language_id: usize,
highlight_property_sheet: OnceCell<Option<PropertySheet<Properties>>>,
highlight_config: OnceCell<Option<HighlightConfiguration>>,
}
pub struct Loader {
@ -134,7 +134,6 @@ impl Loader {
if configuration_ids.len() == 1 {
configuration = &self.language_configurations[configuration_ids[0]];
}
// If multiple language configurations match, then determine which
// one to use by applying the configurations' content regexes.
else {
@ -151,7 +150,6 @@ impl Loader {
if let Some(mat) = content_regex.find(&file_contents) {
score = (mat.end() - mat.start()) as isize;
}
// If the content regex does not match, then *penalize* this
// language configuration, so that language configurations
// without content regexes are preferred over those with
@ -394,6 +392,7 @@ impl Loader {
});
let configuration = LanguageConfiguration {
root_path: parser_path.to_path_buf(),
scope: config_json.scope,
language_id,
file_types: config_json.file_types.unwrap_or(Vec::new()),
@ -406,10 +405,7 @@ impl Loader {
injection_regex: config_json
.injection_regex
.and_then(|r| RegexBuilder::new(&r).multi_line(true).build().ok()),
highlight_property_sheet_path: config_json
.highlights
.map(|h| parser_path.join(h)),
highlight_property_sheet: OnceCell::new(),
highlight_config: OnceCell::new(),
};
for file_type in &configuration.file_types {
@ -428,14 +424,14 @@ impl Loader {
&& parser_path.join("src").join("grammar.json").exists()
{
self.language_configurations.push(LanguageConfiguration {
root_path: parser_path.to_owned(),
language_id: self.languages_by_id.len(),
scope: None,
content_regex: None,
injection_regex: None,
file_types: Vec::new(),
_first_line_regex: None,
highlight_property_sheet_path: None,
highlight_property_sheet: OnceCell::new(),
highlight_config: OnceCell::new(),
});
self.languages_by_id
.push((parser_path.to_owned(), OnceCell::new()));
@ -446,30 +442,41 @@ impl Loader {
}
impl LanguageConfiguration {
pub fn highlight_property_sheet(
pub fn highlight_config(
&self,
highlighter: &Highlighter,
language: Language,
) -> Result<Option<&PropertySheet<Properties>>> {
self.highlight_property_sheet
) -> Result<Option<&HighlightConfiguration>> {
self.highlight_config
.get_or_try_init(|| {
if let Some(path) = &self.highlight_property_sheet_path {
let sheet_json = fs::read_to_string(path).map_err(Error::wrap(|| {
format!(
"Failed to read property sheet {:?}",
path.file_name().unwrap()
)
}))?;
let sheet =
load_property_sheet(language, &sheet_json).map_err(Error::wrap(|| {
format!(
"Failed to parse property sheet {:?}",
path.file_name().unwrap()
)
}))?;
Ok(Some(sheet))
} else {
Ok(None)
let queries_path = self.root_path.join("queries");
let highlights_path = queries_path.join("highlights.scm");
let injections_path = queries_path.join("injections.scm");
let locals_path = queries_path.join("locals.scm");
if !highlights_path.exists() {
return Ok(None);
}
let highlights_query = fs::read_to_string(highlights_path)?;
let injections_query = if injections_path.exists() {
fs::read_to_string(injections_path)?
} else {
String::new()
};
let locals_query = if locals_path.exists() {
fs::read_to_string(locals_path)?
} else {
String::new()
};
Ok(Some(highlighter.load_configuration(
language,
&highlights_query,
&injections_query,
&locals_query,
)?))
})
.map(Option::as_ref)
}

19
cli/src/main.rs
View file

@ -110,7 +110,8 @@ fn run() -> error::Result<()> {
)
.arg(Arg::with_name("scope").long("scope").takes_value(true))
.arg(Arg::with_name("html").long("html").short("h"))
.arg(Arg::with_name("time").long("time").short("t")),
.arg(Arg::with_name("time").long("time").short("t"))
.arg(Arg::with_name("q").short("q")),
)
.subcommand(
SubCommand::with_name("build-wasm")
@ -260,15 +261,18 @@ fn run() -> error::Result<()> {
},
};
if let Some(sheet) = language_config.highlight_property_sheet(language)? {
let source = fs::read(path)?;
let source = fs::read(path)?;
if let Some(highlight_config) =
language_config.highlight_config(&config.theme.highlighter, language)?
{
if html_mode {
highlight::html(&loader, &config.theme, &source, language, sheet)?;
highlight::html(&loader, &config.theme, &source, highlight_config, time)?;
} else {
highlight::ansi(&loader, &config.theme, &source, language, sheet, time)?;
highlight::ansi(&loader, &config.theme, &source, highlight_config, time)?;
}
} else {
return Error::err(format!("No syntax highlighting property sheet specified"));
return Error::err(format!("No syntax highlighting query found"));
}
}
} else if let Some(matches) = matches.subcommand_matches("build-wasm") {
@ -280,10 +284,9 @@ fn run() -> error::Result<()> {
loader.find_all_languages(&config.parser_directories)?;
for (configuration, language_path) in loader.get_all_language_configurations() {
println!(
"scope: {}\nparser: {:?}\nproperties: {:?}\nfile_types: {:?}\ncontent_regex: {:?}\ninjection_regex: {:?}\n",
"scope: {}\nparser: {:?}\nfile_types: {:?}\ncontent_regex: {:?}\ninjection_regex: {:?}\n",
configuration.scope.as_ref().unwrap_or(&String::new()),
language_path,
configuration.highlight_property_sheet_path,
configuration.file_types,
configuration.content_regex,
configuration.injection_regex,

46
cli/src/tests/helpers/fixtures.rs
View file

@ -2,8 +2,8 @@ use crate::loader::Loader;
use lazy_static::lazy_static;
use std::fs;
use std::path::{Path, PathBuf};
use tree_sitter::{Language, PropertySheet};
use tree_sitter_highlight::{load_property_sheet, Properties};
use tree_sitter::Language;
use tree_sitter_highlight::{HighlightConfiguration, Highlighter};
include!("./dirs.rs");
@ -21,18 +21,42 @@ pub fn get_language(name: &str) -> Language {
.unwrap()
}
pub fn get_property_sheet_json(language_name: &str, sheet_name: &str) -> String {
let path = GRAMMARS_DIR
.join(language_name)
.join("src")
.join(sheet_name);
fs::read_to_string(path).unwrap()
pub fn get_highlight_query_sources(language_name: &str) -> (String, String, String) {
let queries_path = GRAMMARS_DIR.join(language_name).join("queries");
let highlights_path = queries_path.join("highlights.scm");
let injections_path = queries_path.join("injections.scm");
let locals_path = queries_path.join("locals.scm");
let highlights_query = fs::read_to_string(highlights_path).unwrap();
let injections_query = if injections_path.exists() {
fs::read_to_string(injections_path).unwrap()
} else {
String::new()
};
let locals_query = if locals_path.exists() {
fs::read_to_string(locals_path).unwrap()
} else {
String::new()
};
(highlights_query, injections_query, locals_query)
}
pub fn get_property_sheet(language_name: &str, sheet_name: &str) -> PropertySheet<Properties> {
let json = get_property_sheet_json(language_name, sheet_name);
pub fn get_highlight_config(
language_name: &str,
highlighter: &Highlighter,
) -> HighlightConfiguration {
let language = get_language(language_name);
load_property_sheet(language, &json).unwrap()
let (highlights_query, injections_query, locals_query) =
get_highlight_query_sources(language_name);
highlighter
.load_configuration(
language,
&highlights_query,
&injections_query,
&locals_query,
)
.unwrap()
}
pub fn get_test_language(name: &str, parser_code: &str, path: Option<&Path>) -> Language {

516
cli/src/tests/highlight_test.rs
View file

@ -1,32 +1,85 @@
use super::helpers::fixtures::{get_language, get_property_sheet, get_property_sheet_json};
use super::helpers::fixtures::{get_highlight_config, get_highlight_query_sources, get_language};
use lazy_static::lazy_static;
use std::ffi::CString;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::{ptr, slice, str};
use tree_sitter::{Language, PropertySheet};
use tree_sitter_highlight::{
c, highlight, highlight_html, Error, Highlight, HighlightEvent, Properties,
c, Error, HighlightConfiguration, HighlightContext, HighlightEvent, Highlighter, HtmlRenderer,
};
lazy_static! {
static ref JS_SHEET: PropertySheet<Properties> =
get_property_sheet("javascript", "highlights.json");
static ref HTML_SHEET: PropertySheet<Properties> =
get_property_sheet("html", "highlights.json");
static ref EJS_SHEET: PropertySheet<Properties> =
get_property_sheet("embedded-template", "highlights-ejs.json");
static ref RUST_SHEET: PropertySheet<Properties> =
get_property_sheet("rust", "highlights.json");
static ref SCOPE_CLASS_STRINGS: Vec<String> = {
let mut result = Vec::new();
let mut i = 0;
while let Some(highlight) = Highlight::from_usize(i) {
result.push(format!("class={:?}", highlight));
i += 1;
}
result
};
static ref JS_HIGHLIGHT: HighlightConfiguration =
get_highlight_config("javascript", &HIGHLIGHTER);
static ref HTML_HIGHLIGHT: HighlightConfiguration = get_highlight_config("html", &HIGHLIGHTER);
static ref EJS_HIGHLIGHT: HighlightConfiguration =
get_highlight_config("embedded-template", &HIGHLIGHTER);
static ref RUST_HIGHLIGHT: HighlightConfiguration = get_highlight_config("rust", &HIGHLIGHTER);
static ref HIGHLIGHTER: Highlighter = Highlighter::new(
[
"attribute",
"constructor",
"function.builtin",
"function",
"embedded",
"keyword",
"operator",
"property.builtin",
"property",
"punctuation",
"punctuation.bracket",
"punctuation.delimiter",
"punctuation.special",
"string",
"tag",
"type.builtin",
"type",
"variable.builtin",
"variable.parameter",
"variable",
]
.iter()
.cloned()
.map(String::from)
.collect()
);
static ref HTML_ATTRS: Vec<String> = HIGHLIGHTER
.highlight_names
.iter()
.map(|s| format!("class={}", s))
.collect();
}
#[test]
fn test_highlighting_javascript() {
let source = "const a = function(b) { return b + c; }";
assert_eq!(
&to_token_vector(&source, &JS_HIGHLIGHT).unwrap(),
&[vec![
("const", vec!["keyword"]),
(" ", vec![]),
("a", vec!["function"]),
(" ", vec![]),
("=", vec!["operator"]),
(" ", vec![]),
("function", vec!["keyword"]),
("(", vec!["punctuation.bracket"]),
("b", vec!["variable.parameter"]),
(")", vec!["punctuation.bracket"]),
(" ", vec![]),
("{", vec!["punctuation.bracket"]),
(" ", vec![]),
("return", vec!["keyword"]),
(" ", vec![]),
("b", vec!["variable.parameter"]),
(" ", vec![]),
("+", vec!["operator"]),
(" ", vec![]),
("c", vec!["variable"]),
(";", vec!["punctuation.delimiter"]),
(" ", vec![]),
("}", vec!["punctuation.bracket"]),
]]
);
}
#[test]
@ -34,57 +87,68 @@ fn test_highlighting_injected_html_in_javascript() {
let source = vec!["const s = html `<div>${a < b}</div>`;"].join("\n");
assert_eq!(
&to_token_vector(&source, get_language("javascript"), &JS_SHEET).unwrap(),
&to_token_vector(&source, &JS_HIGHLIGHT).unwrap(),
&[vec![
("const", vec![Highlight::Keyword]),
("const", vec!["keyword"]),
(" ", vec![]),
("s", vec![Highlight::Variable]),
("s", vec!["variable"]),
(" ", vec![]),
("=", vec![Highlight::Operator]),
("=", vec!["operator"]),
(" ", vec![]),
("html", vec![Highlight::Function]),
("html", vec!["function"]),
(" ", vec![]),
("`<", vec![Highlight::String]),
("div", vec![Highlight::String, Highlight::Tag]),
(">", vec![Highlight::String]),
(
"${",
vec![
Highlight::String,
Highlight::Embedded,
Highlight::PunctuationSpecial
]
),
(
"a",
vec![Highlight::String, Highlight::Embedded, Highlight::Variable]
),
(" ", vec![Highlight::String, Highlight::Embedded]),
(
"<",
vec![Highlight::String, Highlight::Embedded, Highlight::Operator]
),
(" ", vec![Highlight::String, Highlight::Embedded]),
(
"b",
vec![Highlight::String, Highlight::Embedded, Highlight::Variable]
),
(
"}",
vec![
Highlight::String,
Highlight::Embedded,
Highlight::PunctuationSpecial
]
),
("</", vec![Highlight::String]),
("div", vec![Highlight::String, Highlight::Tag]),
(">`", vec![Highlight::String]),
(";", vec![Highlight::PunctuationDelimiter]),
("`", vec!["string"]),
("<", vec!["string", "punctuation.bracket"]),
("div", vec!["string", "tag"]),
(">", vec!["string", "punctuation.bracket"]),
("${", vec!["string", "embedded", "punctuation.special"]),
("a", vec!["string", "embedded", "variable"]),
(" ", vec!["string", "embedded"]),
("<", vec!["string", "embedded", "operator"]),
(" ", vec!["string", "embedded"]),
("b", vec!["string", "embedded", "variable"]),
("}", vec!["string", "embedded", "punctuation.special"]),
("</", vec!["string", "punctuation.bracket"]),
("div", vec!["string", "tag"]),
(">", vec!["string", "punctuation.bracket"]),
("`", vec!["string"]),
(";", vec!["punctuation.delimiter"]),
]]
);
}
#[test]
fn test_highlighting_injected_javascript_in_html_mini() {
let source = "<script>const x = new Thing();</script>";
eprintln!("HTML {:?}", HTML_HIGHLIGHT.language);
eprintln!("JavaScript {:?}", JS_HIGHLIGHT.language);
assert_eq!(
&to_token_vector(source, &HTML_HIGHLIGHT).unwrap(),
&[vec![
("<", vec!["punctuation.bracket"]),
("script", vec!["tag"]),
(">", vec!["punctuation.bracket"]),
("const", vec!["keyword"]),
(" ", vec![]),
("x", vec!["variable"]),
(" ", vec![]),
("=", vec!["operator"]),
(" ", vec![]),
("new", vec!["keyword"]),
(" ", vec![]),
("Thing", vec!["constructor"]),
("(", vec!["punctuation.bracket"]),
(")", vec!["punctuation.bracket"]),
(";", vec!["punctuation.delimiter"]),
("</", vec!["punctuation.bracket"]),
("script", vec!["tag"]),
(">", vec!["punctuation.bracket"]),
],]
);
}
#[test]
fn test_highlighting_injected_javascript_in_html() {
let source = vec![
@ -97,38 +161,44 @@ fn test_highlighting_injected_javascript_in_html() {
.join("\n");
assert_eq!(
&to_token_vector(&source, get_language("html"), &HTML_SHEET).unwrap(),
&to_token_vector(&source, &HTML_HIGHLIGHT).unwrap(),
&[
vec![("<", vec![]), ("body", vec![Highlight::Tag]), (">", vec![]),],
vec![
(" <", vec![]),
("script", vec![Highlight::Tag]),
(">", vec![]),
("<", vec!["punctuation.bracket"]),
("body", vec!["tag"]),
(">", vec!["punctuation.bracket"]),
],
vec![
(" ", vec![]),
("<", vec!["punctuation.bracket"]),
("script", vec!["tag"]),
(">", vec!["punctuation.bracket"]),
],
vec![
(" ", vec![]),
("const", vec![Highlight::Keyword]),
("const", vec!["keyword"]),
(" ", vec![]),
("x", vec![Highlight::Variable]),
("x", vec!["variable"]),
(" ", vec![]),
("=", vec![Highlight::Operator]),
("=", vec!["operator"]),
(" ", vec![]),
("new", vec![Highlight::Keyword]),
("new", vec!["keyword"]),
(" ", vec![]),
("Thing", vec![Highlight::Constructor]),
("(", vec![Highlight::PunctuationBracket]),
(")", vec![Highlight::PunctuationBracket]),
(";", vec![Highlight::PunctuationDelimiter]),
("Thing", vec!["constructor"]),
("(", vec!["punctuation.bracket"]),
(")", vec!["punctuation.bracket"]),
(";", vec!["punctuation.delimiter"]),
],
vec![
(" </", vec![]),
("script", vec![Highlight::Tag]),
(">", vec![]),
(" ", vec![]),
("</", vec!["punctuation.bracket"]),
("script", vec!["tag"]),
(">", vec!["punctuation.bracket"]),
],
vec![
("</", vec![]),
("body", vec![Highlight::Tag]),
(">", vec![]),
("</", vec!["punctuation.bracket"]),
("body", vec!["tag"]),
(">", vec!["punctuation.bracket"]),
],
]
);
@ -147,7 +217,7 @@ fn test_highlighting_multiline_nodes_to_html() {
.join("\n");
assert_eq!(
&to_html(&source, get_language("javascript"), &JS_SHEET,).unwrap(),
&to_html(&source, &JS_HIGHLIGHT).unwrap(),
&[
"<span class=Keyword>const</span> <span class=Constant>SOMETHING</span> <span class=Operator>=</span> <span class=String>`</span>\n".to_string(),
"<span class=String> one <span class=Embedded><span class=PunctuationSpecial>${</span></span></span>\n".to_string(),
@ -169,51 +239,51 @@ fn test_highlighting_with_local_variable_tracking() {
.join("\n");
assert_eq!(
&to_token_vector(&source, get_language("javascript"), &JS_SHEET).unwrap(),
&to_token_vector(&source, &JS_HIGHLIGHT).unwrap(),
&[
vec![
("module", vec![Highlight::VariableBuiltin]),
(".", vec![Highlight::PunctuationDelimiter]),
("exports", vec![Highlight::Property]),
("module", vec!["variable.builtin"]),
(".", vec!["punctuation.delimiter"]),
("exports", vec!["function"]),
(" ", vec![]),
("=", vec![Highlight::Operator]),
("=", vec!["operator"]),
(" ", vec![]),
("function", vec![Highlight::Keyword]),
("function", vec!["keyword"]),
(" ", vec![]),
("a", vec![Highlight::Function]),
("(", vec![Highlight::PunctuationBracket]),
("b", vec![Highlight::VariableParameter]),
(")", vec![Highlight::PunctuationBracket]),
("a", vec!["function"]),
("(", vec!["punctuation.bracket"]),
("b", vec!["variable.parameter"]),
(")", vec!["punctuation.bracket"]),
(" ", vec![]),
("{", vec![Highlight::PunctuationBracket])
("{", vec!["punctuation.bracket"])
],
vec![
(" ", vec![]),
("const", vec![Highlight::Keyword]),
("const", vec!["keyword"]),
(" ", vec![]),
("module", vec![Highlight::Variable]),
("module", vec!["variable"]),
(" ", vec![]),
("=", vec![Highlight::Operator]),
("=", vec!["operator"]),
(" ", vec![]),
("c", vec![Highlight::Variable]),
(";", vec![Highlight::PunctuationDelimiter])
("c", vec!["variable"]),
(";", vec!["punctuation.delimiter"])
],
vec![
(" ", vec![]),
("console", vec![Highlight::VariableBuiltin]),
(".", vec![Highlight::PunctuationDelimiter]),
("log", vec![Highlight::Function]),
("(", vec![Highlight::PunctuationBracket]),
("console", vec!["variable.builtin"]),
(".", vec!["punctuation.delimiter"]),
("log", vec!["function"]),
("(", vec!["punctuation.bracket"]),
// Not a builtin, because `module` was defined as a variable above.
("module", vec![Highlight::Variable]),
(",", vec![Highlight::PunctuationDelimiter]),
("module", vec!["variable"]),
(",", vec!["punctuation.delimiter"]),
(" ", vec![]),
// A parameter, because `b` was defined as a parameter above.
("b", vec![Highlight::VariableParameter]),
(")", vec![Highlight::PunctuationBracket]),
(";", vec![Highlight::PunctuationDelimiter]),
("b", vec!["variable.parameter"]),
(")", vec!["punctuation.bracket"]),
(";", vec!["punctuation.delimiter"]),
],
vec![("}", vec![Highlight::PunctuationBracket])]
vec![("}", vec!["punctuation.bracket"])]
],
);
}
@ -234,17 +304,17 @@ fn test_highlighting_empty_lines() {
.join("\n");
assert_eq!(
&to_html(&source, get_language("javascript"), &JS_SHEET,).unwrap(),
&to_html(&source, &JS_HIGHLIGHT,).unwrap(),
&[
"<span class=Keyword>class</span> <span class=Constructor>A</span> <span class=PunctuationBracket>{</span>\n".to_string(),
"<span class=keyword>class</span> <span class=constructor>A</span> <span class=punctuation.bracket>{</span>\n".to_string(),
"\n".to_string(),
" <span class=Function>b</span><span class=PunctuationBracket>(</span><span class=VariableParameter>c</span><span class=PunctuationBracket>)</span> <span class=PunctuationBracket>{</span>\n".to_string(),
" <span class=function>b</span><span class=punctuation.bracket>(</span><span class=variable.parameter>c</span><span class=punctuation.bracket>)</span> <span class=punctuation.bracket>{</span>\n".to_string(),
"\n".to_string(),
" <span class=Function>d</span><span class=PunctuationBracket>(</span><span class=Variable>e</span><span class=PunctuationBracket>)</span>\n".to_string(),
" <span class=function>d</span><span class=punctuation.bracket>(</span><span class=variable>e</span><span class=punctuation.bracket>)</span>\n".to_string(),
"\n".to_string(),
" <span class=PunctuationBracket>}</span>\n".to_string(),
" <span class=punctuation.bracket>}</span>\n".to_string(),
"\n".to_string(),
"<span class=PunctuationBracket>}</span>\n".to_string(),
"<span class=punctuation.bracket>}</span>\n".to_string(),
]
);
}
@ -254,20 +324,20 @@ fn test_highlighting_ejs() {
let source = vec!["<div><% foo() %></div>"].join("\n");
assert_eq!(
&to_token_vector(&source, get_language("embedded-template"), &EJS_SHEET).unwrap(),
&to_token_vector(&source, &EJS_HIGHLIGHT).unwrap(),
&[[
("<", vec![]),
("div", vec![Highlight::Tag]),
("div", vec!["tag"]),
(">", vec![]),
("<%", vec![Highlight::Keyword]),
("<%", vec!["keyword"]),
(" ", vec![]),
("foo", vec![Highlight::Function]),
("(", vec![Highlight::PunctuationBracket]),
(")", vec![Highlight::PunctuationBracket]),
("foo", vec!["function"]),
("(", vec!["punctuation.bracket"]),
(")", vec!["punctuation.bracket"]),
(" ", vec![]),
("%>", vec![Highlight::Keyword]),
("%>", vec!["keyword"]),
("</", vec![]),
("div", vec![Highlight::Tag]),
("div", vec!["tag"]),
(">", vec![])
]],
);
@ -278,33 +348,33 @@ fn test_highlighting_with_content_children_included() {
let source = vec!["assert!(", " a.b.c() < D::e::<F>()", ");"].join("\n");
assert_eq!(
&to_token_vector(&source, get_language("rust"), &RUST_SHEET).unwrap(),
&to_token_vector(&source, &RUST_HIGHLIGHT).unwrap(),
&[
vec![
("assert", vec![Highlight::Function]),
("!", vec![Highlight::Function]),
("(", vec![Highlight::PunctuationBracket]),
("assert", vec!["function"]),
("!", vec!["function"]),
("(", vec!["punctuation.bracket"]),
],
vec![
(" a", vec![]),
(".", vec![Highlight::PunctuationDelimiter]),
("b", vec![Highlight::Property]),
(".", vec![Highlight::PunctuationDelimiter]),
("c", vec![Highlight::Function]),
("(", vec![Highlight::PunctuationBracket]),
(")", vec![Highlight::PunctuationBracket]),
(".", vec!["punctuation.delimiter"]),
("b", vec!["property"]),
(".", vec!["punctuation.delimiter"]),
("c", vec!["function"]),
("(", vec!["punctuation.bracket"]),
(")", vec!["punctuation.bracket"]),
(" < ", vec![]),
("D", vec![Highlight::Type]),
("::", vec![Highlight::PunctuationDelimiter]),
("e", vec![Highlight::Function]),
("::", vec![Highlight::PunctuationDelimiter]),
("<", vec![Highlight::PunctuationBracket]),
("F", vec![Highlight::Type]),
(">", vec![Highlight::PunctuationBracket]),
("(", vec![Highlight::PunctuationBracket]),
(")", vec![Highlight::PunctuationBracket]),
("D", vec!["type"]),
("::", vec!["punctuation.delimiter"]),
("e", vec!["function"]),
("::", vec!["punctuation.delimiter"]),
("<", vec!["punctuation.bracket"]),
("F", vec!["type"]),
(">", vec!["punctuation.bracket"]),
("(", vec!["punctuation.bracket"]),
(")", vec!["punctuation.bracket"]),
],
vec![(")", vec![Highlight::PunctuationBracket]), (";", vec![]),]
vec![(")", vec!["punctuation.bracket"]), (";", vec![]),]
],
);
}
@ -327,18 +397,20 @@ fn test_highlighting_cancellation() {
// Constructing the highlighter, which eagerly parses the outer document,
// should not fail.
let highlighter = highlight(
source.as_bytes(),
get_language("html"),
&HTML_SHEET,
Some(&cancellation_flag),
injection_callback,
)
.unwrap();
let mut context = HighlightContext::new();
let events = HIGHLIGHTER
.highlight(
&mut context,
&HTML_HIGHLIGHT,
source.as_bytes(),
Some(&cancellation_flag),
injection_callback,
)
.unwrap();
// Iterating the scopes should not panic. It should return an error
// once the cancellation is detected.
for event in highlighter {
for event in events {
if let Err(e) = event {
assert_eq!(e, Error::Cancelled);
return;
@ -349,49 +421,68 @@ fn test_highlighting_cancellation() {
#[test]
fn test_highlighting_via_c_api() {
let js_lang = get_language("javascript");
let html_lang = get_language("html");
let js_sheet = get_property_sheet_json("javascript", "highlights.json");
let js_sheet = c_string(&js_sheet);
let html_sheet = get_property_sheet_json("html", "highlights.json");
let html_sheet = c_string(&html_sheet);
let highlights = vec![
"class=tag\0",
"class=function\0",
"class=string\0",
"class=keyword\0",
];
let highlight_names = highlights
.iter()
.map(|h| h["class=".len()..].as_ptr() as *const i8)
.collect::<Vec<_>>();
let highlight_attrs = highlights
.iter()
.map(|h| h.as_bytes().as_ptr() as *const i8)
.collect::<Vec<_>>();
let highlighter = c::ts_highlighter_new(
&highlight_names[0] as *const *const i8,
&highlight_attrs[0] as *const *const i8,
highlights.len() as u32,
);
let class_tag = c_string("class=tag");
let class_function = c_string("class=function");
let class_string = c_string("class=string");
let class_keyword = c_string("class=keyword");
let js_scope_name = c_string("source.js");
let html_scope_name = c_string("text.html.basic");
let injection_regex = c_string("^(javascript|js)$");
let source_code = c_string("<script>\nconst a = b('c');\nc.d();\n</script>");
let attribute_strings = &mut [ptr::null(); Highlight::Unknown as usize + 1];
attribute_strings[Highlight::Tag as usize] = class_tag.as_ptr();
attribute_strings[Highlight::String as usize] = class_string.as_ptr();
attribute_strings[Highlight::Keyword as usize] = class_keyword.as_ptr();
attribute_strings[Highlight::Function as usize] = class_function.as_ptr();
let js_scope = c_string("source.js");
let js_injection_regex = c_string("^javascript");
let language = get_language("javascript");
let (highlights_query, injections_query, locals_query) =
get_highlight_query_sources("javascript");
c::ts_highlighter_add_language(
highlighter,
js_scope.as_ptr(),
js_injection_regex.as_ptr(),
language,
highlights_query.as_ptr() as *const i8,
injections_query.as_ptr() as *const i8,
locals_query.as_ptr() as *const i8,
highlights_query.len() as u32,
injections_query.len() as u32,
locals_query.len() as u32,
);
let html_scope = c_string("text.html.basic");
let html_injection_regex = c_string("^html");
let language = get_language("html");
let (highlights_query, injections_query, locals_query) = get_highlight_query_sources("html");
c::ts_highlighter_add_language(
highlighter,
html_scope.as_ptr(),
html_injection_regex.as_ptr(),
language,
highlights_query.as_ptr() as *const i8,
injections_query.as_ptr() as *const i8,
locals_query.as_ptr() as *const i8,
highlights_query.len() as u32,
injections_query.len() as u32,
locals_query.len() as u32,
);
let highlighter = c::ts_highlighter_new(attribute_strings.as_ptr());
let buffer = c::ts_highlight_buffer_new();
c::ts_highlighter_add_language(
highlighter,
html_scope_name.as_ptr(),
html_lang,
html_sheet.as_ptr(),
ptr::null_mut(),
);
c::ts_highlighter_add_language(
highlighter,
js_scope_name.as_ptr(),
js_lang,
js_sheet.as_ptr(),
injection_regex.as_ptr(),
);
c::ts_highlighter_highlight(
highlighter,
html_scope_name.as_ptr(),
html_scope.as_ptr(),
source_code.as_ptr(),
source_code.as_bytes().len() as u32,
buffer,
@ -452,50 +543,57 @@ fn c_string(s: &str) -> CString {
CString::new(s.as_bytes().to_vec()).unwrap()
}
fn test_language_for_injection_string<'a>(
string: &str,
) -> Option<(Language, &'a PropertySheet<Properties>)> {
fn test_language_for_injection_string<'a>(string: &str) -> Option<&'a HighlightConfiguration> {
match string {
"javascript" => Some((get_language("javascript"), &JS_SHEET)),
"html" => Some((get_language("html"), &HTML_SHEET)),
"rust" => Some((get_language("rust"), &RUST_SHEET)),
"javascript" => Some(&JS_HIGHLIGHT),
"html" => Some(&HTML_HIGHLIGHT),
"rust" => Some(&RUST_HIGHLIGHT),
_ => None,
}
}
fn to_html<'a>(
src: &'a str,
language: Language,
property_sheet: &'a PropertySheet<Properties>,
language_config: &'a HighlightConfiguration,
) -> Result<Vec<String>, Error> {
highlight_html(
src.as_bytes(),
language,
property_sheet,
let src = src.as_bytes();
let mut renderer = HtmlRenderer::new();
let mut context = HighlightContext::new();
let events = HIGHLIGHTER.highlight(
&mut context,
language_config,
src,
None,
&test_language_for_injection_string,
&|highlight| SCOPE_CLASS_STRINGS[highlight as usize].as_str(),
)
)?;
renderer
.render(events, src, &|highlight| HTML_ATTRS[highlight.0].as_bytes())
.unwrap();
Ok(renderer.lines().map(|s| s.to_string()).collect())
}
fn to_token_vector<'a>(
src: &'a str,
language: Language,
property_sheet: &'a PropertySheet<Properties>,
) -> Result<Vec<Vec<(&'a str, Vec<Highlight>)>>, Error> {
language_config: &'a HighlightConfiguration,
) -> Result<Vec<Vec<(&'a str, Vec<&'static str>)>>, Error> {
let src = src.as_bytes();
let mut context = HighlightContext::new();
let mut lines = Vec::new();
let mut highlights = Vec::new();
let mut line = Vec::new();
for event in highlight(
let events = HIGHLIGHTER.highlight(
&mut context,
language_config,
src,
language,
property_sheet,
None,
&test_language_for_injection_string,
)? {
)?;
for event in events {
match event? {
HighlightEvent::HighlightStart(s) => highlights.push(s),
HighlightEvent::HighlightStart(s) => {
highlights.push(HIGHLIGHTER.highlight_names[s.0].as_str())
}
HighlightEvent::HighlightEnd => {
highlights.pop();
}

47
highlight/include/tree_sitter/highlight.h
View file

@ -14,47 +14,15 @@ typedef enum {
TSHighlightInvalidLanguage,
} TSHighlightError;
// The list of scopes which can be styled for syntax highlighting.
// When constructing a `TSHighlighter`, you need to construct an
// `attribute_strings` array whose elements correspond to these values.
enum TSHighlightValue {
TSHighlightValueAttribute,
TSHighlightValueComment,
TSHighlightValueConstant,
TSHighlightValueConstantBuiltin,
TSHighlightValueConstructor,
TSHighlightValueConstructorBuiltin,
TSHighlightValueEmbedded,
TSHighlightValueEscape,
TSHighlightValueFunction,
TSHighlightValueFunctionBuiltin,
TSHighlightValueKeyword,
TSHighlightValueNumber,
TSHighlightValueOperator,
TSHighlightValueProperty,
TSHighlightValuePropertyBuiltin,
TSHighlightValuePunctuation,
TSHighlightValuePunctuationBracket,
TSHighlightValuePunctuationDelimiter,
TSHighlightValuePunctuationSpecial,
TSHighlightValueString,
TSHighlightValueStringSpecial,
TSHighlightValueTag,
TSHighlightValueType,
TSHighlightValueTypeBuiltin,
TSHighlightValueVariable,
TSHighlightValueVariableBuiltin,
TSHighlightValueVariableParameter,
TSHighlightValueUnknown,
};
typedef struct TSHighlighter TSHighlighter;
typedef struct TSHighlightBuffer TSHighlightBuffer;
// Construct a `TSHighlighter` by providing a list of strings containing
// the HTML attributes that should be applied for each highlight value.
TSHighlighter *ts_highlighter_new(
const char **attribute_strings
const char **highlight_names,
const char **attribute_strings,
uint32_t highlight_count
);
// Delete a syntax highlighter.
@ -70,9 +38,14 @@ void ts_highlighter_delete(TSHighlighter *);
int ts_highlighter_add_language(
TSHighlighter *self,
const char *scope_name,
const char *injection_regex,
const TSLanguage *language,
const char *property_sheet_json,
const char *injection_regex
const char *highlight_query,
const char *injection_query,
const char *locals_query,
uint32_t highlight_query_len,
uint32_t injection_query_len,
uint32_t locals_query_len
);
// Compute syntax highlighting for a given document. You must first

192
highlight/src/c_lib.rs
View file

@ -1,25 +1,23 @@
use super::{load_property_sheet, Error, Highlight, Highlighter, HtmlRenderer, Properties};
use super::{Error, HighlightConfiguration, HighlightContext, Highlighter, HtmlRenderer};
use regex::Regex;
use std::collections::HashMap;
use std::ffi::CStr;
use std::os::raw::c_char;
use std::process::abort;
use std::sync::atomic::AtomicUsize;
use std::{fmt, slice};
use tree_sitter::{Language, PropertySheet};
struct LanguageConfiguration {
language: Language,
property_sheet: PropertySheet<Properties>,
injection_regex: Option<Regex>,
}
use std::{fmt, slice, str};
use tree_sitter::Language;
pub struct TSHighlighter {
languages: HashMap<String, LanguageConfiguration>,
languages: HashMap<String, (Option<Regex>, HighlightConfiguration)>,
attribute_strings: Vec<&'static [u8]>,
highlighter: Highlighter,
}
pub struct TSHighlightBuffer(HtmlRenderer);
pub struct TSHighlightBuffer {
context: HighlightContext,
renderer: HtmlRenderer,
}
#[repr(C)]
pub enum ErrorCode {
@ -27,33 +25,113 @@ pub enum ErrorCode {
UnknownScope,
Timeout,
InvalidLanguage,
InvalidUtf8,
InvalidRegex,
InvalidQuery,
}
#[no_mangle]
pub extern "C" fn ts_highlighter_new(
highlight_names: *const *const c_char,
attribute_strings: *const *const c_char,
highlight_count: u32,
) -> *mut TSHighlighter {
let highlight_names =
unsafe { slice::from_raw_parts(highlight_names, highlight_count as usize) };
let attribute_strings =
unsafe { slice::from_raw_parts(attribute_strings, Highlight::Unknown as usize + 1) };
unsafe { slice::from_raw_parts(attribute_strings, highlight_count as usize) };
let highlight_names = highlight_names
.into_iter()
.map(|s| unsafe { CStr::from_ptr(*s).to_string_lossy().to_string() })
.collect();
let attribute_strings = attribute_strings
.into_iter()
.map(|s| {
if s.is_null() {
&[]
} else {
unsafe { CStr::from_ptr(*s).to_bytes() }
}
})
.map(|s| unsafe { CStr::from_ptr(*s).to_bytes() })
.collect();
let highlighter = Highlighter::new(highlight_names);
Box::into_raw(Box::new(TSHighlighter {
languages: HashMap::new(),
attribute_strings,
highlighter,
}))
}
#[no_mangle]
pub extern "C" fn ts_highlighter_add_language(
this: *mut TSHighlighter,
scope_name: *const c_char,
injection_regex: *const c_char,
language: Language,
highlight_query: *const c_char,
injection_query: *const c_char,
locals_query: *const c_char,
highlight_query_len: u32,
injection_query_len: u32,
locals_query_len: u32,
) -> ErrorCode {
let f = move || {
let this = unwrap_mut_ptr(this);
let scope_name = unsafe { CStr::from_ptr(scope_name) };
let scope_name = scope_name
.to_str()
.or(Err(ErrorCode::InvalidUtf8))?
.to_string();
let injection_regex = if injection_regex.is_null() {
None
} else {
let pattern = unsafe { CStr::from_ptr(injection_regex) };
let pattern = pattern.to_str().or(Err(ErrorCode::InvalidUtf8))?;
Some(Regex::new(pattern).or(Err(ErrorCode::InvalidRegex))?)
};
let highlight_query = unsafe {
slice::from_raw_parts(highlight_query as *const u8, highlight_query_len as usize)
};
let highlight_query = str::from_utf8(highlight_query).or(Err(ErrorCode::InvalidUtf8))?;
let injection_query = if injection_query_len > 0 {
let query = unsafe {
slice::from_raw_parts(injection_query as *const u8, injection_query_len as usize)
};
str::from_utf8(query).or(Err(ErrorCode::InvalidUtf8))?
} else {
""
};
let locals_query = if locals_query_len > 0 {
let query = unsafe {
slice::from_raw_parts(locals_query as *const u8, locals_query_len as usize)
};
str::from_utf8(query).or(Err(ErrorCode::InvalidUtf8))?
} else {
""
};
this.languages.insert(
scope_name,
(
injection_regex,
this.highlighter
.load_configuration(language, highlight_query, injection_query, locals_query)
.or(Err(ErrorCode::InvalidQuery))?,
),
);
Ok(())
};
match f() {
Ok(()) => ErrorCode::Ok,
Err(e) => e,
}
}
#[no_mangle]
pub extern "C" fn ts_highlight_buffer_new() -> *mut TSHighlightBuffer {
Box::into_raw(Box::new(TSHighlightBuffer(HtmlRenderer::new())))
Box::into_raw(Box::new(TSHighlightBuffer {
context: HighlightContext::new(),
renderer: HtmlRenderer::new(),
}))
}
#[no_mangle]
@ -69,59 +147,25 @@ pub extern "C" fn ts_highlight_buffer_delete(this: *mut TSHighlightBuffer) {
#[no_mangle]
pub extern "C" fn ts_highlight_buffer_content(this: *const TSHighlightBuffer) -> *const u8 {
let this = unwrap_ptr(this);
this.0.html.as_slice().as_ptr()
this.renderer.html.as_slice().as_ptr()
}
#[no_mangle]
pub extern "C" fn ts_highlight_buffer_line_offsets(this: *const TSHighlightBuffer) -> *const u32 {
let this = unwrap_ptr(this);
this.0.line_offsets.as_slice().as_ptr()
this.renderer.line_offsets.as_slice().as_ptr()
}
#[no_mangle]
pub extern "C" fn ts_highlight_buffer_len(this: *const TSHighlightBuffer) -> u32 {
let this = unwrap_ptr(this);
this.0.html.len() as u32
this.renderer.html.len() as u32
}
#[no_mangle]
pub extern "C" fn ts_highlight_buffer_line_count(this: *const TSHighlightBuffer) -> u32 {
let this = unwrap_ptr(this);
this.0.line_offsets.len() as u32
}
#[no_mangle]
pub extern "C" fn ts_highlighter_add_language(
this: *mut TSHighlighter,
scope_name: *const c_char,
language: Language,
property_sheet_json: *const c_char,
injection_regex: *const c_char,
) -> ErrorCode {
let this = unwrap_mut_ptr(this);
let scope_name = unsafe { CStr::from_ptr(scope_name) };
let scope_name = unwrap(scope_name.to_str()).to_string();
let property_sheet_json = unsafe { CStr::from_ptr(property_sheet_json) };
let property_sheet_json = unwrap(property_sheet_json.to_str());
let property_sheet = unwrap(load_property_sheet(language, property_sheet_json));
let injection_regex = if injection_regex.is_null() {
None
} else {
let pattern = unsafe { CStr::from_ptr(injection_regex) };
Some(unwrap(Regex::new(unwrap(pattern.to_str()))))
};
this.languages.insert(
scope_name,
LanguageConfiguration {
language,
property_sheet,
injection_regex,
},
);
ErrorCode::Ok
this.renderer.line_offsets.len() as u32
}
#[no_mangle]
@ -150,36 +194,36 @@ impl TSHighlighter {
output: &mut TSHighlightBuffer,
cancellation_flag: Option<&AtomicUsize>,
) -> ErrorCode {
let configuration = self.languages.get(scope_name);
if configuration.is_none() {
let entry = self.languages.get(scope_name);
if entry.is_none() {
return ErrorCode::UnknownScope;
}
let configuration = configuration.unwrap();
let (_, configuration) = entry.unwrap();
let languages = &self.languages;
let highlighter = Highlighter::new(
let highlights = self.highlighter.highlight(
&mut output.context,
configuration,
source_code,
configuration.language,
&configuration.property_sheet,
|injection_string| {
languages.values().find_map(|conf| {
conf.injection_regex.as_ref().and_then(|regex| {
cancellation_flag,
move |injection_string| {
languages.values().find_map(|(injection_regex, config)| {
injection_regex.as_ref().and_then(|regex| {
if regex.is_match(injection_string) {
Some((conf.language, &conf.property_sheet))
Some(config)
} else {
None
}
})
})
},
cancellation_flag,
);
if let Ok(highlighter) = highlighter {
output.0.reset();
let result = output.0.render(highlighter, source_code, &|s| {
self.attribute_strings[s as usize]
});
if let Ok(highlights) = highlights {
output.renderer.reset();
let result = output
.renderer
.render(highlights, source_code, &|s| self.attribute_strings[s.0]);
match result {
Err(Error::Cancelled) => {
return ErrorCode::Timeout;

1514
highlight/src/lib.rs
View file

File diff suppressed because it is too large Load diff