traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #499 from tree-sitter/highlight-test

Browse source 
Add a system for testing syntax highlighting queries
This commit is contained in:
Max Brunsfeld 2019-12-05 15:55:24 -08:00 • committed by GitHub
commit d6c7b243a7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 750 additions and 365 deletions
Download patch file Download diff file Expand all files Collapse all files

7
cli/src/error.rs
View file

@ -1,3 +1,4 @@
use super::test_highlight;
use std::fmt::Write;
use std::io;
use tree_sitter::QueryError;
@ -98,6 +99,12 @@ impl From<regex_syntax::ast::Error> for Error {
}
}
impl From<test_highlight::Failure> for Error {
fn from(error: test_highlight::Failure) -> Self {
Error::new(error.message())
}
}
impl From<String> for Error {
fn from(error: String) -> Self {
Error::new(error)

74
cli/src/highlight.rs
View file

@ -10,9 +10,7 @@ use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::Instant;
use std::{fs, io, path, str, thread, usize};
use tree_sitter_highlight::{
HighlightConfiguration, HighlightContext, HighlightEvent, Highlighter, HtmlRenderer,
};
use tree_sitter_highlight::{HighlightConfiguration, HighlightEvent, Highlighter, HtmlRenderer};
pub const HTML_HEADER: &'static str = "
<!doctype HTML>
@ -53,8 +51,8 @@ pub struct Style {
#[derive(Debug)]
pub struct Theme {
pub highlighter: Highlighter,
styles: Vec<Style>,
pub styles: Vec<Style>,
pub highlight_names: Vec<String>,
}
impl Theme {
@ -73,21 +71,21 @@ impl<'de> Deserialize<'de> for Theme {
where
D: Deserializer<'de>,
{
let mut names = Vec::new();
let mut styles = Vec::new();
let mut highlight_names = Vec::new();
if let Ok(colors) = HashMap::<String, Value>::deserialize(deserializer) {
names.reserve(colors.len());
highlight_names.reserve(colors.len());
styles.reserve(colors.len());
for (name, style_value) in colors {
let mut style = Style::default();
parse_style(&mut style, style_value);
names.push(name);
highlight_names.push(name);
styles.push(style);
}
}
Ok(Self {
highlighter: Highlighter::new(names),
styles,
highlight_names,
})
}
}
@ -98,7 +96,7 @@ impl Serialize for Theme {
S: Serializer,
{
let mut map = serializer.serialize_map(Some(self.styles.len()))?;
for (name, style) in self.highlighter.names().iter().zip(&self.styles) {
for (name, style) in self.highlight_names.iter().zip(&self.styles) {
let style = &style.ansi;
let color = style.foreground.map(|color| match color {
Color::Black => json!("black"),
@ -284,15 +282,11 @@ pub fn ansi(
let mut stdout = stdout.lock();
let time = Instant::now();
let cancellation_flag = cancel_on_stdin();
let mut context = HighlightContext::new();
let mut highlighter = Highlighter::new();
let events = theme.highlighter.highlight(
&mut context,
config,
source,
Some(&cancellation_flag),
|string| language_for_injection_string(loader, theme, string),
)?;
let events = highlighter.highlight(config, source, Some(&cancellation_flag), |string| {
loader.highlight_config_for_injection_string(string)
})?;
let mut style_stack = vec![theme.default_style().ansi];
for event in events {
@ -333,15 +327,11 @@ pub fn html(
let mut stdout = stdout.lock();
let time = Instant::now();
let cancellation_flag = cancel_on_stdin();
let mut context = HighlightContext::new();
let mut highlighter = Highlighter::new();
let events = theme.highlighter.highlight(
&mut context,
config,
source,
Some(&cancellation_flag),
|string| language_for_injection_string(loader, theme, string),
)?;
let events = highlighter.highlight(config, source, Some(&cancellation_flag), |string| {
loader.highlight_config_for_injection_string(string)
})?;
let mut renderer = HtmlRenderer::new();
renderer.render(events, source, &move |highlight| {
@ -370,35 +360,3 @@ pub fn html(
Ok(())
}
fn language_for_injection_string<'a>(
loader: &'a Loader,
theme: &Theme,
string: &str,
) -> Option<&'a HighlightConfiguration> {
match loader.language_configuration_for_injection_string(string) {
Err(e) => {
eprintln!(
"Failed to load language for injection string '{}': {}",
string,
e.message()
);
None
}
Ok(None) => None,
Ok(Some((language, configuration))) => {
match configuration.highlight_config(&theme.highlighter, language) {
Err(e) => {
eprintln!(
"Failed to load property sheet for injection string '{}': {}",
string,
e.message()
);
None
}
Ok(None) => None,
Ok(Some(config)) => Some(config),
}
}
}
}

1
cli/src/lib.rs
View file

@ -7,6 +7,7 @@ pub mod logger;
pub mod parse;
pub mod query;
pub mod test;
pub mod test_highlight;
pub mod util;
pub mod wasm;
pub mod web_ui;

139
cli/src/loader.rs
View file

@ -7,10 +7,11 @@ use std::collections::HashMap;
use std::io::BufReader;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::sync::Mutex;
use std::time::SystemTime;
use std::{fs, mem};
use tree_sitter::Language;
use tree_sitter_highlight::{HighlightConfiguration, Highlighter};
use tree_sitter_highlight::HighlightConfiguration;
#[cfg(unix)]
const DYLIB_EXTENSION: &'static str = "so";
@ -20,8 +21,7 @@ const DYLIB_EXTENSION: &'static str = "dll";
const BUILD_TARGET: &'static str = env!("BUILD_TARGET");
#[derive(Default)]
pub struct LanguageConfiguration {
pub struct LanguageConfiguration<'a> {
pub scope: Option<String>,
pub content_regex: Option<Regex>,
pub _first_line_regex: Option<Regex>,
@ -33,13 +33,17 @@ pub struct LanguageConfiguration {
pub locals_filenames: Option<Vec<String>>,
language_id: usize,
highlight_config: OnceCell<Option<HighlightConfiguration>>,
highlight_names: &'a Mutex<Vec<String>>,
use_all_highlight_names: bool,
}
pub struct Loader {
parser_lib_path: PathBuf,
languages_by_id: Vec<(PathBuf, OnceCell<Language>)>,
language_configurations: Vec<LanguageConfiguration>,
language_configurations: Vec<LanguageConfiguration<'static>>,
language_configuration_ids_by_file_type: HashMap<String, Vec<usize>>,
highlight_names: Box<Mutex<Vec<String>>>,
use_all_highlight_names: bool,
}
unsafe impl Send for Loader {}
@ -52,9 +56,22 @@ impl Loader {
languages_by_id: Vec::new(),
language_configurations: Vec::new(),
language_configuration_ids_by_file_type: HashMap::new(),
highlight_names: Box::new(Mutex::new(Vec::new())),
use_all_highlight_names: true,
}
}
pub fn configure_highlights(&mut self, names: &Vec<String>) {
self.use_all_highlight_names = false;
let mut highlights = self.highlight_names.lock().unwrap();
highlights.clear();
highlights.extend(names.iter().cloned());
}
pub fn highlight_names(&self) -> Vec<String> {
self.highlight_names.lock().unwrap().clone()
}
pub fn find_all_languages(&mut self, parser_src_paths: &Vec<PathBuf>) -> Result<()> {
for parser_container_dir in parser_src_paths.iter() {
if let Ok(entries) = fs::read_dir(parser_container_dir) {
@ -339,7 +356,36 @@ impl Loader {
Ok(language)
}
fn find_language_configurations_at_path<'a>(
pub fn highlight_config_for_injection_string<'a>(
&'a self,
string: &str,
) -> Option<&'a HighlightConfiguration> {
match self.language_configuration_for_injection_string(string) {
Err(e) => {
eprintln!(
"Failed to load language for injection string '{}': {}",
string,
e.message()
);
None
}
Ok(None) => None,
Ok(Some((language, configuration))) => match configuration.highlight_config(language) {
Err(e) => {
eprintln!(
"Failed to load property sheet for injection string '{}': {}",
string,
e.message()
);
None
}
Ok(None) => None,
Ok(Some(config)) => Some(config),
},
}
}
pub fn find_language_configurations_at_path<'a>(
&'a mut self,
parser_path: &Path,
) -> Result<&[LanguageConfiguration]> {
@ -428,19 +474,15 @@ impl Loader {
scope: config_json.scope,
language_id,
file_types: config_json.file_types.unwrap_or(Vec::new()),
content_regex: config_json
.content_regex
.and_then(|r| RegexBuilder::new(&r).multi_line(true).build().ok()),
_first_line_regex: config_json
.first_line_regex
.and_then(|r| RegexBuilder::new(&r).multi_line(true).build().ok()),
injection_regex: config_json
.injection_regex
.and_then(|r| RegexBuilder::new(&r).multi_line(true).build().ok()),
highlight_config: OnceCell::new(),
content_regex: Self::regex(config_json.content_regex),
_first_line_regex: Self::regex(config_json.first_line_regex),
injection_regex: Self::regex(config_json.injection_regex),
injections_filenames: config_json.injections.into_vec(),
locals_filenames: config_json.locals.into_vec(),
highlights_filenames: config_json.highlights.into_vec(),
highlight_config: OnceCell::new(),
highlight_names: &*self.highlight_names,
use_all_highlight_names: self.use_all_highlight_names,
};
for file_type in &configuration.file_types {
@ -450,7 +492,8 @@ impl Loader {
.push(self.language_configurations.len());
}
self.language_configurations.push(configuration);
self.language_configurations
.push(unsafe { mem::transmute(configuration) });
}
}
}
@ -458,24 +501,37 @@ impl Loader {
if self.language_configurations.len() == initial_language_configuration_count
&& parser_path.join("src").join("grammar.json").exists()
{
let mut configuration = LanguageConfiguration::default();
configuration.root_path = parser_path.to_owned();
configuration.language_id = self.languages_by_id.len();
self.language_configurations.push(configuration);
let configuration = LanguageConfiguration {
root_path: parser_path.to_owned(),
language_id: self.languages_by_id.len(),
file_types: Vec::new(),
scope: None,
content_regex: None,
_first_line_regex: None,
injection_regex: None,
injections_filenames: None,
locals_filenames: None,
highlights_filenames: None,
highlight_config: OnceCell::new(),
highlight_names: &*self.highlight_names,
use_all_highlight_names: self.use_all_highlight_names,
};
self.language_configurations
.push(unsafe { mem::transmute(configuration) });
self.languages_by_id
.push((parser_path.to_owned(), OnceCell::new()));
}
Ok(&self.language_configurations[initial_language_configuration_count..])
}
fn regex(pattern: Option<String>) -> Option<Regex> {
pattern.and_then(|r| RegexBuilder::new(&r).multi_line(true).build().ok())
}
}
impl LanguageConfiguration {
pub fn highlight_config(
&self,
highlighter: &Highlighter,
language: Language,
) -> Result<Option<&HighlightConfiguration>> {
impl<'a> LanguageConfiguration<'a> {
pub fn highlight_config(&self, language: Language) -> Result<Option<&HighlightConfiguration>> {
self.highlight_config
.get_or_try_init(|| {
let queries_path = self.root_path.join("queries");
@ -508,18 +564,25 @@ impl LanguageConfiguration {
if highlights_query.is_empty() {
Ok(None)
} else {
Ok(Some(
highlighter
.load_configuration(
language,
&highlights_query,
&injections_query,
&locals_query,
)
.map_err(Error::wrap(|| {
format!("Failed to load queries in {:?}", self.root_path)
}))?,
))
let mut result = HighlightConfiguration::new(
language,
&highlights_query,
&injections_query,
&locals_query,
)
.map_err(Error::wrap(|| {
format!("Failed to load queries in {:?}", self.root_path)
}))?;
let mut all_highlight_names = self.highlight_names.lock().unwrap();
if self.use_all_highlight_names {
for capture_name in result.query.capture_names() {
if !all_highlight_names.contains(capture_name) {
all_highlight_names.push(capture_name.clone());
}
}
}
result.configure(&all_highlight_names);
Ok(Some(result))
}
})
.map(Option::as_ref)

47
cli/src/main.rs
View file

@ -5,7 +5,8 @@ use std::process::exit;
use std::{env, fs, u64};
use tree_sitter::Language;
use tree_sitter_cli::{
config, error, generate, highlight, loader, logger, parse, query, test, wasm, web_ui,
config, error, generate, highlight, loader, logger, parse, query, test, test_highlight, wasm,
web_ui,
};
const BUILD_VERSION: &'static str = env!("CARGO_PKG_VERSION");
@ -162,17 +163,28 @@ fn run() -> error::Result<()> {
let debug = matches.is_present("debug");
let debug_graph = matches.is_present("debug-graph");
let filter = matches.value_of("filter");
if let Some(language) = loader.languages_at_path(&current_dir)?.first() {
test::run_tests_at_path(
*language,
&current_dir.join("corpus"),
debug,
debug_graph,
filter,
)?;
test::check_queries_at_path(*language, &current_dir.join("queries"))?;
} else {
eprintln!("No language found");
let languages = loader.languages_at_path(&current_dir)?;
let language = languages
.first()
.ok_or_else(|| "No language found".to_string())?;
let test_dir = current_dir.join("test");
// Run the corpus tests. Look for them at two paths: `test/corpus` and `corpus`.
let mut test_corpus_dir = test_dir.join("corpus");
if !test_corpus_dir.is_dir() {
test_corpus_dir = current_dir.join("corpus");
}
if test_corpus_dir.is_dir() {
test::run_tests_at_path(*language, &test_corpus_dir, debug, debug_graph, filter)?;
}
// Check that all of the queries are valid.
test::check_queries_at_path(*language, &current_dir.join("queries"))?;
// Run the syntax highlighting tests.
let test_highlight_dir = test_dir.join("highlight");
if test_highlight_dir.is_dir() {
test_highlight::test_highlights(&loader, &test_highlight_dir)?;
}
} else if let Some(matches) = matches.subcommand_matches("parse") {
let debug = matches.is_present("debug");
@ -232,11 +244,12 @@ fn run() -> error::Result<()> {
let query_path = Path::new(matches.value_of("query-path").unwrap());
query::query_files_at_paths(language, paths, query_path, ordered_captures)?;
} else if let Some(matches) = matches.subcommand_matches("highlight") {
let paths = matches.values_of("path").unwrap().into_iter();
let html_mode = matches.is_present("html");
let time = matches.is_present("time");
loader.configure_highlights(&config.theme.highlight_names);
loader.find_all_languages(&config.parser_directories)?;
let time = matches.is_present("time");
let paths = matches.values_of("path").unwrap().into_iter();
let html_mode = matches.is_present("html");
if html_mode {
println!("{}", highlight::HTML_HEADER);
}
@ -266,9 +279,7 @@ fn run() -> error::Result<()> {
let source = fs::read(path)?;
if let Some(highlight_config) =
language_config.highlight_config(&config.theme.highlighter, language)?
{
if let Some(highlight_config) = language_config.highlight_config(language)? {
if html_mode {
highlight::html(&loader, &config.theme, &source, highlight_config, time)?;
} else {

305
cli/src/test_highlight.rs Normal file
View file

@ -0,0 +1,305 @@
use super::error::Result;
use crate::loader::Loader;
use ansi_term::Colour;
use lazy_static::lazy_static;
use regex::Regex;
use std::fs;
use std::path::Path;
use tree_sitter::{Language, Parser, Point};
use tree_sitter_highlight::{Highlight, HighlightConfiguration, HighlightEvent, Highlighter};
lazy_static! {
static ref HIGHLIGHT_NAME_REGEX: Regex = Regex::new("[\\w_\\-.]+").unwrap();
}
pub struct Failure {
row: usize,
column: usize,
expected_highlight: String,
actual_highlights: Vec<String>,
}
impl Failure {
pub fn message(&self) -> String {
let mut result = format!(
"Failure - row: {}, column: {}, expected highlight '{}', actual highlights: ",
self.row, self.column, self.expected_highlight
);
if self.actual_highlights.is_empty() {
result += "none.";
} else {
for (i, actual_highlight) in self.actual_highlights.iter().enumerate() {
if i > 0 {
result += ", ";
}
result += "'";
result += actual_highlight;
result += "'";
}
}
result
}
}
pub fn test_highlights(loader: &Loader, directory: &Path) -> Result<()> {
let mut failed = false;
let mut highlighter = Highlighter::new();
println!("syntax highlighting:");
for highlight_test_file in fs::read_dir(directory)? {
let highlight_test_file = highlight_test_file?;
let test_file_path = highlight_test_file.path();
let test_file_name = highlight_test_file.file_name();
let (language, language_config) = loader
.language_configuration_for_file_name(&test_file_path)?
.ok_or_else(|| format!("No language found for path {:?}", test_file_path))?;
let highlight_config = language_config
.highlight_config(language)?
.ok_or_else(|| format!("No highlighting config found for {:?}", test_file_path))?;
match test_highlight(
&loader,
&mut highlighter,
highlight_config,
fs::read(&test_file_path)?.as_slice(),
) {
Ok(assertion_count) => {
println!(
" ✓ {} ({} assertions)",
Colour::Green.paint(test_file_name.to_string_lossy().as_ref()),
assertion_count
);
}
Err(e) => {
println!(
" ✗ {}",
Colour::Red.paint(test_file_name.to_string_lossy().as_ref())
);
println!(" {}", e.message());
failed = true;
}
}
}
if failed {
Err(String::new().into())
} else {
Ok(())
}
}
pub fn test_highlight(
loader: &Loader,
highlighter: &mut Highlighter,
highlight_config: &HighlightConfiguration,
source: &[u8],
) -> Result<usize> {
// Highlight the file, and parse out all of the highlighting assertions.
let highlight_names = loader.highlight_names();
let highlights = get_highlight_positions(loader, highlighter, highlight_config, source)?;
let assertions = parse_highlight_test(highlighter.parser(), highlight_config.language, source)?;
// Iterate through all of the highlighting assertions, checking each one against the
// actual highlights.
let mut i = 0;
let mut actual_highlights = Vec::<&String>::new();
for (position, expected_highlight) in &assertions {
let mut passed = false;
actual_highlights.clear();
'highlight_loop: loop {
// The assertions are ordered by position, so skip past all of the highlights that
// end at or before this assertion's position.
if let Some(highlight) = highlights.get(i) {
if highlight.1 <= *position {
i += 1;
continue;
}
// Iterate through all of the highlights that start at or before this assertion's,
// position, looking for one that matches the assertion.
let mut j = i;
while let (false, Some(highlight)) = (passed, highlights.get(j)) {
if highlight.0 > *position {
break 'highlight_loop;
}
// If the highlight matches the assertion, this test passes. Otherwise,
// add this highlight to the list of actual highlights that span the
// assertion's position, in order to generate an error message in the event
// of a failure.
let highlight_name = &highlight_names[(highlight.2).0];
if *highlight_name == *expected_highlight {
passed = true;
break 'highlight_loop;
} else {
actual_highlights.push(highlight_name);
}
j += 1;
}
} else {
break;
}
}
if !passed {
return Err(Failure {
row: position.row,
column: position.column,
expected_highlight: expected_highlight.clone(),
actual_highlights: actual_highlights.into_iter().cloned().collect(),
}
.into());
}
}
Ok(assertions.len())
}
/// Parse the given source code, finding all of the comments that contain
/// highlighting assertions. Return a vector of (position, expected highlight name)
/// pairs.
pub fn parse_highlight_test(
parser: &mut Parser,
language: Language,
source: &[u8],
) -> Result<Vec<(Point, String)>> {
let mut result = Vec::new();
let mut assertion_ranges = Vec::new();
// Parse the code.
parser.set_language(language).unwrap();
let tree = parser.parse(source, None).unwrap();
// Walk the tree, finding comment nodes that contain assertions.
let mut ascending = false;
let mut cursor = tree.root_node().walk();
loop {
if ascending {
let node = cursor.node();
// Find every comment node.
if node.kind().contains("comment") {
if let Ok(text) = node.utf8_text(source) {
let mut position = node.start_position();
if position.row == 0 {
continue;
}
// Find the arrow character ("^" or '<-") in the comment. A left arrow
// refers to the column where the comment node starts. An up arrow refers
// to its own column.
let mut has_left_caret = false;
let mut has_arrow = false;
let mut arrow_end = 0;
for (i, c) in text.char_indices() {
arrow_end = i + 1;
if c == '-' && has_left_caret {
has_arrow = true;
break;
}
if c == '^' {
has_arrow = true;
position.column += i;
break;
}
has_left_caret = c == '<';
}
// If the comment node contains an arrow and a highlight name, record the
// highlight name and the position.
if let (true, Some(mat)) =
(has_arrow, HIGHLIGHT_NAME_REGEX.find(&text[arrow_end..]))
{
assertion_ranges.push((node.start_position(), node.end_position()));
result.push((position, mat.as_str().to_string()));
}
}
}
// Continue walking the tree.
if cursor.goto_next_sibling() {
ascending = false;
} else if !cursor.goto_parent() {
break;
}
} else if !cursor.goto_first_child() {
ascending = true;
}
}
// Adjust the row number in each assertion's position to refer to the line of
// code *above* the assertion. There can be multiple lines of assertion comments,
// so the positions may have to be decremented by more than one row.
let mut i = 0;
for (position, _) in result.iter_mut() {
loop {
let on_assertion_line = assertion_ranges[i..]
.iter()
.any(|(start, _)| start.row == position.row);
if on_assertion_line {
position.row -= 1;
} else {
while i < assertion_ranges.len() && assertion_ranges[i].0.row < position.row {
i += 1;
}
break;
}
}
}
// The assertions can end up out of order due to the line adjustments.
result.sort_unstable_by_key(|a| a.0);
Ok(result)
}
pub fn get_highlight_positions(
loader: &Loader,
highlighter: &mut Highlighter,
highlight_config: &HighlightConfiguration,
source: &[u8],
) -> Result<Vec<(Point, Point, Highlight)>> {
let mut row = 0;
let mut column = 0;
let mut byte_offset = 0;
let mut was_newline = false;
let mut result = Vec::new();
let mut highlight_stack = Vec::new();
let source = String::from_utf8_lossy(source);
let mut char_indices = source.char_indices();
for event in highlighter.highlight(highlight_config, source.as_bytes(), None, |string| {
loader.highlight_config_for_injection_string(string)
})? {
match event? {
HighlightEvent::HighlightStart(h) => highlight_stack.push(h),
HighlightEvent::HighlightEnd => {
highlight_stack.pop();
}
HighlightEvent::Source { start, end } => {
let mut start_position = Point::new(row, column);
while byte_offset < end {
if byte_offset <= start {
start_position = Point::new(row, column);
}
if let Some((i, c)) = char_indices.next() {
if was_newline {
row += 1;
column = 0;
} else {
column += i - byte_offset;
}
was_newline = c == '\n';
byte_offset = i;
} else {
break;
}
}
if let Some(highlight) = highlight_stack.last() {
result.push((start_position, Point::new(row, column), *highlight))
}
}
}
}
Ok(result)
}

6
cli/src/tests/corpus_test.rs
View file

@ -58,7 +58,11 @@ fn test_real_language_corpus_files() {
}
let language = get_language(language_name);
let corpus_dir = grammars_dir.join(language_name).join("corpus");
let mut corpus_dir = grammars_dir.join(language_name).join("corpus");
if !corpus_dir.is_dir() {
corpus_dir = grammars_dir.join(language_name).join("test").join("corpus");
}
let error_corpus_file = error_corpus_dir.join(&format!("{}_errors.txt", language_name));
let main_tests = parse_tests(&corpus_dir).unwrap();
let error_tests = parse_tests(&error_corpus_file).unwrap_or(TestEntry::default());

25
cli/src/tests/helpers/fixtures.rs
View file

@ -3,7 +3,7 @@ use lazy_static::lazy_static;
use std::fs;
use std::path::{Path, PathBuf};
use tree_sitter::Language;
use tree_sitter_highlight::{HighlightConfiguration, Highlighter};
use tree_sitter_highlight::HighlightConfiguration;
include!("./dirs.rs");
@ -11,6 +11,10 @@ lazy_static! {
static ref TEST_LOADER: Loader = Loader::new(SCRATCH_DIR.clone());
}
pub fn test_loader<'a>() -> &'a Loader {
&*TEST_LOADER
}
pub fn fixtures_dir<'a>() -> &'static Path {
&FIXTURES_DIR
}
@ -26,23 +30,24 @@ pub fn get_language_queries_path(language_name: &str) -> PathBuf {
}
pub fn get_highlight_config(
highlighter: &Highlighter,
language_name: &str,
injection_query_filename: &str,
highlight_names: &[String],
) -> HighlightConfiguration {
let language = get_language(language_name);
let queries_path = get_language_queries_path(language_name);
let highlights_query = fs::read_to_string(queries_path.join("highlights.scm")).unwrap();
let injections_query = fs::read_to_string(queries_path.join(injection_query_filename)).unwrap();
let locals_query = fs::read_to_string(queries_path.join("locals.scm")).unwrap_or(String::new());
highlighter
.load_configuration(
language,
&highlights_query,
&injections_query,
&locals_query,
)
.unwrap()
let mut result = HighlightConfiguration::new(
language,
&highlights_query,
&injections_query,
&locals_query,
)
.unwrap();
result.configure(highlight_names);
result
}
pub fn get_test_language(name: &str, parser_code: &str, path: Option<&Path>) -> Language {

2
cli/src/tests/helpers/mod.rs
View file

@ -1,5 +1,5 @@
pub(super) mod allocations;
pub(super) mod edits;
pub(super) mod fixtures;
pub(super) mod random;
pub(super) mod scope_sequence;
pub(super) mod edits;

94
cli/src/tests/highlight_test.rs
View file

@ -4,49 +4,46 @@ use std::ffi::CString;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::{fs, ptr, slice, str};
use tree_sitter_highlight::{
c, Error, HighlightConfiguration, HighlightContext, HighlightEvent, Highlighter, HtmlRenderer,
c, Error, HighlightConfiguration, HighlightEvent, Highlighter, HtmlRenderer,
};
lazy_static! {
static ref JS_HIGHLIGHT: HighlightConfiguration =
get_highlight_config(&HIGHLIGHTER, "javascript", "injections.scm");
get_highlight_config("javascript", "injections.scm", &HIGHLIGHT_NAMES);
static ref HTML_HIGHLIGHT: HighlightConfiguration =
get_highlight_config(&HIGHLIGHTER, "html", "injections.scm");
get_highlight_config("html", "injections.scm", &HIGHLIGHT_NAMES);
static ref EJS_HIGHLIGHT: HighlightConfiguration =
get_highlight_config(&HIGHLIGHTER, "embedded-template", "injections-ejs.scm");
get_highlight_config("embedded-template", "injections-ejs.scm", &HIGHLIGHT_NAMES);
static ref RUST_HIGHLIGHT: HighlightConfiguration =
get_highlight_config(&HIGHLIGHTER, "rust", "injections.scm");
static ref HIGHLIGHTER: Highlighter = Highlighter::new(
[
"attribute",
"constant",
"constructor",
"function.builtin",
"function",
"embedded",
"keyword",
"operator",
"property.builtin",
"property",
"punctuation",
"punctuation.bracket",
"punctuation.delimiter",
"punctuation.special",
"string",
"tag",
"type.builtin",
"type",
"variable.builtin",
"variable.parameter",
"variable",
]
.iter()
.cloned()
.map(String::from)
.collect()
);
static ref HTML_ATTRS: Vec<String> = HIGHLIGHTER
.names()
get_highlight_config("rust", "injections.scm", &HIGHLIGHT_NAMES);
static ref HIGHLIGHT_NAMES: Vec<String> = [
"attribute",
"constant",
"constructor",
"function.builtin",
"function",
"embedded",
"keyword",
"operator",
"property.builtin",
"property",
"punctuation",
"punctuation.bracket",
"punctuation.delimiter",
"punctuation.special",
"string",
"tag",
"type.builtin",
"type",
"variable.builtin",
"variable.parameter",
"variable",
]
.iter()
.cloned()
.map(String::from)
.collect();
static ref HTML_ATTRS: Vec<String> = HIGHLIGHT_NAMES
.iter()
.map(|s| format!("class={}", s))
.collect();
@ -398,12 +395,10 @@ fn test_highlighting_cancellation() {
test_language_for_injection_string(name)
};
// Constructing the highlighter, which eagerly parses the outer document,
// should not fail.
let mut context = HighlightContext::new();
let events = HIGHLIGHTER
// The initial `highlight` call, which eagerly parses the outer document, should not fail.
let mut highlighter = Highlighter::new();
let events = highlighter
.highlight(
&mut context,
&HTML_HIGHLIGHT,
source.as_bytes(),
Some(&cancellation_flag),
@ -411,14 +406,15 @@ fn test_highlighting_cancellation() {
)
.unwrap();
// Iterating the scopes should not panic. It should return an error
// once the cancellation is detected.
// Iterating the scopes should not panic. It should return an error once the
// cancellation is detected.
for event in events {
if let Err(e) = event {
assert_eq!(e, Error::Cancelled);
return;
}
}
panic!("Expected an error while iterating highlighter");
}
@ -565,9 +561,8 @@ fn to_html<'a>(
) -> Result<Vec<String>, Error> {
let src = src.as_bytes();
let mut renderer = HtmlRenderer::new();
let mut context = HighlightContext::new();
let events = HIGHLIGHTER.highlight(
&mut context,
let mut highlighter = Highlighter::new();
let events = highlighter.highlight(
language_config,
src,
None,
@ -585,12 +580,11 @@ fn to_token_vector<'a>(
language_config: &'a HighlightConfiguration,
) -> Result<Vec<Vec<(&'a str, Vec<&'static str>)>>, Error> {
let src = src.as_bytes();
let mut context = HighlightContext::new();
let mut highlighter = Highlighter::new();
let mut lines = Vec::new();
let mut highlights = Vec::new();
let mut line = Vec::new();
let events = HIGHLIGHTER.highlight(
&mut context,
let events = highlighter.highlight(
language_config,
src,
None,
@ -598,7 +592,7 @@ fn to_token_vector<'a>(
)?;
for event in events {
match event? {
HighlightEvent::HighlightStart(s) => highlights.push(HIGHLIGHTER.names()[s.0].as_str()),
HighlightEvent::HighlightStart(s) => highlights.push(HIGHLIGHT_NAMES[s.0].as_str()),
HighlightEvent::HighlightEnd => {
highlights.pop();
}

1
cli/src/tests/mod.rs
View file

@ -4,4 +4,5 @@ mod highlight_test;
mod node_test;
mod parser_test;
mod query_test;
mod test_highlight_test;
mod tree_test;

53
cli/src/tests/test_highlight_test.rs Normal file
View file

@ -0,0 +1,53 @@
use super::helpers::fixtures::{get_highlight_config, get_language, test_loader};
use crate::test_highlight::{get_highlight_positions, parse_highlight_test};
use tree_sitter::{Parser, Point};
use tree_sitter_highlight::{Highlight, Highlighter};
#[test]
fn test_highlight_test_with_basic_test() {
let language = get_language("javascript");
let config = get_highlight_config(
"javascript",
"injections.scm",
&[
"function.definition".to_string(),
"variable.parameter".to_string(),
"keyword".to_string(),
],
);
let source = [
"var abc = function(d) {",
" // ^ function.definition",
" // ^ keyword",
" return d + e;",
" // ^ variable.parameter",
"};",
]
.join("\n");
let assertions = parse_highlight_test(&mut Parser::new(), language, source.as_bytes()).unwrap();
assert_eq!(
assertions,
&[
(Point::new(0, 5), "function.definition".to_string()),
(Point::new(0, 11), "keyword".to_string()),
(Point::new(3, 9), "variable.parameter".to_string()),
]
);
let mut highlighter = Highlighter::new();
let highlight_positions =
get_highlight_positions(test_loader(), &mut highlighter, &config, source.as_bytes())
.unwrap();
assert_eq!(
highlight_positions,
&[
(Point::new(0, 0), Point::new(0, 3), Highlight(2)), // "var"
(Point::new(0, 4), Point::new(0, 7), Highlight(0)), // "abc"
(Point::new(0, 10), Point::new(0, 18), Highlight(2)), // "function"
(Point::new(0, 19), Point::new(0, 20), Highlight(1)), // "d"
(Point::new(3, 2), Point::new(3, 8), Highlight(2)), // "return"
(Point::new(3, 9), Point::new(3, 10), Highlight(1)), // "d"
]
);
}

77
highlight/README.md
View file

@ -14,66 +14,64 @@ extern "C" tree_sitter_html();
extern "C" tree_sitter_javascript();
```
Create a highlighter. You only need one of these:
Define the list of highlight names that you will recognize:
```rust
let highlight_names = [
"attribute",
"constant",
"function.builtin",
"function",
"keyword",
"operator",
"property",
"punctuation",
"punctuation.bracket",
"punctuation.delimiter",
"string",
"string.special",
"tag",
"type",
"type.builtin",
"variable",
"variable.builtin",
"variable.parameter",
]
.iter()
.cloned()
.map(String::from)
.collect();
```
Create a highlighter. You need one of these for each thread that you're using for syntax highlighting:
```rust
use tree_sitter_highlight::Highlighter;
let highlighter = Highlighter::new(
[
"attribute",
"constant",
"function.builtin",
"function",
"keyword",
"operator",
"property",
"punctuation",
"punctuation.bracket",
"punctuation.delimiter",
"string",
"string.special",
"tag",
"type",
"type.builtin",
"variable",
"variable.builtin",
"variable.parameter",
]
.iter()
.cloned()
.map(String::from)
.collect()
);
```
Create a highlight context. You need one of these for each thread that you're using for syntax highlighting:
```rust
use tree_sitter_highlight::HighlightContext;
let context = HighlightContext::new();
let highlighter = Highlighter::new();
```
Load some highlighting queries from the `queries` directory of some language repositories:
```rust
use tree_sitter_highlight::HighlightConfiguration;
let html_language = unsafe { tree_sitter_html() };
let javascript_language = unsafe { tree_sitter_javascript() };
let html_config = highlighter.load_configuration(
let html_config = HighlightConfiguration::new(
html_language,
&fs::read_to_string("./tree-sitter-html/queries/highlights.scm").unwrap(),
&fs::read_to_string("./tree-sitter-html/queries/injections.scm").unwrap(),
"",
);
).unwrap();
let javascript_config = highlighter.load_configuration(
let javascript_config = HighlightConfiguration::new(
javascript_language,
&fs::read_to_string("./tree-sitter-javascript/queries/highlights.scm").unwrap(),
&fs::read_to_string("./tree-sitter-javascript/queries/injections.scm").unwrap(),
&fs::read_to_string("./tree-sitter-javascript/queries/locals.scm").unwrap(),
);
).unwrap();
```
Highlight some code:
@ -82,7 +80,6 @@ Highlight some code:
use tree_sitter_highlight::HighlightEvent;
let highlights = highlighter.highlight(
&mut context,
javascript_config,
b"const x = new Y();",
None,

28
highlight/src/c_lib.rs
View file

@ -1,4 +1,4 @@
use super::{Error, HighlightConfiguration, HighlightContext, Highlighter, HtmlRenderer};
use super::{Error, HighlightConfiguration, Highlighter, HtmlRenderer};
use regex::Regex;
use std::collections::HashMap;
use std::ffi::CStr;
@ -11,11 +11,11 @@ use tree_sitter::Language;
pub struct TSHighlighter {
languages: HashMap<String, (Option<Regex>, HighlightConfiguration)>,
attribute_strings: Vec<&'static [u8]>,
highlighter: Highlighter,
highlight_names: Vec<String>,
}
pub struct TSHighlightBuffer {
context: HighlightContext,
highlighter: Highlighter,
renderer: HtmlRenderer,
}
@ -48,11 +48,10 @@ pub extern "C" fn ts_highlighter_new(
.into_iter()
.map(|s| unsafe { CStr::from_ptr(*s).to_bytes() })
.collect();
let highlighter = Highlighter::new(highlight_names);
Box::into_raw(Box::new(TSHighlighter {
languages: HashMap::new(),
attribute_strings,
highlighter,
highlight_names,
}))
}
@ -107,15 +106,11 @@ pub extern "C" fn ts_highlighter_add_language(
""
};
this.languages.insert(
scope_name,
(
injection_regex,
this.highlighter
.load_configuration(language, highlight_query, injection_query, locals_query)
.or(Err(ErrorCode::InvalidQuery))?,
),
);
let mut config =
HighlightConfiguration::new(language, highlight_query, injection_query, locals_query)
.or(Err(ErrorCode::InvalidQuery))?;
config.configure(&this.highlight_names);
this.languages.insert(scope_name, (injection_regex, config));
Ok(())
};
@ -129,7 +124,7 @@ pub extern "C" fn ts_highlighter_add_language(
#[no_mangle]
pub extern "C" fn ts_highlight_buffer_new() -> *mut TSHighlightBuffer {
Box::into_raw(Box::new(TSHighlightBuffer {
context: HighlightContext::new(),
highlighter: Highlighter::new(),
renderer: HtmlRenderer::new(),
}))
}
@ -201,8 +196,7 @@ impl TSHighlighter {
let (_, configuration) = entry.unwrap();
let languages = &self.languages;
let highlights = self.highlighter.highlight(
&mut output.context,
let highlights = output.highlighter.highlight(
configuration,
source_code,
cancellation_flag,

256
highlight/src/lib.rs
View file

@ -11,7 +11,7 @@ use tree_sitter::{
const CANCELLATION_CHECK_INTERVAL: usize = 100;
/// Indicates which highlight should be applied to a region of source code.
#[derive(Copy, Clone, Debug)]
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub struct Highlight(pub usize);
/// Represents the reason why syntax highlighting failed.
@ -52,29 +52,10 @@ pub struct HighlightConfiguration {
/// Performs syntax highlighting, recognizing a given list of highlight names.
///
/// Tree-sitter syntax-highlighting queries specify highlights in the form of dot-separated
/// highlight names like `punctuation.bracket` and `function.method.builtin`. Consumers of
/// these queries can choose to recognize highlights with different levels of specificity.
/// For example, the string `function.builtin` will match against `function.method.builtin`
/// and `function.builtin.constructor`, but will not match `function.method`.
///
/// The `Highlight` struct is instantiated with an ordered list of recognized highlight names
/// and is then used for loading highlight queries and performing syntax highlighting.
/// Highlighting results are returned as `Highlight` values, which contain the index of the
/// matched highlight this list of highlight names.
///
/// The `Highlighter` struct is immutable and can be shared between threads.
#[derive(Clone, Debug)]
pub struct Highlighter {
highlight_names: Vec<String>,
}
/// Carries the mutable state required for syntax highlighting.
///
/// For the best performance `HighlightContext` values should be reused between
/// syntax highlighting calls. A separate context is needed for each thread that
/// For the best performance `Highlighter` values should be reused between
/// syntax highlighting calls. A separate highlighter is needed for each thread that
/// is performing highlighting.
pub struct HighlightContext {
pub struct Highlighter {
parser: Parser,
cursors: Vec<QueryCursor>,
}
@ -101,11 +82,11 @@ struct LocalScope<'a> {
struct HighlightIter<'a, F>
where
F: Fn(&str) -> Option<&'a HighlightConfiguration> + 'a,
F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
{
source: &'a [u8],
byte_offset: usize,
context: &'a mut HighlightContext,
highlighter: &'a mut Highlighter,
injections_cursor: QueryCursor,
injection_callback: F,
cancellation_flag: Option<&'a AtomicUsize>,
@ -126,26 +107,58 @@ struct HighlightIterLayer<'a> {
depth: usize,
}
impl HighlightContext {
impl Highlighter {
pub fn new() -> Self {
HighlightContext {
Highlighter {
parser: Parser::new(),
cursors: Vec::new(),
}
}
pub fn parser(&mut self) -> &mut Parser {
&mut self.parser
}
/// Iterate over the highlighted regions for a given slice of source code.
pub fn highlight<'a>(
&'a mut self,
config: &'a HighlightConfiguration,
source: &'a [u8],
cancellation_flag: Option<&'a AtomicUsize>,
injection_callback: impl FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
) -> Result<impl Iterator<Item = Result<HighlightEvent, Error>> + 'a, Error> {
let layer = HighlightIterLayer::new(
config,
source,
self,
cancellation_flag,
0,
vec![Range {
start_byte: 0,
end_byte: usize::MAX,
start_point: Point::new(0, 0),
end_point: Point::new(usize::MAX, usize::MAX),
}],
)?;
let injections_cursor = self.cursors.pop().unwrap_or(QueryCursor::new());
Ok(HighlightIter {
source,
byte_offset: 0,
injection_callback,
cancellation_flag,
injections_cursor,
highlighter: self,
iter_count: 0,
layers: vec![layer],
next_event: None,
last_highlight_range: None,
})
}
}
impl Highlighter {
/// Creates a highlighter with a given list of recognized highlight names.
pub fn new(highlight_names: Vec<String>) -> Self {
Highlighter { highlight_names }
}
/// Returns the list of highlight names with which this Highlighter was constructed.
pub fn names(&self) -> &[String] {
&self.highlight_names
}
impl HighlightConfiguration {
/// Creates a `HighlightConfiguration` for a given `Language` and set of highlighting
/// queries.
///
@ -160,13 +173,12 @@ impl Highlighter {
/// definitions and references. This can be empty if local variable tracking is not needed.
///
/// Returns a `HighlightConfiguration` that can then be used with the `highlight` method.
pub fn load_configuration(
&self,
pub fn new(
language: Language,
highlights_query: &str,
injection_query: &str,
locals_query: &str,
) -> Result<HighlightConfiguration, QueryError> {
) -> Result<Self, QueryError> {
// Concatenate the query strings, keeping track of the start offset of each section.
let mut query_source = String::new();
query_source.push_str(injection_query);
@ -200,38 +212,6 @@ impl Highlighter {
}
}
let mut capture_parts = Vec::new();
// Compute a mapping from the query's capture ids to the indices of the highlighter's
// recognized highlight names.
let highlight_indices = query
.capture_names()
.iter()
.map(move |capture_name| {
capture_parts.clear();
capture_parts.extend(capture_name.split('.'));
let mut best_index = None;
let mut best_match_len = 0;
for (i, highlight_name) in self.highlight_names.iter().enumerate() {
let mut len = 0;
let mut matches = true;
for part in highlight_name.split('.') {
len += 1;
if !capture_parts.contains(&part) {
matches = false;
break;
}
}
if matches && len > best_match_len {
best_index = Some(i);
best_match_len = len;
}
}
best_index.map(Highlight)
})
.collect();
let non_local_variable_patterns = (0..query.pattern_count())
.map(|i| {
query
@ -262,6 +242,8 @@ impl Highlighter {
}
}
let highlight_indices = vec![None; query.capture_names().len()];
Ok(HighlightConfiguration {
language,
query,
@ -280,43 +262,48 @@ impl Highlighter {
})
}
/// Iterate over the highlighted regions for a given slice of source code.
pub fn highlight<'a>(
&'a self,
context: &'a mut HighlightContext,
config: &'a HighlightConfiguration,
source: &'a [u8],
cancellation_flag: Option<&'a AtomicUsize>,
injection_callback: impl Fn(&str) -> Option<&'a HighlightConfiguration> + 'a,
) -> Result<impl Iterator<Item = Result<HighlightEvent, Error>> + 'a, Error> {
let layer = HighlightIterLayer::new(
config,
source,
context,
cancellation_flag,
0,
vec![Range {
start_byte: 0,
end_byte: usize::MAX,
start_point: Point::new(0, 0),
end_point: Point::new(usize::MAX, usize::MAX),
}],
)?;
/// Get a slice containing all of the highlight names used in the configuration.
pub fn names(&self) -> &[String] {
self.query.capture_names()
}
let injections_cursor = context.cursors.pop().unwrap_or(QueryCursor::new());
/// Set the list of recognized highlight names.
///
/// Tree-sitter syntax-highlighting queries specify highlights in the form of dot-separated
/// highlight names like `punctuation.bracket` and `function.method.builtin`. Consumers of
/// these queries can choose to recognize highlights with different levels of specificity.
/// For example, the string `function.builtin` will match against `function.method.builtin`
/// and `function.builtin.constructor`, but will not match `function.method`.
///
/// When highlighting, results are returned as `Highlight` values, which contain the index
/// of the matched highlight this list of highlight names.
pub fn configure(&mut self, recognized_names: &[String]) {
let mut capture_parts = Vec::new();
self.highlight_indices.clear();
self.highlight_indices
.extend(self.query.capture_names().iter().map(move |capture_name| {
capture_parts.clear();
capture_parts.extend(capture_name.split('.'));
Ok(HighlightIter {
source,
byte_offset: 0,
injection_callback,
cancellation_flag,
injections_cursor,
context,
iter_count: 0,
layers: vec![layer],
next_event: None,
last_highlight_range: None,
})
let mut best_index = None;
let mut best_match_len = 0;
for (i, recognized_name) in recognized_names.iter().enumerate() {
let mut len = 0;
let mut matches = true;
for part in recognized_name.split('.') {
len += 1;
if !capture_parts.contains(&part) {
matches = false;
break;
}
}
if matches && len > best_match_len {
best_index = Some(i);
best_match_len = len;
}
}
best_index.map(Highlight)
}));
}
}
@ -324,21 +311,24 @@ impl<'a> HighlightIterLayer<'a> {
fn new(
config: &'a HighlightConfiguration,
source: &'a [u8],
context: &mut HighlightContext,
highlighter: &mut Highlighter,
cancellation_flag: Option<&'a AtomicUsize>,
depth: usize,
ranges: Vec<Range>,
) -> Result<Self, Error> {
context
highlighter
.parser
.set_language(config.language)
.map_err(|_| Error::InvalidLanguage)?;
unsafe { context.parser.set_cancellation_flag(cancellation_flag) };
unsafe { highlighter.parser.set_cancellation_flag(cancellation_flag) };
context.parser.set_included_ranges(&ranges);
highlighter.parser.set_included_ranges(&ranges);
let tree = context.parser.parse(source, None).ok_or(Error::Cancelled)?;
let mut cursor = context.cursors.pop().unwrap_or(QueryCursor::new());
let tree = highlighter
.parser
.parse(source, None)
.ok_or(Error::Cancelled)?;
let mut cursor = highlighter.cursors.pop().unwrap_or(QueryCursor::new());
// The `captures` iterator borrows the `Tree` and the `QueryCursor`, which
// prevents them from being moved. But both of these values are really just
@ -486,7 +476,7 @@ impl<'a> HighlightIterLayer<'a> {
impl<'a, F> HighlightIter<'a, F>
where
F: Fn(&str) -> Option<&'a HighlightConfiguration> + 'a,
F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
{
fn emit_event(
&mut self,
@ -525,7 +515,7 @@ where
}
} else {
let layer = self.layers.remove(0);
self.context.cursors.push(layer.cursor);
self.highlighter.cursors.push(layer.cursor);
}
}
@ -545,7 +535,7 @@ where
impl<'a, F> Iterator for HighlightIter<'a, F>
where
F: Fn(&str) -> Option<&'a HighlightConfiguration> + 'a,
F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
{
type Item = Result<HighlightEvent, Error>;
@ -711,21 +701,23 @@ where
for (_, language, content_nodes, include_children) in injections {
// If a language is found with the given name, then add a new language layer
// to the highlighted document.
if let Some(config) = language.and_then(&self.injection_callback) {
if !content_nodes.is_empty() {
let ranges = self.layers[0]
.intersect_ranges(&content_nodes, include_children);
if !ranges.is_empty() {
match HighlightIterLayer::new(
config,
self.source,
self.context,
self.cancellation_flag,
self.layers[0].depth + 1,
ranges,
) {
Ok(layer) => self.insert_layer(layer),
Err(e) => return Some(Err(e)),
if let Some(language) = language {
if let Some(config) = (self.injection_callback)(language) {
if !content_nodes.is_empty() {
let ranges = self.layers[0]
.intersect_ranges(&content_nodes, include_children);
if !ranges.is_empty() {
match HighlightIterLayer::new(
config,
self.source,
self.highlighter,
self.cancellation_flag,
self.layers[0].depth + 1,
ranges,
) {
Ok(layer) => self.insert_layer(layer),
Err(e) => return Some(Err(e)),
}
}
}
}