traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #499 from tree-sitter/highlight-test

Browse source 
Add a system for testing syntax highlighting queries
This commit is contained in:
Max Brunsfeld 2019-12-05 15:55:24 -08:00 • committed by GitHub
commit d6c7b243a7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 750 additions and 365 deletions
Download patch file Download diff file Expand all files Collapse all files

6
cli/src/tests/corpus_test.rs
View file

@ -58,7 +58,11 @@ fn test_real_language_corpus_files() {
}
let language = get_language(language_name);
let corpus_dir = grammars_dir.join(language_name).join("corpus");
let mut corpus_dir = grammars_dir.join(language_name).join("corpus");
if !corpus_dir.is_dir() {
corpus_dir = grammars_dir.join(language_name).join("test").join("corpus");
}
let error_corpus_file = error_corpus_dir.join(&format!("{}_errors.txt", language_name));
let main_tests = parse_tests(&corpus_dir).unwrap();
let error_tests = parse_tests(&error_corpus_file).unwrap_or(TestEntry::default());

25
cli/src/tests/helpers/fixtures.rs
View file

@ -3,7 +3,7 @@ use lazy_static::lazy_static;
use std::fs;
use std::path::{Path, PathBuf};
use tree_sitter::Language;
use tree_sitter_highlight::{HighlightConfiguration, Highlighter};
use tree_sitter_highlight::HighlightConfiguration;
include!("./dirs.rs");
@ -11,6 +11,10 @@ lazy_static! {
static ref TEST_LOADER: Loader = Loader::new(SCRATCH_DIR.clone());
}
pub fn test_loader<'a>() -> &'a Loader {
&*TEST_LOADER
}
pub fn fixtures_dir<'a>() -> &'static Path {
&FIXTURES_DIR
}
@ -26,23 +30,24 @@ pub fn get_language_queries_path(language_name: &str) -> PathBuf {
}
pub fn get_highlight_config(
highlighter: &Highlighter,
language_name: &str,
injection_query_filename: &str,
highlight_names: &[String],
) -> HighlightConfiguration {
let language = get_language(language_name);
let queries_path = get_language_queries_path(language_name);
let highlights_query = fs::read_to_string(queries_path.join("highlights.scm")).unwrap();
let injections_query = fs::read_to_string(queries_path.join(injection_query_filename)).unwrap();
let locals_query = fs::read_to_string(queries_path.join("locals.scm")).unwrap_or(String::new());
highlighter
.load_configuration(
language,
&highlights_query,
&injections_query,
&locals_query,
)
.unwrap()
let mut result = HighlightConfiguration::new(
language,
&highlights_query,
&injections_query,
&locals_query,
)
.unwrap();
result.configure(highlight_names);
result
}
pub fn get_test_language(name: &str, parser_code: &str, path: Option<&Path>) -> Language {

2
cli/src/tests/helpers/mod.rs
View file

@ -1,5 +1,5 @@
pub(super) mod allocations;
pub(super) mod edits;
pub(super) mod fixtures;
pub(super) mod random;
pub(super) mod scope_sequence;
pub(super) mod edits;

94
cli/src/tests/highlight_test.rs
View file

@ -4,49 +4,46 @@ use std::ffi::CString;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::{fs, ptr, slice, str};
use tree_sitter_highlight::{
c, Error, HighlightConfiguration, HighlightContext, HighlightEvent, Highlighter, HtmlRenderer,
c, Error, HighlightConfiguration, HighlightEvent, Highlighter, HtmlRenderer,
};
lazy_static! {
static ref JS_HIGHLIGHT: HighlightConfiguration =
get_highlight_config(&HIGHLIGHTER, "javascript", "injections.scm");
get_highlight_config("javascript", "injections.scm", &HIGHLIGHT_NAMES);
static ref HTML_HIGHLIGHT: HighlightConfiguration =
get_highlight_config(&HIGHLIGHTER, "html", "injections.scm");
get_highlight_config("html", "injections.scm", &HIGHLIGHT_NAMES);
static ref EJS_HIGHLIGHT: HighlightConfiguration =
get_highlight_config(&HIGHLIGHTER, "embedded-template", "injections-ejs.scm");
get_highlight_config("embedded-template", "injections-ejs.scm", &HIGHLIGHT_NAMES);
static ref RUST_HIGHLIGHT: HighlightConfiguration =
get_highlight_config(&HIGHLIGHTER, "rust", "injections.scm");
static ref HIGHLIGHTER: Highlighter = Highlighter::new(
[
"attribute",
"constant",
"constructor",
"function.builtin",
"function",
"embedded",
"keyword",
"operator",
"property.builtin",
"property",
"punctuation",
"punctuation.bracket",
"punctuation.delimiter",
"punctuation.special",
"string",
"tag",
"type.builtin",
"type",
"variable.builtin",
"variable.parameter",
"variable",
]
.iter()
.cloned()
.map(String::from)
.collect()
);
static ref HTML_ATTRS: Vec<String> = HIGHLIGHTER
.names()
get_highlight_config("rust", "injections.scm", &HIGHLIGHT_NAMES);
static ref HIGHLIGHT_NAMES: Vec<String> = [
"attribute",
"constant",
"constructor",
"function.builtin",
"function",
"embedded",
"keyword",
"operator",
"property.builtin",
"property",
"punctuation",
"punctuation.bracket",
"punctuation.delimiter",
"punctuation.special",
"string",
"tag",
"type.builtin",
"type",
"variable.builtin",
"variable.parameter",
"variable",
]
.iter()
.cloned()
.map(String::from)
.collect();
static ref HTML_ATTRS: Vec<String> = HIGHLIGHT_NAMES
.iter()
.map(|s| format!("class={}", s))
.collect();
@ -398,12 +395,10 @@ fn test_highlighting_cancellation() {
test_language_for_injection_string(name)
};
// Constructing the highlighter, which eagerly parses the outer document,
// should not fail.
let mut context = HighlightContext::new();
let events = HIGHLIGHTER
// The initial `highlight` call, which eagerly parses the outer document, should not fail.
let mut highlighter = Highlighter::new();
let events = highlighter
.highlight(
&mut context,
&HTML_HIGHLIGHT,
source.as_bytes(),
Some(&cancellation_flag),
@ -411,14 +406,15 @@ fn test_highlighting_cancellation() {
)
.unwrap();
// Iterating the scopes should not panic. It should return an error
// once the cancellation is detected.
// Iterating the scopes should not panic. It should return an error once the
// cancellation is detected.
for event in events {
if let Err(e) = event {
assert_eq!(e, Error::Cancelled);
return;
}
}
panic!("Expected an error while iterating highlighter");
}
@ -565,9 +561,8 @@ fn to_html<'a>(
) -> Result<Vec<String>, Error> {
let src = src.as_bytes();
let mut renderer = HtmlRenderer::new();
let mut context = HighlightContext::new();
let events = HIGHLIGHTER.highlight(
&mut context,
let mut highlighter = Highlighter::new();
let events = highlighter.highlight(
language_config,
src,
None,
@ -585,12 +580,11 @@ fn to_token_vector<'a>(
language_config: &'a HighlightConfiguration,
) -> Result<Vec<Vec<(&'a str, Vec<&'static str>)>>, Error> {
let src = src.as_bytes();
let mut context = HighlightContext::new();
let mut highlighter = Highlighter::new();
let mut lines = Vec::new();
let mut highlights = Vec::new();
let mut line = Vec::new();
let events = HIGHLIGHTER.highlight(
&mut context,
let events = highlighter.highlight(
language_config,
src,
None,
@ -598,7 +592,7 @@ fn to_token_vector<'a>(
)?;
for event in events {
match event? {
HighlightEvent::HighlightStart(s) => highlights.push(HIGHLIGHTER.names()[s.0].as_str()),
HighlightEvent::HighlightStart(s) => highlights.push(HIGHLIGHT_NAMES[s.0].as_str()),
HighlightEvent::HighlightEnd => {
highlights.pop();
}

1
cli/src/tests/mod.rs
View file

@ -4,4 +4,5 @@ mod highlight_test;
mod node_test;
mod parser_test;
mod query_test;
mod test_highlight_test;
mod tree_test;

53
cli/src/tests/test_highlight_test.rs Normal file
View file

@ -0,0 +1,53 @@
use super::helpers::fixtures::{get_highlight_config, get_language, test_loader};
use crate::test_highlight::{get_highlight_positions, parse_highlight_test};
use tree_sitter::{Parser, Point};
use tree_sitter_highlight::{Highlight, Highlighter};
#[test]
fn test_highlight_test_with_basic_test() {
let language = get_language("javascript");
let config = get_highlight_config(
"javascript",
"injections.scm",
&[
"function.definition".to_string(),
"variable.parameter".to_string(),
"keyword".to_string(),
],
);
let source = [
"var abc = function(d) {",
" // ^ function.definition",
" // ^ keyword",
" return d + e;",
" // ^ variable.parameter",
"};",
]
.join("\n");
let assertions = parse_highlight_test(&mut Parser::new(), language, source.as_bytes()).unwrap();
assert_eq!(
assertions,
&[
(Point::new(0, 5), "function.definition".to_string()),
(Point::new(0, 11), "keyword".to_string()),
(Point::new(3, 9), "variable.parameter".to_string()),
]
);
let mut highlighter = Highlighter::new();
let highlight_positions =
get_highlight_positions(test_loader(), &mut highlighter, &config, source.as_bytes())
.unwrap();
assert_eq!(
highlight_positions,
&[
(Point::new(0, 0), Point::new(0, 3), Highlight(2)), // "var"
(Point::new(0, 4), Point::new(0, 7), Highlight(0)), // "abc"
(Point::new(0, 10), Point::new(0, 18), Highlight(2)), // "function"
(Point::new(0, 19), Point::new(0, 20), Highlight(1)), // "d"
(Point::new(3, 2), Point::new(3, 8), Highlight(2)), // "return"
(Point::new(3, 9), Point::new(3, 10), Highlight(1)), // "d"
]
);
}