traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into HEAD

Browse source
This commit is contained in:
Max Brunsfeld 2020-12-03 09:44:33 -08:00
commit 026231e93d
173 changed files with 22878 additions and 6961 deletions
Download patch file Download diff file Expand all files Collapse all files

13
cli/src/tests/corpus_test.rs
View file

@ -21,7 +21,11 @@ const LANGUAGES: &'static [&'static str] = &[
"go",
"html",
"javascript",
"json",
"php",
"python",
"ruby",
"rust",
];
lazy_static! {
@ -57,7 +61,11 @@ fn test_real_language_corpus_files() {
}
let language = get_language(language_name);
let corpus_dir = grammars_dir.join(language_name).join("corpus");
let mut corpus_dir = grammars_dir.join(language_name).join("corpus");
if !corpus_dir.is_dir() {
corpus_dir = grammars_dir.join(language_name).join("test").join("corpus");
}
let error_corpus_file = error_corpus_dir.join(&format!("{}_errors.txt", language_name));
let main_tests = parse_tests(&corpus_dir).unwrap();
let error_tests = parse_tests(&error_corpus_file).unwrap_or(TestEntry::default());
@ -300,7 +308,8 @@ fn check_consistent_sizes(tree: &Tree, input: &Vec<u8>) {
let mut last_child_end_point = start_point;
let mut some_child_has_changes = false;
let mut actual_named_child_count = 0;
for child in node.children() {
for i in 0..node.child_count() {
let child = node.child(i).unwrap();
assert!(child.start_byte() >= last_child_end_byte);
assert!(child.start_position() >= last_child_end_point);
check(child, line_offsets);

6
cli/src/tests/helpers/allocations.rs
View file

@ -51,6 +51,12 @@ pub fn stop_recording() {
}
}
pub fn record(f: impl FnOnce()) {
start_recording();
f();
stop_recording();
}
fn record_alloc(ptr: *mut c_void) {
let mut recorder = RECORDER.lock();
if recorder.enabled {

41
cli/src/tests/helpers/fixtures.rs
View file

@ -2,8 +2,8 @@ use crate::loader::Loader;
use lazy_static::lazy_static;
use std::fs;
use std::path::{Path, PathBuf};
use tree_sitter::{Language, PropertySheet};
use tree_sitter_highlight::{load_property_sheet, Properties};
use tree_sitter::Language;
use tree_sitter_highlight::HighlightConfiguration;
include!("./dirs.rs");
@ -11,6 +11,10 @@ lazy_static! {
static ref TEST_LOADER: Loader = Loader::new(SCRATCH_DIR.clone());
}
pub fn test_loader<'a>() -> &'a Loader {
&*TEST_LOADER
}
pub fn fixtures_dir<'a>() -> &'static Path {
&FIXTURES_DIR
}
@ -21,18 +25,33 @@ pub fn get_language(name: &str) -> Language {
.unwrap()
}
pub fn get_property_sheet_json(language_name: &str, sheet_name: &str) -> String {
let path = GRAMMARS_DIR
.join(language_name)
.join("src")
.join(sheet_name);
fs::read_to_string(path).unwrap()
pub fn get_language_queries_path(language_name: &str) -> PathBuf {
GRAMMARS_DIR.join(language_name).join("queries")
}
pub fn get_property_sheet(language_name: &str, sheet_name: &str) -> PropertySheet<Properties> {
let json = get_property_sheet_json(language_name, sheet_name);
pub fn get_highlight_config(
language_name: &str,
injection_query_filename: Option<&str>,
highlight_names: &[String],
) -> HighlightConfiguration {
let language = get_language(language_name);
load_property_sheet(language, &json).unwrap()
let queries_path = get_language_queries_path(language_name);
let highlights_query = fs::read_to_string(queries_path.join("highlights.scm")).unwrap();
let injections_query = if let Some(injection_query_filename) = injection_query_filename {
fs::read_to_string(queries_path.join(injection_query_filename)).unwrap()
} else {
String::new()
};
let locals_query = fs::read_to_string(queries_path.join("locals.scm")).unwrap_or(String::new());
let mut result = HighlightConfiguration::new(
language,
&highlights_query,
&injections_query,
&locals_query,
)
.unwrap();
result.configure(highlight_names);
result
}
pub fn get_test_language(name: &str, parser_code: &str, path: Option<&Path>) -> Language {

2
cli/src/tests/helpers/mod.rs
View file

@ -1,5 +1,5 @@
pub(super) mod allocations;
pub(super) mod edits;
pub(super) mod fixtures;
pub(super) mod random;
pub(super) mod scope_sequence;
pub(super) mod edits;

621
cli/src/tests/highlight_test.rs
View file

@ -1,32 +1,92 @@
use super::helpers::fixtures::{get_language, get_property_sheet, get_property_sheet_json};
use super::helpers::fixtures::{get_highlight_config, get_language, get_language_queries_path};
use lazy_static::lazy_static;
use std::ffi::CString;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::{ptr, slice, str};
use tree_sitter::{Language, PropertySheet};
use std::{fs, ptr, slice, str};
use tree_sitter_highlight::{
c, highlight, highlight_html, Error, Highlight, HighlightEvent, Properties,
c, Error, Highlight, HighlightConfiguration, HighlightEvent, Highlighter, HtmlRenderer,
};
lazy_static! {
static ref JS_SHEET: PropertySheet<Properties> =
get_property_sheet("javascript", "highlights.json");
static ref HTML_SHEET: PropertySheet<Properties> =
get_property_sheet("html", "highlights.json");
static ref EJS_SHEET: PropertySheet<Properties> =
get_property_sheet("embedded-template", "highlights-ejs.json");
static ref RUST_SHEET: PropertySheet<Properties> =
get_property_sheet("rust", "highlights.json");
static ref SCOPE_CLASS_STRINGS: Vec<String> = {
let mut result = Vec::new();
let mut i = 0;
while let Some(highlight) = Highlight::from_usize(i) {
result.push(format!("class={:?}", highlight));
i += 1;
}
result
};
static ref JS_HIGHLIGHT: HighlightConfiguration =
get_highlight_config("javascript", Some("injections.scm"), &HIGHLIGHT_NAMES);
static ref JSDOC_HIGHLIGHT: HighlightConfiguration =
get_highlight_config("jsdoc", None, &HIGHLIGHT_NAMES);
static ref HTML_HIGHLIGHT: HighlightConfiguration =
get_highlight_config("html", Some("injections.scm"), &HIGHLIGHT_NAMES);
static ref EJS_HIGHLIGHT: HighlightConfiguration = get_highlight_config(
"embedded-template",
Some("injections-ejs.scm"),
&HIGHLIGHT_NAMES
);
static ref RUST_HIGHLIGHT: HighlightConfiguration =
get_highlight_config("rust", Some("injections.scm"), &HIGHLIGHT_NAMES);
static ref HIGHLIGHT_NAMES: Vec<String> = [
"attribute",
"carriage-return",
"comment",
"constant",
"constructor",
"function.builtin",
"function",
"embedded",
"keyword",
"operator",
"property.builtin",
"property",
"punctuation",
"punctuation.bracket",
"punctuation.delimiter",
"punctuation.special",
"string",
"tag",
"type.builtin",
"type",
"variable.builtin",
"variable.parameter",
"variable",
]
.iter()
.cloned()
.map(String::from)
.collect();
static ref HTML_ATTRS: Vec<String> = HIGHLIGHT_NAMES
.iter()
.map(|s| format!("class={}", s))
.collect();
}
#[test]
fn test_highlighting_javascript() {
let source = "const a = function(b) { return b + c; }";
assert_eq!(
&to_token_vector(&source, &JS_HIGHLIGHT).unwrap(),
&[vec![
("const", vec!["keyword"]),
(" ", vec![]),
("a", vec!["function"]),
(" ", vec![]),
("=", vec!["operator"]),
(" ", vec![]),
("function", vec!["keyword"]),
("(", vec!["punctuation.bracket"]),
("b", vec!["variable.parameter"]),
(")", vec!["punctuation.bracket"]),
(" ", vec![]),
("{", vec!["punctuation.bracket"]),
(" ", vec![]),
("return", vec!["keyword"]),
(" ", vec![]),
("b", vec!["variable.parameter"]),
(" ", vec![]),
("+", vec!["operator"]),
(" ", vec![]),
("c", vec!["variable"]),
(";", vec!["punctuation.delimiter"]),
(" ", vec![]),
("}", vec!["punctuation.bracket"]),
]]
);
}
#[test]
@ -34,57 +94,65 @@ fn test_highlighting_injected_html_in_javascript() {
let source = vec!["const s = html `<div>${a < b}</div>`;"].join("\n");
assert_eq!(
&to_token_vector(&source, get_language("javascript"), &JS_SHEET).unwrap(),
&to_token_vector(&source, &JS_HIGHLIGHT).unwrap(),
&[vec![
("const", vec![Highlight::Keyword]),
("const", vec!["keyword"]),
(" ", vec![]),
("s", vec![Highlight::Variable]),
("s", vec!["variable"]),
(" ", vec![]),
("=", vec![Highlight::Operator]),
("=", vec!["operator"]),
(" ", vec![]),
("html", vec![Highlight::Function]),
("html", vec!["function"]),
(" ", vec![]),
("`<", vec![Highlight::String]),
("div", vec![Highlight::String, Highlight::Tag]),
(">", vec![Highlight::String]),
(
"${",
vec![
Highlight::String,
Highlight::Embedded,
Highlight::PunctuationSpecial
]
),
(
"a",
vec![Highlight::String, Highlight::Embedded, Highlight::Variable]
),
(" ", vec![Highlight::String, Highlight::Embedded]),
(
"<",
vec![Highlight::String, Highlight::Embedded, Highlight::Operator]
),
(" ", vec![Highlight::String, Highlight::Embedded]),
(
"b",
vec![Highlight::String, Highlight::Embedded, Highlight::Variable]
),
(
"}",
vec![
Highlight::String,
Highlight::Embedded,
Highlight::PunctuationSpecial
]
),
("</", vec![Highlight::String]),
("div", vec![Highlight::String, Highlight::Tag]),
(">`", vec![Highlight::String]),
(";", vec![Highlight::PunctuationDelimiter]),
("`", vec!["string"]),
("<", vec!["string", "punctuation.bracket"]),
("div", vec!["string", "tag"]),
(">", vec!["string", "punctuation.bracket"]),
("${", vec!["string", "embedded", "punctuation.special"]),
("a", vec!["string", "embedded", "variable"]),
(" ", vec!["string", "embedded"]),
("<", vec!["string", "embedded", "operator"]),
(" ", vec!["string", "embedded"]),
("b", vec!["string", "embedded", "variable"]),
("}", vec!["string", "embedded", "punctuation.special"]),
("</", vec!["string", "punctuation.bracket"]),
("div", vec!["string", "tag"]),
(">", vec!["string", "punctuation.bracket"]),
("`", vec!["string"]),
(";", vec!["punctuation.delimiter"]),
]]
);
}
#[test]
fn test_highlighting_injected_javascript_in_html_mini() {
let source = "<script>const x = new Thing();</script>";
assert_eq!(
&to_token_vector(source, &HTML_HIGHLIGHT).unwrap(),
&[vec![
("<", vec!["punctuation.bracket"]),
("script", vec!["tag"]),
(">", vec!["punctuation.bracket"]),
("const", vec!["keyword"]),
(" ", vec![]),
("x", vec!["variable"]),
(" ", vec![]),
("=", vec!["operator"]),
(" ", vec![]),
("new", vec!["keyword"]),
(" ", vec![]),
("Thing", vec!["constructor"]),
("(", vec!["punctuation.bracket"]),
(")", vec!["punctuation.bracket"]),
(";", vec!["punctuation.delimiter"]),
("</", vec!["punctuation.bracket"]),
("script", vec!["tag"]),
(">", vec!["punctuation.bracket"]),
],]
);
}
#[test]
fn test_highlighting_injected_javascript_in_html() {
let source = vec![
@ -97,38 +165,44 @@ fn test_highlighting_injected_javascript_in_html() {
.join("\n");
assert_eq!(
&to_token_vector(&source, get_language("html"), &HTML_SHEET).unwrap(),
&to_token_vector(&source, &HTML_HIGHLIGHT).unwrap(),
&[
vec![("<", vec![]), ("body", vec![Highlight::Tag]), (">", vec![]),],
vec![
(" <", vec![]),
("script", vec![Highlight::Tag]),
(">", vec![]),
("<", vec!["punctuation.bracket"]),
("body", vec!["tag"]),
(">", vec!["punctuation.bracket"]),
],
vec![
(" ", vec![]),
("<", vec!["punctuation.bracket"]),
("script", vec!["tag"]),
(">", vec!["punctuation.bracket"]),
],
vec![
(" ", vec![]),
("const", vec![Highlight::Keyword]),
("const", vec!["keyword"]),
(" ", vec![]),
("x", vec![Highlight::Variable]),
("x", vec!["variable"]),
(" ", vec![]),
("=", vec![Highlight::Operator]),
("=", vec!["operator"]),
(" ", vec![]),
("new", vec![Highlight::Keyword]),
("new", vec!["keyword"]),
(" ", vec![]),
("Thing", vec![Highlight::Constructor]),
("(", vec![Highlight::PunctuationBracket]),
(")", vec![Highlight::PunctuationBracket]),
(";", vec![Highlight::PunctuationDelimiter]),
("Thing", vec!["constructor"]),
("(", vec!["punctuation.bracket"]),
(")", vec!["punctuation.bracket"]),
(";", vec!["punctuation.delimiter"]),
],
vec![
(" </", vec![]),
("script", vec![Highlight::Tag]),
(">", vec![]),
(" ", vec![]),
("</", vec!["punctuation.bracket"]),
("script", vec!["tag"]),
(">", vec!["punctuation.bracket"]),
],
vec![
("</", vec![]),
("body", vec![Highlight::Tag]),
(">", vec![]),
("</", vec!["punctuation.bracket"]),
("body", vec!["tag"]),
(">", vec!["punctuation.bracket"]),
],
]
);
@ -147,13 +221,13 @@ fn test_highlighting_multiline_nodes_to_html() {
.join("\n");
assert_eq!(
&to_html(&source, get_language("javascript"), &JS_SHEET,).unwrap(),
&to_html(&source, &JS_HIGHLIGHT).unwrap(),
&[
"<span class=Keyword>const</span> <span class=Constant>SOMETHING</span> <span class=Operator>=</span> <span class=String>`</span>\n".to_string(),
"<span class=String> one <span class=Embedded><span class=PunctuationSpecial>${</span></span></span>\n".to_string(),
"<span class=String><span class=Embedded> <span class=Function>two</span><span class=PunctuationBracket>(</span><span class=PunctuationBracket>)</span></span></span>\n".to_string(),
"<span class=String><span class=Embedded> <span class=PunctuationSpecial>}</span></span> three</span>\n".to_string(),
"<span class=String>`</span>\n".to_string(),
"<span class=keyword>const</span> <span class=constant>SOMETHING</span> <span class=operator>=</span> <span class=string>`</span>\n".to_string(),
"<span class=string> one <span class=embedded><span class=punctuation.special>${</span></span></span>\n".to_string(),
"<span class=string><span class=embedded> <span class=function>two</span><span class=punctuation.bracket>(</span><span class=punctuation.bracket>)</span></span></span>\n".to_string(),
"<span class=string><span class=embedded> <span class=punctuation.special>}</span></span> three</span>\n".to_string(),
"<span class=string>`</span>\n".to_string(),
]
);
}
@ -169,51 +243,51 @@ fn test_highlighting_with_local_variable_tracking() {
.join("\n");
assert_eq!(
&to_token_vector(&source, get_language("javascript"), &JS_SHEET).unwrap(),
&to_token_vector(&source, &JS_HIGHLIGHT).unwrap(),
&[
vec![
("module", vec![Highlight::VariableBuiltin]),
(".", vec![Highlight::PunctuationDelimiter]),
("exports", vec![Highlight::Property]),
("module", vec!["variable.builtin"]),
(".", vec!["punctuation.delimiter"]),
("exports", vec!["function"]),
(" ", vec![]),
("=", vec![Highlight::Operator]),
("=", vec!["operator"]),
(" ", vec![]),
("function", vec![Highlight::Keyword]),
("function", vec!["keyword"]),
(" ", vec![]),
("a", vec![Highlight::Function]),
("(", vec![Highlight::PunctuationBracket]),
("b", vec![Highlight::VariableParameter]),
(")", vec![Highlight::PunctuationBracket]),
("a", vec!["function"]),
("(", vec!["punctuation.bracket"]),
("b", vec!["variable.parameter"]),
(")", vec!["punctuation.bracket"]),
(" ", vec![]),
("{", vec![Highlight::PunctuationBracket])
("{", vec!["punctuation.bracket"])
],
vec![
(" ", vec![]),
("const", vec![Highlight::Keyword]),
("const", vec!["keyword"]),
(" ", vec![]),
("module", vec![Highlight::Variable]),
("module", vec!["variable"]),
(" ", vec![]),
("=", vec![Highlight::Operator]),
("=", vec!["operator"]),
(" ", vec![]),
("c", vec![Highlight::Variable]),
(";", vec![Highlight::PunctuationDelimiter])
("c", vec!["variable"]),
(";", vec!["punctuation.delimiter"])
],
vec![
(" ", vec![]),
("console", vec![Highlight::VariableBuiltin]),
(".", vec![Highlight::PunctuationDelimiter]),
("log", vec![Highlight::Function]),
("(", vec![Highlight::PunctuationBracket]),
("console", vec!["variable.builtin"]),
(".", vec!["punctuation.delimiter"]),
("log", vec!["function"]),
("(", vec!["punctuation.bracket"]),
// Not a builtin, because `module` was defined as a variable above.
("module", vec![Highlight::Variable]),
(",", vec![Highlight::PunctuationDelimiter]),
("module", vec!["variable"]),
(",", vec!["punctuation.delimiter"]),
(" ", vec![]),
// A parameter, because `b` was defined as a parameter above.
("b", vec![Highlight::VariableParameter]),
(")", vec![Highlight::PunctuationBracket]),
(";", vec![Highlight::PunctuationDelimiter]),
("b", vec!["variable.parameter"]),
(")", vec!["punctuation.bracket"]),
(";", vec!["punctuation.delimiter"]),
],
vec![("}", vec![Highlight::PunctuationBracket])]
vec![("}", vec!["punctuation.bracket"])]
],
);
}
@ -234,41 +308,95 @@ fn test_highlighting_empty_lines() {
.join("\n");
assert_eq!(
&to_html(&source, get_language("javascript"), &JS_SHEET,).unwrap(),
&to_html(&source, &JS_HIGHLIGHT,).unwrap(),
&[
"<span class=Keyword>class</span> <span class=Constructor>A</span> <span class=PunctuationBracket>{</span>\n".to_string(),
"<span class=keyword>class</span> <span class=constructor>A</span> <span class=punctuation.bracket>{</span>\n".to_string(),
"\n".to_string(),
" <span class=Function>b</span><span class=PunctuationBracket>(</span><span class=VariableParameter>c</span><span class=PunctuationBracket>)</span> <span class=PunctuationBracket>{</span>\n".to_string(),
" <span class=function>b</span><span class=punctuation.bracket>(</span><span class=variable.parameter>c</span><span class=punctuation.bracket>)</span> <span class=punctuation.bracket>{</span>\n".to_string(),
"\n".to_string(),
" <span class=Function>d</span><span class=PunctuationBracket>(</span><span class=Variable>e</span><span class=PunctuationBracket>)</span>\n".to_string(),
" <span class=function>d</span><span class=punctuation.bracket>(</span><span class=variable>e</span><span class=punctuation.bracket>)</span>\n".to_string(),
"\n".to_string(),
" <span class=PunctuationBracket>}</span>\n".to_string(),
" <span class=punctuation.bracket>}</span>\n".to_string(),
"\n".to_string(),
"<span class=PunctuationBracket>}</span>\n".to_string(),
"<span class=punctuation.bracket>}</span>\n".to_string(),
]
);
}
#[test]
fn test_highlighting_ejs() {
let source = vec!["<div><% foo() %></div>"].join("\n");
fn test_highlighting_carriage_returns() {
let source = "a = \"a\rb\"\r\nb\r";
assert_eq!(
&to_token_vector(&source, get_language("embedded-template"), &EJS_SHEET).unwrap(),
&to_html(&source, &JS_HIGHLIGHT).unwrap(),
&[
"<span class=variable>a</span> <span class=operator>=</span> <span class=string>&quot;a<span class=carriage-return></span>b&quot;</span>\n",
"<span class=variable>b</span>\n",
],
);
}
#[test]
fn test_highlighting_ejs_with_html_and_javascript() {
let source = vec!["<div><% foo() %></div><script> bar() </script>"].join("\n");
assert_eq!(
&to_token_vector(&source, &EJS_HIGHLIGHT).unwrap(),
&[[
("<", vec![]),
("div", vec![Highlight::Tag]),
(">", vec![]),
("<%", vec![Highlight::Keyword]),
("<", vec!["punctuation.bracket"]),
("div", vec!["tag"]),
(">", vec!["punctuation.bracket"]),
("<%", vec!["keyword"]),
(" ", vec![]),
("foo", vec![Highlight::Function]),
("(", vec![Highlight::PunctuationBracket]),
(")", vec![Highlight::PunctuationBracket]),
("foo", vec!["function"]),
("(", vec!["punctuation.bracket"]),
(")", vec!["punctuation.bracket"]),
(" ", vec![]),
("%>", vec![Highlight::Keyword]),
("</", vec![]),
("div", vec![Highlight::Tag]),
(">", vec![])
("%>", vec!["keyword"]),
("</", vec!["punctuation.bracket"]),
("div", vec!["tag"]),
(">", vec!["punctuation.bracket"]),
("<", vec!["punctuation.bracket"]),
("script", vec!["tag"]),
(">", vec!["punctuation.bracket"]),
(" ", vec![]),
("bar", vec!["function"]),
("(", vec!["punctuation.bracket"]),
(")", vec!["punctuation.bracket"]),
(" ", vec![]),
("</", vec!["punctuation.bracket"]),
("script", vec!["tag"]),
(">", vec!["punctuation.bracket"]),
]],
);
}
#[test]
fn test_highlighting_javascript_with_jsdoc() {
// Regression test: the middle comment has no highlights. This should not prevent
// later injections from highlighting properly.
let source = vec!["a /* @see a */ b; /* nothing */ c; /* @see b */"].join("\n");
assert_eq!(
&to_token_vector(&source, &JS_HIGHLIGHT).unwrap(),
&[[
("a", vec!["variable"]),
(" ", vec![]),
("/* ", vec!["comment"]),
("@see", vec!["comment", "keyword"]),
(" a */", vec!["comment"]),
(" ", vec![]),
("b", vec!["variable"]),
(";", vec!["punctuation.delimiter"]),
(" ", vec![]),
("/* nothing */", vec!["comment"]),
(" ", vec![]),
("c", vec!["variable"]),
(";", vec!["punctuation.delimiter"]),
(" ", vec![]),
("/* ", vec!["comment"]),
("@see", vec!["comment", "keyword"]),
(" b */", vec!["comment"])
]],
);
}
@ -278,33 +406,36 @@ fn test_highlighting_with_content_children_included() {
let source = vec!["assert!(", " a.b.c() < D::e::<F>()", ");"].join("\n");
assert_eq!(
&to_token_vector(&source, get_language("rust"), &RUST_SHEET).unwrap(),
&to_token_vector(&source, &RUST_HIGHLIGHT).unwrap(),
&[
vec![
("assert", vec![Highlight::Function]),
("!", vec![Highlight::Function]),
("(", vec![Highlight::PunctuationBracket]),
("assert", vec!["function"]),
("!", vec!["function"]),
("(", vec!["punctuation.bracket"]),
],
vec![
(" a", vec![]),
(".", vec![Highlight::PunctuationDelimiter]),
("b", vec![Highlight::Property]),
(".", vec![Highlight::PunctuationDelimiter]),
("c", vec![Highlight::Function]),
("(", vec![Highlight::PunctuationBracket]),
(")", vec![Highlight::PunctuationBracket]),
(".", vec!["punctuation.delimiter"]),
("b", vec!["property"]),
(".", vec!["punctuation.delimiter"]),
("c", vec!["function"]),
("(", vec!["punctuation.bracket"]),
(")", vec!["punctuation.bracket"]),
(" < ", vec![]),
("D", vec![Highlight::Type]),
("::", vec![Highlight::PunctuationDelimiter]),
("e", vec![Highlight::Function]),
("::", vec![Highlight::PunctuationDelimiter]),
("<", vec![Highlight::PunctuationBracket]),
("F", vec![Highlight::Type]),
(">", vec![Highlight::PunctuationBracket]),
("(", vec![Highlight::PunctuationBracket]),
(")", vec![Highlight::PunctuationBracket]),
("D", vec!["type"]),
("::", vec!["punctuation.delimiter"]),
("e", vec!["function"]),
("::", vec!["punctuation.delimiter"]),
("<", vec!["punctuation.bracket"]),
("F", vec!["type"]),
(">", vec!["punctuation.bracket"]),
("(", vec!["punctuation.bracket"]),
(")", vec!["punctuation.bracket"]),
],
vec![(")", vec![Highlight::PunctuationBracket]), (";", vec![]),]
vec![
(")", vec!["punctuation.bracket"]),
(";", vec!["punctuation.delimiter"]),
]
],
);
}
@ -325,73 +456,97 @@ fn test_highlighting_cancellation() {
test_language_for_injection_string(name)
};
// Constructing the highlighter, which eagerly parses the outer document,
// should not fail.
let highlighter = highlight(
source.as_bytes(),
get_language("html"),
&HTML_SHEET,
Some(&cancellation_flag),
injection_callback,
)
.unwrap();
// The initial `highlight` call, which eagerly parses the outer document, should not fail.
let mut highlighter = Highlighter::new();
let events = highlighter
.highlight(
&HTML_HIGHLIGHT,
source.as_bytes(),
Some(&cancellation_flag),
injection_callback,
)
.unwrap();
// Iterating the scopes should not panic. It should return an error
// once the cancellation is detected.
for event in highlighter {
// Iterating the scopes should not panic. It should return an error once the
// cancellation is detected.
for event in events {
if let Err(e) = event {
assert_eq!(e, Error::Cancelled);
return;
}
}
panic!("Expected an error while iterating highlighter");
}
#[test]
fn test_highlighting_via_c_api() {
let js_lang = get_language("javascript");
let html_lang = get_language("html");
let js_sheet = get_property_sheet_json("javascript", "highlights.json");
let js_sheet = c_string(&js_sheet);
let html_sheet = get_property_sheet_json("html", "highlights.json");
let html_sheet = c_string(&html_sheet);
let highlights = vec![
"class=tag\0",
"class=function\0",
"class=string\0",
"class=keyword\0",
];
let highlight_names = highlights
.iter()
.map(|h| h["class=".len()..].as_ptr() as *const i8)
.collect::<Vec<_>>();
let highlight_attrs = highlights
.iter()
.map(|h| h.as_bytes().as_ptr() as *const i8)
.collect::<Vec<_>>();
let highlighter = c::ts_highlighter_new(
&highlight_names[0] as *const *const i8,
&highlight_attrs[0] as *const *const i8,
highlights.len() as u32,
);
let class_tag = c_string("class=tag");
let class_function = c_string("class=function");
let class_string = c_string("class=string");
let class_keyword = c_string("class=keyword");
let js_scope_name = c_string("source.js");
let html_scope_name = c_string("text.html.basic");
let injection_regex = c_string("^(javascript|js)$");
let source_code = c_string("<script>\nconst a = b('c');\nc.d();\n</script>");
let attribute_strings = &mut [ptr::null(); Highlight::Unknown as usize + 1];
attribute_strings[Highlight::Tag as usize] = class_tag.as_ptr();
attribute_strings[Highlight::String as usize] = class_string.as_ptr();
attribute_strings[Highlight::Keyword as usize] = class_keyword.as_ptr();
attribute_strings[Highlight::Function as usize] = class_function.as_ptr();
let js_scope = c_string("source.js");
let js_injection_regex = c_string("^javascript");
let language = get_language("javascript");
let queries = get_language_queries_path("javascript");
let highlights_query = fs::read_to_string(queries.join("highlights.scm")).unwrap();
let injections_query = fs::read_to_string(queries.join("injections.scm")).unwrap();
let locals_query = fs::read_to_string(queries.join("locals.scm")).unwrap();
c::ts_highlighter_add_language(
highlighter,
js_scope.as_ptr(),
js_injection_regex.as_ptr(),
language,
highlights_query.as_ptr() as *const i8,
injections_query.as_ptr() as *const i8,
locals_query.as_ptr() as *const i8,
highlights_query.len() as u32,
injections_query.len() as u32,
locals_query.len() as u32,
);
let html_scope = c_string("text.html.basic");
let html_injection_regex = c_string("^html");
let language = get_language("html");
let queries = get_language_queries_path("html");
let highlights_query = fs::read_to_string(queries.join("highlights.scm")).unwrap();
let injections_query = fs::read_to_string(queries.join("injections.scm")).unwrap();
c::ts_highlighter_add_language(
highlighter,
html_scope.as_ptr(),
html_injection_regex.as_ptr(),
language,
highlights_query.as_ptr() as *const i8,
injections_query.as_ptr() as *const i8,
ptr::null(),
highlights_query.len() as u32,
injections_query.len() as u32,
0,
);
let highlighter = c::ts_highlighter_new(attribute_strings.as_ptr());
let buffer = c::ts_highlight_buffer_new();
c::ts_highlighter_add_language(
highlighter,
html_scope_name.as_ptr(),
html_lang,
html_sheet.as_ptr(),
ptr::null_mut(),
);
c::ts_highlighter_add_language(
highlighter,
js_scope_name.as_ptr(),
js_lang,
js_sheet.as_ptr(),
injection_regex.as_ptr(),
);
c::ts_highlighter_highlight(
highlighter,
html_scope_name.as_ptr(),
html_scope.as_ptr(),
source_code.as_ptr(),
source_code.as_bytes().len() as u32,
buffer,
@ -421,8 +576,8 @@ fn test_highlighting_via_c_api() {
lines,
vec![
"&lt;<span class=tag>script</span>&gt;\n",
"<span class=keyword>const</span> <span>a</span> <span>=</span> <span class=function>b</span><span>(</span><span class=string>&#39;c&#39;</span><span>)</span><span>;</span>\n",
"<span>c</span><span>.</span><span class=function>d</span><span>(</span><span>)</span><span>;</span>\n",
"<span class=keyword>const</span> a = <span class=function>b</span>(<span class=string>&#39;c&#39;</span>);\n",
"c.<span class=function>d</span>();\n",
"&lt;/<span class=tag>script</span>&gt;\n",
]
);
@ -433,7 +588,7 @@ fn test_highlighting_via_c_api() {
#[test]
fn test_decode_utf8_lossy() {
use tree_sitter_highlight::util::LossyUtf8;
use tree_sitter::LossyUtf8;
let parts = LossyUtf8::new(b"hi").collect::<Vec<_>>();
assert_eq!(parts, vec!["hi"]);
@ -452,50 +607,60 @@ fn c_string(s: &str) -> CString {
CString::new(s.as_bytes().to_vec()).unwrap()
}
fn test_language_for_injection_string<'a>(
string: &str,
) -> Option<(Language, &'a PropertySheet<Properties>)> {
fn test_language_for_injection_string<'a>(string: &str) -> Option<&'a HighlightConfiguration> {
match string {
"javascript" => Some((get_language("javascript"), &JS_SHEET)),
"html" => Some((get_language("html"), &HTML_SHEET)),
"rust" => Some((get_language("rust"), &RUST_SHEET)),
"javascript" => Some(&JS_HIGHLIGHT),
"html" => Some(&HTML_HIGHLIGHT),
"rust" => Some(&RUST_HIGHLIGHT),
"jsdoc" => Some(&JSDOC_HIGHLIGHT),
_ => None,
}
}
fn to_html<'a>(
src: &'a str,
language: Language,
property_sheet: &'a PropertySheet<Properties>,
language_config: &'a HighlightConfiguration,
) -> Result<Vec<String>, Error> {
highlight_html(
src.as_bytes(),
language,
property_sheet,
let src = src.as_bytes();
let mut renderer = HtmlRenderer::new();
let mut highlighter = Highlighter::new();
let events = highlighter.highlight(
language_config,
src,
None,
&test_language_for_injection_string,
&|highlight| SCOPE_CLASS_STRINGS[highlight as usize].as_str(),
)
)?;
renderer.set_carriage_return_highlight(
HIGHLIGHT_NAMES
.iter()
.position(|s| s == "carriage-return")
.map(Highlight),
);
renderer
.render(events, src, &|highlight| HTML_ATTRS[highlight.0].as_bytes())
.unwrap();
Ok(renderer.lines().map(|s| s.to_string()).collect())
}
fn to_token_vector<'a>(
src: &'a str,
language: Language,
property_sheet: &'a PropertySheet<Properties>,
) -> Result<Vec<Vec<(&'a str, Vec<Highlight>)>>, Error> {
language_config: &'a HighlightConfiguration,
) -> Result<Vec<Vec<(&'a str, Vec<&'static str>)>>, Error> {
let src = src.as_bytes();
let mut highlighter = Highlighter::new();
let mut lines = Vec::new();
let mut highlights = Vec::new();
let mut line = Vec::new();
for event in highlight(
let events = highlighter.highlight(
language_config,
src,
language,
property_sheet,
None,
&test_language_for_injection_string,
)? {
)?;
for event in events {
match event? {
HighlightEvent::HighlightStart(s) => highlights.push(s),
HighlightEvent::HighlightStart(s) => highlights.push(HIGHLIGHT_NAMES[s.0].as_str()),
HighlightEvent::HighlightEnd => {
highlights.pop();
}

5
cli/src/tests/mod.rs
View file

@ -3,5 +3,8 @@ mod helpers;
mod highlight_test;
mod node_test;
mod parser_test;
mod properties_test;
mod pathological_test;
mod query_test;
mod tags_test;
mod test_highlight_test;
mod tree_test;

62
cli/src/tests/node_refs.rs
View file

@ -1,62 +0,0 @@
use super::helpers::fixtures::get_test_language;
use crate::generate::generate_parser_for_grammar;
use tree_sitter::Parser;
#[test]
fn test_basic_node_refs() {
let (parser_name, parser_code) = generate_parser_for_grammar(
r#"
{
"name": "test_grammar_with_refs",
"extras": [
{"type": "PATTERN", "value": "\\s+"}
],
"rules": {
"rule_a": {
"type": "SEQ",
"members": [
{
"type": "REF",
"value": "ref_1",
"content": {
"type": "STRING",
"value": "child-1"
}
},
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "child-2"
},
{
"type": "BLANK"
}
]
},
{
"type": "REF",
"value": "ref_2",
"content": {
"type": "STRING",
"value": "child-3"
}
}
]
}
}
}
"#,
)
.unwrap();
let mut parser = Parser::new();
let language = get_test_language(&parser_name, &parser_code, None);
parser.set_language(language).unwrap();
let tree = parser.parse("child-1 child-2 child-3", None).unwrap();
let root_node = tree.root_node();
assert_eq!(root_node.child_by_ref("ref_1"), root_node.child(0));
assert_eq!(root_node.child_by_ref("ref_2"), root_node.child(2));
}

130
cli/src/tests/node_test.rs
View file

@ -167,6 +167,79 @@ fn test_node_child() {
assert_eq!(tree.root_node().parent(), None);
}
#[test]
fn test_node_children() {
let tree = parse_json_example();
let mut cursor = tree.walk();
let array_node = tree.root_node().child(0).unwrap();
assert_eq!(
array_node
.children(&mut cursor)
.map(|n| n.kind())
.collect::<Vec<_>>(),
&["[", "number", ",", "false", ",", "object", "]",]
);
assert_eq!(
array_node
.named_children(&mut cursor)
.map(|n| n.kind())
.collect::<Vec<_>>(),
&["number", "false", "object"]
);
let object_node = array_node
.named_children(&mut cursor)
.find(|n| n.kind() == "object")
.unwrap();
assert_eq!(
object_node
.children(&mut cursor)
.map(|n| n.kind())
.collect::<Vec<_>>(),
&["{", "pair", "}",]
);
}
#[test]
fn test_node_children_by_field_name() {
let mut parser = Parser::new();
parser.set_language(get_language("python")).unwrap();
let source = "
if one:
a()
elif two:
b()
elif three:
c()
elif four:
d()
";
let tree = parser.parse(source, None).unwrap();
let node = tree.root_node().child(0).unwrap();
assert_eq!(node.kind(), "if_statement");
let mut cursor = tree.walk();
let alternatives = node.children_by_field_name("alternative", &mut cursor);
let alternative_texts =
alternatives.map(|n| &source[n.child_by_field_name("condition").unwrap().byte_range()]);
assert_eq!(
alternative_texts.collect::<Vec<_>>(),
&["two", "three", "four",]
);
}
#[test]
fn test_node_parent_of_child_by_field_name() {
let mut parser = Parser::new();
parser.set_language(get_language("javascript")).unwrap();
let tree = parser.parse("foo(a().b[0].c.d.e())", None).unwrap();
let call_node = tree.root_node().named_child(0).unwrap().named_child(0).unwrap();
assert_eq!(call_node.kind(), "call_expression");
// Regression test - when a field points to a hidden node (in this case, `_expression`)
// the hidden node should not be added to the node parent cache.
assert_eq!(call_node.child_by_field_name("function").unwrap().parent(), Some(call_node));
}
#[test]
fn test_node_named_child() {
let tree = parse_json_example();
@ -627,6 +700,63 @@ fn test_node_is_named_but_aliased_as_anonymous() {
assert_eq!(root_node.named_child(0).unwrap().kind(), "c");
}
#[test]
fn test_node_numeric_symbols_respect_simple_aliases() {
let mut parser = Parser::new();
parser.set_language(get_language("python")).unwrap();
// Example 1:
// Python argument lists can contain "splat" arguments, which are not allowed within
// other expressions. This includes `parenthesized_list_splat` nodes like `(*b)`. These
// `parenthesized_list_splat` nodes are aliased as `parenthesized_expression`. Their numeric
// `symbol`, aka `kind_id` should match that of a normal `parenthesized_expression`.
let tree = parser.parse("(a((*b)))", None).unwrap();
let root = tree.root_node();
assert_eq!(
root.to_sexp(),
"(module (expression_statement (parenthesized_expression (call function: (identifier) arguments: (argument_list (parenthesized_expression (list_splat (identifier))))))))",
);
let outer_expr_node = root.child(0).unwrap().child(0).unwrap();
assert_eq!(outer_expr_node.kind(), "parenthesized_expression");
let inner_expr_node = outer_expr_node
.named_child(0)
.unwrap()
.child_by_field_name("arguments")
.unwrap()
.named_child(0)
.unwrap();
assert_eq!(inner_expr_node.kind(), "parenthesized_expression");
assert_eq!(inner_expr_node.kind_id(), outer_expr_node.kind_id());
// Example 2:
// Ruby handles the unary (negative) and binary (minus) `-` operators using two different
// tokens. One or more of these is an external token that's aliased as `-`. Their numeric
// kind ids should match.
parser.set_language(get_language("ruby")).unwrap();
let tree = parser.parse("-a - b", None).unwrap();
let root = tree.root_node();
assert_eq!(
root.to_sexp(),
"(program (binary left: (unary operand: (identifier)) right: (identifier)))",
);
let binary_node = root.child(0).unwrap();
assert_eq!(binary_node.kind(), "binary");
let unary_minus_node = binary_node
.child_by_field_name("left")
.unwrap()
.child(0)
.unwrap();
assert_eq!(unary_minus_node.kind(), "-");
let binary_minus_node = binary_node.child_by_field_name("operator").unwrap();
assert_eq!(binary_minus_node.kind(), "-");
assert_eq!(unary_minus_node.kind_id(), binary_minus_node.kind_id());
}
fn get_all_nodes(tree: &Tree) -> Vec<Node> {
let mut result = Vec::new();
let mut visited_children = false;

431
cli/src/tests/parser_test.rs
View file

@ -1,13 +1,14 @@
use super::helpers::allocations;
use super::helpers::edits::ReadRecorder;
use super::helpers::fixtures::{get_language, get_test_language};
use crate::generate::generate_parser_for_grammar;
use crate::parse::{perform_edit, Edit};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::{thread, time};
use tree_sitter::{InputEdit, LogType, Parser, Point, Range};
use tree_sitter::{IncludedRangesError, InputEdit, LogType, Parser, Point, Range};
#[test]
fn test_basic_parsing() {
fn test_parsing_simple_string() {
let mut parser = Parser::new();
parser.set_language(get_language("rust")).unwrap();
@ -26,7 +27,11 @@ fn test_basic_parsing() {
assert_eq!(
root_node.to_sexp(),
"(source_file (struct_item (type_identifier) (field_declaration_list)) (function_item (identifier) (parameters) (block)))"
concat!(
"(source_file ",
"(struct_item name: (type_identifier) body: (field_declaration_list)) ",
"(function_item name: (identifier) parameters: (parameters) body: (block)))"
)
);
let struct_node = root_node.child(0).unwrap();
@ -118,7 +123,17 @@ fn test_parsing_with_custom_utf8_input() {
.unwrap();
let root = tree.root_node();
assert_eq!(root.to_sexp(), "(source_file (function_item (visibility_modifier) (identifier) (parameters) (block (integer_literal))))");
assert_eq!(
root.to_sexp(),
concat!(
"(source_file ",
"(function_item ",
"(visibility_modifier) ",
"name: (identifier) ",
"parameters: (parameters) ",
"body: (block (integer_literal))))"
)
);
assert_eq!(root.kind(), "source_file");
assert_eq!(root.has_error(), false);
assert_eq!(root.child(0).unwrap().kind(), "function_item");
@ -154,7 +169,10 @@ fn test_parsing_with_custom_utf16_input() {
.unwrap();
let root = tree.root_node();
assert_eq!(root.to_sexp(), "(source_file (function_item (visibility_modifier) (identifier) (parameters) (block (integer_literal))))");
assert_eq!(
root.to_sexp(),
"(source_file (function_item (visibility_modifier) name: (identifier) parameters: (parameters) body: (block (integer_literal))))"
);
assert_eq!(root.kind(), "source_file");
assert_eq!(root.has_error(), false);
assert_eq!(root.child(0).unwrap().kind(), "function_item");
@ -175,7 +193,10 @@ fn test_parsing_with_callback_returning_owned_strings() {
.unwrap();
let root = tree.root_node();
assert_eq!(root.to_sexp(), "(source_file (function_item (visibility_modifier) (identifier) (parameters) (block (integer_literal))))");
assert_eq!(
root.to_sexp(),
"(source_file (function_item (visibility_modifier) name: (identifier) parameters: (parameters) body: (block (integer_literal))))"
);
}
#[test]
@ -192,7 +213,7 @@ fn test_parsing_text_with_byte_order_mark() {
.unwrap();
assert_eq!(
tree.root_node().to_sexp(),
"(source_file (function_item (identifier) (parameters) (block)))"
"(source_file (function_item name: (identifier) parameters: (parameters) body: (block)))"
);
assert_eq!(tree.root_node().start_byte(), 2);
@ -200,7 +221,7 @@ fn test_parsing_text_with_byte_order_mark() {
let mut tree = parser.parse("\u{FEFF}fn a() {}", None).unwrap();
assert_eq!(
tree.root_node().to_sexp(),
"(source_file (function_item (identifier) (parameters) (block)))"
"(source_file (function_item name: (identifier) parameters: (parameters) body: (block)))"
);
assert_eq!(tree.root_node().start_byte(), 3);
@ -216,7 +237,7 @@ fn test_parsing_text_with_byte_order_mark() {
let mut tree = parser.parse(" \u{FEFF}fn a() {}", Some(&tree)).unwrap();
assert_eq!(
tree.root_node().to_sexp(),
"(source_file (ERROR (UNEXPECTED 65279)) (function_item (identifier) (parameters) (block)))"
"(source_file (ERROR (UNEXPECTED 65279)) (function_item name: (identifier) parameters: (parameters) body: (block)))"
);
assert_eq!(tree.root_node().start_byte(), 1);
@ -232,11 +253,52 @@ fn test_parsing_text_with_byte_order_mark() {
let tree = parser.parse("\u{FEFF}fn a() {}", Some(&tree)).unwrap();
assert_eq!(
tree.root_node().to_sexp(),
"(source_file (function_item (identifier) (parameters) (block)))"
"(source_file (function_item name: (identifier) parameters: (parameters) body: (block)))"
);
assert_eq!(tree.root_node().start_byte(), 3);
}
#[test]
fn test_parsing_invalid_chars_at_eof() {
let mut parser = Parser::new();
parser.set_language(get_language("json")).unwrap();
let tree = parser.parse(b"\xdf", None).unwrap();
assert_eq!(tree.root_node().to_sexp(), "(ERROR (UNEXPECTED INVALID))");
}
#[test]
fn test_parsing_unexpected_null_characters_within_source() {
let mut parser = Parser::new();
parser.set_language(get_language("javascript")).unwrap();
let tree = parser.parse(b"var \0 something;", None).unwrap();
assert_eq!(
tree.root_node().to_sexp(),
"(program (variable_declaration (ERROR (UNEXPECTED '\\0')) (variable_declarator name: (identifier))))"
);
}
#[test]
fn test_parsing_ends_when_input_callback_returns_empty() {
let mut parser = Parser::new();
parser.set_language(get_language("javascript")).unwrap();
let mut i = 0;
let source = b"abcdefghijklmnoqrs";
let tree = parser
.parse_with(
&mut |offset, _| {
i += 1;
if offset >= 6 {
b""
} else {
&source[offset..usize::min(source.len(), offset + 3)]
}
},
None,
)
.unwrap();
assert_eq!(tree.root_node().end_byte(), 6);
}
// Incremental parsing
#[test]
@ -333,6 +395,18 @@ fn test_parsing_after_editing_end_of_code() {
assert_eq!(recorder.strings_read(), vec![" * ", "abc.d)",]);
}
#[test]
fn test_parsing_empty_file_with_reused_tree() {
let mut parser = Parser::new();
parser.set_language(get_language("rust")).unwrap();
let tree = parser.parse("", None);
parser.parse("", tree.as_ref());
let tree = parser.parse("\n ", None);
parser.parse("\n ", tree.as_ref());
}
// Thread safety
#[test]
@ -388,7 +462,7 @@ fn test_parsing_on_multiple_threads() {
#[test]
fn test_parsing_cancelled_by_another_thread() {
let cancellation_flag = Box::new(AtomicUsize::new(0));
let cancellation_flag = std::sync::Arc::new(AtomicUsize::new(0));
let mut parser = Parser::new();
parser.set_language(get_language("javascript")).unwrap();
@ -409,9 +483,10 @@ fn test_parsing_cancelled_by_another_thread() {
);
assert!(tree.is_some());
let flag = cancellation_flag.clone();
let cancel_thread = thread::spawn(move || {
thread::sleep(time::Duration::from_millis(100));
cancellation_flag.store(1, Ordering::SeqCst);
flag.store(1, Ordering::SeqCst);
});
// Infinite input
@ -547,6 +622,56 @@ fn test_parsing_with_a_timeout_and_a_reset() {
);
}
#[test]
fn test_parsing_with_a_timeout_and_implicit_reset() {
allocations::record(|| {
let mut parser = Parser::new();
parser.set_language(get_language("javascript")).unwrap();
parser.set_timeout_micros(5);
let tree = parser.parse(
"[\"ok\", 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]",
None,
);
assert!(tree.is_none());
// Changing the parser's language implicitly resets, discarding
// the previous partial parse.
parser.set_language(get_language("json")).unwrap();
parser.set_timeout_micros(0);
let tree = parser.parse(
"[null, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]",
None,
).unwrap();
assert_eq!(
tree.root_node()
.named_child(0)
.unwrap()
.named_child(0)
.unwrap()
.kind(),
"null"
);
});
}
#[test]
fn test_parsing_with_timeout_and_no_completion() {
allocations::record(|| {
let mut parser = Parser::new();
parser.set_language(get_language("javascript")).unwrap();
parser.set_timeout_micros(5);
let tree = parser.parse(
"[\"ok\", 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]",
None,
);
assert!(tree.is_none());
// drop the parser when it has an unfinished parse
});
}
// Included Ranges
#[test]
@ -559,7 +684,9 @@ fn test_parsing_with_one_included_range() {
let script_content_node = html_tree.root_node().child(1).unwrap().child(1).unwrap();
assert_eq!(script_content_node.kind(), "raw_text");
parser.set_included_ranges(&[script_content_node.range()]);
parser
.set_included_ranges(&[script_content_node.range()])
.unwrap();
parser.set_language(get_language("javascript")).unwrap();
let js_tree = parser.parse(source_code, None).unwrap();
@ -599,26 +726,28 @@ fn test_parsing_with_multiple_included_ranges() {
let close_quote_node = template_string_node.child(3).unwrap();
parser.set_language(get_language("html")).unwrap();
parser.set_included_ranges(&[
Range {
start_byte: open_quote_node.end_byte(),
start_point: open_quote_node.end_position(),
end_byte: interpolation_node1.start_byte(),
end_point: interpolation_node1.start_position(),
},
Range {
start_byte: interpolation_node1.end_byte(),
start_point: interpolation_node1.end_position(),
end_byte: interpolation_node2.start_byte(),
end_point: interpolation_node2.start_position(),
},
Range {
start_byte: interpolation_node2.end_byte(),
start_point: interpolation_node2.end_position(),
end_byte: close_quote_node.start_byte(),
end_point: close_quote_node.start_position(),
},
]);
parser
.set_included_ranges(&[
Range {
start_byte: open_quote_node.end_byte(),
start_point: open_quote_node.end_position(),
end_byte: interpolation_node1.start_byte(),
end_point: interpolation_node1.start_position(),
},
Range {
start_byte: interpolation_node1.end_byte(),
start_point: interpolation_node1.end_position(),
end_byte: interpolation_node2.start_byte(),
end_point: interpolation_node2.start_position(),
},
Range {
start_byte: interpolation_node2.end_byte(),
start_point: interpolation_node2.end_position(),
end_byte: close_quote_node.start_byte(),
end_point: close_quote_node.start_position(),
},
])
.unwrap();
let html_tree = parser.parse(source_code, None).unwrap();
assert_eq!(
@ -667,6 +796,47 @@ fn test_parsing_with_multiple_included_ranges() {
);
}
#[test]
fn test_parsing_error_in_invalid_included_ranges() {
let mut parser = Parser::new();
// Ranges are not ordered
let error = parser
.set_included_ranges(&[
Range {
start_byte: 23,
end_byte: 29,
start_point: Point::new(0, 23),
end_point: Point::new(0, 29),
},
Range {
start_byte: 0,
end_byte: 5,
start_point: Point::new(0, 0),
end_point: Point::new(0, 5),
},
Range {
start_byte: 50,
end_byte: 60,
start_point: Point::new(0, 50),
end_point: Point::new(0, 60),
},
])
.unwrap_err();
assert_eq!(error, IncludedRangesError(1));
// Range ends before it starts
let error = parser
.set_included_ranges(&[Range {
start_byte: 10,
end_byte: 5,
start_point: Point::new(0, 10),
end_point: Point::new(0, 5),
}])
.unwrap_err();
assert_eq!(error, IncludedRangesError(0));
}
#[test]
fn test_parsing_utf16_code_with_errors_at_the_end_of_an_included_range() {
let source_code = "<script>a.</script>";
@ -677,12 +847,14 @@ fn test_parsing_utf16_code_with_errors_at_the_end_of_an_included_range() {
let mut parser = Parser::new();
parser.set_language(get_language("javascript")).unwrap();
parser.set_included_ranges(&[Range {
start_byte,
end_byte,
start_point: Point::new(0, start_byte),
end_point: Point::new(0, end_byte),
}]);
parser
.set_included_ranges(&[Range {
start_byte,
end_byte,
start_point: Point::new(0, start_byte),
end_point: Point::new(0, end_byte),
}])
.unwrap();
let tree = parser.parse_utf16(&utf16_source_code, None).unwrap();
assert_eq!(tree.root_node().to_sexp(), "(program (ERROR (identifier)))");
}
@ -697,20 +869,22 @@ fn test_parsing_with_external_scanner_that_uses_included_range_boundaries() {
let mut parser = Parser::new();
parser.set_language(get_language("javascript")).unwrap();
parser.set_included_ranges(&[
Range {
start_byte: range1_start_byte,
end_byte: range1_end_byte,
start_point: Point::new(0, range1_start_byte),
end_point: Point::new(0, range1_end_byte),
},
Range {
start_byte: range2_start_byte,
end_byte: range2_end_byte,
start_point: Point::new(0, range2_start_byte),
end_point: Point::new(0, range2_end_byte),
},
]);
parser
.set_included_ranges(&[
Range {
start_byte: range1_start_byte,
end_byte: range1_end_byte,
start_point: Point::new(0, range1_start_byte),
end_point: Point::new(0, range1_end_byte),
},
Range {
start_byte: range2_start_byte,
end_byte: range2_end_byte,
start_point: Point::new(0, range2_start_byte),
end_point: Point::new(0, range2_end_byte),
},
])
.unwrap();
let tree = parser.parse(source_code, None).unwrap();
let root = tree.root_node();
@ -758,20 +932,22 @@ fn test_parsing_with_a_newly_excluded_range() {
let directive_start = source_code.find("<%=").unwrap();
let directive_end = source_code.find("</span>").unwrap();
let source_code_end = source_code.len();
parser.set_included_ranges(&[
Range {
start_byte: 0,
end_byte: directive_start,
start_point: Point::new(0, 0),
end_point: Point::new(0, directive_start),
},
Range {
start_byte: directive_end,
end_byte: source_code_end,
start_point: Point::new(0, directive_end),
end_point: Point::new(0, source_code_end),
},
]);
parser
.set_included_ranges(&[
Range {
start_byte: 0,
end_byte: directive_start,
start_point: Point::new(0, 0),
end_point: Point::new(0, directive_start),
},
Range {
start_byte: directive_end,
end_byte: source_code_end,
start_point: Point::new(0, directive_end),
end_point: Point::new(0, source_code_end),
},
])
.unwrap();
let tree = parser.parse(&source_code, Some(&first_tree)).unwrap();
assert_eq!(
@ -809,59 +985,73 @@ fn test_parsing_with_a_newly_excluded_range() {
#[test]
fn test_parsing_with_a_newly_included_range() {
let source_code = "<div><%= foo() %></div><div><%= bar() %>";
let first_code_start_index = source_code.find(" foo").unwrap();
let first_code_end_index = first_code_start_index + 7;
let second_code_start_index = source_code.find(" bar").unwrap();
let second_code_end_index = second_code_start_index + 7;
let ranges = [
Range {
start_byte: first_code_start_index,
end_byte: first_code_end_index,
start_point: Point::new(0, first_code_start_index),
end_point: Point::new(0, first_code_end_index),
},
Range {
start_byte: second_code_start_index,
end_byte: second_code_end_index,
start_point: Point::new(0, second_code_start_index),
end_point: Point::new(0, second_code_end_index),
},
];
let source_code = "<div><%= foo() %></div><span><%= bar() %></span><%= baz() %>";
let range1_start = source_code.find(" foo").unwrap();
let range2_start = source_code.find(" bar").unwrap();
let range3_start = source_code.find(" baz").unwrap();
let range1_end = range1_start + 7;
let range2_end = range2_start + 7;
let range3_end = range3_start + 7;
// Parse only the first code directive as JavaScript
let mut parser = Parser::new();
parser.set_language(get_language("javascript")).unwrap();
parser.set_included_ranges(&ranges[0..1]);
let first_tree = parser.parse(source_code, None).unwrap();
parser
.set_included_ranges(&[simple_range(range1_start, range1_end)])
.unwrap();
let tree = parser.parse(source_code, None).unwrap();
assert_eq!(
first_tree.root_node().to_sexp(),
tree.root_node().to_sexp(),
concat!(
"(program",
" (expression_statement (call_expression function: (identifier) arguments: (arguments))))",
)
);
// Parse both the code directives as JavaScript, using the old tree as a reference.
parser.set_included_ranges(&ranges);
let tree = parser.parse(&source_code, Some(&first_tree)).unwrap();
// Parse both the first and third code directives as JavaScript, using the old tree as a
// reference.
parser
.set_included_ranges(&[
simple_range(range1_start, range1_end),
simple_range(range3_start, range3_end),
])
.unwrap();
let tree2 = parser.parse(&source_code, Some(&tree)).unwrap();
assert_eq!(
tree.root_node().to_sexp(),
tree2.root_node().to_sexp(),
concat!(
"(program",
" (expression_statement (call_expression function: (identifier) arguments: (arguments)))",
" (expression_statement (call_expression function: (identifier) arguments: (arguments))))",
)
);
assert_eq!(
tree.changed_ranges(&first_tree).collect::<Vec<_>>(),
vec![Range {
start_byte: first_code_end_index + 1,
end_byte: second_code_end_index + 1,
start_point: Point::new(0, first_code_end_index + 1),
end_point: Point::new(0, second_code_end_index + 1),
}]
tree2.changed_ranges(&tree).collect::<Vec<_>>(),
&[simple_range(range1_end, range3_end)]
);
// Parse all three code directives as JavaScript, using the old tree as a
// reference.
parser
.set_included_ranges(&[
simple_range(range1_start, range1_end),
simple_range(range2_start, range2_end),
simple_range(range3_start, range3_end),
])
.unwrap();
let tree3 = parser.parse(&source_code, Some(&tree)).unwrap();
assert_eq!(
tree3.root_node().to_sexp(),
concat!(
"(program",
" (expression_statement (call_expression function: (identifier) arguments: (arguments)))",
" (expression_statement (call_expression function: (identifier) arguments: (arguments)))",
" (expression_statement (call_expression function: (identifier) arguments: (arguments))))",
)
);
assert_eq!(
tree3.changed_ranges(&tree2).collect::<Vec<_>>(),
&[simple_range(range2_start + 1, range2_end - 1)]
);
}
@ -899,20 +1089,22 @@ fn test_parsing_with_included_ranges_and_missing_tokens() {
// There's a missing `a` token at the beginning of the code. It must be inserted
// at the beginning of the first included range, not at {0, 0}.
let source_code = "__bc__bc__";
parser.set_included_ranges(&[
Range {
start_byte: 2,
end_byte: 4,
start_point: Point::new(0, 2),
end_point: Point::new(0, 4),
},
Range {
start_byte: 6,
end_byte: 8,
start_point: Point::new(0, 6),
end_point: Point::new(0, 8),
},
]);
parser
.set_included_ranges(&[
Range {
start_byte: 2,
end_byte: 4,
start_point: Point::new(0, 2),
end_point: Point::new(0, 4),
},
Range {
start_byte: 6,
end_byte: 8,
start_point: Point::new(0, 6),
end_point: Point::new(0, 8),
},
])
.unwrap();
let tree = parser.parse(source_code, None).unwrap();
let root = tree.root_node();
@ -923,3 +1115,12 @@ fn test_parsing_with_included_ranges_and_missing_tokens() {
assert_eq!(root.start_byte(), 2);
assert_eq!(root.child(3).unwrap().start_byte(), 4);
}
fn simple_range(start: usize, end: usize) -> Range {
Range {
start_byte: start,
end_byte: end,
start_point: Point::new(0, start),
end_point: Point::new(0, end),
}
}

15
cli/src/tests/pathological_test.rs Normal file
View file

@ -0,0 +1,15 @@
use super::helpers::allocations;
use super::helpers::fixtures::get_language;
use tree_sitter::Parser;
#[test]
fn test_pathological_example_1() {
let language = "cpp";
let source = r#"*ss<s"ss<sqXqss<s._<s<sq<(qqX<sqss<s.ss<sqsssq<(qss<qssqXqss<s._<s<sq<(qqX<sqss<s.ss<sqsssq<(qss<sqss<sqss<s._<s<sq>(qqX<sqss<s.ss<sqsssq<(qss<sq&=ss<s<sqss<s._<s<sq<(qqX<sqss<s.ss<sqs"#;
allocations::record(|| {
let mut parser = Parser::new();
parser.set_language(get_language(language)).unwrap();
parser.parse(source, None).unwrap();
});
}

265
cli/src/tests/properties_test.rs
View file

@ -1,265 +0,0 @@
use super::helpers::fixtures::get_language;
use crate::generate::properties;
use serde_derive::Deserialize;
use serde_json;
use std::collections::HashSet;
use tree_sitter::{Parser, PropertySheet};
#[derive(Debug, Default, Deserialize, PartialEq, Eq)]
struct Properties {
a: Option<String>,
b: Option<String>,
}
#[test]
fn test_walk_with_properties_with_nth_child() {
let language = get_language("javascript");
let property_sheet = PropertySheet::<Properties>::new(
language,
&generate_property_sheet_string(
"/some/path.css",
"
binary_expression > identifier:nth-child(2) {
a: x;
}
binary_expression > identifier {
a: y;
}
identifier {
a: z;
}
",
),
)
.unwrap();
let source_code = "a = b || c;";
let mut parser = Parser::new();
parser.set_language(language).unwrap();
let tree = parser.parse(source_code, None).unwrap();
let mut cursor = tree.walk_with_properties(&property_sheet, source_code.as_bytes());
assert_eq!(cursor.node().kind(), "program");
assert!(cursor.goto_first_child());
assert_eq!(cursor.node().kind(), "expression_statement");
assert!(cursor.goto_first_child());
assert_eq!(cursor.node().kind(), "assignment_expression");
assert!(cursor.goto_first_child());
assert_eq!(cursor.node().kind(), "identifier");
assert_eq!(
*cursor.node_properties(),
Properties {
a: Some("z".to_string()),
b: None
}
);
assert!(cursor.goto_next_sibling());
assert_eq!(cursor.node().kind(), "=");
assert!(cursor.goto_next_sibling());
assert_eq!(cursor.node().kind(), "binary_expression");
assert!(cursor.goto_first_child());
assert_eq!(cursor.node().kind(), "identifier");
assert_eq!(
*cursor.node_properties(),
Properties {
a: Some("y".to_string()),
b: None
}
);
assert!(cursor.goto_next_sibling());
assert_eq!(cursor.node().kind(), "||");
assert!(cursor.goto_next_sibling());
assert_eq!(cursor.node().kind(), "identifier");
assert_eq!(
*cursor.node_properties(),
Properties {
a: Some("x".to_string()),
b: None
}
);
}
#[test]
fn test_walk_with_properties_with_regexes() {
let language = get_language("javascript");
let property_sheet = PropertySheet::<Properties>::new(
language,
&generate_property_sheet_string(
"/some/path.css",
"
identifier {
&[text='^[A-Z]'] {
a: y;
}
&[text='^[A-Z_]+$'] {
a: z;
}
a: x;
}
",
),
)
.unwrap();
let source_code = "const ABC = Def(ghi);";
let mut parser = Parser::new();
parser.set_language(language).unwrap();
let tree = parser.parse(source_code, None).unwrap();
let mut cursor = tree.walk_with_properties(&property_sheet, source_code.as_bytes());
assert_eq!(cursor.node().kind(), "program");
assert!(cursor.goto_first_child());
assert_eq!(cursor.node().kind(), "lexical_declaration");
assert!(cursor.goto_first_child());
assert_eq!(cursor.node().kind(), "const");
assert!(cursor.goto_next_sibling());
assert_eq!(cursor.node().kind(), "variable_declarator");
// The later selector with a text regex overrides the earlier one.
assert!(cursor.goto_first_child());
assert_eq!(cursor.node().kind(), "identifier");
assert_eq!(
*cursor.node_properties(),
Properties {
a: Some("z".to_string()),
b: None
}
);
assert!(cursor.goto_next_sibling());
assert_eq!(cursor.node().kind(), "=");
assert!(cursor.goto_next_sibling());
assert_eq!(cursor.node().kind(), "call_expression");
// The selectors with text regexes override the selector without one.
assert!(cursor.goto_first_child());
assert_eq!(cursor.node().kind(), "identifier");
assert_eq!(
*cursor.node_properties(),
Properties {
a: Some("y".to_string()),
b: None
}
);
assert!(cursor.goto_next_sibling());
assert_eq!(cursor.node().kind(), "arguments");
assert!(cursor.goto_first_child());
assert_eq!(cursor.node().kind(), "(");
// This node doesn't match either of the regexes.
assert!(cursor.goto_next_sibling());
assert_eq!(cursor.node().kind(), "identifier");
assert_eq!(
*cursor.node_properties(),
Properties {
a: Some("x".to_string()),
b: None
}
);
}
#[test]
fn test_walk_with_properties_based_on_fields() {
let language = get_language("javascript");
let property_sheet = PropertySheet::<Properties>::new(
language,
&generate_property_sheet_string(
"/some/path.css",
"
arrow_function > .parameter {
a: x;
}
function_declaration {
& > .parameters > identifier {
a: y;
}
& > .name {
b: z;
}
}
identifier {
a: w;
}
",
),
)
.unwrap();
let source_code = "function a(b) { return c => c + b; }";
let mut parser = Parser::new();
parser.set_language(language).unwrap();
let tree = parser.parse(source_code, None).unwrap();
let mut cursor = tree.walk_with_properties(&property_sheet, source_code.as_bytes());
assert!(cursor.goto_first_child());
assert_eq!(cursor.node().kind(), "function_declaration");
assert!(cursor.goto_first_child());
assert_eq!(cursor.node().kind(), "function");
assert_eq!(*cursor.node_properties(), Properties::default());
assert!(cursor.goto_next_sibling());
assert_eq!(cursor.node().kind(), "identifier");
assert_eq!(
*cursor.node_properties(),
Properties {
a: Some("w".to_string()),
b: Some("z".to_string())
}
);
assert!(cursor.goto_next_sibling());
assert_eq!(cursor.node().kind(), "formal_parameters");
assert_eq!(*cursor.node_properties(), Properties::default());
assert!(cursor.goto_first_child());
assert_eq!(cursor.node().kind(), "(");
assert_eq!(*cursor.node_properties(), Properties::default());
assert!(cursor.goto_next_sibling());
assert_eq!(cursor.node().kind(), "identifier");
assert_eq!(
*cursor.node_properties(),
Properties {
a: Some("y".to_string()),
b: None,
}
);
assert!(cursor.goto_parent());
assert!(cursor.goto_next_sibling());
assert_eq!(cursor.node().kind(), "statement_block");
assert!(cursor.goto_first_child());
assert!(cursor.goto_next_sibling());
assert_eq!(cursor.node().kind(), "return_statement");
assert!(cursor.goto_first_child());
assert!(cursor.goto_next_sibling());
assert_eq!(cursor.node().kind(), "arrow_function");
assert!(cursor.goto_first_child());
assert_eq!(cursor.node().kind(), "identifier");
assert_eq!(
*cursor.node_properties(),
Properties {
a: Some("x".to_string()),
b: None,
}
);
}
fn generate_property_sheet_string(path: &str, css: &str) -> String {
serde_json::to_string(&properties::generate_property_sheet(path, css, &HashSet::new()).unwrap())
.unwrap()
}

3027
cli/src/tests/query_test.rs Normal file
View file

File diff suppressed because it is too large Load diff

437
cli/src/tests/tags_test.rs Normal file
View file

@ -0,0 +1,437 @@
use super::helpers::allocations;
use super::helpers::fixtures::{get_language, get_language_queries_path};
use std::ffi::CStr;
use std::ffi::CString;
use std::{fs, ptr, slice, str};
use tree_sitter::Point;
use tree_sitter_tags::c_lib as c;
use tree_sitter_tags::{Error, TagsConfiguration, TagsContext};
const PYTHON_TAG_QUERY: &'static str = r#"
(
(function_definition
name: (identifier) @name
body: (block . (expression_statement (string) @doc))) @definition.function
(#strip! @doc "(^['\"\\s]*)|(['\"\\s]*$)")
)
(function_definition
name: (identifier) @name) @definition.function
(
(class_definition
name: (identifier) @name
body: (block
. (expression_statement (string) @doc))) @definition.class
(#strip! @doc "(^['\"\\s]*)|(['\"\\s]*$)")
)
(class_definition
name: (identifier) @name) @definition.class
(call
function: (identifier) @name) @reference.call
(call
function: (attribute
attribute: (identifier) @name)) @reference.call
"#;
const JS_TAG_QUERY: &'static str = r#"
(
(comment)* @doc .
(class_declaration
name: (identifier) @name) @definition.class
(#select-adjacent! @doc @definition.class)
(#strip! @doc "(^[/\\*\\s]*)|([/\\*\\s]*$)")
)
(
(comment)* @doc .
(method_definition
name: (property_identifier) @name) @definition.method
(#select-adjacent! @doc @definition.method)
(#strip! @doc "(^[/\\*\\s]*)|([/\\*\\s]*$)")
)
(
(comment)* @doc .
(function_declaration
name: (identifier) @name) @definition.function
(#select-adjacent! @doc @definition.function)
(#strip! @doc "(^[/\\*\\s]*)|([/\\*\\s]*$)")
)
(call_expression
function: (identifier) @name) @reference.call
"#;
const RUBY_TAG_QUERY: &'static str = r#"
(method
name: (_) @name) @definition.method
(method_call
method: (identifier) @name) @reference.call
(setter (identifier) @ignore)
((identifier) @name @reference.call
(#is-not? local))
"#;
#[test]
fn test_tags_python() {
let language = get_language("python");
let tags_config = TagsConfiguration::new(language, PYTHON_TAG_QUERY, "").unwrap();
let mut tag_context = TagsContext::new();
let source = br#"
class Customer:
"""
Data about a customer
"""
def age(self):
'''
Get the customer's age
'''
compute_age(self.id)
}
"#;
let tags = tag_context
.generate_tags(&tags_config, source, None)
.unwrap()
.0
.collect::<Result<Vec<_>, _>>()
.unwrap();
assert_eq!(
tags.iter()
.map(|t| (
substr(source, &t.name_range),
tags_config.syntax_type_name(t.syntax_type_id)
))
.collect::<Vec<_>>(),
&[
("Customer", "class"),
("age", "function"),
("compute_age", "call"),
]
);
assert_eq!(substr(source, &tags[0].line_range), "class Customer:");
assert_eq!(substr(source, &tags[1].line_range), "def age(self):");
assert_eq!(tags[0].docs.as_ref().unwrap(), "Data about a customer");
assert_eq!(tags[1].docs.as_ref().unwrap(), "Get the customer's age");
}
#[test]
fn test_tags_javascript() {
let language = get_language("javascript");
let tags_config = TagsConfiguration::new(language, JS_TAG_QUERY, "").unwrap();
let source = br#"
// hi
// Data about a customer.
// bla bla bla
class Customer {
/*
* Get the customer's age
*/
getAge() {
}
}
// ok
class Agent {
}
"#;
let mut tag_context = TagsContext::new();
let tags = tag_context
.generate_tags(&tags_config, source, None)
.unwrap()
.0
.collect::<Result<Vec<_>, _>>()
.unwrap();
assert_eq!(
tags.iter()
.map(|t| (
substr(source, &t.name_range),
t.span.clone(),
tags_config.syntax_type_name(t.syntax_type_id)
))
.collect::<Vec<_>>(),
&[
("Customer", Point::new(5, 10)..Point::new(5, 18), "class",),
("getAge", Point::new(9, 8)..Point::new(9, 14), "method",),
("Agent", Point::new(15, 10)..Point::new(15, 15), "class",)
]
);
assert_eq!(
tags[0].docs.as_ref().unwrap(),
"Data about a customer.\nbla bla bla"
);
assert_eq!(tags[1].docs.as_ref().unwrap(), "Get the customer's age");
assert_eq!(tags[2].docs, None);
}
#[test]
fn test_tags_columns_measured_in_utf16_code_units() {
let language = get_language("python");
let tags_config = TagsConfiguration::new(language, PYTHON_TAG_QUERY, "").unwrap();
let mut tag_context = TagsContext::new();
let source = r#""❤️❤️❤️".hello_α_ω()"#.as_bytes();
let tag = tag_context
.generate_tags(&tags_config, source, None)
.unwrap()
.0
.next()
.unwrap()
.unwrap();
assert_eq!(substr(source, &tag.name_range), "hello_α_ω");
assert_eq!(tag.span, Point::new(0, 21)..Point::new(0, 32));
assert_eq!(tag.utf16_column_range, 9..18);
}
#[test]
fn test_tags_ruby() {
let language = get_language("ruby");
let locals_query =
fs::read_to_string(get_language_queries_path("ruby").join("locals.scm")).unwrap();
let tags_config = TagsConfiguration::new(language, RUBY_TAG_QUERY, &locals_query).unwrap();
let source = strip_whitespace(
8,
"
b = 1
def foo=()
c = 1
# a is a method because it is not in scope
# b is a method because `b` doesn't capture variables from its containing scope
bar a, b, c
[1, 2, 3].each do |a|
# a is a parameter
# b is a method
# c is a variable, because the block captures variables from its containing scope.
baz a, b, c
end
end",
);
let mut tag_context = TagsContext::new();
let tags = tag_context
.generate_tags(&tags_config, source.as_bytes(), None)
.unwrap()
.0
.collect::<Result<Vec<_>, _>>()
.unwrap();
assert_eq!(
tags.iter()
.map(|t| (
substr(source.as_bytes(), &t.name_range),
tags_config.syntax_type_name(t.syntax_type_id),
(t.span.start.row, t.span.start.column),
))
.collect::<Vec<_>>(),
&[
("foo=", "method", (2, 4)),
("bar", "call", (7, 4)),
("a", "call", (7, 8)),
("b", "call", (7, 11)),
("each", "call", (9, 14)),
("baz", "call", (13, 8)),
("b", "call", (13, 15),),
]
);
}
#[test]
fn test_tags_cancellation() {
use std::sync::atomic::{AtomicUsize, Ordering};
allocations::record(|| {
// Large javascript document
let source = (0..500)
.map(|_| "/* hi */ class A { /* ok */ b() {} }\n")
.collect::<String>();
let cancellation_flag = AtomicUsize::new(0);
let language = get_language("javascript");
let tags_config = TagsConfiguration::new(language, JS_TAG_QUERY, "").unwrap();
let mut tag_context = TagsContext::new();
let tags = tag_context
.generate_tags(&tags_config, source.as_bytes(), Some(&cancellation_flag))
.unwrap();
for (i, tag) in tags.0.enumerate() {
if i == 150 {
cancellation_flag.store(1, Ordering::SeqCst);
}
if let Err(e) = tag {
assert_eq!(e, Error::Cancelled);
return;
}
}
panic!("Expected to halt tagging with an error");
});
}
#[test]
fn test_invalid_capture() {
let language = get_language("python");
let e = TagsConfiguration::new(language, "(identifier) @method", "")
.expect_err("expected InvalidCapture error");
assert_eq!(e, Error::InvalidCapture("method".to_string()));
}
#[test]
fn test_tags_with_parse_error() {
let language = get_language("python");
let tags_config = TagsConfiguration::new(language, PYTHON_TAG_QUERY, "").unwrap();
let mut tag_context = TagsContext::new();
let source = br#"
class Fine: pass
class Bad
"#;
let (tags, failed) = tag_context
.generate_tags(&tags_config, source, None)
.unwrap();
let newtags = tags.collect::<Result<Vec<_>, _>>().unwrap();
assert!(failed, "syntax error should have been detected");
assert_eq!(
newtags.iter()
.map(|t| (
substr(source, &t.name_range),
tags_config.syntax_type_name(t.syntax_type_id)
))
.collect::<Vec<_>>(),
&[
("Fine", "class"),
]
);
}
#[test]
fn test_tags_via_c_api() {
allocations::record(|| {
let tagger = c::ts_tagger_new();
let buffer = c::ts_tags_buffer_new();
let scope_name = "source.js";
let language = get_language("javascript");
let source_code = strip_whitespace(
12,
"
var a = 1;
// one
// two
// three
function b() {
}
// four
// five
class C extends D {
}
b(a);",
);
let c_scope_name = CString::new(scope_name).unwrap();
let result = c::ts_tagger_add_language(
tagger,
c_scope_name.as_ptr(),
language,
JS_TAG_QUERY.as_ptr(),
ptr::null(),
JS_TAG_QUERY.len() as u32,
0,
);
assert_eq!(result, c::TSTagsError::Ok);
let result = c::ts_tagger_tag(
tagger,
c_scope_name.as_ptr(),
source_code.as_ptr(),
source_code.len() as u32,
buffer,
ptr::null(),
);
assert_eq!(result, c::TSTagsError::Ok);
let tags = unsafe {
slice::from_raw_parts(
c::ts_tags_buffer_tags(buffer),
c::ts_tags_buffer_tags_len(buffer) as usize,
)
};
let docs = str::from_utf8(unsafe {
slice::from_raw_parts(
c::ts_tags_buffer_docs(buffer) as *const u8,
c::ts_tags_buffer_docs_len(buffer) as usize,
)
})
.unwrap();
let syntax_types: Vec<&str> = unsafe {
let mut len: u32 = 0;
let ptr =
c::ts_tagger_syntax_kinds_for_scope_name(tagger, c_scope_name.as_ptr(), &mut len);
slice::from_raw_parts(ptr, len as usize)
.iter()
.map(|i| CStr::from_ptr(*i).to_str().unwrap())
.collect()
};
assert_eq!(
tags.iter()
.map(|tag| (
syntax_types[tag.syntax_type_id as usize],
&source_code[tag.name_start_byte as usize..tag.name_end_byte as usize],
&source_code[tag.line_start_byte as usize..tag.line_end_byte as usize],
&docs[tag.docs_start_byte as usize..tag.docs_end_byte as usize],
))
.collect::<Vec<_>>(),
&[
("function", "b", "function b() {", "one\ntwo\nthree"),
("class", "C", "class C extends D {", "four\nfive"),
("call", "b", "b(a);", "")
]
);
c::ts_tags_buffer_delete(buffer);
c::ts_tagger_delete(tagger);
});
}
fn substr<'a>(source: &'a [u8], range: &std::ops::Range<usize>) -> &'a str {
std::str::from_utf8(&source[range.clone()]).unwrap()
}
fn strip_whitespace(indent: usize, s: &str) -> String {
s.lines()
.skip(1)
.map(|line| &line[line.len().min(indent)..])
.collect::<Vec<_>>()
.join("\n")
}

64
cli/src/tests/test_highlight_test.rs Normal file
View file

@ -0,0 +1,64 @@
use super::helpers::fixtures::{get_highlight_config, get_language, test_loader};
use crate::query_testing::{parse_position_comments, Assertion};
use crate::test_highlight::get_highlight_positions;
use tree_sitter::{Parser, Point};
use tree_sitter_highlight::{Highlight, Highlighter};
#[test]
fn test_highlight_test_with_basic_test() {
let language = get_language("javascript");
let config = get_highlight_config(
"javascript",
Some("injections.scm"),
&[
"function".to_string(),
"variable.parameter".to_string(),
"keyword".to_string(),
],
);
let source = [
"var abc = function(d) {",
" // ^ function",
" // ^ keyword",
" return d + e;",
" // ^ variable.parameter",
"};",
]
.join("\n");
let assertions =
parse_position_comments(&mut Parser::new(), language, source.as_bytes()).unwrap();
assert_eq!(
assertions,
&[
Assertion {
position: Point::new(0, 5),
expected_capture_name: "function".to_string()
},
Assertion {
position: Point::new(0, 11),
expected_capture_name: "keyword".to_string()
},
Assertion {
position: Point::new(3, 9),
expected_capture_name: "variable.parameter".to_string()
},
]
);
let mut highlighter = Highlighter::new();
let highlight_positions =
get_highlight_positions(test_loader(), &mut highlighter, &config, source.as_bytes())
.unwrap();
assert_eq!(
highlight_positions,
&[
(Point::new(0, 0), Point::new(0, 3), Highlight(2)), // "var"
(Point::new(0, 4), Point::new(0, 7), Highlight(0)), // "abc"
(Point::new(0, 10), Point::new(0, 18), Highlight(2)), // "function"
(Point::new(0, 19), Point::new(0, 20), Highlight(1)), // "d"
(Point::new(3, 2), Point::new(3, 8), Highlight(2)), // "return"
(Point::new(3, 9), Point::new(3, 10), Highlight(1)), // "d"
]
);
}