traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into node-fields

Browse source
This commit is contained in:
Max Brunsfeld 2019-03-26 11:58:21 -07:00
commit 5035e194ff
34 changed files with 1178 additions and 240 deletions
Download patch file Download diff file Expand all files Collapse all files

6
cli/src/tests/helpers/allocations.rs
View file

@ -98,7 +98,9 @@ extern "C" fn ts_record_free(ptr: *mut c_void) {
}
#[no_mangle]
extern "C" fn ts_record_allocations_toggle() {
extern "C" fn ts_toggle_allocation_recording(enabled: bool) -> bool {
let mut recorder = RECORDER.lock();
recorder.enabled = !recorder.enabled;
let was_enabled = recorder.enabled;
recorder.enabled = enabled;
was_enabled
}

8
cli/src/tests/helpers/fixtures.rs
View file

@ -21,12 +21,16 @@ pub fn get_language(name: &str) -> Language {
.unwrap()
}
pub fn get_property_sheet(language_name: &str, sheet_name: &str) -> PropertySheet<Properties> {
pub fn get_property_sheet_json(language_name: &str, sheet_name: &str) -> String {
let path = GRAMMARS_DIR
.join(language_name)
.join("src")
.join(sheet_name);
let json = fs::read_to_string(path).unwrap();
fs::read_to_string(path).unwrap()
}
pub fn get_property_sheet(language_name: &str, sheet_name: &str) -> PropertySheet<Properties> {
let json = get_property_sheet_json(language_name, sheet_name);
let language = get_language(language_name);
load_property_sheet(language, &json).unwrap()
}

120
cli/src/tests/highlight_test.rs
View file

@ -1,13 +1,17 @@
use super::helpers::fixtures::{get_language, get_property_sheet};
use super::helpers::fixtures::{get_language, get_property_sheet, get_property_sheet_json};
use lazy_static::lazy_static;
use std::ffi::CString;
use std::{ptr, slice, str};
use tree_sitter::{Language, PropertySheet};
use tree_sitter_highlight::{highlight, highlight_html, HighlightEvent, Properties, Scope};
use tree_sitter_highlight::{c, highlight, highlight_html, HighlightEvent, Properties, Scope};
lazy_static! {
static ref JS_SHEET: PropertySheet<Properties> =
get_property_sheet("javascript", "highlights.json");
static ref HTML_SHEET: PropertySheet<Properties> =
get_property_sheet("html", "highlights.json");
static ref EJS_SHEET: PropertySheet<Properties> =
get_property_sheet("embedded-template", "highlights-ejs.json");
static ref SCOPE_CLASS_STRINGS: Vec<String> = {
let mut result = Vec::new();
let mut i = 0;
@ -153,6 +157,118 @@ fn test_highlighting_empty_lines() {
);
}
#[test]
fn test_highlighting_ejs() {
let source = vec!["<div><% foo() %></div>"].join("\n");
assert_eq!(
&to_token_vector(&source, get_language("embedded-template"), &EJS_SHEET).unwrap(),
&[[
("<", vec![]),
("div", vec![Scope::Tag]),
(">", vec![]),
("<%", vec![Scope::Keyword]),
(" ", vec![]),
("foo", vec![Scope::Function]),
("(", vec![Scope::PunctuationBracket]),
(")", vec![Scope::PunctuationBracket]),
(" ", vec![]),
("%>", vec![Scope::Keyword]),
("</", vec![]),
("div", vec![Scope::Tag]),
(">", vec![])
]],
);
}
#[test]
fn test_highlighting_via_c_api() {
let js_lang = get_language("javascript");
let html_lang = get_language("html");
let js_sheet = get_property_sheet_json("javascript", "highlights.json");
let js_sheet = c_string(&js_sheet);
let html_sheet = get_property_sheet_json("html", "highlights.json");
let html_sheet = c_string(&html_sheet);
let class_tag = c_string("class=tag");
let class_function = c_string("class=function");
let class_string = c_string("class=string");
let class_keyword = c_string("class=keyword");
let js_scope_name = c_string("source.js");
let html_scope_name = c_string("text.html.basic");
let injection_regex = c_string("^(javascript|js)$");
let source_code = c_string("<script>\nconst a = b('c');\nc.d();\n</script>");
let attribute_strings = &mut [ptr::null(); Scope::Unknown as usize + 1];
attribute_strings[Scope::Tag as usize] = class_tag.as_ptr();
attribute_strings[Scope::String as usize] = class_string.as_ptr();
attribute_strings[Scope::Keyword as usize] = class_keyword.as_ptr();
attribute_strings[Scope::Function as usize] = class_function.as_ptr();
let highlighter = c::ts_highlighter_new(attribute_strings.as_ptr());
let buffer = c::ts_highlight_buffer_new();
c::ts_highlighter_add_language(
highlighter,
html_scope_name.as_ptr(),
html_lang,
html_sheet.as_ptr(),
ptr::null_mut(),
);
c::ts_highlighter_add_language(
highlighter,
js_scope_name.as_ptr(),
js_lang,
js_sheet.as_ptr(),
injection_regex.as_ptr(),
);
c::ts_highlighter_highlight(
highlighter,
html_scope_name.as_ptr(),
source_code.as_ptr(),
source_code.as_bytes().len() as u32,
buffer,
ptr::null_mut(),
);
let output_bytes = c::ts_highlight_buffer_content(buffer);
let output_line_offsets = c::ts_highlight_buffer_line_offsets(buffer);
let output_len = c::ts_highlight_buffer_len(buffer);
let output_line_count = c::ts_highlight_buffer_line_count(buffer);
let output_bytes = unsafe { slice::from_raw_parts(output_bytes, output_len as usize) };
let output_line_offsets =
unsafe { slice::from_raw_parts(output_line_offsets, output_line_count as usize) };
let mut lines = Vec::new();
for i in 0..(output_line_count as usize) {
let line_start = output_line_offsets[i] as usize;
let line_end = output_line_offsets
.get(i + 1)
.map(|x| *x as usize)
.unwrap_or(output_bytes.len());
lines.push(str::from_utf8(&output_bytes[line_start..line_end]).unwrap());
}
assert_eq!(
lines,
vec![
"&lt;<span class=tag>script</span>&gt;",
"<span class=keyword>const</span> <span>a</span> <span>=</span> <span class=function>b</span><span>(</span><span class=string>&#39;c&#39;</span><span>)</span><span>;</span>",
"<span>c</span><span>.</span><span class=function>d</span><span>(</span><span>)</span><span>;</span>",
"&lt;/<span class=tag>script</span>&gt;",
]
);
c::ts_highlighter_delete(highlighter);
c::ts_highlight_buffer_delete(buffer);
}
fn c_string(s: &str) -> CString {
CString::new(s.as_bytes().to_vec()).unwrap()
}
fn test_language_for_injection_string<'a>(
string: &str,
) -> Option<(Language, &'a PropertySheet<Properties>)> {

188
cli/src/tests/parser_test.rs
View file

@ -1,7 +1,8 @@
use super::helpers::edits::{perform_edit, Edit, ReadRecorder};
use super::helpers::fixtures::{get_language, get_test_language};
use crate::generate::generate_parser_for_grammar;
use std::{thread, usize};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::{thread, time};
use tree_sitter::{InputEdit, LogType, Parser, Point, Range};
#[test]
@ -56,6 +57,37 @@ fn test_parsing_with_logging() {
"reduce sym:struct_item, child_count:3".to_string()
)));
assert!(messages.contains(&(LogType::Lex, "skip character:' '".to_string())));
for (_, m) in &messages {
assert!(!m.contains("row:0"));
}
}
#[test]
#[cfg(unix)]
fn test_parsing_with_debug_graph_enabled() {
use std::io::{BufRead, BufReader, Seek};
let has_zero_indexed_row = |s: &str| s.contains("position: 0,");
let mut parser = Parser::new();
parser.set_language(get_language("javascript")).unwrap();
let mut debug_graph_file = tempfile::tempfile().unwrap();
parser.print_dot_graphs(&debug_graph_file);
parser.parse("const zero = 0", None).unwrap();
debug_graph_file.seek(std::io::SeekFrom::Start(0)).unwrap();
let log_reader = BufReader::new(debug_graph_file)
.lines()
.map(|l| l.expect("Failed to read line from graph log"));
for line in log_reader {
assert!(
!has_zero_indexed_row(&line),
"Graph log output includes zero-indexed row: {}",
line
);
}
}
#[test]
@ -269,84 +301,164 @@ fn test_parsing_on_multiple_threads() {
assert_eq!(child_count_differences, &[1, 2, 3, 4]);
}
// Operation limits
#[test]
fn test_parsing_cancelled_by_another_thread() {
let cancellation_flag = Box::new(AtomicUsize::new(0));
let mut parser = Parser::new();
parser.set_language(get_language("javascript")).unwrap();
unsafe { parser.set_cancellation_flag(Some(&cancellation_flag)) };
// Long input - parsing succeeds
let tree = parser.parse_with(
&mut |offset, _| {
if offset == 0 {
b" ["
} else if offset >= 20000 {
b""
} else {
b"0,"
}
},
None,
);
assert!(tree.is_some());
let cancel_thread = thread::spawn(move || {
thread::sleep(time::Duration::from_millis(100));
cancellation_flag.store(1, Ordering::SeqCst);
});
// Infinite input
let tree = parser.parse_with(
&mut |offset, _| {
thread::yield_now();
thread::sleep(time::Duration::from_millis(10));
if offset == 0 {
b" ["
} else {
b"0,"
}
},
None,
);
// Parsing returns None because it was cancelled.
cancel_thread.join().unwrap();
assert!(tree.is_none());
}
// Timeouts
#[test]
fn test_parsing_with_an_operation_limit() {
fn test_parsing_with_a_timeout() {
let mut parser = Parser::new();
parser.set_language(get_language("json")).unwrap();
// Start parsing from an infinite input. Parsing should abort after 5 "operations".
parser.set_operation_limit(5);
let mut call_count = 0;
// Parse an infinitely-long array, but pause after 100 microseconds of processing.
parser.set_timeout_micros(100);
let start_time = time::Instant::now();
let tree = parser.parse_with(
&mut |_, _| {
if call_count == 0 {
call_count += 1;
b"[0"
&mut |offset, _| {
if offset == 0 {
b" ["
} else {
call_count += 1;
b", 0"
b",0"
}
},
None,
);
assert!(tree.is_none());
assert!(call_count >= 3);
assert!(call_count <= 8);
assert!(start_time.elapsed().as_micros() < 500);
// Resume parsing from the previous state.
call_count = 0;
parser.set_operation_limit(20);
// Continue parsing, but pause after 300 microseconds of processing.
parser.set_timeout_micros(1000);
let start_time = time::Instant::now();
let tree = parser.parse_with(
&mut |offset, _| {
if offset == 0 {
b" ["
} else {
b",0"
}
},
None,
);
assert!(tree.is_none());
assert!(start_time.elapsed().as_micros() > 500);
assert!(start_time.elapsed().as_micros() < 1500);
// Finish parsing
parser.set_timeout_micros(0);
let tree = parser
.parse_with(
&mut |_, _| {
if call_count == 0 {
call_count += 1;
&mut |offset, _| {
if offset > 5000 {
b""
} else if offset == 5000 {
b"]"
} else {
b""
b",0"
}
},
None,
)
.unwrap();
assert_eq!(
tree.root_node().to_sexp(),
"(value (array (number) (number) (number)))"
);
assert_eq!(tree.root_node().child(0).unwrap().kind(), "array");
}
#[test]
fn test_parsing_with_a_reset_after_reaching_an_operation_limit() {
fn test_parsing_with_a_timeout_and_a_reset() {
let mut parser = Parser::new();
parser.set_language(get_language("json")).unwrap();
parser.set_operation_limit(3);
let tree = parser.parse("[1234, 5, 6, 7, 8]", None);
parser.set_timeout_micros(30);
let tree = parser.parse(
"[\"ok\", 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]",
None,
);
assert!(tree.is_none());
// Without calling reset, the parser continues from where it left off, so
// it does not see the changes to the beginning of the source code.
parser.set_operation_limit(usize::MAX);
let tree = parser.parse("[null, 5, 6, 4, 5]", None).unwrap();
parser.set_timeout_micros(0);
let tree = parser.parse(
"[null, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]",
None,
).unwrap();
assert_eq!(
tree.root_node().to_sexp(),
"(value (array (number) (number) (number) (number) (number)))"
tree.root_node()
.named_child(0)
.unwrap()
.named_child(0)
.unwrap()
.kind(),
"string"
);
parser.set_operation_limit(3);
let tree = parser.parse("[1234, 5, 6, 7, 8]", None);
parser.set_timeout_micros(30);
let tree = parser.parse(
"[\"ok\", 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]",
None,
);
assert!(tree.is_none());
// By calling reset, we force the parser to start over from scratch so
// that it sees the changes to the beginning of the source code.
parser.set_operation_limit(usize::MAX);
parser.set_timeout_micros(0);
parser.reset();
let tree = parser.parse("[null, 5, 6, 4, 5]", None).unwrap();
let tree = parser.parse(
"[null, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]",
None,
).unwrap();
assert_eq!(
tree.root_node().to_sexp(),
"(value (array (null) (number) (number) (number) (number)))"
tree.root_node()
.named_child(0)
.unwrap()
.named_child(0)
.unwrap()
.kind(),
"null"
);
}