traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #301 from tree-sitter/clock-based-timeouts

Browse source 
Replace operation limit API with a clock-based timeout API
This commit is contained in:
Max Brunsfeld 2019-03-14 16:50:44 -07:00 • committed by GitHub
commit 1aaad66a03
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 281 additions and 164 deletions
Download patch file Download diff file Expand all files Collapse all files

31
cli/src/main.rs
View file

@ -3,7 +3,7 @@ use std::env;
use std::fs;
use std::path::Path;
use std::process::exit;
use std::usize;
use std::{u64, usize};
use tree_sitter_cli::{
config, error, generate, highlight, loader, logger, parse, properties, test,
};
@ -49,10 +49,12 @@ fn run() -> error::Result<()> {
.multiple(true)
.required(true),
)
.arg(Arg::with_name("scope").long("scope").takes_value(true))
.arg(Arg::with_name("debug").long("debug").short("d"))
.arg(Arg::with_name("debug-graph").long("debug-graph").short("D"))
.arg(Arg::with_name("quiet").long("quiet").short("q"))
.arg(Arg::with_name("time").long("time").short("t")),
.arg(Arg::with_name("time").long("time").short("t"))
.arg(Arg::with_name("timeout").long("timeout").takes_value(true)),
)
.subcommand(
SubCommand::with_name("test")
@ -132,6 +134,9 @@ fn run() -> error::Result<()> {
let debug_graph = matches.is_present("debug-graph");
let quiet = matches.is_present("quiet");
let time = matches.is_present("time");
let timeout = matches
.value_of("timeout")
.map_or(0, |t| u64::from_str_radix(t, 10).unwrap());
loader.find_all_languages(&config.parser_directories)?;
let paths = matches
.values_of("path")
@ -142,21 +147,27 @@ fn run() -> error::Result<()> {
let mut has_error = false;
for path in paths {
let path = Path::new(path);
let language =
if let Some((l, _)) = loader.language_configuration_for_file_name(path)? {
l
} else if let Some(l) = loader.language_at_path(&current_dir)? {
l
let language = if let Some(scope) = matches.value_of("scope") {
if let Some(config) = loader.language_configuration_for_scope(scope)? {
config.0
} else {
eprintln!("No language found");
return Ok(());
};
return Err(error::Error(format!("Unknown scope '{}'", scope)));
}
} else if let Some((l, _)) = loader.language_configuration_for_file_name(path)? {
l
} else if let Some(l) = loader.language_at_path(&current_dir)? {
l
} else {
eprintln!("No language found");
return Ok(());
};
has_error |= parse::parse_file_at_path(
language,
path,
max_path_length,
quiet,
time,
timeout,
debug,
debug_graph,
)?;

176
cli/src/parse.rs
View file

@ -12,6 +12,7 @@ pub fn parse_file_at_path(
max_path_length: usize,
quiet: bool,
print_time: bool,
timeout: u64,
debug: bool,
debug_graph: bool,
) -> Result<bool> {
@ -32,111 +33,122 @@ pub fn parse_file_at_path(
})));
}
parser.set_timeout_micros(timeout);
let time = Instant::now();
let tree = parser
.parse(&source_code, None)
.expect("Incompatible language version");
let tree = parser.parse(&source_code, None);
let duration = time.elapsed();
let duration_ms = duration.as_secs() * 1000 + duration.subsec_nanos() as u64 / 1000000;
let mut cursor = tree.walk();
let stdout = io::stdout();
let mut stdout = stdout.lock();
if !quiet {
let mut needs_newline = false;
let mut indent_level = 0;
let mut did_visit_children = false;
if let Some(tree) = tree {
let mut cursor = tree.walk();
if !quiet {
let mut needs_newline = false;
let mut indent_level = 0;
let mut did_visit_children = false;
loop {
let node = cursor.node();
let is_named = node.is_named();
if did_visit_children {
if is_named {
stdout.write(b")")?;
needs_newline = true;
}
if cursor.goto_next_sibling() {
did_visit_children = false;
} else if cursor.goto_parent() {
did_visit_children = true;
indent_level -= 1;
} else {
break;
}
} else {
if is_named {
if needs_newline {
stdout.write(b"\n")?;
}
for _ in 0..indent_level {
stdout.write(b" ")?;
}
let start = node.start_position();
let end = node.end_position();
write!(
&mut stdout,
"({} [{}, {}] - [{}, {}]",
node.kind(),
start.row,
start.column,
end.row,
end.column
)?;
needs_newline = true;
}
if cursor.goto_first_child() {
did_visit_children = false;
indent_level += 1;
} else {
did_visit_children = true;
}
}
}
cursor.reset(tree.root_node());
println!("");
}
let mut first_error = None;
loop {
let node = cursor.node();
let is_named = node.is_named();
if did_visit_children {
if is_named {
stdout.write(b")")?;
needs_newline = true;
}
if cursor.goto_next_sibling() {
did_visit_children = false;
} else if cursor.goto_parent() {
did_visit_children = true;
indent_level -= 1;
if node.has_error() {
if node.is_error() || node.is_missing() {
first_error = Some(node);
break;
} else {
cursor.goto_first_child();
}
} else if !cursor.goto_next_sibling() {
if !cursor.goto_parent() {
break;
}
} else {
if is_named {
if needs_newline {
stdout.write(b"\n")?;
}
for _ in 0..indent_level {
stdout.write(b" ")?;
}
let start = node.start_position();
let end = node.end_position();
write!(
&mut stdout,
"({} [{}, {}] - [{}, {}]",
node.kind(),
start.row,
start.column,
end.row,
end.column
)?;
needs_newline = true;
}
if cursor.goto_first_child() {
did_visit_children = false;
indent_level += 1;
} else {
did_visit_children = true;
}
}
}
cursor.reset(tree.root_node());
println!("");
}
let mut first_error = None;
loop {
let node = cursor.node();
if node.has_error() {
if node.is_error() || node.is_missing() {
first_error = Some(node);
break;
} else {
cursor.goto_first_child();
}
} else if !cursor.goto_next_sibling() {
if !cursor.goto_parent() {
break;
if first_error.is_some() || print_time {
write!(
&mut stdout,
"{:width$}\t{} ms",
path.to_str().unwrap(),
duration_ms,
width = max_path_length
)?;
if let Some(node) = first_error {
let start = node.start_position();
let end = node.end_position();
write!(
&mut stdout,
"\t({} [{}, {}] - [{}, {}])",
node.kind(),
start.row,
start.column,
end.row,
end.column
)?;
}
write!(&mut stdout, "\n")?;
}
}
if first_error.is_some() || print_time {
write!(
return Ok(first_error.is_some())
} else if print_time {
writeln!(
&mut stdout,
"{:width$}\t{} ms",
"{:width$}\t{} ms (timed out)",
path.to_str().unwrap(),
duration_ms,
width = max_path_length
)?;
if let Some(node) = first_error {
let start = node.start_position();
let end = node.end_position();
write!(
&mut stdout,
"\t({} [{}, {}] - [{}, {}])",
node.kind(),
start.row,
start.column,
end.row,
end.column
)?;
}
write!(&mut stdout, "\n")?;
}
Ok(first_error.is_some())
Ok(false)
}

109
cli/src/tests/parser_test.rs
View file

@ -1,7 +1,7 @@
use super::helpers::edits::{perform_edit, Edit, ReadRecorder};
use super::helpers::fixtures::{get_language, get_test_language};
use crate::generate::generate_parser_for_grammar;
use std::{thread, usize};
use std::{thread, time};
use tree_sitter::{InputEdit, LogType, Parser, Point, Range};
#[test]
@ -269,84 +269,117 @@ fn test_parsing_on_multiple_threads() {
assert_eq!(child_count_differences, &[1, 2, 3, 4]);
}
// Operation limits
// Timeouts
#[test]
fn test_parsing_with_an_operation_limit() {
fn test_parsing_with_a_timeout() {
let mut parser = Parser::new();
parser.set_language(get_language("json")).unwrap();
// Start parsing from an infinite input. Parsing should abort after 5 "operations".
parser.set_operation_limit(5);
let mut call_count = 0;
// Parse an infinitely-long array, but pause after 100 microseconds of processing.
parser.set_timeout_micros(100);
let start_time = time::Instant::now();
let tree = parser.parse_with(
&mut |_, _| {
if call_count == 0 {
call_count += 1;
b"[0"
&mut |offset, _| {
if offset == 0 {
b" ["
} else {
call_count += 1;
b", 0"
b",0"
}
},
None,
);
assert!(tree.is_none());
assert!(call_count >= 3);
assert!(call_count <= 8);
assert!(start_time.elapsed().as_micros() < 500);
// Resume parsing from the previous state.
call_count = 0;
parser.set_operation_limit(20);
// Continue parsing, but pause after 300 microseconds of processing.
parser.set_timeout_micros(1000);
let start_time = time::Instant::now();
let tree = parser.parse_with(
&mut |offset, _| {
if offset == 0 {
b" ["
} else {
b",0"
}
},
None,
);
assert!(tree.is_none());
assert!(start_time.elapsed().as_micros() > 500);
assert!(start_time.elapsed().as_micros() < 1500);
// Finish parsing
parser.set_timeout_micros(0);
let tree = parser
.parse_with(
&mut |_, _| {
if call_count == 0 {
call_count += 1;
&mut |offset, _| {
if offset > 5000 {
b""
} else if offset == 5000 {
b"]"
} else {
b""
b",0"
}
},
None,
)
.unwrap();
assert_eq!(
tree.root_node().to_sexp(),
"(value (array (number) (number) (number)))"
);
assert_eq!(tree.root_node().child(0).unwrap().kind(), "array");
}
#[test]
fn test_parsing_with_a_reset_after_reaching_an_operation_limit() {
fn test_parsing_with_a_timeout_and_a_reset() {
let mut parser = Parser::new();
parser.set_language(get_language("json")).unwrap();
parser.set_operation_limit(3);
let tree = parser.parse("[1234, 5, 6, 7, 8]", None);
parser.set_timeout_micros(30);
let tree = parser.parse(
"[\"ok\", 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]",
None,
);
assert!(tree.is_none());
// Without calling reset, the parser continues from where it left off, so
// it does not see the changes to the beginning of the source code.
parser.set_operation_limit(usize::MAX);
let tree = parser.parse("[null, 5, 6, 4, 5]", None).unwrap();
parser.set_timeout_micros(0);
let tree = parser.parse(
"[null, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]",
None,
).unwrap();
assert_eq!(
tree.root_node().to_sexp(),
"(value (array (number) (number) (number) (number) (number)))"
tree.root_node()
.named_child(0)
.unwrap()
.named_child(0)
.unwrap()
.kind(),
"string"
);
parser.set_operation_limit(3);
let tree = parser.parse("[1234, 5, 6, 7, 8]", None);
parser.set_timeout_micros(30);
let tree = parser.parse(
"[\"ok\", 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]",
None,
);
assert!(tree.is_none());
// By calling reset, we force the parser to start over from scratch so
// that it sees the changes to the beginning of the source code.
parser.set_operation_limit(usize::MAX);
parser.set_timeout_micros(0);
parser.reset();
let tree = parser.parse("[null, 5, 6, 4, 5]", None).unwrap();
let tree = parser.parse(
"[null, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]",
None,
).unwrap();
assert_eq!(
tree.root_node().to_sexp(),
"(value (array (null) (number) (number) (number) (number)))"
tree.root_node()
.named_child(0)
.unwrap()
.named_child(0)
.unwrap()
.kind(),
"null"
);
}