traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(rust): misc fixes & tidying

Browse source
This commit is contained in:
Amaan Qureshi 2024-04-09 21:42:59 -04:00
parent 5825e24d56
commit abc7910381
23 changed files with 137 additions and 127 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Cargo.lock generated
View file

@ -1447,6 +1447,7 @@ dependencies = [
"regex",
"serde",
"serde_json",
"tempfile",
"tree-sitter",
"tree-sitter-highlight",
"tree-sitter-tags",

1
cli/loader/Cargo.toml
View file

@ -26,6 +26,7 @@ once_cell.workspace = true
regex.workspace = true
serde.workspace = true
serde_json.workspace = true
tempfile.workspace = true
tree-sitter.workspace = true
tree-sitter-highlight.workspace = true

28
cli/loader/src/lib.rs
View file

@ -130,11 +130,12 @@ pub struct CompileConfig<'a> {
}
impl<'a> CompileConfig<'a> {
#[must_use]
pub fn new(
src_path: &'a Path,
externals: Option<&'a [PathBuf]>,
output_path: Option<PathBuf>,
) -> CompileConfig<'a> {
) -> Self {
Self {
src_path,
header_paths: vec![src_path],
@ -449,7 +450,7 @@ impl Loader {
let parser_path = config.src_path.join("parser.c");
config.scanner_path = self.get_scanner_path(config.src_path);
let mut paths_to_check = vec![parser_path.clone()];
let mut paths_to_check = vec![parser_path];
if let Some(scanner_path) = config.scanner_path.as_ref() {
paths_to_check.push(scanner_path.clone());
@ -488,7 +489,9 @@ impl Loader {
}
let lock_path = if env::var("CROSS_RUNNER").is_ok() {
PathBuf::from("/tmp")
tempfile::tempdir()
.unwrap()
.path()
.join("tree-sitter")
.join("lock")
.join(format!("{}.lock", config.name))
@ -1021,7 +1024,7 @@ impl Loader {
language_name: grammar_json.name.clone(),
scope: config_json.scope,
language_id,
file_types: config_json.file_types.unwrap_or(Vec::new()),
file_types: config_json.file_types.unwrap_or_default(),
content_regex: Self::regex(config_json.content_regex.as_deref()),
first_line_regex: Self::regex(config_json.first_line_regex.as_deref()),
injection_regex: Self::regex(config_json.injection_regex.as_deref()),
@ -1048,8 +1051,11 @@ impl Loader {
.push(self.language_configurations.len());
}
self.language_configurations
.push(unsafe { mem::transmute(configuration) });
self.language_configurations.push(unsafe {
mem::transmute::<LanguageConfiguration<'_>, LanguageConfiguration<'static>>(
configuration,
)
});
if set_current_path_config
&& self.language_configuration_in_current_path.is_none()
@ -1088,8 +1094,11 @@ impl Loader {
highlight_names: &self.highlight_names,
use_all_highlight_names: self.use_all_highlight_names,
};
self.language_configurations
.push(unsafe { mem::transmute(configuration) });
self.language_configurations.push(unsafe {
mem::transmute::<LanguageConfiguration<'_>, LanguageConfiguration<'static>>(
configuration,
)
});
self.languages_by_id
.push((parser_path.to_owned(), OnceCell::new(), None));
}
@ -1327,8 +1336,7 @@ impl<'a> LanguageConfiguration<'a> {
.unwrap_or_else(|| ranges.last().unwrap());
error.offset = offset_within_section - range.start;
error.row = source[range.start..offset_within_section]
.chars()
.filter(|c| *c == '\n')
.matches(|c| c == '\n')
.count();
Error::from(error).context(format!("Error in query file {path:?}"))
}

12
cli/src/generate/build_tables/build_parse_table.rs
View file

@ -298,7 +298,7 @@ impl<'a> ParseTableBuilder<'a> {
}
}
reduction_info.precedence = precedence.clone();
reduction_info.precedence.clone_from(precedence);
if let Err(i) = reduction_info.symbols.binary_search(&symbol) {
reduction_info.symbols.insert(i, symbol);
}
@ -604,13 +604,13 @@ impl<'a> ParseTableBuilder<'a> {
write!(&mut msg, " {}", self.symbol_name(symbol)).unwrap();
}
write!(
writeln!(
&mut msg,
" • {} …\n\n",
" • {} …\n",
self.symbol_name(&conflicting_lookahead)
)
.unwrap();
write!(&mut msg, "Possible interpretations:\n\n").unwrap();
writeln!(&mut msg, "Possible interpretations:\n").unwrap();
let mut interpretations = conflicting_items
.iter()
@ -685,7 +685,7 @@ impl<'a> ParseTableBuilder<'a> {
}
let mut resolution_count = 0;
write!(&mut msg, "\nPossible resolutions:\n\n").unwrap();
writeln!(&mut msg, "\nPossible resolutions:\n").unwrap();
let mut shift_items = Vec::new();
let mut reduce_items = Vec::new();
for item in conflicting_items {
@ -961,7 +961,7 @@ fn populate_following_tokens(
for entry in result.iter_mut() {
entry.insert(*extra);
}
result[extra.index] = all_tokens.clone();
result[extra.index].clone_from(&all_tokens);
}
}
}

2
cli/src/generate/build_tables/item.rs
View file

@ -133,7 +133,7 @@ impl<'a> ParseItem<'a> {
/// Create an item like this one, but advanced by one step.
#[must_use]
pub const fn successor(&self) -> ParseItem<'a> {
pub const fn successor(&self) -> Self {
ParseItem {
variable_index: self.variable_index,
production: self.production,

23
cli/src/generate/parse_grammar.rs
View file

@ -166,19 +166,18 @@ fn parse_rule(json: RuleJSON) -> Rule {
RuleJSON::PATTERN { value, flags } => Rule::Pattern(
value,
flags.map_or(String::new(), |f| {
f.chars()
.filter(|c| {
if *c == 'i' {
true
} else {
// silently ignore unicode flags
if *c != 'u' && *c != 'v' {
eprintln!("Warning: unsupported flag {c}");
}
false
f.matches(|c| {
if c == 'i' {
true
} else {
// silently ignore unicode flags
if c != 'u' && c != 'v' {
eprintln!("Warning: unsupported flag {c}");
}
})
.collect()
false
}
})
.collect()
}),
),
RuleJSON::SYMBOL { name } => Rule::NamedSymbol(name),

2
cli/src/generate/prepare_grammar/process_inlines.rs
View file

@ -154,7 +154,7 @@ impl InlinedProductionMapBuilder {
self.productions
.iter()
.position(|p| *p == production)
.unwrap_or({
.unwrap_or_else(|| {
self.productions.push(production);
self.productions.len() - 1
})

17
cli/src/generate/render.rs
View file

@ -29,7 +29,7 @@ macro_rules! add {
}
macro_rules! add_whitespace {
($this: tt) => {{
($this:tt) => {{
for _ in 0..$this.indent_level {
write!(&mut $this.buffer, " ").unwrap();
}
@ -45,13 +45,13 @@ macro_rules! add_line {
}
macro_rules! indent {
($this: tt) => {
($this:tt) => {
$this.indent_level += 1;
};
}
macro_rules! dedent {
($this: tt) => {
($this:tt) => {
assert_ne!($this.indent_level, 0);
$this.indent_level -= 1;
};
@ -221,9 +221,8 @@ impl Generator {
});
// Some aliases match an existing symbol in the grammar.
let alias_id;
if let Some(existing_symbol) = existing_symbol {
alias_id = self.symbol_ids[&self.symbol_map[&existing_symbol]].clone();
let alias_id = if let Some(existing_symbol) = existing_symbol {
self.symbol_ids[&self.symbol_map[&existing_symbol]].clone()
}
// Other aliases don't match any existing symbol, and need their own
// identifiers.
@ -232,12 +231,12 @@ impl Generator {
self.unique_aliases.insert(i, alias.clone());
}
alias_id = if alias.is_named {
if alias.is_named {
format!("alias_sym_{}", self.sanitize_identifier(&alias.value))
} else {
format!("anon_alias_sym_{}", self.sanitize_identifier(&alias.value))
};
}
}
};
self.alias_ids.entry(alias.clone()).or_insert(alias_id);
}

3
cli/src/main.rs
View file

@ -870,8 +870,7 @@ fn run() -> Result<()> {
let open_in_browser = !playground_options.quiet;
let grammar_path = playground_options
.grammar_path
.map(PathBuf::from)
.unwrap_or(current_dir);
.map_or(current_dir, PathBuf::from);
playground::serve(&grammar_path, open_in_browser)?;
}

2
cli/src/playground.rs
View file

@ -12,7 +12,7 @@ use tiny_http::{Header, Response, Server};
use super::wasm;
macro_rules! optional_resource {
($name: tt, $path: tt) => {
($name:tt, $path:tt) => {
#[cfg(TREE_SITTER_EMBED_WASM_BINDING)]
fn $name(tree_sitter_dir: Option<&Path>) -> Cow<'static, [u8]> {
if let Some(tree_sitter_dir) = tree_sitter_dir {

6
cli/src/test.rs
View file

@ -432,9 +432,9 @@ fn write_tests_to_buffer(
if i > 0 {
writeln!(buffer)?;
}
write!(
writeln!(
buffer,
"{}\n{name}\n{}\n{input}\n{}\n\n{}\n",
"{}\n{name}\n{}\n{input}\n{}\n\n{}",
"=".repeat(*header_delim_len),
"=".repeat(*header_delim_len),
"-".repeat(*divider_delim_len),
@ -662,7 +662,7 @@ fn parse_test_content(name: String, content: &str, file_path: Option<PathBuf>) -
}
}
prev_attributes = attributes;
prev_name = test_name.unwrap_or(String::new());
prev_name = test_name.unwrap_or_default();
prev_header_len = header_delim_len;
prev_header_end = header_range.end;
}

2
cli/src/tests/corpus_test.rs
View file

@ -217,7 +217,7 @@ fn test_language_corpus(
// Perform a random series of edits and reparse.
let mut undo_stack = Vec::new();
for _ in 0..1 + rand.unsigned(*EDIT_COUNT) {
for _ in 0..=rand.unsigned(*EDIT_COUNT) {
let edit = get_random_edit(&mut rand, &input);
undo_stack.push(invert_edit(&input, &edit));
perform_edit(&mut tree, &mut input, &edit).unwrap();

23
cli/src/tests/detect_language.rs
View file

@ -33,25 +33,25 @@ fn detect_language_by_first_line_regex() {
assert_eq!(config[0].scope.as_ref().unwrap(), "source.strace");
let file_name = strace_dir.path().join("strace.log");
std::fs::write(&file_name, "execve\nworld").unwrap();
fs::write(&file_name, "execve\nworld").unwrap();
assert_eq!(
get_lang_scope(&loader, &file_name),
Some("source.strace".into())
);
let file_name = strace_dir.path().join("strace.log");
std::fs::write(&file_name, "447845 execve\nworld").unwrap();
fs::write(&file_name, "447845 execve\nworld").unwrap();
assert_eq!(
get_lang_scope(&loader, &file_name),
Some("source.strace".into())
);
let file_name = strace_dir.path().join("strace.log");
std::fs::write(&file_name, "hello\nexecve").unwrap();
fs::write(&file_name, "hello\nexecve").unwrap();
assert!(get_lang_scope(&loader, &file_name).is_none());
let file_name = strace_dir.path().join("strace.log");
std::fs::write(&file_name, "").unwrap();
fs::write(&file_name, "").unwrap();
assert!(get_lang_scope(&loader, &file_name).is_none());
let dummy_dir = tree_sitter_dir(
@ -76,7 +76,7 @@ fn detect_language_by_first_line_regex() {
.find_language_configurations_at_path(dummy_dir.path(), false)
.unwrap();
let file_name = dummy_dir.path().join("strace.dummy");
std::fs::write(&file_name, "execve").unwrap();
fs::write(&file_name, "execve").unwrap();
assert_eq!(
get_lang_scope(&loader, &file_name),
Some("source.dummy".into())
@ -85,15 +85,14 @@ fn detect_language_by_first_line_regex() {
fn tree_sitter_dir(package_json: &str, name: &str) -> tempfile::TempDir {
let temp_dir = tempfile::tempdir().unwrap();
std::fs::write(temp_dir.path().join("package.json"), package_json).unwrap();
std::fs::create_dir(temp_dir.path().join("src")).unwrap();
std::fs::create_dir(temp_dir.path().join("src/tree_sitter")).unwrap();
std::fs::write(
fs::write(temp_dir.path().join("package.json"), package_json).unwrap();
fs::create_dir_all(temp_dir.path().join("src/tree_sitter")).unwrap();
fs::write(
temp_dir.path().join("src/grammar.json"),
format!(r#"{{"name":"{name}"}}"#),
)
.unwrap();
std::fs::write(
fs::write(
temp_dir.path().join("src/parser.c"),
format!(
r##"
@ -108,7 +107,7 @@ fn tree_sitter_dir(package_json: &str, name: &str) -> tempfile::TempDir {
),
)
.unwrap();
std::fs::write(
fs::write(
temp_dir.path().join("src/tree_sitter/parser.h"),
include_str!("../../../lib/src/parser.h"),
)
@ -116,7 +115,7 @@ fn tree_sitter_dir(package_json: &str, name: &str) -> tempfile::TempDir {
temp_dir
}
// if we manage to get the language scope, it means we correctly detected the file-type
// If we manage to get the language scope, it means we correctly detected the file-type
fn get_lang_scope(loader: &Loader, file_name: &Path) -> Option<String> {
loader
.language_configuration_for_file_name(file_name)

2
cli/src/tests/helpers/edits.rs
View file

@ -31,7 +31,7 @@ impl<'a> ReadRecorder<'a> {
pub fn strings_read(&self) -> Vec<&'a str> {
let mut result = Vec::new();
let mut last_range: Option<Range<usize>> = None;
let mut last_range = Option::<Range<usize>>::None;
for index in &self.indices_read {
if let Some(ref mut range) = &mut last_range {
if range.end == *index {

2
cli/src/tests/helpers/random.rs
View file

@ -15,7 +15,7 @@ impl Rand {
}
pub fn unsigned(&mut self, max: usize) -> usize {
self.0.gen_range(0..max + 1)
self.0.gen_range(0..=max)
}
pub fn words(&mut self, max_count: usize) -> Vec<u8> {

3
cli/src/tests/highlight_test.rs
View file

@ -753,8 +753,7 @@ fn to_token_vector<'a>(
for (i, l) in s.split('\n').enumerate() {
let l = l.trim_end_matches('\r');
if i > 0 {
lines.push(line);
line = Vec::new();
lines.push(std::mem::take(&mut line));
}
if !l.is_empty() {
line.push((l, highlights.clone()));

95
cli/src/tests/parser_hang_test.rs
View file

@ -37,7 +37,7 @@ fn test_grammar_that_should_hang_and_not_segfault() {
let tests_exec_path = std::env::args()
.next()
.expect("Failed get get tests executable path");
.expect("Failed to get tests executable path");
match std::env::var(test_var) {
Ok(v) if v == test_name => {
@ -47,60 +47,59 @@ fn test_grammar_that_should_hang_and_not_segfault() {
Err(VarError::NotPresent) => {
eprintln!(" parent process id {}", std::process::id());
if true {
let mut command = Command::new(tests_exec_path);
command.arg(test_name).env(test_var, test_name);
if std::env::args().any(|x| x == "--nocapture") {
command.arg("--nocapture");
} else {
command.stdout(Stdio::null()).stderr(Stdio::null());
}
match command.spawn() {
Ok(mut child) => {
std::thread::sleep(std::time::Duration::from_millis(parent_sleep_millis));
match child.try_wait() {
Ok(Some(status)) if status.success() => {
panic!("Child wasn't hang and exited successfully")
}
Ok(Some(status)) => panic!(
"Child wasn't hang and exited with status code: {:?}",
status.code()
),
_ => (),
}
if let Err(e) = child.kill() {
eprintln!(
"Failed to kill hang test sub process id: {}, error: {e}",
child.id()
);
let mut command = Command::new(tests_exec_path);
command.arg(test_name).env(test_var, test_name);
if std::env::args().any(|x| x == "--nocapture") {
command.arg("--nocapture");
} else {
command.stdout(Stdio::null()).stderr(Stdio::null());
}
match command.spawn() {
Ok(mut child) => {
std::thread::sleep(std::time::Duration::from_millis(parent_sleep_millis));
match child.try_wait() {
Ok(Some(status)) if status.success() => {
panic!("Child didn't hang and exited successfully")
}
Ok(Some(status)) => panic!(
"Child didn't hang and exited with status code: {:?}",
status.code()
),
_ => (),
}
if let Err(e) = child.kill() {
eprintln!(
"Failed to kill hang test's process id: {}, error: {e}",
child.id()
);
}
Err(e) => panic!("{e}"),
}
Err(e) => panic!("{e}"),
}
}
Err(e) => panic!("Env var error: {e}"),
_ => unreachable!(),
}
fn hang_test() {
let test_grammar_dir = fixtures_dir()
.join("test_grammars")
.join("get_col_should_hang_not_crash");
let grammar_json = load_grammar_file(&test_grammar_dir.join("grammar.js"), None).unwrap();
let (parser_name, parser_code) =
generate_parser_for_grammar(grammar_json.as_str()).unwrap();
let language =
get_test_language(&parser_name, &parser_code, Some(test_grammar_dir.as_path()));
let mut parser = Parser::new();
parser.set_language(&language).unwrap();
let code_that_should_hang = "\nHello";
parser.parse(code_that_should_hang, None).unwrap();
}
}
fn hang_test() {
let test_grammar_dir = fixtures_dir()
.join("test_grammars")
.join("get_col_should_hang_not_crash");
let grammar_json = load_grammar_file(&test_grammar_dir.join("grammar.js"), None).unwrap();
let (parser_name, parser_code) = generate_parser_for_grammar(grammar_json.as_str()).unwrap();
let language = get_test_language(&parser_name, &parser_code, Some(test_grammar_dir.as_path()));
let mut parser = Parser::new();
parser.set_language(&language).unwrap();
let code_that_should_hang = "\nHello";
parser.parse(code_that_should_hang, None).unwrap();
}

2
cli/src/tests/parser_test.rs
View file

@ -99,7 +99,7 @@ fn test_parsing_with_debug_graph_enabled() {
parser.print_dot_graphs(&debug_graph_file);
parser.parse("const zero = 0", None).unwrap();
debug_graph_file.seek(std::io::SeekFrom::Start(0)).unwrap();
debug_graph_file.rewind().unwrap();
let log_reader = BufReader::new(debug_graph_file)
.lines()
.map(|l| l.expect("Failed to read line from graph log"));

8
cli/src/tests/proc_macro/src/lib.rs
View file

@ -10,8 +10,8 @@ use syn::{
pub fn retry(args: TokenStream, input: TokenStream) -> TokenStream {
let count = parse_macro_input!(args as LitInt);
let input = parse_macro_input!(input as ItemFn);
let attrs = input.attrs.clone();
let name = input.sig.ident.clone();
let attrs = &input.attrs;
let name = &input.sig.ident;
TokenStream::from(quote! {
#(#attrs),*
@ -98,8 +98,8 @@ pub fn test_with_seed(args: TokenStream, input: TokenStream) -> TokenStream {
let seed_fn = seed_fn.iter();
let func = parse_macro_input!(input as ItemFn);
let attrs = func.attrs.clone();
let name = func.sig.ident.clone();
let attrs = &func.attrs;
let name = &func.sig.ident;
TokenStream::from(quote! {
#[test]

4
lib/binding_rust/ffi.rs
View file

@ -91,7 +91,7 @@ impl<'tree> Node<'tree> {
///
/// `ptr` must be non-null.
#[must_use]
pub const unsafe fn from_raw(raw: TSNode) -> Node<'tree> {
pub const unsafe fn from_raw(raw: TSNode) -> Self {
Self(raw, PhantomData)
}
@ -109,7 +109,7 @@ impl<'a> TreeCursor<'a> {
///
/// `ptr` must be non-null.
#[must_use]
pub const unsafe fn from_raw(raw: TSTreeCursor) -> TreeCursor<'a> {
pub const unsafe fn from_raw(raw: TSTreeCursor) -> Self {
Self(raw, PhantomData)
}

9
lib/binding_rust/lib.rs
View file

@ -1486,7 +1486,7 @@ impl fmt::Display for Node<'_> {
if sexp.is_empty() {
write!(f, "")
} else if !f.alternate() {
write!(f, "{}", sexp)
write!(f, "{sexp}")
} else {
write!(f, "{}", format_sexp(&sexp, f.width().unwrap_or(0)))
}
@ -1781,7 +1781,7 @@ impl Query {
let mut line_start = 0;
let mut row = 0;
let mut line_containing_error = None;
for line in source.split('\n') {
for line in source.lines() {
let line_end = line_start + line.len() + 1;
if line_end > offset {
line_containing_error = Some(line);
@ -2511,7 +2511,7 @@ impl<'tree> QueryMatch<'_, 'tree> {
} else if let Some(ref first_chunk) = self.first_chunk {
first_chunk.as_ref()
} else {
Default::default()
&[]
}
}
}
@ -2823,7 +2823,7 @@ impl<'a> Iterator for LossyUtf8<'a> {
}
match std::str::from_utf8(self.bytes) {
Ok(valid) => {
self.bytes = Default::default();
self.bytes = &[];
Some(valid)
}
Err(error) => {
@ -2901,6 +2901,7 @@ impl fmt::Display for QueryError {
}
#[doc(hidden)]
#[must_use]
pub fn format_sexp(sexp: &str, initial_indent_level: usize) -> String {
let mut indent_level = initial_indent_level;
let mut formatted = String::new();

6
rustfmt.toml Normal file
View file

@ -0,0 +1,6 @@
comment_width = 100
format_code_in_doc_comments = true
format_macro_matchers = true
imports_granularity = "Crate"
group_imports = "StdExternalCrate"
wrap_comments = true

11
tags/src/lib.rs
View file

@ -489,7 +489,6 @@ where
// Compute tag properties that depend on the text of the containing line. If
// the previous tag occurred on the same line, then
// reuse results from the previous tag.
let line_range;
let mut prev_utf16_column = 0;
let mut prev_utf8_byte = name_range.start - span.start.column;
let line_info = self.prev_line_info.as_ref().and_then(|info| {
@ -499,20 +498,20 @@ where
None
}
});
if let Some(line_info) = line_info {
line_range = line_info.line_range.clone();
let line_range = if let Some(line_info) = line_info {
if line_info.utf8_position.column <= span.start.column {
prev_utf8_byte = line_info.utf8_byte;
prev_utf16_column = line_info.utf16_column;
}
line_info.line_range.clone()
} else {
line_range = self::line_range(
self::line_range(
self.source,
name_range.start,
span.start,
MAX_LINE_LEN,
);
}
)
};
let utf16_start_column = prev_utf16_column
+ utf16_len(&self.source[prev_utf8_byte..name_range.start]);