From 5ff5ab3a42df03e54b033fcb1b7d6dc643abe054 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Thu, 29 Aug 2024 21:49:46 -0400 Subject: [PATCH 01/35] fix(generate): remove necessary files from gitignore template (cherry picked from commit 253a112dd49519f229e4be27a915f6bbd03c385f) --- cli/src/generate/templates/gitignore | 2 -- 1 file changed, 2 deletions(-) diff --git a/cli/src/generate/templates/gitignore b/cli/src/generate/templates/gitignore index dd5cc848..2fd9daca 100644 --- a/cli/src/generate/templates/gitignore +++ b/cli/src/generate/templates/gitignore @@ -1,5 +1,4 @@ # Rust artifacts -Cargo.lock target/ # Node artifacts @@ -13,7 +12,6 @@ node_modules/ Package.resolved # Go artifacts -go.sum _obj/ # Python artifacts From 8c8271875ad8b84333d279065a176726640ffcf3 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Thu, 29 Aug 2024 21:50:56 -0400 Subject: [PATCH 02/35] fix(cli): remove conflicting short flags in the `fuzz` subcommand (cherry picked from commit 278526ef75dec32e97cfbf6f18f4d299dd2c279e) --- cli/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index ce4aae95..81add51f 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -269,7 +269,7 @@ struct Fuzz { pub iterations: Option, #[arg(long, short, help = "Regex pattern to filter tests")] pub filter: Option, - #[arg(long, short, help = "Enable logging of graphs and input")] + #[arg(long, help = "Enable logging of graphs and input")] pub log_graphs: bool, #[arg(long, short, help = "Enable parser logging")] pub log: bool, From 366ffc9b3e85e7264b93d1ea87f30e546a37c694 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Thu, 29 Aug 2024 22:01:02 -0400 Subject: [PATCH 03/35] feat(generate): bump `tree-sitter` dev dependency to `0.23` (cherry picked from commit b5a91a4a853b201ada4ed838ac6234012722ef7e) --- cli/src/generate/templates/_cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/src/generate/templates/_cargo.toml b/cli/src/generate/templates/_cargo.toml index 91701b4b..eea4eb4f 100644 --- a/cli/src/generate/templates/_cargo.toml +++ b/cli/src/generate/templates/_cargo.toml @@ -19,8 +19,8 @@ path = "bindings/rust/lib.rs" [dependencies] tree-sitter-language = "0.1" -[dev-dependencies] -tree-sitter = { version = "0.22" } - [build-dependencies] cc = "1.0.87" + +[dev-dependencies] +tree-sitter = "0.23" From a9455a2cc7523a2edcfadadd12d86e3d6b3a0953 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Thu, 29 Aug 2024 22:50:01 -0400 Subject: [PATCH 04/35] feat(bindings): bump `go-tree-sitter` version (cherry picked from commit d0125ef3870e30e57126188f931917a564894666) --- cli/src/generate/templates/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/generate/templates/go.mod b/cli/src/generate/templates/go.mod index d13d1563..26d2dbf1 100644 --- a/cli/src/generate/templates/go.mod +++ b/cli/src/generate/templates/go.mod @@ -2,4 +2,4 @@ module github.com/tree-sitter/tree-sitter-LOWER_PARSER_NAME go 1.23 -require github.com/tree-sitter/go-tree-sitter v0.23 +require github.com/tree-sitter/go-tree-sitter v0.23.1 From 1a3f486059be7602b37d7525b667b9608c113626 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Fri, 30 Aug 2024 22:46:03 -0400 Subject: [PATCH 05/35] fix(lib): correct extra node creation from non-zero root-alias cursors (cherry picked from commit ee06325f67beba7413a4179018a87a417e2e4a49) --- cli/src/tests/tree_test.rs | 27 +++++++++++++++++++++++++++ lib/src/tree_cursor.c | 5 +++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/cli/src/tests/tree_test.rs b/cli/src/tests/tree_test.rs index 3e4b2775..62cc23cd 100644 --- a/cli/src/tests/tree_test.rs +++ b/cli/src/tests/tree_test.rs @@ -702,6 +702,33 @@ fn test_consistency_with_mid_codepoint_edit() { assert_eq!(tree3.root_node().to_sexp(), tree.root_node().to_sexp()); } +#[test] +fn test_tree_cursor_on_aliased_root_with_extra_child() { + let source = r#" +fn main() { + C/* hi */::::E; +} +"#; + + let mut parser = Parser::new(); + parser.set_language(&get_language("rust")).unwrap(); + + let tree = parser.parse(source, None).unwrap(); + + let function = tree.root_node().child(0).unwrap(); + let block = function.child(3).unwrap(); + let expression_statement = block.child(1).unwrap(); + let scoped_identifier = expression_statement.child(0).unwrap(); + let generic_type = scoped_identifier.child(0).unwrap(); + assert_eq!(generic_type.kind(), "generic_type"); + + let mut cursor = generic_type.walk(); + assert!(cursor.goto_first_child()); + assert_eq!(cursor.node().kind(), "type_identifier"); + assert!(cursor.goto_next_sibling()); + assert_eq!(cursor.node().kind(), "block_comment"); +} + fn index_of(text: &[u8], substring: &str) -> usize { str::from_utf8(text).unwrap().find(substring).unwrap() } diff --git a/lib/src/tree_cursor.c b/lib/src/tree_cursor.c index ddd7d66b..24416663 100644 --- a/lib/src/tree_cursor.c +++ b/lib/src/tree_cursor.c @@ -475,8 +475,9 @@ uint32_t ts_tree_cursor_current_descendant_index(const TSTreeCursor *_self) { TSNode ts_tree_cursor_current_node(const TSTreeCursor *_self) { const TreeCursor *self = (const TreeCursor *)_self; TreeCursorEntry *last_entry = array_back(&self->stack); - TSSymbol alias_symbol = self->root_alias_symbol; - if (self->stack.size > 1 && !ts_subtree_extra(*last_entry->subtree)) { + bool is_extra = ts_subtree_extra(*last_entry->subtree); + TSSymbol alias_symbol = is_extra ? 0 : self->root_alias_symbol; + if (self->stack.size > 1 && !is_extra) { TreeCursorEntry *parent_entry = &self->stack.contents[self->stack.size - 2]; alias_symbol = ts_language_alias_at( self->tree->language, From 55fb817dc8c3bfe926ea6b815144923217a06b87 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sat, 31 Aug 2024 22:34:10 -0400 Subject: [PATCH 06/35] fix(rust): add missing TSNode functions (cherry picked from commit 4387e44b9857d3ec7b17d661de3b40cbea4d6b09) --- lib/binding_rust/lib.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/binding_rust/lib.rs b/lib/binding_rust/lib.rs index c97fd5ca..46f319d8 100644 --- a/lib/binding_rust/lib.rs +++ b/lib/binding_rust/lib.rs @@ -1381,6 +1381,20 @@ impl<'tree> Node<'tree> { Self::new(unsafe { ffi::ts_node_prev_named_sibling(self.0) }) } + /// Get the node's first child that extends beyond the given byte offset. + #[doc(alias = "ts_node_first_child_for_byte")] + #[must_use] + pub fn first_child_for_byte(&self, byte: usize) -> Option { + Self::new(unsafe { ffi::ts_node_first_child_for_byte(self.0, byte as u32) }) + } + + /// Get the node's first named child that extends beyond the given byte offset. + #[doc(alias = "ts_node_first_named_child_for_point")] + #[must_use] + pub fn first_named_child_for_byte(&self, byte: usize) -> Option { + Self::new(unsafe { ffi::ts_node_first_named_child_for_byte(self.0, byte as u32) }) + } + /// Get the node's number of descendants, including one for the node itself. #[doc(alias = "ts_node_descendant_count")] #[must_use] From 9a7048bf1454d06e4addf09a73965d94465a199d Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sun, 1 Sep 2024 04:31:59 -0400 Subject: [PATCH 07/35] fix(test): exit with an error if a test marked with `:error` has no error (cherry picked from commit 0a486d508fe14c9f6a7a94c478dbfdb340aae33e) --- cli/src/test.rs | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/cli/src/test.rs b/cli/src/test.rs index 250c8efe..a3bf5e88 100644 --- a/cli/src/test.rs +++ b/cli/src/test.rs @@ -59,6 +59,7 @@ pub enum TestEntry { header_delim_len: usize, divider_delim_len: usize, has_fields: bool, + attributes_str: String, attributes: TestAttributes, }, } @@ -171,10 +172,22 @@ pub fn run_tests_at_path(parser: &mut Parser, opts: &mut TestOptions) -> Result< print_diff_key(); } for (i, (name, actual, expected)) in failures.iter().enumerate() { - println!("\n {}. {name}:", i + 1); - let actual = format_sexp(actual, 2); - let expected = format_sexp(expected, 2); - print_diff(&actual, &expected, opts.color); + if expected == "NO ERROR" { + println!("\n {}. {name}:\n", i + 1); + println!(" Expected an ERROR node, but got:"); + println!( + " {}", + paint( + opts.color.then_some(AnsiColor::Red), + &format_sexp(actual, 2) + ) + ); + } else { + println!("\n {}. {name}:", i + 1); + let actual = format_sexp(actual, 2); + let expected = format_sexp(expected, 2); + print_diff(&actual, &expected, opts.color); + } } if has_parse_errors { @@ -382,6 +395,11 @@ fn run_tests( opts.test_num, paint(opts.color.then_some(AnsiColor::Red), &name) ); + failures.push(( + name.clone(), + tree.root_node().to_sexp(), + "NO ERROR".to_string(), + )); } if attributes.fail_fast { From f312e2c5f58d7bde403510a900f41b849131b3da Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sun, 1 Sep 2024 04:32:10 -0400 Subject: [PATCH 08/35] fix(test): retain attributes when running `test -u` (cherry picked from commit 272ebf77b9e2b24a0ba1e30599794a77e434c159) --- cli/src/test.rs | 106 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 88 insertions(+), 18 deletions(-) diff --git a/cli/src/test.rs b/cli/src/test.rs index a3bf5e88..cd570053 100644 --- a/cli/src/test.rs +++ b/cli/src/test.rs @@ -339,7 +339,7 @@ fn run_tests( opts: &mut TestOptions, mut indent_level: i32, failures: &mut Vec<(String, String, String)>, - corrected_entries: &mut Vec<(String, String, String, usize, usize)>, + corrected_entries: &mut Vec<(String, String, String, String, usize, usize)>, has_parse_errors: &mut bool, ) -> Result { match test_entry { @@ -350,6 +350,7 @@ fn run_tests( header_delim_len, divider_delim_len, has_fields, + attributes_str, attributes, } => { print!("{}", " ".repeat(indent_level as usize)); @@ -389,7 +390,32 @@ fn run_tests( opts.test_num, paint(opts.color.then_some(AnsiColor::Green), &name) ); + if opts.update { + let input = String::from_utf8(input.clone()).unwrap(); + let output = format_sexp(&output, 0); + corrected_entries.push(( + name.clone(), + input, + output, + attributes_str.clone(), + header_delim_len, + divider_delim_len, + )); + } } else { + if opts.update { + let input = String::from_utf8(input.clone()).unwrap(); + // Keep the original `expected` output if the actual output has no error + let output = format_sexp(&output, 0); + corrected_entries.push(( + name.clone(), + input, + output, + attributes_str.clone(), + header_delim_len, + divider_delim_len, + )); + } println!( "{:>3}.  {}", opts.test_num, @@ -424,6 +450,7 @@ fn run_tests( name.clone(), input, output, + attributes_str.clone(), header_delim_len, divider_delim_len, )); @@ -447,6 +474,7 @@ fn run_tests( name.clone(), input, expected_output, + attributes_str.clone(), header_delim_len, divider_delim_len, )); @@ -455,6 +483,7 @@ fn run_tests( name.clone(), input, actual_output, + attributes_str.clone(), header_delim_len, divider_delim_len, )); @@ -584,7 +613,7 @@ fn count_subtests(test_entry: &TestEntry) -> usize { fn write_tests( file_path: &Path, - corrected_entries: &[(String, String, String, usize, usize)], + corrected_entries: &[(String, String, String, String, usize, usize)], ) -> Result<()> { let mut buffer = fs::File::create(file_path)?; write_tests_to_buffer(&mut buffer, corrected_entries) @@ -592,9 +621,9 @@ fn write_tests( fn write_tests_to_buffer( buffer: &mut impl Write, - corrected_entries: &[(String, String, String, usize, usize)], + corrected_entries: &[(String, String, String, String, usize, usize)], ) -> Result<()> { - for (i, (name, input, output, header_delim_len, divider_delim_len)) in + for (i, (name, input, output, attributes_str, header_delim_len, divider_delim_len)) in corrected_entries.iter().enumerate() { if i > 0 { @@ -602,8 +631,13 @@ fn write_tests_to_buffer( } writeln!( buffer, - "{}\n{name}\n{}\n{input}\n{}\n\n{}", + "{}\n{name}\n{}{}\n{input}\n{}\n\n{}", "=".repeat(*header_delim_len), + if attributes_str.is_empty() { + attributes_str.clone() + } else { + format!("{}\n", attributes_str) + }, "=".repeat(*header_delim_len), "-".repeat(*divider_delim_len), output.trim() @@ -661,6 +695,7 @@ fn parse_test_content(name: String, content: &str, file_path: Option) - let mut children = Vec::new(); let bytes = content.as_bytes(); let mut prev_name = String::new(); + let mut prev_attributes_str = String::new(); let mut prev_header_end = 0; // Find the first test header in the file, and determine if it has a @@ -691,17 +726,20 @@ fn parse_test_content(name: String, content: &str, file_path: Option) - .map_or("".as_bytes(), |m| m.as_bytes()); let mut test_name = String::new(); + let mut attributes_str = String::new(); + let mut seen_marker = false; - for line in str::from_utf8(test_name_and_markers) - .unwrap() - .lines() + let test_name_and_markers = str::from_utf8(test_name_and_markers).unwrap(); + for line in test_name_and_markers + .split_inclusive('\n') .filter(|s| !s.is_empty()) { - match line.split('(').next().unwrap() { + let trimmed_line = line.trim(); + match trimmed_line.split('(').next().unwrap() { ":skip" => (seen_marker, skip) = (true, true), ":platform" => { - if let Some(platforms) = line.strip_prefix(':').and_then(|s| { + if let Some(platforms) = trimmed_line.strip_prefix(':').and_then(|s| { s.strip_prefix("platform(") .and_then(|s| s.strip_suffix(')')) }) { @@ -714,7 +752,7 @@ fn parse_test_content(name: String, content: &str, file_path: Option) - ":fail-fast" => (seen_marker, fail_fast) = (true, true), ":error" => (seen_marker, error) = (true, true), ":language" => { - if let Some(lang) = line.strip_prefix(':').and_then(|s| { + if let Some(lang) = trimmed_line.strip_prefix(':').and_then(|s| { s.strip_prefix("language(") .and_then(|s| s.strip_suffix(')')) }) { @@ -724,11 +762,11 @@ fn parse_test_content(name: String, content: &str, file_path: Option) - } _ if !seen_marker => { test_name.push_str(line); - test_name.push('\n'); } _ => {} } } + attributes_str.push_str(test_name_and_markers.strip_prefix(&test_name).unwrap()); // prefer skip over error, both shouldn't be set if skip { @@ -747,10 +785,16 @@ fn parse_test_content(name: String, content: &str, file_path: Option) - } else { Some(test_name.trim_end().to_string()) }; + let attributes_str = if attributes_str.is_empty() { + None + } else { + Some(attributes_str.trim_end().to_string()) + }; Some(( header_delim_len, header_range, test_name, + attributes_str, TestAttributes { skip, platform: platform.unwrap_or(true), @@ -765,12 +809,15 @@ fn parse_test_content(name: String, content: &str, file_path: Option) - }); let (mut prev_header_len, mut prev_attributes) = (80, TestAttributes::default()); - for (header_delim_len, header_range, test_name, attributes) in header_matches.chain(Some(( - 80, - bytes.len()..bytes.len(), - None, - TestAttributes::default(), - ))) { + for (header_delim_len, header_range, test_name, attributes_str, attributes) in header_matches + .chain(Some(( + 80, + bytes.len()..bytes.len(), + None, + None, + TestAttributes::default(), + ))) + { // Find the longest line of dashes following each test description. That line // separates the input from the expected output. Ignore any matches whose suffix // does not match the first suffix in the file. @@ -822,6 +869,7 @@ fn parse_test_content(name: String, content: &str, file_path: Option) - header_delim_len: prev_header_len, divider_delim_len, has_fields, + attributes_str: prev_attributes_str, attributes: prev_attributes, }; @@ -831,6 +879,7 @@ fn parse_test_content(name: String, content: &str, file_path: Option) - } prev_attributes = attributes; prev_name = test_name.unwrap_or_default(); + prev_attributes_str = attributes_str.unwrap_or_default(); prev_header_len = header_delim_len; prev_header_end = header_range.end; } @@ -884,6 +933,7 @@ d header_delim_len: 15, divider_delim_len: 3, has_fields: false, + attributes_str: String::new(), attributes: TestAttributes::default(), }, TestEntry::Example { @@ -893,6 +943,7 @@ d header_delim_len: 16, divider_delim_len: 3, has_fields: false, + attributes_str: String::new(), attributes: TestAttributes::default(), }, ], @@ -943,6 +994,7 @@ abc header_delim_len: 18, divider_delim_len: 7, has_fields: false, + attributes_str: String::new(), attributes: TestAttributes::default(), }, TestEntry::Example { @@ -952,6 +1004,7 @@ abc header_delim_len: 25, divider_delim_len: 19, has_fields: false, + attributes_str: String::new(), attributes: TestAttributes::default(), }, ], @@ -1017,6 +1070,7 @@ abc "title 1".to_string(), "input 1".to_string(), "output 1".to_string(), + String::new(), 80, 80, ), @@ -1024,6 +1078,7 @@ abc "title 2".to_string(), "input 2".to_string(), "output 2".to_string(), + String::new(), 80, 80, ), @@ -1104,6 +1159,7 @@ code header_delim_len: 18, divider_delim_len: 3, has_fields: false, + attributes_str: String::new(), attributes: TestAttributes::default(), }, TestEntry::Example { @@ -1113,6 +1169,7 @@ code header_delim_len: 18, divider_delim_len: 3, has_fields: false, + attributes_str: String::new(), attributes: TestAttributes::default(), }, TestEntry::Example { @@ -1122,6 +1179,7 @@ code header_delim_len: 25, divider_delim_len: 3, has_fields: false, + attributes_str: String::new(), attributes: TestAttributes::default(), } ], @@ -1195,6 +1253,7 @@ NOT A TEST HEADER header_delim_len: 18, divider_delim_len: 3, has_fields: false, + attributes_str: String::new(), attributes: TestAttributes::default(), }, TestEntry::Example { @@ -1204,6 +1263,7 @@ NOT A TEST HEADER header_delim_len: 18, divider_delim_len: 3, has_fields: false, + attributes_str: String::new(), attributes: TestAttributes::default(), }, TestEntry::Example { @@ -1213,6 +1273,7 @@ NOT A TEST HEADER header_delim_len: 25, divider_delim_len: 3, has_fields: false, + attributes_str: String::new(), attributes: TestAttributes::default(), } ], @@ -1258,6 +1319,7 @@ code with ---- header_delim_len: 15, divider_delim_len: 3, has_fields: false, + attributes_str: String::new(), attributes: TestAttributes::default(), }, TestEntry::Example { @@ -1267,6 +1329,7 @@ code with ---- header_delim_len: 20, divider_delim_len: 3, has_fields: false, + attributes_str: String::new(), attributes: TestAttributes::default(), } ] @@ -1304,6 +1367,7 @@ a header_delim_len: 21, divider_delim_len: 3, has_fields: false, + attributes_str: ":skip".to_string(), attributes: TestAttributes { skip: true, platform: true, @@ -1360,6 +1424,7 @@ a header_delim_len: 25, divider_delim_len: 3, has_fields: false, + attributes_str: format!(":platform({})\n:fail-fast", std::env::consts::OS), attributes: TestAttributes { skip: false, platform: true, @@ -1375,6 +1440,11 @@ a header_delim_len: 29, divider_delim_len: 3, has_fields: false, + attributes_str: if std::env::consts::OS == "linux" { + ":platform(macos)\n:language(foo)".to_string() + } else { + ":platform(linux)\n:language(foo)".to_string() + }, attributes: TestAttributes { skip: false, platform: false, From 22b38a083c4f481d83d00d6515c6bf589fbcef23 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sat, 31 Aug 2024 15:21:55 -0400 Subject: [PATCH 09/35] fix(generate): disallow inline variables referencing themselves This fixes an infinite loop bug (cherry picked from commit 53cc93c267f473d82c22dbd6d4250653652d3eba) --- .../prepare_grammar/flatten_grammar.rs | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/cli/src/generate/prepare_grammar/flatten_grammar.rs b/cli/src/generate/prepare_grammar/flatten_grammar.rs index 4b707bee..765ab3b0 100644 --- a/cli/src/generate/prepare_grammar/flatten_grammar.rs +++ b/cli/src/generate/prepare_grammar/flatten_grammar.rs @@ -192,8 +192,10 @@ pub(super) fn flatten_grammar(grammar: ExtractedSyntaxGrammar) -> Result Date: Mon, 2 Sep 2024 21:12:17 -0400 Subject: [PATCH 10/35] feat(language): derive Clone and Copy on LanguageFn Allows a LanguageFn to be passed around and create multiple languages since Language::new consumes a LanguageFn LanguageFn just wraps a function pointer, which already conforms to Copy so this is a simple addition. (cherry picked from commit d60789afdcfcd70b7f78449d011355b3e30b1e3e) --- lib/language/language.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/language/language.rs b/lib/language/language.rs index 4c194da7..1997b1f3 100644 --- a/lib/language/language.rs +++ b/lib/language/language.rs @@ -1,6 +1,7 @@ #![no_std] /// LanguageFn wraps a C function that returns a pointer to a tree-sitter grammer. #[repr(transparent)] +#[derive(Clone, Copy)] pub struct LanguageFn(unsafe extern "C" fn() -> *const ()); impl LanguageFn { From 43e16dd75ca284d333f9b0a94f9dc83b2a5303d8 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sat, 31 Aug 2024 20:37:18 -0400 Subject: [PATCH 11/35] fix(lib): backtrack to the last relevant iterator if no child was found (cherry picked from commit 9b398c2b84f0e62bf6c7d6833848e7808faf9d3c) --- lib/src/node.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/src/node.c b/lib/src/node.c index 1c0eea73..ef04d17e 100644 --- a/lib/src/node.c +++ b/lib/src/node.c @@ -304,22 +304,36 @@ static inline TSNode ts_node__first_child_for_byte( TSNode node = self; bool did_descend = true; + NodeChildIterator last_iterator; + bool has_last_iterator = false; + while (did_descend) { did_descend = false; TSNode child; NodeChildIterator iterator = ts_node_iterate_children(&node); + loop: while (ts_node_child_iterator_next(&iterator, &child)) { if (ts_node_end_byte(child) > goal) { if (ts_node__is_relevant(child, include_anonymous)) { return child; } else if (ts_node_child_count(child) > 0) { + if (iterator.child_index < ts_subtree_child_count(ts_node__subtree(child))) { + last_iterator = iterator; + has_last_iterator = true; + } did_descend = true; node = child; break; } } } + + if (!did_descend && has_last_iterator) { + iterator = last_iterator; + has_last_iterator = false; + goto loop; + } } return ts_node__null(); From dce9bedc48ad83916c83339ece7c87a50c964579 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sat, 7 Sep 2024 17:43:46 -0400 Subject: [PATCH 12/35] fix(generate): add `tree-sitter` to the `dev-dependencies` of the Cargo.toml (cherry picked from commit 4d3d1f0df2a985a983b1196c5c347577c0298283) --- cli/src/generate/grammar_files.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cli/src/generate/grammar_files.rs b/cli/src/generate/grammar_files.rs index f45cf7f7..42690c36 100644 --- a/cli/src/generate/grammar_files.rs +++ b/cli/src/generate/grammar_files.rs @@ -334,10 +334,12 @@ pub fn generate_grammar_files( .ok_or_else(|| anyhow!("Failed to find the end of the `tree-sitter` version in Cargo.toml"))?; let cargo_toml = format!( - "{}{}{}", + "{}{}{}\n{}\n{}", &cargo_toml[..start_index], "tree-sitter-language = \"0.1.0\"", &cargo_toml[version_end_index + 1..], + "[dev-dependencies]", + "tree-sitter = \"0.23\"", ); write_file(path, cargo_toml)?; From 513f19d0998f93e3fb49da7f693d3c5714148f9f Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sat, 7 Sep 2024 17:52:48 -0400 Subject: [PATCH 13/35] fix(binding_web): correct `edit` signature (cherry picked from commit fcbd67b3fab8499bc1baaa051a2741d2c0ff6ce0) --- lib/binding_web/tree-sitter-web.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/binding_web/tree-sitter-web.d.ts b/lib/binding_web/tree-sitter-web.d.ts index 97a48077..ad6b76c5 100644 --- a/lib/binding_web/tree-sitter-web.d.ts +++ b/lib/binding_web/tree-sitter-web.d.ts @@ -150,7 +150,7 @@ declare module 'web-tree-sitter' { rootNodeWithOffset(offsetBytes: number, offsetExtent: Point): SyntaxNode; copy(): Tree; delete(): void; - edit(edit: Edit): Tree; + edit(edit: Edit): void; walk(): TreeCursor; getChangedRanges(other: Tree): Range[]; getIncludedRanges(): Range[]; From 5f51550a8ccf7f7e9ac4a5fcbac2f2668769075a Mon Sep 17 00:00:00 2001 From: Jinser Kafka Date: Fri, 6 Sep 2024 10:51:29 +0800 Subject: [PATCH 14/35] fix(cli): keep skipped tests unchanged in the test/corpus (cherry picked from commit fd190f1d9d10ecc8f8ed259c95e97a8e8fca3343) --- cli/src/test.rs | 110 +++++++++++++++++++++++++++--------------------- 1 file changed, 61 insertions(+), 49 deletions(-) diff --git a/cli/src/test.rs b/cli/src/test.rs index cd570053..f1f82a43 100644 --- a/cli/src/test.rs +++ b/cli/src/test.rs @@ -1,5 +1,5 @@ use std::{ - collections::{BTreeMap, HashSet}, + collections::BTreeMap, ffi::OsStr, fs, io::{self, Write}, @@ -517,64 +517,76 @@ fn run_tests( } TestEntry::Group { name, - mut children, + children, file_path, } => { - // track which tests are being skipped to maintain consistent numbering while using - // filters - let mut skipped_tests = HashSet::new(); - let mut advance_counter = opts.test_num; - children.retain(|child| match child { - TestEntry::Example { name, .. } => { - if let Some(filter) = opts.filter { - if !name.contains(filter) { - skipped_tests.insert(advance_counter); - advance_counter += 1; - return false; - } - } - if let Some(include) = &opts.include { - if !include.is_match(name) { - skipped_tests.insert(advance_counter); - advance_counter += 1; - return false; - } - } - if let Some(exclude) = &opts.exclude { - if exclude.is_match(name) { - skipped_tests.insert(advance_counter); - advance_counter += 1; - return false; - } - } - advance_counter += 1; - true - } - TestEntry::Group { .. } => { - advance_counter += count_subtests(child); - true - } - }); - if children.is_empty() { - opts.test_num = advance_counter; return Ok(true); } - if indent_level > 0 { - print!("{}", " ".repeat(indent_level as usize)); - println!("{name}:"); - } - - let failure_count = failures.len(); - indent_level += 1; + let mut advance_counter = opts.test_num; + let failure_count = failures.len(); + let mut has_printed = false; + let mut skipped_tests = 0; + + let matches_filter = |name: &str, opts: &TestOptions| { + if let Some(filter) = opts.filter { + name.contains(filter) + } else if let Some(include) = &opts.include { + include.is_match(name) + } else if let Some(exclude) = &opts.exclude { + !exclude.is_match(name) + } else { + true + } + }; + + let mut should_skip = |entry: &TestEntry, opts: &TestOptions| match entry { + TestEntry::Example { name, .. } => { + advance_counter += 1; + !matches_filter(name, opts) + } + TestEntry::Group { .. } => { + advance_counter += count_subtests(entry); + false + } + }; + for child in children { - if let TestEntry::Example { .. } = child { - while skipped_tests.remove(&opts.test_num) { + if let TestEntry::Example { + ref name, + ref input, + ref output, + ref attributes_str, + header_delim_len, + divider_delim_len, + .. + } = child + { + if should_skip(&child, opts) { + let input = String::from_utf8(input.clone()).unwrap(); + let output = format_sexp(output, 0); + corrected_entries.push(( + name.clone(), + input, + output, + attributes_str.clone(), + header_delim_len, + divider_delim_len, + )); + opts.test_num += 1; + skipped_tests += 1; + + continue; } } + if !has_printed && indent_level > 1 { + has_printed = true; + print!("{}", " ".repeat((indent_level - 1) as usize)); + println!("{name}:"); + } if !run_tests( parser, child, @@ -589,7 +601,7 @@ fn run_tests( } } - opts.test_num += skipped_tests.len(); + opts.test_num += skipped_tests; if let Some(file_path) = file_path { if opts.update && failures.len() - failure_count > 0 { From 831c144d9b118935fc56e03876963cd2a97e8f92 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sat, 7 Sep 2024 21:15:18 -0400 Subject: [PATCH 15/35] fix(generate): do not generate large character sets for unused variables (cherry picked from commit d8ab779df4a8aa1e2a6fac3fc2551345752e033c) --- cli/src/generate/build_tables/build_lex_table.rs | 4 ++++ cli/src/generate/prepare_grammar/flatten_grammar.rs | 2 +- cli/src/generate/prepare_grammar/mod.rs | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cli/src/generate/build_tables/build_lex_table.rs b/cli/src/generate/build_tables/build_lex_table.rs index f7bff0d9..215e45b1 100644 --- a/cli/src/generate/build_tables/build_lex_table.rs +++ b/cli/src/generate/build_tables/build_lex_table.rs @@ -10,6 +10,7 @@ use crate::generate::{ dedup::split_state_id_groups, grammars::{LexicalGrammar, SyntaxGrammar}, nfa::{CharacterSet, NfaCursor}, + prepare_grammar::symbol_is_used, rules::{Symbol, TokenSet}, tables::{AdvanceAction, LexState, LexTable, ParseStateId, ParseTable}, }; @@ -93,6 +94,9 @@ pub fn build_lex_table( let mut large_character_sets = Vec::new(); for (variable_ix, _variable) in lexical_grammar.variables.iter().enumerate() { let symbol = Symbol::terminal(variable_ix); + if !symbol_is_used(&syntax_grammar.variables, symbol) { + continue; + } builder.reset(); builder.add_state_for_tokens(&TokenSet::from_iter([symbol])); for state in &builder.table.states { diff --git a/cli/src/generate/prepare_grammar/flatten_grammar.rs b/cli/src/generate/prepare_grammar/flatten_grammar.rs index 765ab3b0..04cfe265 100644 --- a/cli/src/generate/prepare_grammar/flatten_grammar.rs +++ b/cli/src/generate/prepare_grammar/flatten_grammar.rs @@ -173,7 +173,7 @@ fn flatten_variable(variable: Variable) -> SyntaxVariable { } } -fn symbol_is_used(variables: &[SyntaxVariable], symbol: Symbol) -> bool { +pub fn symbol_is_used(variables: &[SyntaxVariable], symbol: Symbol) -> bool { for variable in variables { for production in &variable.productions { for step in &production.steps { diff --git a/cli/src/generate/prepare_grammar/mod.rs b/cli/src/generate/prepare_grammar/mod.rs index 7e4800c1..81288b82 100644 --- a/cli/src/generate/prepare_grammar/mod.rs +++ b/cli/src/generate/prepare_grammar/mod.rs @@ -13,6 +13,7 @@ use std::{ }; use anyhow::{anyhow, Result}; +pub(super) use flatten_grammar::symbol_is_used; pub use self::expand_tokens::expand_tokens; use self::{ From 0eb5b0a029d53d7fb0ed1ad9f2b8497a72f1c3f0 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sun, 8 Sep 2024 16:54:12 -0400 Subject: [PATCH 16/35] fix(binding_web): remove nonexistent function definition (cherry picked from commit 8667e3ea0c10a7d6803240c2c611cbfaa4ca8605) --- lib/binding_web/tree-sitter-web.d.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/binding_web/tree-sitter-web.d.ts b/lib/binding_web/tree-sitter-web.d.ts index ad6b76c5..fa780336 100644 --- a/lib/binding_web/tree-sitter-web.d.ts +++ b/lib/binding_web/tree-sitter-web.d.ts @@ -154,7 +154,6 @@ declare module 'web-tree-sitter' { walk(): TreeCursor; getChangedRanges(other: Tree): Range[]; getIncludedRanges(): Range[]; - getEditedRange(other: Tree): Range; getLanguage(): Language; } From d5a1266b25700109b1c5ceb6f54d975115a69513 Mon Sep 17 00:00:00 2001 From: Will Lillis Date: Sun, 15 Sep 2024 04:04:12 -0400 Subject: [PATCH 17/35] fix(cli): remove duplicate short options from `fuzz` command (#3635) - Remove short option from fuzz command edits option - Remove short option from fuzz command iterations option (cherry picked from commit b0e8e50a1936b977ab720d4a4d38e1c343e879a3) --- cli/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 81add51f..0f0d911c 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -263,9 +263,9 @@ struct Fuzz { pub skip: Option>, #[arg(long, help = "Subdirectory to the language")] pub subdir: Option, - #[arg(long, short, help = "Maximum number of edits to perform per fuzz test")] + #[arg(long, help = "Maximum number of edits to perform per fuzz test")] pub edits: Option, - #[arg(long, short, help = "Number of fuzzing iterations to run per test")] + #[arg(long, help = "Number of fuzzing iterations to run per test")] pub iterations: Option, #[arg(long, short, help = "Regex pattern to filter tests")] pub filter: Option, From 28b71cea27c3f3fffe52cde7f431a9a00d0e6bba Mon Sep 17 00:00:00 2001 From: Dave Abrahams Date: Mon, 16 Sep 2024 18:40:21 -0700 Subject: [PATCH 18/35] fix(generate): remove excludes in `Package.swift` (cherry picked from commit 112acd5b930d0d2febc46b8073614734dce41e89) --- Package.swift | 21 --------------------- cli/src/generate/templates/package.swift | 23 ----------------------- 2 files changed, 44 deletions(-) diff --git a/Package.swift b/Package.swift index 6135da0c..3a4b9744 100644 --- a/Package.swift +++ b/Package.swift @@ -14,27 +14,6 @@ let package = Package( targets: [ .target(name: "TreeSitter", path: "lib", - exclude: [ - "binding_rust", - "binding_web", - "node_modules", - "Cargo.toml", - "README.md", - "src/unicode/README.md", - "src/unicode/LICENSE", - "src/unicode/ICU_SHA", - "src/get_changed_ranges.c", - "src/tree_cursor.c", - "src/stack.c", - "src/node.c", - "src/lexer.c", - "src/parser.c", - "src/language.c", - "src/alloc.c", - "src/subtree.c", - "src/tree.c", - "src/query.c" - ], sources: ["src/lib.c"], cSettings: [.headerSearchPath("src")]), ], diff --git a/cli/src/generate/templates/package.swift b/cli/src/generate/templates/package.swift index c1be93db..d1053b46 100644 --- a/cli/src/generate/templates/package.swift +++ b/cli/src/generate/templates/package.swift @@ -14,29 +14,6 @@ let package = Package( name: "TreeSitterCAMEL_PARSER_NAME", dependencies: [], path: ".", - exclude: [ - "Cargo.toml", - "Makefile", - "binding.gyp", - "bindings/c", - "bindings/go", - "bindings/node", - "bindings/python", - "bindings/rust", - "prebuilds", - "grammar.js", - "package.json", - "package-lock.json", - "pyproject.toml", - "setup.py", - "test", - "examples", - ".editorconfig", - ".github", - ".gitignore", - ".gitattributes", - ".gitmodules", - ], sources: [ "src/parser.c", // NOTE: if your language has an external scanner, add it here. From 7688c6fa2fe8f162031074724d621d7f223abcfe Mon Sep 17 00:00:00 2001 From: ObserverOfTime Date: Wed, 11 Sep 2024 17:32:18 +0300 Subject: [PATCH 19/35] chore(bindings): update rust lib docs (cherry picked from commit 6e19fccf39421492fa61c31cc9ad4a39bf1029c7) --- cli/src/generate/templates/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/src/generate/templates/lib.rs b/cli/src/generate/templates/lib.rs index adb8e481..4e3522ae 100644 --- a/cli/src/generate/templates/lib.rs +++ b/cli/src/generate/templates/lib.rs @@ -1,6 +1,6 @@ //! This crate provides CAMEL_PARSER_NAME language support for the [tree-sitter][] parsing library. //! -//! Typically, you will use the [language][language func] function to add this language to a +//! Typically, you will use the [LANGUAGE][] constant to add this language to a //! tree-sitter [Parser][], and then use the parser to parse some code: //! //! ``` @@ -15,8 +15,6 @@ //! assert!(!tree.root_node().has_error()); //! ``` //! -//! [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html -//! [language func]: fn.language.html //! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html //! [tree-sitter]: https://tree-sitter.github.io/ @@ -26,7 +24,9 @@ extern "C" { fn tree_sitter_PARSER_NAME() -> *const (); } -/// The tree-sitter [`LanguageFn`] for this grammar. +/// The tree-sitter [`LanguageFn`][LanguageFn] for this grammar. +/// +/// [LanguageFn]: https://docs.rs/tree-sitter-language/*/tree_sitter_language/struct.LanguageFn.html pub const LANGUAGE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_PARSER_NAME) }; /// The content of the [`node-types.json`][] file for this grammar. From b4d42514276fb3faf99802cd4d9ae34132ce4e1f Mon Sep 17 00:00:00 2001 From: Hanlu Date: Tue, 17 Sep 2024 09:39:49 +0800 Subject: [PATCH 20/35] fix: correct comment quote (cherry picked from commit ff813a311ba6f5d797a956f310528b29f27f299d) --- cli/src/query_testing.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/query_testing.rs b/cli/src/query_testing.rs index cdf2e988..3456dbe6 100644 --- a/cli/src/query_testing.rs +++ b/cli/src/query_testing.rs @@ -62,7 +62,7 @@ pub fn parse_position_comments( if let Ok(text) = node.utf8_text(source) { let mut position = node.start_position(); if position.row > 0 { - // Find the arrow character ("^" or '<-") in the comment. A left arrow + // Find the arrow character ("^" or "<-") in the comment. A left arrow // refers to the column where the comment node starts. An up arrow refers // to its own column. let mut has_left_caret = false; From 6813e8d5c1fcf151151299f40ae66ac8191f230f Mon Sep 17 00:00:00 2001 From: Firas al-Khalil Date: Tue, 17 Sep 2024 10:01:54 +0200 Subject: [PATCH 21/35] build(make): support darwin cross-compile (cherry picked from commit 4f0d463d49a3f1f2a6eaa75ecc284a5719aad94e) --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 465f940d..d5a7a1eb 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ SONAME_MAJOR := $(word 1,$(subst ., ,$(VERSION))) SONAME_MINOR := $(word 2,$(subst ., ,$(VERSION))) # OS-specific bits -ifeq ($(shell uname),Darwin) +ifneq ($(findstring darwin,$(shell $(CC) -dumpmachine)),) SOEXT = dylib SOEXTVER_MAJOR = $(SONAME_MAJOR).$(SOEXT) SOEXTVER = $(SONAME_MAJOR).$(SONAME_MINOR).$(SOEXT) From a6b248c1ad086b98f704da212dc89469068f49c6 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 26 Aug 2024 23:26:45 -0400 Subject: [PATCH 22/35] fix(lib): peek at the next sibling when iterating to find the child that contains a given descendant This issue shows up when we have a zero-width token that is the target descendant node, previously the previous sibling would be returned as the child that contains the descendant, which is incorrect. (cherry picked from commit 0a85744ebad81a04b61d11e4efd949d148d562f6) --- cli/src/tests/node_test.rs | 10 ++++++++++ lib/src/node.c | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/cli/src/tests/node_test.rs b/cli/src/tests/node_test.rs index e05ed932..c153df99 100644 --- a/cli/src/tests/node_test.rs +++ b/cli/src/tests/node_test.rs @@ -290,6 +290,16 @@ fn test_parent_of_zero_width_node() { function_definition ); assert_eq!(function_definition.child_containing_descendant(block), None); + + let code = ""; + parser.set_language(&get_language("html")).unwrap(); + + let tree = parser.parse(code, None).unwrap(); + let root = tree.root_node(); + let script_element = root.child(0).unwrap(); + let raw_text = script_element.child(1).unwrap(); + let parent = raw_text.parent().unwrap(); + assert_eq!(parent, script_element); } #[test] diff --git a/lib/src/node.c b/lib/src/node.c index ef04d17e..237df6ad 100644 --- a/lib/src/node.c +++ b/lib/src/node.c @@ -12,6 +12,8 @@ typedef struct { const TSSymbol *alias_sequence; } NodeChildIterator; +static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous); + // TSNode - constructors TSNode ts_node_new( @@ -101,6 +103,21 @@ static inline bool ts_node_child_iterator_next( return true; } +// This will return true if the next sibling is a zero-width token that is adjacent to the current node and is relevant +static inline bool ts_node_child_iterator_next_sibling_is_empty_adjacent(NodeChildIterator *self, TSNode previous) { + if (!self->parent.ptr || ts_node_child_iterator_done(self)) return false; + if (self->child_index == 0) return false; + const Subtree *child = &ts_subtree_children(self->parent)[self->child_index]; + TSSymbol alias = 0; + if (!ts_subtree_extra(*child)) { + if (self->alias_sequence) { + alias = self->alias_sequence[self->structural_child_index]; + } + } + TSNode next = ts_node_new(self->tree, child, self->position, alias); + return ts_node_end_byte(previous) == ts_node_end_byte(next) && ts_node__is_relevant(next, true); +} + // TSNode - private static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous) { @@ -544,6 +561,24 @@ TSNode ts_node_child_containing_descendant(TSNode self, TSNode subnode) { ) { return ts_node__null(); } + + // Here we check the current self node and *all* of its zero-width token siblings that follow. + // If any of these nodes contain the target subnode, we return that node. Otherwise, we restore the node we started at + // for the loop condition, and that will continue with the next *non-zero-width* sibling. + TSNode old = self; + // While the next sibling is a zero-width token + while (ts_node_child_iterator_next_sibling_is_empty_adjacent(&iter, self)) { + TSNode current_node = ts_node_child_containing_descendant(self, subnode); + // If the target child is in self, return it + if (!ts_node_is_null(current_node)) { + return current_node; + } + ts_node_child_iterator_next(&iter, &self); + if (self.id == subnode.id) { + return ts_node__null(); + } + } + self = old; } while (iter.position.bytes < end_byte || ts_node_child_count(self) == 0); } while (!ts_node__is_relevant(self, true)); From 1e9df802fb7936cd191a27ad3fff6b554168345c Mon Sep 17 00:00:00 2001 From: PanGan21 Date: Sun, 1 Sep 2024 14:16:20 +0300 Subject: [PATCH 23/35] fix(docs): fix highlight readme example using compatible versions (cherry picked from commit 1a6af3fafe991990fd385eae4c79f41fc4744f76) --- highlight/README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/highlight/README.md b/highlight/README.md index 982e510a..4ca76d6c 100644 --- a/highlight/README.md +++ b/highlight/README.md @@ -12,8 +12,8 @@ to parse, to your `Cargo.toml`: ```toml [dependencies] -tree-sitter-highlight = "^0.21.0" -tree-sitter-javascript = "0.20.3" +tree-sitter-highlight = "0.22.0" +tree-sitter-javascript = "0.21.3" ``` Define the list of highlight names that you will recognize: @@ -61,9 +61,8 @@ let mut javascript_config = HighlightConfiguration::new( javascript_language, "javascript", tree_sitter_javascript::HIGHLIGHT_QUERY, - tree_sitter_javascript::INJECTION_QUERY, + tree_sitter_javascript::INJECTIONS_QUERY, tree_sitter_javascript::LOCALS_QUERY, - false, ).unwrap(); ``` From 3492bee2f747feb3c917b24c21123fbd707d0bde Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 21:37:01 +0000 Subject: [PATCH 24/35] build(deps): bump the cargo group across 1 directory with 11 updates Bumps the cargo group with 10 updates in the / directory: | Package | From | To | | --- | --- | --- | | [anyhow](https://github.com/dtolnay/anyhow) | `1.0.86` | `1.0.89` | | [cc](https://github.com/rust-lang/cc-rs) | `1.1.14` | `1.1.19` | | [clap](https://github.com/clap-rs/clap) | `4.5.16` | `4.5.17` | | [filetime](https://github.com/alexcrichton/filetime) | `0.2.24` | `0.2.25` | | [indexmap](https://github.com/indexmap-rs/indexmap) | `2.4.0` | `2.5.0` | | [pretty_assertions](https://github.com/rust-pretty-assertions/rust-pretty-assertions) | `1.4.0` | `1.4.1` | | [serde](https://github.com/serde-rs/serde) | `1.0.209` | `1.0.210` | | [serde_json](https://github.com/serde-rs/json) | `1.0.127` | `1.0.128` | | [webbrowser](https://github.com/amodm/webbrowser-rs) | `1.0.1` | `1.0.2` | | [bindgen](https://github.com/rust-lang/rust-bindgen) | `0.69.4` | `0.70.1` | Updates `anyhow` from 1.0.86 to 1.0.89 - [Release notes](https://github.com/dtolnay/anyhow/releases) - [Commits](https://github.com/dtolnay/anyhow/compare/1.0.86...1.0.89) Updates `cc` from 1.1.14 to 1.1.19 - [Release notes](https://github.com/rust-lang/cc-rs/releases) - [Changelog](https://github.com/rust-lang/cc-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/rust-lang/cc-rs/compare/cc-v1.1.14...cc-v1.1.19) Updates `clap` from 4.5.16 to 4.5.17 - [Release notes](https://github.com/clap-rs/clap/releases) - [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md) - [Commits](https://github.com/clap-rs/clap/compare/clap_complete-v4.5.16...clap_complete-v4.5.17) Updates `filetime` from 0.2.24 to 0.2.25 - [Commits](https://github.com/alexcrichton/filetime/compare/0.2.24...0.2.25) Updates `indexmap` from 2.4.0 to 2.5.0 - [Changelog](https://github.com/indexmap-rs/indexmap/blob/master/RELEASES.md) - [Commits](https://github.com/indexmap-rs/indexmap/compare/2.4.0...2.5.0) Updates `pretty_assertions` from 1.4.0 to 1.4.1 - [Release notes](https://github.com/rust-pretty-assertions/rust-pretty-assertions/releases) - [Changelog](https://github.com/rust-pretty-assertions/rust-pretty-assertions/blob/main/CHANGELOG.md) - [Commits](https://github.com/rust-pretty-assertions/rust-pretty-assertions/compare/v1.4.0...v1.4.1) Updates `serde` from 1.0.209 to 1.0.210 - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.209...v1.0.210) Updates `serde_derive` from 1.0.209 to 1.0.210 - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.209...v1.0.210) Updates `serde_json` from 1.0.127 to 1.0.128 - [Release notes](https://github.com/serde-rs/json/releases) - [Commits](https://github.com/serde-rs/json/compare/1.0.127...1.0.128) Updates `webbrowser` from 1.0.1 to 1.0.2 - [Release notes](https://github.com/amodm/webbrowser-rs/releases) - [Changelog](https://github.com/amodm/webbrowser-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/amodm/webbrowser-rs/compare/v1.0.1...v1.0.2) Updates `bindgen` from 0.69.4 to 0.70.1 - [Release notes](https://github.com/rust-lang/rust-bindgen/releases) - [Changelog](https://github.com/rust-lang/rust-bindgen/blob/main/CHANGELOG.md) - [Commits](https://github.com/rust-lang/rust-bindgen/compare/v0.69.4...v0.70.1) --- updated-dependencies: - dependency-name: anyhow dependency-type: direct:production update-type: version-update:semver-patch dependency-group: cargo - dependency-name: cc dependency-type: direct:production update-type: version-update:semver-patch dependency-group: cargo - dependency-name: clap dependency-type: direct:production update-type: version-update:semver-patch dependency-group: cargo - dependency-name: filetime dependency-type: direct:production update-type: version-update:semver-patch dependency-group: cargo - dependency-name: indexmap dependency-type: direct:production update-type: version-update:semver-minor dependency-group: cargo - dependency-name: pretty_assertions dependency-type: direct:production update-type: version-update:semver-patch dependency-group: cargo - dependency-name: serde dependency-type: direct:production update-type: version-update:semver-patch dependency-group: cargo - dependency-name: serde_derive dependency-type: direct:production update-type: version-update:semver-patch dependency-group: cargo - dependency-name: serde_json dependency-type: direct:production update-type: version-update:semver-patch dependency-group: cargo - dependency-name: webbrowser dependency-type: direct:production update-type: version-update:semver-patch dependency-group: cargo - dependency-name: bindgen dependency-type: direct:production update-type: version-update:semver-minor dependency-group: cargo ... Signed-off-by: dependabot[bot] (cherry picked from commit c6faeb948e8bd35054ab84c99bcc7c036233ce6c) --- Cargo.lock | 77 ++++++++++++++++++-------------------------------- Cargo.toml | 18 ++++++------ lib/Cargo.toml | 2 +- 3 files changed, 38 insertions(+), 59 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 96982282..08f33326 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -74,9 +74,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.86" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" [[package]] name = "arbitrary" @@ -92,16 +92,14 @@ checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" [[package]] name = "bindgen" -version = "0.69.4" +version = "0.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f" dependencies = [ "bitflags", "cexpr", "clang-sys", "itertools", - "lazy_static", - "lazycell", "log", "prettyplease", "proc-macro2", @@ -110,7 +108,6 @@ dependencies = [ "rustc-hash", "shlex", "syn", - "which", ] [[package]] @@ -148,9 +145,9 @@ checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" [[package]] name = "cc" -version = "1.1.14" +version = "1.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d2eb3cd3d1bf4529e31c215ee6f93ec5a3d536d9f578f93d9d33ee19562932" +checksum = "2d74707dde2ba56f86ae90effb3b43ddd369504387e718014de010cec7959800" dependencies = [ "jobserver", "libc", @@ -203,9 +200,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.16" +version = "4.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed6719fffa43d0d87e5fd8caeab59be1554fb028cd30edc88fc4369b17971019" +checksum = "3e5a21b8495e732f1b3c364c9949b201ca7bae518c502c80256c96ad79eaf6ac" dependencies = [ "clap_builder", "clap_derive", @@ -213,9 +210,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.15" +version = "4.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216aec2b177652e3846684cbfe25c9964d18ec45234f0f5da5157b207ed1aab6" +checksum = "8cf2dd12af7a047ad9d6da2b6b249759a22a7abc0f474c1dae1777afa4b21a73" dependencies = [ "anstream", "anstyle", @@ -265,9 +262,9 @@ dependencies = [ [[package]] name = "core-foundation" -version = "0.9.4" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63" dependencies = [ "core-foundation-sys", "libc", @@ -505,9 +502,9 @@ checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" [[package]] name = "filetime" -version = "0.2.24" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf401df4a4e3872c4fe8151134cf483738e74b67fc934d6532c882b3d24a4550" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" dependencies = [ "cfg-if", "libc", @@ -650,9 +647,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93ead53efc7ea8ed3cfb0c79fc8023fbb782a5432b52830b6518941cebe6505c" +checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" dependencies = [ "equivalent", "hashbrown 0.14.5", @@ -732,12 +729,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "leb128" version = "0.2.5" @@ -1010,9 +1001,9 @@ dependencies = [ [[package]] name = "pretty_assertions" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" dependencies = [ "diff", "yansi", @@ -1189,18 +1180,18 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.209" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.209" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", @@ -1209,9 +1200,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.127" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad" +checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ "indexmap", "itoa", @@ -1933,9 +1924,9 @@ dependencies = [ [[package]] name = "webbrowser" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "425ba64c1e13b1c6e8c5d2541c8fac10022ca584f33da781db01b5756aef1f4e" +checksum = "2e5f07fb9bc8de2ddfe6b24a71a75430673fd679e568c48b52716cef1cfae923" dependencies = [ "block2", "core-foundation", @@ -1949,18 +1940,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix", -] - [[package]] name = "winapi-util" version = "0.1.9" @@ -2225,9 +2204,9 @@ dependencies = [ [[package]] name = "yansi" -version = "0.5.1" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "zerocopy" diff --git a/Cargo.toml b/Cargo.toml index 6f1f984a..d0ff77a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,9 +40,9 @@ strip = false [workspace.dependencies] anstyle = "1.0.8" -anyhow = "1.0.86" -cc = "1.1.13" -clap = { version = "4.5.16", features = [ +anyhow = "1.0.89" +cc = "1.1.19" +clap = { version = "4.5.17", features = [ "cargo", "derive", "env", @@ -52,28 +52,28 @@ clap = { version = "4.5.16", features = [ ctor = "0.2.8" ctrlc = { version = "3.4.5", features = ["termination"] } dirs = "5.0.1" -filetime = "0.2.24" +filetime = "0.2.25" fs4 = "0.8.4" git2 = "0.18.3" glob = "0.3.1" heck = "0.5.0" html-escape = "0.2.13" -indexmap = "2.4.0" +indexmap = "2.5.0" indoc = "2.0.5" lazy_static = "1.5.0" libloading = "0.8.5" log = { version = "0.4.22", features = ["std"] } memchr = "2.7.4" once_cell = "1.19.0" -pretty_assertions = "1.4.0" +pretty_assertions = "1.4.1" rand = "0.8.5" regex = "1.10.6" regex-syntax = "0.8.4" rustc-hash = "1.1.0" semver = "1.0.23" -serde = { version = "1.0.208", features = ["derive"] } +serde = { version = "1.0.210", features = ["derive"] } serde_derive = "1.0.197" -serde_json = { version = "1.0.125", features = ["preserve_order"] } +serde_json = { version = "1.0.128", features = ["preserve_order"] } similar = "2.6.0" smallbitvec = "2.5.3" tempfile = "3.12.0" @@ -83,7 +83,7 @@ toml = "0.8.19" unindent = "0.2.3" walkdir = "2.5.0" wasmparser = "0.215.0" -webbrowser = "1.0.1" +webbrowser = "1.0.2" tree-sitter = { version = "0.23.0", path = "./lib" } tree-sitter-loader = { version = "0.23.0", path = "./cli/loader" } diff --git a/lib/Cargo.toml b/lib/Cargo.toml index c5dc919d..2884d411 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -43,7 +43,7 @@ default-features = false features = ["cranelift"] [build-dependencies] -bindgen = { version = "0.69.4", optional = true } +bindgen = { version = "0.70.1", optional = true } cc.workspace = true [lib] From 3ad82e67728609286a83a786451fb45e73286624 Mon Sep 17 00:00:00 2001 From: Joel Spadin Date: Sat, 21 Sep 2024 14:34:30 -0500 Subject: [PATCH 25/35] fix(wasm): use / paths for workdir Reimplemented the fix from #2183 to fix building WASM files with Docker on Windows again. The --workdir argument gives a path inside the Docker container, so it must use forward slashes regardless of the default path separator on the host OS. (cherry picked from commit 755e49e21221bd3add0dee57c3fc9b20427ddadb) --- Cargo.lock | 7 +++++++ Cargo.toml | 1 + cli/loader/Cargo.toml | 1 + cli/loader/src/lib.rs | 3 ++- 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 08f33326..8d6182c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -960,6 +960,12 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "path-slash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e91099d4268b0e11973f036e885d652fb0b21fedcf69738c627f94db6a44f42" + [[package]] name = "percent-encoding" version = "2.3.1" @@ -1514,6 +1520,7 @@ dependencies = [ "indoc", "libloading", "once_cell", + "path-slash", "regex", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index d0ff77a0..1c95c66a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,6 +65,7 @@ libloading = "0.8.5" log = { version = "0.4.22", features = ["std"] } memchr = "2.7.4" once_cell = "1.19.0" +path-slash = "0.2.1" pretty_assertions = "1.4.1" rand = "0.8.5" regex = "1.10.6" diff --git a/cli/loader/Cargo.toml b/cli/loader/Cargo.toml index 5179b36d..0c4df0af 100644 --- a/cli/loader/Cargo.toml +++ b/cli/loader/Cargo.toml @@ -26,6 +26,7 @@ fs4.workspace = true indoc.workspace = true libloading.workspace = true once_cell.workspace = true +path-slash.workspace = true regex.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/cli/loader/src/lib.rs b/cli/loader/src/lib.rs index 610d0602..9ecd716c 100644 --- a/cli/loader/src/lib.rs +++ b/cli/loader/src/lib.rs @@ -23,6 +23,7 @@ use fs4::FileExt; use indoc::indoc; use libloading::{Library, Symbol}; use once_cell::unsync::OnceCell; +use path_slash::PathBufExt as _; use regex::{Regex, RegexBuilder}; use serde::{Deserialize, Deserializer, Serialize}; use tree_sitter::Language; @@ -823,7 +824,7 @@ impl Loader { path.push(src_path.strip_prefix(root_path).unwrap()); path }; - command.args(["--workdir", &workdir.to_string_lossy()]); + command.args(["--workdir", &workdir.to_slash_lossy()]); // Mount the root directory as a volume, which is the repo root let mut volume_string = OsString::from(&root_path); From 8e1dbb4617cffbb79e05c446c4bda7eb7922d6cc Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sun, 8 Sep 2024 14:26:34 -0400 Subject: [PATCH 26/35] fix: properly handle utf8 code points for highlight and tag assertions (cherry picked from commit 6f050f0da51db6a97a8e59210a35d8d9feb541e6) --- Cargo.lock | 57 +++++++++++++++++++--------- Cargo.toml | 1 + cli/Cargo.toml | 1 + cli/src/query.rs | 10 ++--- cli/src/query_testing.rs | 51 ++++++++++++++++++++++--- cli/src/test_highlight.rs | 11 ++++-- cli/src/test_tags.rs | 11 ++++-- cli/src/tests/test_highlight_test.rs | 26 ++++++++----- cli/src/tests/test_tags_test.rs | 16 ++++---- 9 files changed, 131 insertions(+), 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8d6182c3..b1d35340 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -99,7 +99,7 @@ dependencies = [ "bitflags", "cexpr", "clang-sys", - "itertools", + "itertools 0.13.0", "log", "prettyplease", "proc-macro2", @@ -125,6 +125,17 @@ dependencies = [ "objc2", ] +[[package]] +name = "bstr" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -145,9 +156,9 @@ checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" [[package]] name = "cc" -version = "1.1.19" +version = "1.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d74707dde2ba56f86ae90effb3b43ddd369504387e718014de010cec7959800" +checksum = "45bcde016d64c21da4be18b655631e5ab6d3107607e71a73a9f53eb48aae23fb" dependencies = [ "jobserver", "libc", @@ -391,7 +402,7 @@ dependencies = [ "cranelift-codegen", "cranelift-entity", "cranelift-frontend", - "itertools", + "itertools 0.12.1", "log", "smallvec", "wasmparser", @@ -677,6 +688,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -914,9 +934,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.3" +version = "0.36.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b64972346851a39438c60b341ebc01bba47464ae329e55cf343eb93964efd9" +checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" dependencies = [ "crc32fast", "hashbrown 0.14.5", @@ -1036,9 +1056,9 @@ dependencies = [ [[package]] name = "psm" -version = "0.1.21" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874" +checksum = "aa37f80ca58604976033fae9515a8a2989fc13797d953f7c04fb8fa36a11f205" dependencies = [ "cc", ] @@ -1084,9 +1104,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" +checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853" dependencies = [ "bitflags", ] @@ -1152,9 +1172,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" dependencies = [ "bitflags", "errno", @@ -1279,9 +1299,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" -version = "2.0.76" +version = "2.0.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578e081a14e0cefc3279b0472138c513f37b41a08d5a3cca9b6e4e8ceb6cd525" +checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" dependencies = [ "proc-macro2", "quote", @@ -1386,9 +1406,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.20" +version = "0.22.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" +checksum = "3b072cee73c449a636ffd6f32bd8de3a9f7119139aff882f44943ce2986dc5cf" dependencies = [ "indexmap", "serde", @@ -1446,6 +1466,7 @@ version = "0.23.0" dependencies = [ "anstyle", "anyhow", + "bstr", "clap", "ctor", "ctrlc", @@ -1558,9 +1579,9 @@ checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-normalization" diff --git a/Cargo.toml b/Cargo.toml index 1c95c66a..7594e80b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,7 @@ strip = false [workspace.dependencies] anstyle = "1.0.8" anyhow = "1.0.89" +bstr = "1.10.0" cc = "1.1.19" clap = { version = "4.5.17", features = [ "cargo", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 493503ce..34fa31db 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -27,6 +27,7 @@ wasm = ["tree-sitter/wasm", "tree-sitter-loader/wasm"] [dependencies] anstyle.workspace = true anyhow.workspace = true +bstr.workspace = true clap.workspace = true ctor.workspace = true ctrlc.workspace = true diff --git a/cli/src/query.rs b/cli/src/query.rs index bffa0588..f32c5450 100644 --- a/cli/src/query.rs +++ b/cli/src/query.rs @@ -9,7 +9,7 @@ use std::{ use anyhow::{Context, Result}; use tree_sitter::{Language, Parser, Point, Query, QueryCursor}; -use crate::query_testing; +use crate::query_testing::{self, to_utf8_point}; #[allow(clippy::too_many_arguments)] pub fn query_files_at_paths( @@ -70,8 +70,8 @@ pub fn query_files_at_paths( } results.push(query_testing::CaptureInfo { name: (*capture_name).to_string(), - start: capture.node.start_position(), - end: capture.node.end_position(), + start: to_utf8_point(capture.node.start_position(), source_code.as_slice()), + end: to_utf8_point(capture.node.end_position(), source_code.as_slice()), }); } } else { @@ -100,8 +100,8 @@ pub fn query_files_at_paths( } results.push(query_testing::CaptureInfo { name: (*capture_name).to_string(), - start: capture.node.start_position(), - end: capture.node.end_position(), + start: to_utf8_point(capture.node.start_position(), source_code.as_slice()), + end: to_utf8_point(capture.node.end_position(), source_code.as_slice()), }); } } diff --git a/cli/src/query_testing.rs b/cli/src/query_testing.rs index 3456dbe6..258770d1 100644 --- a/cli/src/query_testing.rs +++ b/cli/src/query_testing.rs @@ -1,6 +1,7 @@ use std::fs; use anyhow::{anyhow, Result}; +use bstr::{BStr, ByteSlice}; use lazy_static::lazy_static; use regex::Regex; use tree_sitter::{Language, Parser, Point}; @@ -9,16 +10,56 @@ lazy_static! { static ref CAPTURE_NAME_REGEX: Regex = Regex::new("[\\w_\\-.]+").unwrap(); } +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct Utf8Point { + pub row: usize, + pub column: usize, +} + +impl std::fmt::Display for Utf8Point { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "({}, {})", self.row, self.column) + } +} + +impl Utf8Point { + pub const fn new(row: usize, column: usize) -> Self { + Self { row, column } + } +} + +pub fn to_utf8_point(point: Point, source: &[u8]) -> Utf8Point { + if point.column == 0 { + return Utf8Point::new(point.row, 0); + } + + let bstr = BStr::new(source); + let line = bstr.lines_with_terminator().nth(point.row).unwrap(); + let mut utf8_column = 0; + + for (_, grapheme_end, _) in line.grapheme_indices() { + utf8_column += 1; + if grapheme_end >= point.column { + break; + } + } + + Utf8Point { + row: point.row, + column: utf8_column, + } +} + #[derive(Debug, Eq, PartialEq)] pub struct CaptureInfo { pub name: String, - pub start: Point, - pub end: Point, + pub start: Utf8Point, + pub end: Utf8Point, } #[derive(Debug, PartialEq, Eq)] pub struct Assertion { - pub position: Point, + pub position: Utf8Point, pub negative: bool, pub expected_capture_name: String, } @@ -27,7 +68,7 @@ impl Assertion { #[must_use] pub fn new(row: usize, col: usize, negative: bool, expected_capture_name: String) -> Self { Self { - position: Point::new(row, col), + position: Utf8Point::new(row, col), negative, expected_capture_name, } @@ -103,7 +144,7 @@ pub fn parse_position_comments( { assertion_ranges.push((node.start_position(), node.end_position())); result.push(Assertion { - position, + position: to_utf8_point(position, source), negative, expected_capture_name: mat.as_str().to_string(), }); diff --git a/cli/src/test_highlight.rs b/cli/src/test_highlight.rs index 541d98fd..34be438f 100644 --- a/cli/src/test_highlight.rs +++ b/cli/src/test_highlight.rs @@ -7,7 +7,7 @@ use tree_sitter_highlight::{Highlight, HighlightConfiguration, HighlightEvent, H use tree_sitter_loader::{Config, Loader}; use super::{ - query_testing::{parse_position_comments, Assertion}, + query_testing::{parse_position_comments, to_utf8_point, Assertion, Utf8Point}, test::paint, util, }; @@ -141,7 +141,7 @@ fn test_highlights_indented( } pub fn iterate_assertions( assertions: &[Assertion], - highlights: &[(Point, Point, Highlight)], + highlights: &[(Utf8Point, Utf8Point, Highlight)], highlight_names: &[String], ) -> Result { // Iterate through all of the highlighting assertions, checking each one against the @@ -224,7 +224,7 @@ pub fn get_highlight_positions( highlighter: &mut Highlighter, highlight_config: &HighlightConfiguration, source: &[u8], -) -> Result> { +) -> Result> { let mut row = 0; let mut column = 0; let mut byte_offset = 0; @@ -261,7 +261,10 @@ pub fn get_highlight_positions( } } if let Some(highlight) = highlight_stack.last() { - result.push((start_position, Point::new(row, column), *highlight)); + let utf8_start_position = to_utf8_point(start_position, source.as_bytes()); + let utf8_end_position = + to_utf8_point(Point::new(row, column), source.as_bytes()); + result.push((utf8_start_position, utf8_end_position, *highlight)); } } } diff --git a/cli/src/test_tags.rs b/cli/src/test_tags.rs index c5a1dc02..5b290bda 100644 --- a/cli/src/test_tags.rs +++ b/cli/src/test_tags.rs @@ -2,12 +2,11 @@ use std::{fs, path::Path}; use anstyle::AnsiColor; use anyhow::{anyhow, Result}; -use tree_sitter::Point; use tree_sitter_loader::{Config, Loader}; use tree_sitter_tags::{TagsConfiguration, TagsContext}; use super::{ - query_testing::{parse_position_comments, Assertion}, + query_testing::{parse_position_comments, to_utf8_point, Assertion, Utf8Point}, test::paint, util, }; @@ -168,7 +167,7 @@ pub fn get_tag_positions( tags_context: &mut TagsContext, tags_config: &TagsConfiguration, source: &[u8], -) -> Result> { +) -> Result> { let (tags_iter, _has_error) = tags_context.generate_tags(tags_config, source, None)?; let tag_positions = tags_iter .filter_map(std::result::Result::ok) @@ -179,7 +178,11 @@ pub fn get_tag_positions( } else { format!("reference.{tag_postfix}") }; - (tag.span.start, tag.span.end, tag_name) + ( + to_utf8_point(tag.span.start, source), + to_utf8_point(tag.span.end, source), + tag_name, + ) }) .collect(); Ok(tag_positions) diff --git a/cli/src/tests/test_highlight_test.rs b/cli/src/tests/test_highlight_test.rs index 8699c2a6..054e33f8 100644 --- a/cli/src/tests/test_highlight_test.rs +++ b/cli/src/tests/test_highlight_test.rs @@ -1,9 +1,9 @@ -use tree_sitter::{Parser, Point}; +use tree_sitter::Parser; use tree_sitter_highlight::{Highlight, Highlighter}; use super::helpers::fixtures::{get_highlight_config, get_language, test_loader}; use crate::{ - query_testing::{parse_position_comments, Assertion}, + query_testing::{parse_position_comments, Assertion, Utf8Point}, test_highlight::get_highlight_positions, }; @@ -28,6 +28,9 @@ fn test_highlight_test_with_basic_test() { " // ^ variable", " // ^ !variable", "};", + "var y̆y̆y̆y̆ = function() {}", + " // ^ function", + " // ^ keyword", ] .join("\n"); @@ -40,6 +43,8 @@ fn test_highlight_test_with_basic_test() { Assertion::new(1, 11, false, String::from("keyword")), Assertion::new(4, 9, false, String::from("variable")), Assertion::new(4, 11, true, String::from("variable")), + Assertion::new(8, 5, false, String::from("function")), + Assertion::new(8, 11, false, String::from("keyword")), ] ); @@ -50,13 +55,16 @@ fn test_highlight_test_with_basic_test() { assert_eq!( highlight_positions, &[ - (Point::new(1, 0), Point::new(1, 3), Highlight(2)), // "var" - (Point::new(1, 4), Point::new(1, 7), Highlight(0)), // "abc" - (Point::new(1, 10), Point::new(1, 18), Highlight(2)), // "function" - (Point::new(1, 19), Point::new(1, 20), Highlight(1)), // "d" - (Point::new(4, 2), Point::new(4, 8), Highlight(2)), // "return" - (Point::new(4, 9), Point::new(4, 10), Highlight(1)), // "d" - (Point::new(4, 13), Point::new(4, 14), Highlight(1)), // "e" + (Utf8Point::new(1, 0), Utf8Point::new(1, 3), Highlight(2)), // "var" + (Utf8Point::new(1, 4), Utf8Point::new(1, 7), Highlight(0)), // "abc" + (Utf8Point::new(1, 10), Utf8Point::new(1, 18), Highlight(2)), // "function" + (Utf8Point::new(1, 19), Utf8Point::new(1, 20), Highlight(1)), // "d" + (Utf8Point::new(4, 2), Utf8Point::new(4, 8), Highlight(2)), // "return" + (Utf8Point::new(4, 9), Utf8Point::new(4, 10), Highlight(1)), // "d" + (Utf8Point::new(4, 13), Utf8Point::new(4, 14), Highlight(1)), // "e" + (Utf8Point::new(8, 0), Utf8Point::new(8, 3), Highlight(2)), // "var" + (Utf8Point::new(8, 4), Utf8Point::new(8, 8), Highlight(0)), // "y̆y̆y̆y̆" + (Utf8Point::new(8, 11), Utf8Point::new(8, 19), Highlight(2)), // "function" ] ); } diff --git a/cli/src/tests/test_tags_test.rs b/cli/src/tests/test_tags_test.rs index 5e7bf9c9..5f7b88fc 100644 --- a/cli/src/tests/test_tags_test.rs +++ b/cli/src/tests/test_tags_test.rs @@ -1,9 +1,9 @@ -use tree_sitter::{Parser, Point}; +use tree_sitter::Parser; use tree_sitter_tags::TagsContext; use super::helpers::fixtures::{get_language, get_tags_config}; use crate::{ - query_testing::{parse_position_comments, Assertion}, + query_testing::{parse_position_comments, Assertion, Utf8Point}, test_tags::get_tag_positions, }; @@ -43,18 +43,18 @@ fn test_tags_test_with_basic_test() { tag_positions, &[ ( - Point::new(1, 4), - Point::new(1, 7), + Utf8Point::new(1, 4), + Utf8Point::new(1, 7), "definition.function".to_string() ), ( - Point::new(3, 8), - Point::new(3, 11), + Utf8Point::new(3, 8), + Utf8Point::new(3, 11), "reference.call".to_string() ), ( - Point::new(5, 11), - Point::new(5, 12), + Utf8Point::new(5, 11), + Utf8Point::new(5, 12), "reference.call".to_string() ), ] From 83509ad4d7b65f060597c059e91e381ff99a36bf Mon Sep 17 00:00:00 2001 From: Will Lillis Date: Sun, 22 Sep 2024 03:46:52 -0400 Subject: [PATCH 27/35] fix(fuzz): skip tests marked with `:skip` & don't report errors on tests marked with `:error` (cherry picked from commit 99dbbbcbe9d7f3c286057e9f3bcc6e0b42690606) --- cli/src/fuzz/mod.rs | 52 +++++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/cli/src/fuzz/mod.rs b/cli/src/fuzz/mod.rs index 14f50c72..e7926df0 100644 --- a/cli/src/fuzz/mod.rs +++ b/cli/src/fuzz/mod.rs @@ -108,11 +108,16 @@ pub fn fuzz_language_corpus( } let tests = flatten_tests(main_tests, options.filter.as_ref()); - let mut skipped = options.skipped.as_ref().map(|x| { - x.iter() - .map(|x| (x.as_str(), 0)) - .collect::>() - }); + let get_test_name = |test: &FlattenedTest| format!("{language_name} - {}", test.name); + + let mut skipped = options + .skipped + .take() + .unwrap_or_default() + .into_iter() + .chain(tests.iter().filter(|x| x.skip).map(get_test_name)) + .map(|x| (x, 0)) + .collect::>(); let mut failure_count = 0; @@ -125,13 +130,11 @@ pub fn fuzz_language_corpus( println!(); for (test_index, test) in tests.iter().enumerate() { - let test_name = format!("{language_name} - {}", test.name); - if let Some(skipped) = skipped.as_mut() { - if let Some(counter) = skipped.get_mut(test_name.as_str()) { - println!(" {test_index}. {test_name} - SKIPPED"); - *counter += 1; - continue; - } + let test_name = get_test_name(test); + if let Some(counter) = skipped.get_mut(test_name.as_str()) { + println!(" {test_index}. {test_name} - SKIPPED"); + *counter += 1; + continue; } println!(" {test_index}. {test_name}"); @@ -143,6 +146,11 @@ pub fn fuzz_language_corpus( set_included_ranges(&mut parser, &test.input, test.template_delimiters); let tree = parser.parse(&test.input, None).unwrap(); + + if test.error { + return true; + } + let mut actual_output = tree.root_node().to_sexp(); if !test.has_fields { actual_output = strip_sexp_fields(&actual_output); @@ -240,7 +248,7 @@ pub fn fuzz_language_corpus( actual_output = strip_sexp_fields(&actual_output); } - if actual_output != test.output { + if actual_output != test.output && !test.error { println!("Incorrect parse for {test_name} - seed {seed}"); print_diff_key(); print_diff(&actual_output, &test.output, true); @@ -272,16 +280,14 @@ pub fn fuzz_language_corpus( eprintln!("{failure_count} {language_name} corpus tests failed fuzzing"); } - if let Some(skipped) = skipped.as_mut() { - skipped.retain(|_, v| *v == 0); + skipped.retain(|_, v| *v == 0); - if !skipped.is_empty() { - println!("Non matchable skip definitions:"); - for k in skipped.keys() { - println!(" {k}"); - } - panic!("Non matchable skip definitions needs to be removed"); + if !skipped.is_empty() { + println!("Non matchable skip definitions:"); + for k in skipped.keys() { + println!(" {k}"); } + panic!("Non matchable skip definitions needs to be removed"); } } @@ -290,6 +296,8 @@ pub struct FlattenedTest { pub input: Vec, pub output: String, pub languages: Vec>, + pub error: bool, + pub skip: bool, pub has_fields: bool, pub template_delimiters: Option<(&'static str, &'static str)>, } @@ -327,6 +335,8 @@ pub fn flatten_tests(test: TestEntry, filter: Option<&Regex>) -> Vec Date: Tue, 24 Sep 2024 13:55:16 -0400 Subject: [PATCH 28/35] fix: do not generate spurious files if the grammar path is not the default path (cherry picked from commit 1708a295a8c0a2a0911277959a1079a7e9aed363) --- cli/src/generate/mod.rs | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/cli/src/generate/mod.rs b/cli/src/generate/mod.rs index 0ce63d1b..3214f46c 100644 --- a/cli/src/generate/mod.rs +++ b/cli/src/generate/mod.rs @@ -68,7 +68,11 @@ pub fn generate_parser_in_directory( } } - if repo_path.is_dir() && !repo_path.join("grammar.js").exists() && !path_in_ignore(&repo_path) { + let grammar_path = grammar_path + .map(PathBuf::from) + .unwrap_or(repo_path.join("grammar.js")); + + if repo_path.is_dir() && !grammar_path.exists() && !path_in_ignore(&repo_path) { if let Some(dir_name) = repo_path .file_name() .map(|x| x.to_string_lossy().to_ascii_lowercase()) @@ -82,14 +86,8 @@ pub fn generate_parser_in_directory( } } - // Read the grammar.json. - let grammar_json = if let Some(path) = grammar_path { - load_grammar_file(path.as_ref(), js_runtime)? - } else { - let grammar_js_path = - grammar_path.map_or(repo_path.join("grammar.js"), std::convert::Into::into); - load_grammar_file(&grammar_js_path, js_runtime)? - }; + // Read the grammar file. + let grammar_json = load_grammar_file(&grammar_path, js_runtime)?; let src_path = repo_path.join("src"); let header_path = src_path.join("tree_sitter"); @@ -98,7 +96,7 @@ pub fn generate_parser_in_directory( fs::create_dir_all(&src_path)?; fs::create_dir_all(&header_path)?; - if grammar_path.is_none() { + if grammar_path.file_name().unwrap() != "grammar.json" { fs::write(src_path.join("grammar.json"), &grammar_json) .with_context(|| format!("Failed to write grammar.json to {src_path:?}"))?; } @@ -118,7 +116,7 @@ pub fn generate_parser_in_directory( write_file(&header_path.join("array.h"), tree_sitter::ARRAY_HEADER)?; write_file(&header_path.join("parser.h"), tree_sitter::PARSER_HEADER)?; - if !path_in_ignore(&repo_path) { + if !path_in_ignore(&repo_path) && grammar_path == repo_path.join("grammar.js") { grammar_files::generate_grammar_files(&repo_path, &input_grammar.name, generate_bindings)?; } From 7561e813b8024c86c973068c846836afd358af3c Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Tue, 24 Sep 2024 15:09:08 -0400 Subject: [PATCH 29/35] fix: disallow empty string literals in rules (cherry picked from commit 86d3a5313d7d1be1e642801ef30533aa5244da01) --- .../prepare_grammar/extract_tokens.rs | 75 +++++++++++++------ 1 file changed, 52 insertions(+), 23 deletions(-) diff --git a/cli/src/generate/prepare_grammar/extract_tokens.rs b/cli/src/generate/prepare_grammar/extract_tokens.rs index 34d99dec..27c4edbe 100644 --- a/cli/src/generate/prepare_grammar/extract_tokens.rs +++ b/cli/src/generate/prepare_grammar/extract_tokens.rs @@ -14,16 +14,17 @@ pub(super) fn extract_tokens( let mut extractor = TokenExtractor { current_variable_name: String::new(), current_variable_token_count: 0, + is_first_rule: false, extracted_variables: Vec::new(), extracted_usage_counts: Vec::new(), }; - for variable in &mut grammar.variables { - extractor.extract_tokens_in_variable(variable); + for (i, variable) in &mut grammar.variables.iter_mut().enumerate() { + extractor.extract_tokens_in_variable(i == 0, variable)?; } for variable in &mut grammar.external_tokens { - extractor.extract_tokens_in_variable(variable); + extractor.extract_tokens_in_variable(false, variable)?; } let mut lexical_variables = Vec::with_capacity(extractor.extracted_variables.len()); @@ -172,6 +173,7 @@ pub(super) fn extract_tokens( struct TokenExtractor { current_variable_name: String, current_variable_token_count: usize, + is_first_rule: bool, extracted_variables: Vec, extracted_usage_counts: Vec, } @@ -181,19 +183,25 @@ struct SymbolReplacer { } impl TokenExtractor { - fn extract_tokens_in_variable(&mut self, variable: &mut Variable) { + fn extract_tokens_in_variable( + &mut self, + is_first: bool, + variable: &mut Variable, + ) -> Result<()> { self.current_variable_name.clear(); self.current_variable_name.push_str(&variable.name); self.current_variable_token_count = 0; + self.is_first_rule = is_first; let mut rule = Rule::Blank; mem::swap(&mut rule, &mut variable.rule); - variable.rule = self.extract_tokens_in_rule(&rule); + variable.rule = self.extract_tokens_in_rule(&rule)?; + Ok(()) } - fn extract_tokens_in_rule(&mut self, input: &Rule) -> Rule { + fn extract_tokens_in_rule(&mut self, input: &Rule) -> Result { match input { - Rule::String(name) => self.extract_token(input, Some(name)).into(), - Rule::Pattern(..) => self.extract_token(input, None).into(), + Rule::String(name) => Ok(self.extract_token(input, Some(name))?.into()), + Rule::Pattern(..) => Ok(self.extract_token(input, None)?.into()), Rule::Metadata { params, rule } => { if params.is_token { let mut params = params.clone(); @@ -210,41 +218,53 @@ impl TokenExtractor { input }; - self.extract_token(rule_to_extract, string_value).into() + Ok(self.extract_token(rule_to_extract, string_value)?.into()) } else { - Rule::Metadata { + Ok(Rule::Metadata { params: params.clone(), - rule: Box::new(self.extract_tokens_in_rule(rule)), - } + rule: Box::new(self.extract_tokens_in_rule(rule)?), + }) } } - Rule::Repeat(content) => Rule::Repeat(Box::new(self.extract_tokens_in_rule(content))), - Rule::Seq(elements) => Rule::Seq( + Rule::Repeat(content) => Ok(Rule::Repeat(Box::new( + self.extract_tokens_in_rule(content)?, + ))), + Rule::Seq(elements) => Ok(Rule::Seq( elements .iter() .map(|e| self.extract_tokens_in_rule(e)) - .collect(), - ), - Rule::Choice(elements) => Rule::Choice( + .collect::>>()?, + )), + Rule::Choice(elements) => Ok(Rule::Choice( elements .iter() .map(|e| self.extract_tokens_in_rule(e)) - .collect(), - ), - _ => input.clone(), + .collect::>>()?, + )), + _ => Ok(input.clone()), } } - fn extract_token(&mut self, rule: &Rule, string_value: Option<&String>) -> Symbol { + fn extract_token(&mut self, rule: &Rule, string_value: Option<&String>) -> Result { for (i, variable) in self.extracted_variables.iter_mut().enumerate() { if variable.rule == *rule { self.extracted_usage_counts[i] += 1; - return Symbol::terminal(i); + return Ok(Symbol::terminal(i)); } } let index = self.extracted_variables.len(); let variable = if let Some(string_value) = string_value { + if string_value.is_empty() && !self.is_first_rule { + return Err(anyhow!( + "The rule `{}` contains an empty string. + +Tree-sitter does not support syntactic rules that contain an empty string +unless they are used only as the grammar's start rule. +", + self.current_variable_name + )); + } Variable { name: string_value.clone(), kind: VariableType::Anonymous, @@ -264,7 +284,7 @@ impl TokenExtractor { self.extracted_variables.push(variable); self.extracted_usage_counts.push(1); - Symbol::terminal(index) + Ok(Symbol::terminal(index)) } } @@ -520,6 +540,15 @@ mod test { ); } + #[test] + fn test_extraction_with_empty_string() { + assert!(extract_tokens(build_grammar(vec![ + Variable::named("rule_0", Rule::non_terminal(1)), + Variable::hidden("_rule_1", Rule::string("")), + ])) + .is_err()); + } + fn build_grammar(variables: Vec) -> InternedGrammar { InternedGrammar { variables, From 865f6595e74639a26f21974ea291b32867487c02 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Thu, 26 Sep 2024 23:15:26 -0400 Subject: [PATCH 30/35] fix(lib): correct descendant-for-range behavior with zero-width tokens (cherry picked from commit 0c43988a5e17eb5feecac103f88cdae49dc5973d) --- cli/src/tests/node_test.rs | 20 ++++++++++++++++++++ lib/src/node.c | 18 ++++++++++++++---- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/cli/src/tests/node_test.rs b/cli/src/tests/node_test.rs index c153df99..559f6f59 100644 --- a/cli/src/tests/node_test.rs +++ b/cli/src/tests/node_test.rs @@ -613,6 +613,26 @@ fn test_node_descendant_for_range() { assert_eq!(pair_node.end_byte(), string_index + 9); assert_eq!(pair_node.start_position(), Point::new(6, 4)); assert_eq!(pair_node.end_position(), Point::new(6, 13)); + + // Zero-width token + { + let code = ""; + let mut parser = Parser::new(); + parser.set_language(&get_language("html")).unwrap(); + + let tree = parser.parse(code, None).unwrap(); + let root = tree.root_node(); + + let child = root + .named_descendant_for_point_range(Point::new(0, 8), Point::new(0, 8)) + .unwrap(); + assert_eq!(child.kind(), "raw_text"); + + let child2 = root.named_descendant_for_byte_range(8, 8).unwrap(); + assert_eq!(child2.kind(), "raw_text"); + + assert_eq!(child, child2); + } } #[test] diff --git a/lib/src/node.c b/lib/src/node.c index 237df6ad..2768efbb 100644 --- a/lib/src/node.c +++ b/lib/src/node.c @@ -375,9 +375,13 @@ static inline TSNode ts_node__descendant_for_byte_range( uint32_t node_end = iterator.position.bytes; // The end of this node must extend far enough forward to touch - // the end of the range and exceed the start of the range. + // the end of the range if (node_end < range_end) continue; - if (node_end <= range_start) continue; + + // ...and exceed the start of the range, unless the node itself is + // empty, in which case it must at least be equal to the start of the range. + bool is_empty = ts_node_start_byte(child) == node_end; + if (is_empty ? node_end < range_start : node_end <= range_start) continue; // The start of this node must extend far enough backward to // touch the start of the range. @@ -414,9 +418,15 @@ static inline TSNode ts_node__descendant_for_point_range( TSPoint node_end = iterator.position.extent; // The end of this node must extend far enough forward to touch - // the end of the range and exceed the start of the range. + // the end of the range if (point_lt(node_end, range_end)) continue; - if (point_lte(node_end, range_start)) continue; + + // ...and exceed the start of the range, unless the node itself is + // empty, in which case it must at least be equal to the start of the range. + bool is_empty = point_eq(ts_node_start_point(child), node_end); + if (is_empty ? point_lt(node_end, range_start) : point_lte(node_end, range_start)) { + continue; + } // The start of this node must extend far enough backward to // touch the start of the range. From 77794c558f130337f1ca1514088233000bb1461b Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Sun, 29 Sep 2024 19:33:49 -0400 Subject: [PATCH 31/35] fix: correct test name parsing when the prior test has equal signs (cherry picked from commit 2fffe036e09086da12687fc89e994d32eca17de5) --- cli/src/test.rs | 51 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/cli/src/test.rs b/cli/src/test.rs index f1f82a43..3ce12a48 100644 --- a/cli/src/test.rs +++ b/cli/src/test.rs @@ -27,7 +27,7 @@ lazy_static! { (?P(?:=+){3,}) (?P[^=\r\n][^\r\n]*)? \r?\n - (?P(?:[^=][^\r\n]*\r?\n)+) + (?P(?:([^=\r\n]|\s+:)[^\r\n]*\r?\n)+) ===+ (?P[^=\r\n][^\r\n]*)?\r?\n" ) @@ -1243,7 +1243,27 @@ NOT A TEST HEADER ---asdf\()[]|{}*+?^$.- (a) - " + +==============================asdf\()[]|{}*+?^$.- +Test containing equals +==============================asdf\()[]|{}*+?^$.- + +=== + +------------------------------asdf\()[]|{}*+?^$.- + +(a) + +==============================asdf\()[]|{}*+?^$.- +Subsequent test containing equals +==============================asdf\()[]|{}*+?^$.- + +=== + +------------------------------asdf\()[]|{}*+?^$.- + +(a) +" .trim(), None, ); @@ -1253,7 +1273,7 @@ NOT A TEST HEADER =========================\n\ -------------------------\n" .to_vec(); - assert_eq!( + pretty_assertions::assert_eq!( entry, TestEntry::Group { name: "the-filename".to_string(), @@ -1287,6 +1307,26 @@ NOT A TEST HEADER has_fields: false, attributes_str: String::new(), attributes: TestAttributes::default(), + }, + TestEntry::Example { + name: "Test containing equals".to_string(), + input: "\n===\n".into(), + output: "(a)".into(), + header_delim_len: 30, + divider_delim_len: 30, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), + }, + TestEntry::Example { + name: "Subsequent test containing equals".to_string(), + input: "\n===\n".into(), + output: "(a)".into(), + header_delim_len: 30, + divider_delim_len: 30, + has_fields: false, + attributes_str: String::new(), + attributes: TestAttributes::default(), } ], file_path: None, @@ -1407,6 +1447,7 @@ a ============================= Test with bad platform marker :platform({}) + :language(foo) ============================= a @@ -1453,9 +1494,9 @@ a divider_delim_len: 3, has_fields: false, attributes_str: if std::env::consts::OS == "linux" { - ":platform(macos)\n:language(foo)".to_string() + ":platform(macos)\n\n:language(foo)".to_string() } else { - ":platform(linux)\n:language(foo)".to_string() + ":platform(linux)\n\n:language(foo)".to_string() }, attributes: TestAttributes { skip: false, From d10308528d2838b2053f9dfa7bf8de872f4eaaef Mon Sep 17 00:00:00 2001 From: Ron Panduwana Date: Wed, 3 Apr 2024 13:10:03 +0700 Subject: [PATCH 32/35] fix: handle more cases of editing subtrees that depend on column values (cherry picked from commit a83b89301639f6e5333d3a1918ffa23a2b0c773c) --- cli/src/tests/parser_test.rs | 61 +++++++++++++++++++ lib/src/subtree.c | 11 +++- .../depends_on_column/corpus.txt | 21 +++++++ .../depends_on_column/grammar.js | 7 +++ .../test_grammars/depends_on_column/scanner.c | 40 ++++++++++++ 5 files changed, 137 insertions(+), 3 deletions(-) create mode 100644 test/fixtures/test_grammars/depends_on_column/corpus.txt create mode 100644 test/fixtures/test_grammars/depends_on_column/grammar.js create mode 100644 test/fixtures/test_grammars/depends_on_column/scanner.c diff --git a/cli/src/tests/parser_test.rs b/cli/src/tests/parser_test.rs index bd19717b..7b534be1 100644 --- a/cli/src/tests/parser_test.rs +++ b/cli/src/tests/parser_test.rs @@ -501,6 +501,67 @@ h + i ); } +#[test] +fn test_parsing_after_editing_tree_that_depends_on_column_position() { + let dir = fixtures_dir() + .join("test_grammars") + .join("depends_on_column"); + + let grammar_json = load_grammar_file(&dir.join("grammar.js"), None).unwrap(); + let (grammar_name, parser_code) = generate_parser_for_grammar(grammar_json.as_str()).unwrap(); + + let mut parser = Parser::new(); + parser + .set_language(&get_test_language(&grammar_name, &parser_code, Some(&dir))) + .unwrap(); + + let mut code = b"\n x".to_vec(); + let mut tree = parser.parse(&code, None).unwrap(); + assert_eq!(tree.root_node().to_sexp(), "(x_is_at (odd_column))"); + + perform_edit( + &mut tree, + &mut code, + &Edit { + position: 1, + deleted_length: 0, + inserted_text: b" ".to_vec(), + }, + ) + .unwrap(); + + assert_eq!(code, b"\n x"); + + let mut recorder = ReadRecorder::new(&code); + let mut tree = parser + .parse_with(&mut |i, _| recorder.read(i), Some(&tree)) + .unwrap(); + + assert_eq!(tree.root_node().to_sexp(), "(x_is_at (even_column))",); + assert_eq!(recorder.strings_read(), vec!["\n x"]); + + perform_edit( + &mut tree, + &mut code, + &Edit { + position: 1, + deleted_length: 0, + inserted_text: b"\n".to_vec(), + }, + ) + .unwrap(); + + assert_eq!(code, b"\n\n x"); + + let mut recorder = ReadRecorder::new(&code); + let tree = parser + .parse_with(&mut |i, _| recorder.read(i), Some(&tree)) + .unwrap(); + + assert_eq!(tree.root_node().to_sexp(), "(x_is_at (even_column))",); + assert_eq!(recorder.strings_read(), vec!["\n\n x"]); +} + #[test] fn test_parsing_after_detecting_error_in_the_middle_of_a_string_token() { let mut parser = Parser::new(); diff --git a/lib/src/subtree.c b/lib/src/subtree.c index 4524e182..2ab8f475 100644 --- a/lib/src/subtree.c +++ b/lib/src/subtree.c @@ -677,7 +677,8 @@ Subtree ts_subtree_edit(Subtree self, const TSInputEdit *input_edit, SubtreePool Edit edit = entry.edit; bool is_noop = edit.old_end.bytes == edit.start.bytes && edit.new_end.bytes == edit.start.bytes; bool is_pure_insertion = edit.old_end.bytes == edit.start.bytes; - bool invalidate_first_row = ts_subtree_depends_on_column(*entry.tree); + bool parent_depends_on_column = ts_subtree_depends_on_column(*entry.tree); + bool column_shifted = edit.new_end.extent.column != edit.old_end.extent.column; Length size = ts_subtree_size(*entry.tree); Length padding = ts_subtree_padding(*entry.tree); @@ -771,8 +772,12 @@ Subtree ts_subtree_edit(Subtree self, const TSInputEdit *input_edit, SubtreePool (child_left.bytes > edit.old_end.bytes) || (child_left.bytes == edit.old_end.bytes && child_size.bytes > 0 && i > 0) ) && ( - !invalidate_first_row || - child_left.extent.row > entry.tree->ptr->padding.extent.row + !parent_depends_on_column || + child_left.extent.row > padding.extent.row + ) && ( + !ts_subtree_depends_on_column(*child) || + !column_shifted || + child_left.extent.row > edit.old_end.extent.row )) { break; } diff --git a/test/fixtures/test_grammars/depends_on_column/corpus.txt b/test/fixtures/test_grammars/depends_on_column/corpus.txt new file mode 100644 index 00000000..5c8dbbe6 --- /dev/null +++ b/test/fixtures/test_grammars/depends_on_column/corpus.txt @@ -0,0 +1,21 @@ +================== +X is at odd column +================== + + x + +--- + +(x_is_at + (odd_column)) + +=================== +X is at even column +=================== + + x + +--- + +(x_is_at + (even_column)) diff --git a/test/fixtures/test_grammars/depends_on_column/grammar.js b/test/fixtures/test_grammars/depends_on_column/grammar.js new file mode 100644 index 00000000..6f74810e --- /dev/null +++ b/test/fixtures/test_grammars/depends_on_column/grammar.js @@ -0,0 +1,7 @@ +module.exports = grammar({ + name: "depends_on_column", + rules: { + x_is_at: ($) => seq(/[ \r\n]*/, choice($.odd_column, $.even_column), "x"), + }, + externals: ($) => [$.odd_column, $.even_column], +}); diff --git a/test/fixtures/test_grammars/depends_on_column/scanner.c b/test/fixtures/test_grammars/depends_on_column/scanner.c new file mode 100644 index 00000000..29c2eb1b --- /dev/null +++ b/test/fixtures/test_grammars/depends_on_column/scanner.c @@ -0,0 +1,40 @@ +#include "tree_sitter/parser.h" + +enum TokenType { ODD_COLUMN, EVEN_COLUMN }; + +// The scanner is stateless + +void *tree_sitter_depends_on_column_external_scanner_create() { + return NULL; +} + +void tree_sitter_depends_on_column_external_scanner_destroy( + void *payload +) { + // no-op +} + +unsigned tree_sitter_depends_on_column_external_scanner_serialize( + void *payload, + char *buffer +) { + return 0; +} + +void tree_sitter_depends_on_column_external_scanner_deserialize( + void *payload, + const char *buffer, + unsigned length +) { + // no-op +} + +bool tree_sitter_depends_on_column_external_scanner_scan( + void *payload, + TSLexer *lexer, + const bool *valid_symbols +) { + lexer->result_symbol = + lexer->get_column(lexer) % 2 ? ODD_COLUMN : EVEN_COLUMN; + return true; +} From bf094bd98a6597b15c752c69be1cf563d6a4208e Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Thu, 18 Jul 2024 15:14:10 +0900 Subject: [PATCH 33/35] fix: exclude APIs that dup given file descriptors from WASI builds WASI doesn't support `dup(2)` system call, so we cannot implement the `print_dot_graph` and `print_dot_graphs` functions with exactly the same semantics as in other platforms. (cherry picked from commit 94a8262110db8352b3797578d61639679e406862) --- lib/binding_rust/ffi.rs | 2 +- lib/binding_rust/lib.rs | 10 ++++++---- lib/src/tree.c | 9 ++++++++- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/lib/binding_rust/ffi.rs b/lib/binding_rust/ffi.rs index ba3bcab6..755d2d52 100644 --- a/lib/binding_rust/ffi.rs +++ b/lib/binding_rust/ffi.rs @@ -8,7 +8,7 @@ include!(concat!(env!("OUT_DIR"), "/bindings.rs")); #[cfg(not(feature = "bindgen"))] include!("./bindings.rs"); -#[cfg(any(unix, target_os = "wasi"))] +#[cfg(unix)] #[cfg(feature = "std")] extern "C" { pub(crate) fn _ts_dup(fd: std::os::raw::c_int) -> std::os::raw::c_int; diff --git a/lib/binding_rust/lib.rs b/lib/binding_rust/lib.rs index 46f319d8..c97b4eaa 100644 --- a/lib/binding_rust/lib.rs +++ b/lib/binding_rust/lib.rs @@ -548,13 +548,14 @@ impl Parser { /// want to pipe these graphs directly to a `dot(1)` process in order to /// generate SVG output. #[doc(alias = "ts_parser_print_dot_graphs")] + #[cfg(not(target_os = "wasi"))] #[cfg(feature = "std")] pub fn print_dot_graphs( &mut self, - #[cfg(any(unix, target_os = "wasi"))] file: &impl AsRawFd, + #[cfg(unix)] file: &impl AsRawFd, #[cfg(windows)] file: &impl AsRawHandle, ) { - #[cfg(any(unix, target_os = "wasi"))] + #[cfg(unix)] { let fd = file.as_raw_fd(); unsafe { @@ -946,13 +947,14 @@ impl Tree { /// graph directly to a `dot(1)` process in order to generate SVG /// output. #[doc(alias = "ts_tree_print_dot_graph")] + #[cfg(not(target_os = "wasi"))] #[cfg(feature = "std")] pub fn print_dot_graph( &self, - #[cfg(any(unix, target_os = "wasi"))] file: &impl AsRawFd, + #[cfg(unix)] file: &impl AsRawFd, #[cfg(windows)] file: &impl AsRawHandle, ) { - #[cfg(any(unix, target_os = "wasi"))] + #[cfg(unix)] { let fd = file.as_raw_fd(); unsafe { ffi::ts_tree_print_dot_graph(self.0.as_ptr(), fd) } diff --git a/lib/src/tree.c b/lib/src/tree.c index 14936732..55e79a7e 100644 --- a/lib/src/tree.c +++ b/lib/src/tree.c @@ -148,7 +148,7 @@ void ts_tree_print_dot_graph(const TSTree *self, int fd) { fclose(file); } -#else +#elif !defined(__wasi__) // WASI doesn't support dup #include @@ -162,4 +162,11 @@ void ts_tree_print_dot_graph(const TSTree *self, int file_descriptor) { fclose(file); } +#else + +void ts_tree_print_dot_graph(const TSTree *self, int file_descriptor) { + (void)self; + (void)file_descriptor; +} + #endif From 78c41e3cedb1906b5dd7cbe82ca41cee30f440b1 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 30 Sep 2024 18:22:29 -0400 Subject: [PATCH 34/35] 0.23.1 --- Cargo.lock | 105 +++++++++++++++++++---------------- Cargo.toml | 12 ++-- Makefile | 2 +- build.zig.zon | 2 +- cli/npm/package.json | 2 +- lib/binding_web/package.json | 2 +- 6 files changed, 67 insertions(+), 58 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b1d35340..fa9176a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -150,15 +150,15 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" +checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" [[package]] name = "cc" -version = "1.1.20" +version = "1.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45bcde016d64c21da4be18b655631e5ab6d3107607e71a73a9f53eb48aae23fb" +checksum = "3bbb537bb4a30b90362caddba8f360c0a56bc13d3a5570028e7197204cb54a17" dependencies = [ "jobserver", "libc", @@ -211,9 +211,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.17" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e5a21b8495e732f1b3c364c9949b201ca7bae518c502c80256c96ad79eaf6ac" +checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3" dependencies = [ "clap_builder", "clap_derive", @@ -221,9 +221,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.17" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cf2dd12af7a047ad9d6da2b6b249759a22a7abc0f474c1dae1777afa4b21a73" +checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b" dependencies = [ "anstream", "anstyle", @@ -233,9 +233,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.13" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -757,9 +757,9 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "libc" -version = "0.2.158" +version = "0.2.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" +checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" [[package]] name = "libgit2-sys" @@ -946,9 +946,12 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1" +dependencies = [ + "portable-atomic", +] [[package]] name = "openssl-probe" @@ -1000,9 +1003,15 @@ checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" [[package]] name = "pkg-config" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" + +[[package]] +name = "portable-atomic" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" [[package]] name = "postcard" @@ -1104,9 +1113,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.4" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" dependencies = [ "bitflags", ] @@ -1137,9 +1146,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.6" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" dependencies = [ "aho-corasick", "memchr", @@ -1149,9 +1158,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick", "memchr", @@ -1160,9 +1169,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "rustc-hash" @@ -1239,9 +1248,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" dependencies = [ "serde", ] @@ -1299,9 +1308,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" -version = "2.0.77" +version = "2.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" +checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" dependencies = [ "proc-macro2", "quote", @@ -1316,9 +1325,9 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "tempfile" -version = "3.12.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" +checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" dependencies = [ "cfg-if", "fastrand", @@ -1338,18 +1347,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" +checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" dependencies = [ "proc-macro2", "quote", @@ -1406,9 +1415,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.21" +version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b072cee73c449a636ffd6f32bd8de3a9f7119139aff882f44943ce2986dc5cf" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ "indexmap", "serde", @@ -1450,7 +1459,7 @@ dependencies = [ [[package]] name = "tree-sitter" -version = "0.23.0" +version = "0.23.1" dependencies = [ "bindgen", "cc", @@ -1462,7 +1471,7 @@ dependencies = [ [[package]] name = "tree-sitter-cli" -version = "0.23.0" +version = "0.23.1" dependencies = [ "anstyle", "anyhow", @@ -1508,7 +1517,7 @@ dependencies = [ [[package]] name = "tree-sitter-config" -version = "0.23.0" +version = "0.23.1" dependencies = [ "anyhow", "dirs", @@ -1518,7 +1527,7 @@ dependencies = [ [[package]] name = "tree-sitter-highlight" -version = "0.23.0" +version = "0.23.1" dependencies = [ "lazy_static", "regex", @@ -1532,7 +1541,7 @@ version = "0.1.0" [[package]] name = "tree-sitter-loader" -version = "0.23.0" +version = "0.23.1" dependencies = [ "anyhow", "cc", @@ -1553,7 +1562,7 @@ dependencies = [ [[package]] name = "tree-sitter-tags" -version = "0.23.0" +version = "0.23.1" dependencies = [ "memchr", "regex", @@ -1585,18 +1594,18 @@ checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-normalization" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" dependencies = [ "tinyvec", ] [[package]] name = "unicode-xid" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "229730647fbc343e3a80e463c1db7f78f3855d3f3739bee0dda773c9a037c90a" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "unindent" @@ -2193,9 +2202,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.18" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" dependencies = [ "memchr", ] diff --git a/Cargo.toml b/Cargo.toml index 7594e80b..d76ec79d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.23.0" +version = "0.23.1" authors = ["Max Brunsfeld "] edition = "2021" rust-version = "1.74.1" @@ -87,8 +87,8 @@ walkdir = "2.5.0" wasmparser = "0.215.0" webbrowser = "1.0.2" -tree-sitter = { version = "0.23.0", path = "./lib" } -tree-sitter-loader = { version = "0.23.0", path = "./cli/loader" } -tree-sitter-config = { version = "0.23.0", path = "./cli/config" } -tree-sitter-highlight = { version = "0.23.0", path = "./highlight" } -tree-sitter-tags = { version = "0.23.0", path = "./tags" } +tree-sitter = { version = "0.23.1", path = "./lib" } +tree-sitter-loader = { version = "0.23.1", path = "./cli/loader" } +tree-sitter-config = { version = "0.23.1", path = "./cli/config" } +tree-sitter-highlight = { version = "0.23.1", path = "./highlight" } +tree-sitter-tags = { version = "0.23.1", path = "./tags" } diff --git a/Makefile b/Makefile index d5a7a1eb..892e1304 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ ifeq ($(OS),Windows_NT) $(error Windows is not supported) endif -VERSION := 0.23.0 +VERSION := 0.23.1 # install directory layout PREFIX ?= /usr/local diff --git a/build.zig.zon b/build.zig.zon index 97b4cdb9..5a00f9cf 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,6 +1,6 @@ .{ .name = "tree-sitter", - .version = "0.23.0", + .version = "0.23.1", .paths = .{ "build.zig", "build.zig.zon", diff --git a/cli/npm/package.json b/cli/npm/package.json index bdab77fc..d33b43b5 100644 --- a/cli/npm/package.json +++ b/cli/npm/package.json @@ -1,6 +1,6 @@ { "name": "tree-sitter-cli", - "version": "0.23.0", + "version": "0.23.1", "author": "Max Brunsfeld", "license": "MIT", "repository": { diff --git a/lib/binding_web/package.json b/lib/binding_web/package.json index 06319dc7..60ef0d4c 100644 --- a/lib/binding_web/package.json +++ b/lib/binding_web/package.json @@ -1,6 +1,6 @@ { "name": "web-tree-sitter", - "version": "0.23.0", + "version": "0.23.1", "description": "Tree-sitter bindings for the web", "main": "tree-sitter.js", "types": "tree-sitter-web.d.ts", From d97db6d63507eb62c536bcb2c4ac7d70c8ec665e Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Tue, 1 Oct 2024 11:53:40 -0400 Subject: [PATCH 35/35] 0.23.2 --- Cargo.lock | 18 +++++++++--------- Cargo.toml | 12 ++++++------ Makefile | 2 +- build.zig.zon | 2 +- cli/npm/package.json | 2 +- lib/Cargo.toml | 2 +- lib/binding_web/package.json | 2 +- lib/language/Cargo.toml | 2 +- 8 files changed, 21 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fa9176a4..eb488156 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -156,9 +156,9 @@ checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" [[package]] name = "cc" -version = "1.1.23" +version = "1.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bbb537bb4a30b90362caddba8f360c0a56bc13d3a5570028e7197204cb54a17" +checksum = "812acba72f0a070b003d3697490d2b55b837230ae7c6c6497f05cc2ddbb8d938" dependencies = [ "jobserver", "libc", @@ -1459,7 +1459,7 @@ dependencies = [ [[package]] name = "tree-sitter" -version = "0.23.1" +version = "0.23.2" dependencies = [ "bindgen", "cc", @@ -1471,7 +1471,7 @@ dependencies = [ [[package]] name = "tree-sitter-cli" -version = "0.23.1" +version = "0.23.2" dependencies = [ "anstyle", "anyhow", @@ -1517,7 +1517,7 @@ dependencies = [ [[package]] name = "tree-sitter-config" -version = "0.23.1" +version = "0.23.2" dependencies = [ "anyhow", "dirs", @@ -1527,7 +1527,7 @@ dependencies = [ [[package]] name = "tree-sitter-highlight" -version = "0.23.1" +version = "0.23.2" dependencies = [ "lazy_static", "regex", @@ -1537,11 +1537,11 @@ dependencies = [ [[package]] name = "tree-sitter-language" -version = "0.1.0" +version = "0.1.1" [[package]] name = "tree-sitter-loader" -version = "0.23.1" +version = "0.23.2" dependencies = [ "anyhow", "cc", @@ -1562,7 +1562,7 @@ dependencies = [ [[package]] name = "tree-sitter-tags" -version = "0.23.1" +version = "0.23.2" dependencies = [ "memchr", "regex", diff --git a/Cargo.toml b/Cargo.toml index d76ec79d..ddaf5b56 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.23.1" +version = "0.23.2" authors = ["Max Brunsfeld "] edition = "2021" rust-version = "1.74.1" @@ -87,8 +87,8 @@ walkdir = "2.5.0" wasmparser = "0.215.0" webbrowser = "1.0.2" -tree-sitter = { version = "0.23.1", path = "./lib" } -tree-sitter-loader = { version = "0.23.1", path = "./cli/loader" } -tree-sitter-config = { version = "0.23.1", path = "./cli/config" } -tree-sitter-highlight = { version = "0.23.1", path = "./highlight" } -tree-sitter-tags = { version = "0.23.1", path = "./tags" } +tree-sitter = { version = "0.23.2", path = "./lib" } +tree-sitter-loader = { version = "0.23.2", path = "./cli/loader" } +tree-sitter-config = { version = "0.23.2", path = "./cli/config" } +tree-sitter-highlight = { version = "0.23.2", path = "./highlight" } +tree-sitter-tags = { version = "0.23.2", path = "./tags" } diff --git a/Makefile b/Makefile index 892e1304..57ac8ae0 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ ifeq ($(OS),Windows_NT) $(error Windows is not supported) endif -VERSION := 0.23.1 +VERSION := 0.23.2 # install directory layout PREFIX ?= /usr/local diff --git a/build.zig.zon b/build.zig.zon index 5a00f9cf..dbebf231 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,6 +1,6 @@ .{ .name = "tree-sitter", - .version = "0.23.1", + .version = "0.23.2", .paths = .{ "build.zig", "build.zig.zon", diff --git a/cli/npm/package.json b/cli/npm/package.json index d33b43b5..33c1194f 100644 --- a/cli/npm/package.json +++ b/cli/npm/package.json @@ -1,6 +1,6 @@ { "name": "tree-sitter-cli", - "version": "0.23.1", + "version": "0.23.2", "author": "Max Brunsfeld", "license": "MIT", "repository": { diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 2884d411..a0f04259 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -33,7 +33,7 @@ wasm = ["wasmtime-c-api"] [dependencies] regex = { version = "1.10.6", default-features = false, features = ["unicode"] } regex-syntax = { version = "0.8.4", default-features = false } -tree-sitter-language = { version = "0.1", path = "language" } +tree-sitter-language = { version = "0.1.1", path = "language" } [dependencies.wasmtime-c-api] version = "24.0.0" diff --git a/lib/binding_web/package.json b/lib/binding_web/package.json index 60ef0d4c..755f1be7 100644 --- a/lib/binding_web/package.json +++ b/lib/binding_web/package.json @@ -1,6 +1,6 @@ { "name": "web-tree-sitter", - "version": "0.23.1", + "version": "0.23.2", "description": "Tree-sitter bindings for the web", "main": "tree-sitter.js", "types": "tree-sitter-web.d.ts", diff --git a/lib/language/Cargo.toml b/lib/language/Cargo.toml index 55361fd0..40524b6f 100644 --- a/lib/language/Cargo.toml +++ b/lib/language/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "tree-sitter-language" description = "The tree-sitter Language type, used by the library and by language implementations" -version = "0.1.0" +version = "0.1.1" authors.workspace = true edition.workspace = true rust-version.workspace = true