traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: properly handle utf8 code points for highlight and tag assertions

Browse source
This commit is contained in:
Amaan Qureshi 2024-09-08 14:26:34 -04:00
parent 755e49e212
commit 6f050f0da5
9 changed files with 131 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

26
cli/src/tests/test_highlight_test.rs
View file

@ -1,9 +1,9 @@
use tree_sitter::{Parser, Point};
use tree_sitter::Parser;
use tree_sitter_highlight::{Highlight, Highlighter};
use super::helpers::fixtures::{get_highlight_config, get_language, test_loader};
use crate::{
query_testing::{parse_position_comments, Assertion},
query_testing::{parse_position_comments, Assertion, Utf8Point},
test_highlight::get_highlight_positions,
};
@ -28,6 +28,9 @@ fn test_highlight_test_with_basic_test() {
" // ^ variable",
" // ^ !variable",
"};",
"var y̆y̆y̆y̆ = function() {}",
" // ^ function",
" // ^ keyword",
]
.join("\n");
@ -40,6 +43,8 @@ fn test_highlight_test_with_basic_test() {
Assertion::new(1, 11, false, String::from("keyword")),
Assertion::new(4, 9, false, String::from("variable")),
Assertion::new(4, 11, true, String::from("variable")),
Assertion::new(8, 5, false, String::from("function")),
Assertion::new(8, 11, false, String::from("keyword")),
]
);
@ -50,13 +55,16 @@ fn test_highlight_test_with_basic_test() {
assert_eq!(
highlight_positions,
&[
(Point::new(1, 0), Point::new(1, 3), Highlight(2)), // "var"
(Point::new(1, 4), Point::new(1, 7), Highlight(0)), // "abc"
(Point::new(1, 10), Point::new(1, 18), Highlight(2)), // "function"
(Point::new(1, 19), Point::new(1, 20), Highlight(1)), // "d"
(Point::new(4, 2), Point::new(4, 8), Highlight(2)), // "return"
(Point::new(4, 9), Point::new(4, 10), Highlight(1)), // "d"
(Point::new(4, 13), Point::new(4, 14), Highlight(1)), // "e"
(Utf8Point::new(1, 0), Utf8Point::new(1, 3), Highlight(2)), // "var"
(Utf8Point::new(1, 4), Utf8Point::new(1, 7), Highlight(0)), // "abc"
(Utf8Point::new(1, 10), Utf8Point::new(1, 18), Highlight(2)), // "function"
(Utf8Point::new(1, 19), Utf8Point::new(1, 20), Highlight(1)), // "d"
(Utf8Point::new(4, 2), Utf8Point::new(4, 8), Highlight(2)), // "return"
(Utf8Point::new(4, 9), Utf8Point::new(4, 10), Highlight(1)), // "d"
(Utf8Point::new(4, 13), Utf8Point::new(4, 14), Highlight(1)), // "e"
(Utf8Point::new(8, 0), Utf8Point::new(8, 3), Highlight(2)), // "var"
(Utf8Point::new(8, 4), Utf8Point::new(8, 8), Highlight(0)), // "y̆y̆y̆y̆"
(Utf8Point::new(8, 11), Utf8Point::new(8, 19), Highlight(2)), // "function"
]
);
}

16
cli/src/tests/test_tags_test.rs
View file

@ -1,9 +1,9 @@
use tree_sitter::{Parser, Point};
use tree_sitter::Parser;
use tree_sitter_tags::TagsContext;
use super::helpers::fixtures::{get_language, get_tags_config};
use crate::{
query_testing::{parse_position_comments, Assertion},
query_testing::{parse_position_comments, Assertion, Utf8Point},
test_tags::get_tag_positions,
};
@ -43,18 +43,18 @@ fn test_tags_test_with_basic_test() {
tag_positions,
&[
(
Point::new(1, 4),
Point::new(1, 7),
Utf8Point::new(1, 4),
Utf8Point::new(1, 7),
"definition.function".to_string()
),
(
Point::new(3, 8),
Point::new(3, 11),
Utf8Point::new(3, 8),
Utf8Point::new(3, 11),
"reference.call".to_string()
),
(
Point::new(5, 11),
Point::new(5, 12),
Utf8Point::new(5, 11),
Utf8Point::new(5, 12),
"reference.call".to_string()
),
]