traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: properly handle utf8 code points for highlight and tag assertions

Browse source
This commit is contained in:
Amaan Qureshi 2024-09-08 14:26:34 -04:00
parent 755e49e212
commit 6f050f0da5
9 changed files with 131 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

11
cli/src/test_highlight.rs
View file

@ -7,7 +7,7 @@ use tree_sitter_highlight::{Highlight, HighlightConfiguration, HighlightEvent, H
use tree_sitter_loader::{Config, Loader};
use super::{
query_testing::{parse_position_comments, Assertion},
query_testing::{parse_position_comments, to_utf8_point, Assertion, Utf8Point},
test::paint,
util,
};
@ -141,7 +141,7 @@ fn test_highlights_indented(
}
pub fn iterate_assertions(
assertions: &[Assertion],
highlights: &[(Point, Point, Highlight)],
highlights: &[(Utf8Point, Utf8Point, Highlight)],
highlight_names: &[String],
) -> Result<usize> {
// Iterate through all of the highlighting assertions, checking each one against the
@ -224,7 +224,7 @@ pub fn get_highlight_positions(
highlighter: &mut Highlighter,
highlight_config: &HighlightConfiguration,
source: &[u8],
) -> Result<Vec<(Point, Point, Highlight)>> {
) -> Result<Vec<(Utf8Point, Utf8Point, Highlight)>> {
let mut row = 0;
let mut column = 0;
let mut byte_offset = 0;
@ -261,7 +261,10 @@ pub fn get_highlight_positions(
}
}
if let Some(highlight) = highlight_stack.last() {
result.push((start_position, Point::new(row, column), *highlight));
let utf8_start_position = to_utf8_point(start_position, source.as_bytes());
let utf8_end_position =
to_utf8_point(Point::new(row, column), source.as_bytes());
result.push((utf8_start_position, utf8_end_position, *highlight));
}
}
}