traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: properly handle utf8 code points for highlight and tag assertions

Browse source 
(cherry picked from commit 6f050f0da5)
This commit is contained in:
Amaan Qureshi 2024-09-08 14:26:34 -04:00
parent 3ad82e6772
commit 8e1dbb4617
9 changed files with 131 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

1
cli/Cargo.toml
View file

@ -27,6 +27,7 @@ wasm = ["tree-sitter/wasm", "tree-sitter-loader/wasm"]
[dependencies]
anstyle.workspace = true
anyhow.workspace = true
bstr.workspace = true
clap.workspace = true
ctor.workspace = true
ctrlc.workspace = true

10
cli/src/query.rs
View file

@ -9,7 +9,7 @@ use std::{
use anyhow::{Context, Result};
use tree_sitter::{Language, Parser, Point, Query, QueryCursor};
use crate::query_testing;
use crate::query_testing::{self, to_utf8_point};
#[allow(clippy::too_many_arguments)]
pub fn query_files_at_paths(
@ -70,8 +70,8 @@ pub fn query_files_at_paths(
}
results.push(query_testing::CaptureInfo {
name: (*capture_name).to_string(),
start: capture.node.start_position(),
end: capture.node.end_position(),
start: to_utf8_point(capture.node.start_position(), source_code.as_slice()),
end: to_utf8_point(capture.node.end_position(), source_code.as_slice()),
});
}
} else {
@ -100,8 +100,8 @@ pub fn query_files_at_paths(
}
results.push(query_testing::CaptureInfo {
name: (*capture_name).to_string(),
start: capture.node.start_position(),
end: capture.node.end_position(),
start: to_utf8_point(capture.node.start_position(), source_code.as_slice()),
end: to_utf8_point(capture.node.end_position(), source_code.as_slice()),
});
}
}

51
cli/src/query_testing.rs
View file

@ -1,6 +1,7 @@
use std::fs;
use anyhow::{anyhow, Result};
use bstr::{BStr, ByteSlice};
use lazy_static::lazy_static;
use regex::Regex;
use tree_sitter::{Language, Parser, Point};
@ -9,16 +10,56 @@ lazy_static! {
static ref CAPTURE_NAME_REGEX: Regex = Regex::new("[\\w_\\-.]+").unwrap();
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct Utf8Point {
pub row: usize,
pub column: usize,
}
impl std::fmt::Display for Utf8Point {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(f, "({}, {})", self.row, self.column)
}
}
impl Utf8Point {
pub const fn new(row: usize, column: usize) -> Self {
Self { row, column }
}
}
pub fn to_utf8_point(point: Point, source: &[u8]) -> Utf8Point {
if point.column == 0 {
return Utf8Point::new(point.row, 0);
}
let bstr = BStr::new(source);
let line = bstr.lines_with_terminator().nth(point.row).unwrap();
let mut utf8_column = 0;
for (_, grapheme_end, _) in line.grapheme_indices() {
utf8_column += 1;
if grapheme_end >= point.column {
break;
}
}
Utf8Point {
row: point.row,
column: utf8_column,
}
}
#[derive(Debug, Eq, PartialEq)]
pub struct CaptureInfo {
pub name: String,
pub start: Point,
pub end: Point,
pub start: Utf8Point,
pub end: Utf8Point,
}
#[derive(Debug, PartialEq, Eq)]
pub struct Assertion {
pub position: Point,
pub position: Utf8Point,
pub negative: bool,
pub expected_capture_name: String,
}
@ -27,7 +68,7 @@ impl Assertion {
#[must_use]
pub fn new(row: usize, col: usize, negative: bool, expected_capture_name: String) -> Self {
Self {
position: Point::new(row, col),
position: Utf8Point::new(row, col),
negative,
expected_capture_name,
}
@ -103,7 +144,7 @@ pub fn parse_position_comments(
{
assertion_ranges.push((node.start_position(), node.end_position()));
result.push(Assertion {
position,
position: to_utf8_point(position, source),
negative,
expected_capture_name: mat.as_str().to_string(),
});

11
cli/src/test_highlight.rs
View file

@ -7,7 +7,7 @@ use tree_sitter_highlight::{Highlight, HighlightConfiguration, HighlightEvent, H
use tree_sitter_loader::{Config, Loader};
use super::{
query_testing::{parse_position_comments, Assertion},
query_testing::{parse_position_comments, to_utf8_point, Assertion, Utf8Point},
test::paint,
util,
};
@ -141,7 +141,7 @@ fn test_highlights_indented(
}
pub fn iterate_assertions(
assertions: &[Assertion],
highlights: &[(Point, Point, Highlight)],
highlights: &[(Utf8Point, Utf8Point, Highlight)],
highlight_names: &[String],
) -> Result<usize> {
// Iterate through all of the highlighting assertions, checking each one against the
@ -224,7 +224,7 @@ pub fn get_highlight_positions(
highlighter: &mut Highlighter,
highlight_config: &HighlightConfiguration,
source: &[u8],
) -> Result<Vec<(Point, Point, Highlight)>> {
) -> Result<Vec<(Utf8Point, Utf8Point, Highlight)>> {
let mut row = 0;
let mut column = 0;
let mut byte_offset = 0;
@ -261,7 +261,10 @@ pub fn get_highlight_positions(
}
}
if let Some(highlight) = highlight_stack.last() {
result.push((start_position, Point::new(row, column), *highlight));
let utf8_start_position = to_utf8_point(start_position, source.as_bytes());
let utf8_end_position =
to_utf8_point(Point::new(row, column), source.as_bytes());
result.push((utf8_start_position, utf8_end_position, *highlight));
}
}
}

11
cli/src/test_tags.rs
View file

@ -2,12 +2,11 @@ use std::{fs, path::Path};
use anstyle::AnsiColor;
use anyhow::{anyhow, Result};
use tree_sitter::Point;
use tree_sitter_loader::{Config, Loader};
use tree_sitter_tags::{TagsConfiguration, TagsContext};
use super::{
query_testing::{parse_position_comments, Assertion},
query_testing::{parse_position_comments, to_utf8_point, Assertion, Utf8Point},
test::paint,
util,
};
@ -168,7 +167,7 @@ pub fn get_tag_positions(
tags_context: &mut TagsContext,
tags_config: &TagsConfiguration,
source: &[u8],
) -> Result<Vec<(Point, Point, String)>> {
) -> Result<Vec<(Utf8Point, Utf8Point, String)>> {
let (tags_iter, _has_error) = tags_context.generate_tags(tags_config, source, None)?;
let tag_positions = tags_iter
.filter_map(std::result::Result::ok)
@ -179,7 +178,11 @@ pub fn get_tag_positions(
} else {
format!("reference.{tag_postfix}")
};
(tag.span.start, tag.span.end, tag_name)
(
to_utf8_point(tag.span.start, source),
to_utf8_point(tag.span.end, source),
tag_name,
)
})
.collect();
Ok(tag_positions)

26
cli/src/tests/test_highlight_test.rs
View file

@ -1,9 +1,9 @@
use tree_sitter::{Parser, Point};
use tree_sitter::Parser;
use tree_sitter_highlight::{Highlight, Highlighter};
use super::helpers::fixtures::{get_highlight_config, get_language, test_loader};
use crate::{
query_testing::{parse_position_comments, Assertion},
query_testing::{parse_position_comments, Assertion, Utf8Point},
test_highlight::get_highlight_positions,
};
@ -28,6 +28,9 @@ fn test_highlight_test_with_basic_test() {
" // ^ variable",
" // ^ !variable",
"};",
"var y̆y̆y̆y̆ = function() {}",
" // ^ function",
" // ^ keyword",
]
.join("\n");
@ -40,6 +43,8 @@ fn test_highlight_test_with_basic_test() {
Assertion::new(1, 11, false, String::from("keyword")),
Assertion::new(4, 9, false, String::from("variable")),
Assertion::new(4, 11, true, String::from("variable")),
Assertion::new(8, 5, false, String::from("function")),
Assertion::new(8, 11, false, String::from("keyword")),
]
);
@ -50,13 +55,16 @@ fn test_highlight_test_with_basic_test() {
assert_eq!(
highlight_positions,
&[
(Point::new(1, 0), Point::new(1, 3), Highlight(2)), // "var"
(Point::new(1, 4), Point::new(1, 7), Highlight(0)), // "abc"
(Point::new(1, 10), Point::new(1, 18), Highlight(2)), // "function"
(Point::new(1, 19), Point::new(1, 20), Highlight(1)), // "d"
(Point::new(4, 2), Point::new(4, 8), Highlight(2)), // "return"
(Point::new(4, 9), Point::new(4, 10), Highlight(1)), // "d"
(Point::new(4, 13), Point::new(4, 14), Highlight(1)), // "e"
(Utf8Point::new(1, 0), Utf8Point::new(1, 3), Highlight(2)), // "var"
(Utf8Point::new(1, 4), Utf8Point::new(1, 7), Highlight(0)), // "abc"
(Utf8Point::new(1, 10), Utf8Point::new(1, 18), Highlight(2)), // "function"
(Utf8Point::new(1, 19), Utf8Point::new(1, 20), Highlight(1)), // "d"
(Utf8Point::new(4, 2), Utf8Point::new(4, 8), Highlight(2)), // "return"
(Utf8Point::new(4, 9), Utf8Point::new(4, 10), Highlight(1)), // "d"
(Utf8Point::new(4, 13), Utf8Point::new(4, 14), Highlight(1)), // "e"
(Utf8Point::new(8, 0), Utf8Point::new(8, 3), Highlight(2)), // "var"
(Utf8Point::new(8, 4), Utf8Point::new(8, 8), Highlight(0)), // "y̆y̆y̆y̆"
(Utf8Point::new(8, 11), Utf8Point::new(8, 19), Highlight(2)), // "function"
]
);
}

16
cli/src/tests/test_tags_test.rs
View file

@ -1,9 +1,9 @@
use tree_sitter::{Parser, Point};
use tree_sitter::Parser;
use tree_sitter_tags::TagsContext;
use super::helpers::fixtures::{get_language, get_tags_config};
use crate::{
query_testing::{parse_position_comments, Assertion},
query_testing::{parse_position_comments, Assertion, Utf8Point},
test_tags::get_tag_positions,
};
@ -43,18 +43,18 @@ fn test_tags_test_with_basic_test() {
tag_positions,
&[
(
Point::new(1, 4),
Point::new(1, 7),
Utf8Point::new(1, 4),
Utf8Point::new(1, 7),
"definition.function".to_string()
),
(
Point::new(3, 8),
Point::new(3, 11),
Utf8Point::new(3, 8),
Utf8Point::new(3, 11),
"reference.call".to_string()
),
(
Point::new(5, 11),
Point::new(5, 12),
Utf8Point::new(5, 11),
Utf8Point::new(5, 12),
"reference.call".to_string()
),
]