fix(lib): advance the lookahead end byte by 4 when there's an invalid code point

This helps in the case where an edit was made in the middle of a code
point, but bytes 1-3 are valid, thus we could advance by at most 4 bytes
This commit is contained in:
Amaan Qureshi 2024-04-30 19:48:04 -04:00
parent 61d0395543
commit 4c083252ec
2 changed files with 24 additions and 1 deletions

View file

@ -679,6 +679,29 @@ fn test_get_changed_ranges() {
}
}
#[test]
fn test_consistency_with_mid_codepoint_edit() {
let mut parser = Parser::new();
parser.set_language(&get_language("php/php")).unwrap();
let mut source_code =
b"\n<?php\n\n<<<'\xE5\xAD\x97\xE6\xBC\xA2'\n T\n\xE5\xAD\x97\xE6\xBC\xA2;".to_vec();
let mut tree = parser.parse(&source_code, None).unwrap();
let edit = Edit {
position: 17,
deleted_length: 0,
inserted_text: vec![46],
};
perform_edit(&mut tree, &mut source_code, &edit).unwrap();
let mut tree2 = parser.parse(&source_code, Some(&tree)).unwrap();
let inverted = invert_edit(&source_code, &edit);
perform_edit(&mut tree2, &mut source_code, &inverted).unwrap();
let tree3 = parser.parse(&source_code, Some(&tree2)).unwrap();
assert_eq!(tree3.root_node().to_sexp(), tree.root_node().to_sexp());
}
fn index_of(text: &[u8], substring: &str) -> usize {
str::from_utf8(text).unwrap().find(substring).unwrap()
}