traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Always invalidate old chunk of text when parsing after an edit

Browse source
This commit is contained in:
Max Brunsfeld 2017-10-04 15:09:46 -07:00
parent c0073c5b72
commit 36c2b685b9
5 changed files with 38 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

3
src/runtime/lexer.c
View file

@ -132,6 +132,9 @@ static inline void ts_lexer__reset(Lexer *self, Length position) {
void ts_lexer_set_input(Lexer *self, TSInput input) {
self->input = input;
self->chunk = 0;
self->chunk_start = 0;
self->chunk_size = 0;
ts_lexer__reset(self, length_zero());
}

2
src/runtime/parser.c
View file

@ -17,7 +17,7 @@
if (self->lexer.logger.log || self->print_debugging_graphs) { \
snprintf(self->lexer.debug_buffer, TREE_SITTER_SERIALIZATION_BUFFER_SIZE, __VA_ARGS__); \
parser__log(self); \
} \
}
#define LOG_STACK() \
if (self->print_debugging_graphs) { \

2
test/helpers/spy_input.cc
View file

@ -9,9 +9,9 @@ using std::string;
using std::vector;
SpyInput::SpyInput(string content, size_t chars_per_chunk) :
chars_per_chunk(chars_per_chunk),
buffer(nullptr),
byte_offset(0),
chars_per_chunk(chars_per_chunk),
content(content),
encoding(TSInputEncodingUTF8),
ranges_read({}) {}

2
test/helpers/spy_input.h
View file

@ -12,7 +12,6 @@ struct SpyInputEdit {
};
class SpyInput {
uint32_t chars_per_chunk;
char *buffer;
uint32_t byte_offset;
std::vector<SpyInputEdit> undo_stack;
@ -31,6 +30,7 @@ class SpyInput {
TSInputEdit undo();
std::vector<std::string> strings_read() const;
uint32_t chars_per_chunk;
std::string content;
TSInputEncoding encoding;
std::vector<std::pair<uint32_t, uint32_t>> ranges_read;

35
test/runtime/document_test.cc
View file

@ -15,8 +15,6 @@ TSPoint point(size_t row, size_t column) {
START_TEST
describe("Document", [&]() {
TSDocument *document;
TSNode root;
@ -259,7 +257,7 @@ describe("Document", [&]() {
before_each([&]() {
ts_document_set_language(document, load_real_language("javascript"));
input = new SpyInput("{a: null};", 3);
input = new SpyInput("{a: null};\n", 3);
ts_document_set_input(document, input->input());
ts_document_parse(document);
assert_node_string_equals(
@ -313,6 +311,37 @@ describe("Document", [&]() {
})));
});
it("reports no changes when leading whitespace has changed (regression)", [&]() {
input->chars_per_chunk = 80;
// Insert leading whitespace
auto ranges = get_invalidated_ranges_for_edit([&]() {
return input->replace(0, 0, "\n");
});
assert_node_string_equals(
ts_document_root_node(document),
"(program (expression_statement (object (pair (property_identifier) (null)))))");
AssertThat(ranges, Equals(vector<TSRange>({})));
// Remove leading whitespace
ranges = get_invalidated_ranges_for_edit([&]() {
return input->undo();
});
assert_node_string_equals(
ts_document_root_node(document),
"(program (expression_statement (object (pair (property_identifier) (null)))))");
AssertThat(ranges, Equals(vector<TSRange>({})));
// Insert leading whitespace again
ranges = get_invalidated_ranges_for_edit([&]() {
return input->replace(0, 0, "\n");
});
assert_node_string_equals(
ts_document_root_node(document),
"(program (expression_statement (object (pair (property_identifier) (null)))))");
AssertThat(ranges, Equals(vector<TSRange>({})));
});
it("reports changes when tokens have been appended", [&]() {
// Add a second key-value pair
auto ranges = get_invalidated_ranges_for_edit([&]() {