traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Represent Lengths in terms of Points

Browse source
This commit is contained in:
Max Brunsfeld 2016-09-09 21:11:02 -07:00
parent eb5dda75c4
commit cc62fe0375
13 changed files with 106 additions and 100 deletions
Download patch file Download diff file Expand all files Collapse all files

3
include/tree_sitter/parser.h
View file

@ -17,8 +17,7 @@ typedef unsigned short TSStateId;
typedef struct {
size_t bytes;
size_t chars;
size_t rows;
size_t columns;
TSPoint extent;
} TSLength;
typedef struct {

23
spec/helpers/point_helpers.cc
View file

@ -1,6 +1,6 @@
#include "./point_helpers.h"
#include <string>
#include <ostream>
#include "runtime/length.h"
using namespace std;
@ -12,12 +12,8 @@ bool operator==(const TSRange &left, const TSRange &right) {
return left.start == right.start && left.end == right.end;
}
std::ostream &operator<<(std::ostream &stream, const TSPoint &point) {
return stream << "{" << point.row << ", " << point.column << "}";
}
std::ostream &operator<<(std::ostream &stream, const TSRange &range) {
return stream << "{" << range.start << ", " << range.end << "}";
bool operator==(const TSLength &left, const TSLength &right) {
return ts_length_eq(left, right);
}
bool operator<(const TSPoint &left, const TSPoint &right) {
@ -30,3 +26,16 @@ bool operator<(const TSPoint &left, const TSPoint &right) {
bool operator>(const TSPoint &left, const TSPoint &right) {
return right < left;
}
std::ostream &operator<<(std::ostream &stream, const TSPoint &point) {
return stream << "{" << point.row << ", " << point.column << "}";
}
std::ostream &operator<<(std::ostream &stream, const TSRange &range) {
return stream << "{" << range.start << ", " << range.end << "}";
}
ostream &operator<<(ostream &stream, const TSLength &length) {
return stream << "{chars:" << length.chars << ", bytes:" <<
length.bytes << ", extent:" << length.extent << "}";
}

7
spec/helpers/point_helpers.h
View file

@ -1,6 +1,9 @@
#ifndef HELPERS_POINT_HELPERS_H_
#define HELPERS_POINT_HELPERS_H_
#include "runtime/length.h"
#include <ostream>
bool operator==(const TSPoint &left, const TSPoint &right);
bool operator<(const TSPoint &left, const TSPoint &right);
@ -9,8 +12,12 @@ bool operator>(const TSPoint &left, const TSPoint &right);
bool operator==(const TSRange &left, const TSRange &right);
bool operator==(const TSLength &left, const TSLength &right);
std::ostream &operator<<(std::ostream &stream, const TSPoint &point);
std::ostream &operator<<(std::ostream &stream, const TSRange &range);
std::ostream &operator<<(std::ostream &stream, const TSLength &length);
#endif // HELPERS_POINT_HELPERS_H_

9
spec/helpers/tree_helpers.cc
View file

@ -40,10 +40,6 @@ bool operator==(const TSNode &left, const TSNode &right) {
return ts_node_eq(left, right);
}
bool operator==(const TSLength &left, const TSLength &right) {
return ts_length_eq(left, right);
}
bool operator==(const std::vector<TSTree *> &vec, const TreeArray &array) {
if (vec.size() != array.size)
return false;
@ -52,8 +48,3 @@ bool operator==(const std::vector<TSTree *> &vec, const TreeArray &array) {
return false;
return true;
}
ostream &operator<<(ostream &stream, const TSLength &length) {
return stream << "{chars:" << length.chars << ", bytes:" <<
length.bytes << ", rows:" << length.rows << ", columns:" << length.columns << "}";
}

2
spec/helpers/tree_helpers.h
View file

@ -10,9 +10,7 @@ TSTree ** tree_array(std::vector<TSTree *> trees);
std::ostream &operator<<(std::ostream &stream, const TSTree *tree);
std::ostream &operator<<(std::ostream &stream, const TSNode &node);
std::ostream &operator<<(std::ostream &stream, const TSLength &length);
bool operator==(const TSNode &left, const TSNode &right);
bool operator==(const TSLength &left, const TSLength &right);
bool operator==(const std::vector<TSTree *> &right, const TreeArray &array);
#endif // HELPERS_TREE_HELPERS_H_

5
spec/runtime/stack_spec.cc
View file

@ -1,5 +1,6 @@
#include "spec_helper.h"
#include "helpers/tree_helpers.h"
#include "helpers/point_helpers.h"
#include "helpers/record_alloc.h"
#include "helpers/stream_methods.h"
#include "runtime/stack.h"
@ -19,7 +20,7 @@ enum {
};
TSLength operator*(const TSLength &length, size_t factor) {
return {length.bytes * factor, length.chars * factor, 0, length.columns * factor};
return {length.bytes * factor, length.chars * factor, {0, length.extent.column * factor}};
}
void free_slice_array(StackSliceArray *slices) {
@ -69,7 +70,7 @@ describe("Stack", [&]() {
Stack *stack;
const size_t tree_count = 11;
TSTree *trees[tree_count];
TSLength tree_len = {2, 3, 0, 3};
TSLength tree_len = {2, 3, {0, 3}};
before_each([&]() {
record_alloc::start();

75
spec/runtime/tree_spec.cc
View file

@ -1,5 +1,6 @@
#include "spec_helper.h"
#include "helpers/tree_helpers.h"
#include "helpers/point_helpers.h"
#include "runtime/tree.h"
#include "runtime/length.h"
@ -36,8 +37,8 @@ describe("Tree", []() {
TSSymbolMetadata invisible = {false, false, false, true};
before_each([&]() {
tree1 = ts_tree_make_leaf(cat, {2, 1, 0, 1}, {5, 4, 0, 4}, visible);
tree2 = ts_tree_make_leaf(cat, {1, 1, 0, 1}, {3, 3, 0, 3}, visible);
tree1 = ts_tree_make_leaf(cat, {2, 1, {0, 1}}, {5, 4, {0, 4}}, visible);
tree2 = ts_tree_make_leaf(cat, {1, 1, {0, 1}}, {3, 3, {0, 3}}, visible);
ts_tree_retain(tree1);
ts_tree_retain(tree2);
@ -166,13 +167,13 @@ describe("Tree", []() {
before_each([&]() {
tree = ts_tree_make_node(cat, 3, tree_array({
ts_tree_make_leaf(dog, {2, 2, 0, 2}, {3, 3, 0, 3}, visible),
ts_tree_make_leaf(eel, {2, 2, 0, 2}, {3, 3, 0, 3}, visible),
ts_tree_make_leaf(fox, {2, 2, 0, 2}, {3, 3, 0, 3}, visible),
ts_tree_make_leaf(dog, {2, 2, {0, 2}}, {3, 3, {0, 3}}, visible),
ts_tree_make_leaf(eel, {2, 2, {0, 2}}, {3, 3, {0, 3}}, visible),
ts_tree_make_leaf(fox, {2, 2, {0, 2}}, {3, 3, {0, 3}}, visible),
}), visible);
AssertThat(tree->padding, Equals<TSLength>({2, 2, 0, 2}));
AssertThat(tree->size, Equals<TSLength>({13, 13, 0, 13}));
AssertThat(tree->padding, Equals<TSLength>({2, 2, {0, 2}}));
AssertThat(tree->size, Equals<TSLength>({13, 13, {0, 13}}));
});
after_each([&]() {
@ -187,16 +188,16 @@ describe("Tree", []() {
assert_consistent(tree);
AssertThat(tree->has_changes, IsTrue());
AssertThat(tree->padding, Equals<TSLength>({0, 3, 0, 0}));
AssertThat(tree->size, Equals<TSLength>({13, 13, 0, 13}));
AssertThat(tree->padding, Equals<TSLength>({0, 3, {0, 0}}));
AssertThat(tree->size, Equals<TSLength>({13, 13, {0, 13}}));
AssertThat(tree->children[0]->has_changes, IsTrue());
AssertThat(tree->children[0]->padding, Equals<TSLength>({0, 3, 0, 0}));
AssertThat(tree->children[0]->size, Equals<TSLength>({3, 3, 0, 3}));
AssertThat(tree->children[0]->padding, Equals<TSLength>({0, 3, {0, 0}}));
AssertThat(tree->children[0]->size, Equals<TSLength>({3, 3, {0, 3}}));
AssertThat(tree->children[1]->has_changes, IsFalse());
AssertThat(tree->children[1]->padding, Equals<TSLength>({2, 2, 0, 2}));
AssertThat(tree->children[1]->size, Equals<TSLength>({3, 3, 0, 3}));
AssertThat(tree->children[1]->padding, Equals<TSLength>({2, 2, {0, 2}}));
AssertThat(tree->children[1]->size, Equals<TSLength>({3, 3, {0, 3}}));
});
});
@ -207,12 +208,12 @@ describe("Tree", []() {
assert_consistent(tree);
AssertThat(tree->has_changes, IsTrue());
AssertThat(tree->padding, Equals<TSLength>({0, 5, 0, 0}));
AssertThat(tree->size, Equals<TSLength>({0, 11, 0, 0}));
AssertThat(tree->padding, Equals<TSLength>({0, 5, {0, 0}}));
AssertThat(tree->size, Equals<TSLength>({0, 11, {0, 0}}));
AssertThat(tree->children[0]->has_changes, IsTrue());
AssertThat(tree->children[0]->padding, Equals<TSLength>({0, 5, 0, 0}));
AssertThat(tree->children[0]->size, Equals<TSLength>({0, 1, 0, 0}));
AssertThat(tree->children[0]->padding, Equals<TSLength>({0, 5, {0, 0}}));
AssertThat(tree->children[0]->size, Equals<TSLength>({0, 1, {0, 0}}));
});
});
@ -223,12 +224,12 @@ describe("Tree", []() {
assert_consistent(tree);
AssertThat(tree->has_changes, IsTrue());
AssertThat(tree->padding, Equals<TSLength>({0, 4, 0, 0}));
AssertThat(tree->size, Equals<TSLength>({13, 13, 0, 13}));
AssertThat(tree->padding, Equals<TSLength>({0, 4, {0, 0}}));
AssertThat(tree->size, Equals<TSLength>({13, 13, {0, 13}}));
AssertThat(tree->children[0]->has_changes, IsTrue());
AssertThat(tree->children[0]->padding, Equals<TSLength>({0, 4, 0, 0}));
AssertThat(tree->children[0]->size, Equals<TSLength>({3, 3, 0, 3}));
AssertThat(tree->children[0]->padding, Equals<TSLength>({0, 4, {0, 0}}));
AssertThat(tree->children[0]->size, Equals<TSLength>({3, 3, {0, 3}}));
AssertThat(tree->children[1]->has_changes, IsFalse());
});
@ -241,12 +242,12 @@ describe("Tree", []() {
assert_consistent(tree);
AssertThat(tree->has_changes, IsTrue());
AssertThat(tree->padding, Equals<TSLength>({2, 2, 0, 2}));
AssertThat(tree->size, Equals<TSLength>({0, 16, 0, 0}));
AssertThat(tree->padding, Equals<TSLength>({2, 2, {0, 2}}));
AssertThat(tree->size, Equals<TSLength>({0, 16, {0, 0}}));
AssertThat(tree->children[0]->has_changes, IsTrue());
AssertThat(tree->children[0]->padding, Equals<TSLength>({2, 2, 0, 2}));
AssertThat(tree->children[0]->size, Equals<TSLength>({0, 6, 0, 0}));
AssertThat(tree->children[0]->padding, Equals<TSLength>({2, 2, {0, 2}}));
AssertThat(tree->children[0]->size, Equals<TSLength>({0, 6, {0, 0}}));
AssertThat(tree->children[1]->has_changes, IsFalse());
});
@ -259,30 +260,30 @@ describe("Tree", []() {
assert_consistent(tree);
AssertThat(tree->has_changes, IsTrue());
AssertThat(tree->padding, Equals<TSLength>({0, 4, 0, 0}));
AssertThat(tree->size, Equals<TSLength>({0, 4, 0, 0}));
AssertThat(tree->padding, Equals<TSLength>({0, 4, {0, 0}}));
AssertThat(tree->size, Equals<TSLength>({0, 4, {0, 0}}));
AssertThat(tree->children[0]->has_changes, IsTrue());
AssertThat(tree->children[0]->padding, Equals<TSLength>({0, 4, 0, 0}));
AssertThat(tree->children[0]->size, Equals<TSLength>({0, 0, 0, 0}));
AssertThat(tree->children[0]->padding, Equals<TSLength>({0, 4, {0, 0}}));
AssertThat(tree->children[0]->size, Equals<TSLength>({0, 0, {0, 0}}));
AssertThat(tree->children[1]->has_changes, IsTrue());
AssertThat(tree->children[1]->padding, Equals<TSLength>({0, 0, 0, 0}));
AssertThat(tree->children[1]->size, Equals<TSLength>({0, 0, 0, 0}));
AssertThat(tree->children[1]->padding, Equals<TSLength>({0, 0, {0, 0}}));
AssertThat(tree->children[1]->size, Equals<TSLength>({0, 0, {0, 0}}));
AssertThat(tree->children[2]->has_changes, IsTrue());
AssertThat(tree->children[2]->padding, Equals<TSLength>({0, 1, 0, 0}));
AssertThat(tree->children[2]->size, Equals<TSLength>({3, 3, 0, 3}));
AssertThat(tree->children[2]->padding, Equals<TSLength>({0, 1, {0, 0}}));
AssertThat(tree->children[2]->size, Equals<TSLength>({3, 3, {0, 3}}));
});
});
});
describe("equality", [&]() {
it("returns true for identical trees", [&]() {
TSTree *tree1_copy = ts_tree_make_leaf(cat, {2, 1, 1, 1}, {5, 4, 1, 4}, visible);
TSTree *tree1_copy = ts_tree_make_leaf(cat, {2, 1, {1, 1}}, {5, 4, {1, 4}}, visible);
AssertThat(ts_tree_eq(tree1, tree1_copy), IsTrue());
TSTree *tree2_copy = ts_tree_make_leaf(cat, {1, 1, 0, 1}, {3, 3, 0, 3}, visible);
TSTree *tree2_copy = ts_tree_make_leaf(cat, {1, 1, {0, 1}}, {3, 3, {0, 3}}, visible);
AssertThat(ts_tree_eq(tree2, tree2_copy), IsTrue());
TSTree *parent2 = ts_tree_make_node(dog, 2, tree_array({
@ -313,11 +314,11 @@ describe("Tree", []() {
});
it("returns false for trees with different sizes", [&]() {
TSTree *tree1_copy = ts_tree_make_leaf(cat, {2, 1, 0, 1}, tree1->size, invisible);
TSTree *tree1_copy = ts_tree_make_leaf(cat, {2, 1, {0, 1}}, tree1->size, invisible);
AssertThat(ts_tree_eq(tree1, tree1_copy), IsFalse());
ts_tree_release(tree1_copy);
tree1_copy = ts_tree_make_leaf(cat, tree1->padding, {5, 4, 1, 10}, invisible);
tree1_copy = ts_tree_make_leaf(cat, tree1->padding, {5, 4, {1, 10}}, invisible);
AssertThat(ts_tree_eq(tree1, tree1_copy), IsFalse());
ts_tree_release(tree1_copy);
});

44
src/runtime/length.h
View file

@ -4,14 +4,27 @@
#include "tree_sitter/parser.h"
#include <stdbool.h>
static inline TSPoint ts_point_add(TSPoint a, TSPoint b) {
if (b.row > 0)
return (TSPoint){a.row + b.row, b.column};
else
return (TSPoint){a.row, a.column + b.column};
}
static inline TSPoint ts_point_sub(TSPoint a, TSPoint b) {
if (a.row > b.row)
return (TSPoint){a.row - b.row, a.column};
else
return (TSPoint){0, a.column - b.column};
}
static inline bool ts_length_is_unknown(TSLength self) {
return self.chars > 0 && self.bytes == 0;
}
static inline void ts_length_set_unknown(TSLength *self) {
self->bytes = 0;
self->rows = 0;
self->columns = 0;
self->extent = (TSPoint){0, 0};
}
static inline TSLength ts_length_min(TSLength len1, TSLength len2) {
@ -24,17 +37,10 @@ static inline TSLength ts_length_add(TSLength len1, TSLength len2) {
if (ts_length_is_unknown(len1) || ts_length_is_unknown(len2)) {
result.bytes = 0;
result.rows = 0;
result.columns = result.chars;
result.extent = (TSPoint){0, result.chars};
} else {
result.bytes = len1.bytes + len2.bytes;
if (len2.rows == 0) {
result.rows = len1.rows;
result.columns = len1.columns + len2.columns;
} else {
result.rows = len1.rows + len2.rows;
result.columns = len2.columns;
}
result.extent = ts_point_add(len1.extent, len2.extent);
}
return result;
@ -46,29 +52,23 @@ static inline TSLength ts_length_sub(TSLength len1, TSLength len2) {
if (ts_length_is_unknown(len1) || ts_length_is_unknown(len2)) {
result.bytes = 0;
result.rows = 0;
result.columns = result.chars;
result.extent = (TSPoint){0, result.chars};
} else {
result.bytes = len1.bytes - len2.bytes;
if (len1.rows == len2.rows) {
result.rows = 0;
result.columns = len1.columns - len2.columns;
} else {
result.rows = len1.rows - len2.rows;
result.columns = len1.columns;
}
result.extent = ts_point_sub(len1.extent, len2.extent);
}
return result;
}
static inline TSLength ts_length_zero() {
return (TSLength){ 0, 0, 0, 0 };
return (TSLength){ 0, 0, {0, 0} };
}
static inline bool ts_length_eq(TSLength self, TSLength other) {
return self.bytes == other.bytes && self.chars == other.chars &&
self.rows == other.rows && self.columns == other.columns;
self.extent.row == other.extent.row &&
self.extent.column == other.extent.column;
}
#endif

6
src/runtime/lexer.c
View file

@ -55,10 +55,10 @@ static void ts_lexer__advance(TSLexer *self, TSStateId state, bool skip) {
self->current_position.bytes += self->lookahead_size;
self->current_position.chars++;
if (self->lookahead == '\n') {
self->current_position.rows++;
self->current_position.columns = 0;
self->current_position.extent.row++;
self->current_position.extent.column = 0;
} else {
self->current_position.columns++;
self->current_position.extent.column++;
}
}

10
src/runtime/node.c
View file

@ -48,7 +48,7 @@ static inline TSNode ts_node__direct_parent(TSNode self, size_t *index) {
return ts_node_make(tree->context.parent,
ts_node__offset_char(self) - tree->context.offset.chars,
ts_node__offset_byte(self) - tree->context.offset.bytes,
ts_node__offset_row(self) - tree->context.offset.rows);
ts_node__offset_row(self) - tree->context.offset.extent.row);
}
static inline TSNode ts_node__direct_child(TSNode self, size_t i) {
@ -56,7 +56,7 @@ static inline TSNode ts_node__direct_child(TSNode self, size_t i) {
return ts_node_make(
child_tree, ts_node__offset_char(self) + child_tree->context.offset.chars,
ts_node__offset_byte(self) + child_tree->context.offset.bytes,
ts_node__offset_row(self) + child_tree->context.offset.rows);
ts_node__offset_row(self) + child_tree->context.offset.extent.row);
}
static inline TSNode ts_node__child(TSNode self, size_t child_index,
@ -246,14 +246,14 @@ size_t ts_node_end_byte(TSNode self) {
TSPoint ts_node_start_point(TSNode self) {
const TSTree *tree = ts_node__tree(self);
return (TSPoint){ ts_node__offset_row(self) + tree->padding.rows,
return (TSPoint){ ts_node__offset_row(self) + tree->padding.extent.row,
ts_tree_start_column(tree) };
}
TSPoint ts_node_end_point(TSNode self) {
const TSTree *tree = ts_node__tree(self);
return (TSPoint){ ts_node__offset_row(self) + tree->padding.rows +
tree->size.rows,
return (TSPoint){ ts_node__offset_row(self) + tree->padding.extent.row +
tree->size.extent.row,
ts_tree_end_column(tree) };
}

4
src/runtime/parser.c
View file

@ -1272,8 +1272,8 @@ TSTree *parser_parse(Parser *self, TSInput input, TSTree *old_tree) {
LOG("process version:%d, version_count:%lu, state:%d, row:%lu, col:%lu",
version, ts_stack_version_count(self->stack),
ts_stack_top_state(self->stack, version),
ts_stack_top_position(self->stack, version).rows + 1,
ts_stack_top_position(self->stack, version).columns + 1);
ts_stack_top_position(self->stack, version).extent.row + 1,
ts_stack_top_position(self->stack, version).extent.column + 1);
CHECK(parser__advance(self, version, &reusable_node));
LOG_STACK();

4
src/runtime/stack.c
View file

@ -126,7 +126,7 @@ static StackNode *stack_node_new(StackNode *next, TSTree *tree, bool is_pending,
ERROR_COST_PER_SKIPPED_CHAR *
(tree->padding.chars + tree->size.chars) +
ERROR_COST_PER_SKIPPED_LINE *
(tree->padding.rows + tree->size.rows);
(tree->padding.extent.row + tree->size.extent.row);
}
}
} else {
@ -606,7 +606,7 @@ bool ts_stack_print_dot_graph(Stack *self, const char **symbol_names, FILE *f) {
fprintf(f,
" tooltip=\"position: %lu,%lu\nerror_count: %u\nerror_cost: %u\"];\n",
node->position.rows, node->position.columns, node->error_count,
node->position.extent.row, node->position.extent.column, node->error_count,
node->error_cost);
for (int j = 0; j < node->link_count; j++) {

14
src/runtime/tree.c
View file

@ -152,7 +152,7 @@ void ts_tree_set_children(TSTree *self, size_t child_count, TSTree **children) {
if (self->symbol == ts_builtin_sym_error) {
self->error_cost += ERROR_COST_PER_SKIPPED_CHAR * self->size.chars +
ERROR_COST_PER_SKIPPED_LINE * self->size.rows;
ERROR_COST_PER_SKIPPED_LINE * self->size.extent.row;
for (size_t i = 0; i < child_count; i++)
if (!self->children[i]->extra)
self->error_cost += ERROR_COST_PER_SKIPPED_TREE;
@ -233,20 +233,20 @@ recur:
}
size_t ts_tree_start_column(const TSTree *self) {
size_t column = self->padding.columns;
if (self->padding.rows > 0)
size_t column = self->padding.extent.column;
if (self->padding.extent.row > 0)
return column;
for (const TSTree *tree = self; tree != NULL; tree = tree->context.parent) {
column += tree->context.offset.columns;
if (tree->context.offset.rows > 0)
column += tree->context.offset.extent.column;
if (tree->context.offset.extent.row > 0)
break;
}
return column;
}
size_t ts_tree_end_column(const TSTree *self) {
size_t result = self->size.columns;
if (self->size.rows == 0)
size_t result = self->size.extent.column;
if (self->size.extent.row == 0)
result += ts_tree_start_column(self);
return result;
}