traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into changed-ranges

Browse source
This commit is contained in:
Max Brunsfeld 2016-10-16 21:10:25 -07:00
commit eed54d95e1
28 changed files with 218 additions and 208 deletions
Download patch file Download diff file Expand all files Collapse all files

60
src/runtime/lexer.c
View file

@ -1,31 +1,30 @@
#include <stdio.h>
#include "runtime/lexer.h"
#include "tree_sitter/parser.h"
#include "runtime/tree.h"
#include "runtime/length.h"
#include "runtime/utf16.h"
#include "utf8proc.h"
#define LOG(...) \
if (self->logger.log) { \
snprintf(self->debug_buffer, TS_DEBUG_BUFFER_SIZE, __VA_ARGS__); \
self->logger.log(self->logger.payload, TSLogTypeLex, \
self->debug_buffer); \
#define LOG(...) \
if (self->logger.log) { \
snprintf(self->debug_buffer, TS_DEBUG_BUFFER_SIZE, __VA_ARGS__); \
self->logger.log(self->logger.payload, TSLogTypeLex, self->debug_buffer); \
}
#define LOG_LOOKAHEAD() \
LOG((0 < self->lookahead && self->lookahead < 256) ? "lookahead char:'%c'" \
: "lookahead char:%d", \
self->lookahead);
#define LOG_LOOKAHEAD() \
LOG((0 < self->data.lookahead && self->data.lookahead < 256) \
? "lookahead char:'%c'" \
: "lookahead char:%d", \
self->data.lookahead);
static const char empty_chunk[2] = { 0, 0 };
static void ts_lexer__get_chunk(TSLexer *self) {
static void ts_lexer__get_chunk(Lexer *self) {
TSInput input = self->input;
if (!self->chunk ||
self->current_position.bytes != self->chunk_start + self->chunk_size)
input.seek(input.payload, self->current_position.chars,
self->current_position.bytes);
self->current_position.bytes);
self->chunk_start = self->current_position.bytes;
self->chunk = input.read(input.payload, &self->chunk_size);
@ -33,28 +32,29 @@ static void ts_lexer__get_chunk(TSLexer *self) {
self->chunk = empty_chunk;
}
static void ts_lexer__get_lookahead(TSLexer *self) {
static void ts_lexer__get_lookahead(Lexer *self) {
size_t position_in_chunk = self->current_position.bytes - self->chunk_start;
const uint8_t *chunk = (const uint8_t *)self->chunk + position_in_chunk;
size_t size = self->chunk_size - position_in_chunk + 1;
if (self->input.encoding == TSInputEncodingUTF8)
self->lookahead_size = utf8proc_iterate(chunk, size, &self->lookahead);
self->lookahead_size =
utf8proc_iterate(chunk, size, &self->data.lookahead);
else
self->lookahead_size = utf16_iterate(chunk, size, &self->lookahead);
self->lookahead_size = utf16_iterate(chunk, size, &self->data.lookahead);
LOG_LOOKAHEAD();
}
static void ts_lexer__advance(TSLexer *self, TSStateId state, bool skip) {
static void ts_lexer__advance(void *payload, TSStateId state, bool skip) {
Lexer *self = (Lexer *)payload;
if (self->chunk == empty_chunk)
return;
if (self->lookahead_size) {
self->current_position.bytes += self->lookahead_size;
self->current_position.chars++;
if (self->lookahead == '\n') {
if (self->data.lookahead == '\n') {
self->current_position.extent.row++;
self->current_position.extent.column = 0;
} else {
@ -80,9 +80,12 @@ static void ts_lexer__advance(TSLexer *self, TSStateId state, bool skip) {
* parsers can call it without needing to be linked against this library.
*/
void ts_lexer_init(TSLexer *self) {
*self = (TSLexer){
.advance = ts_lexer__advance,
void ts_lexer_init(Lexer *self) {
*self = (Lexer){
.data =
{
.advance = ts_lexer__advance, .lookahead = 0, .result_symbol = 0,
},
.chunk = NULL,
.chunk_start = 0,
.logger = {},
@ -90,36 +93,37 @@ void ts_lexer_init(TSLexer *self) {
ts_lexer_reset(self, ts_length_zero());
}
static inline void ts_lexer__reset(TSLexer *self, TSLength position) {
static inline void ts_lexer__reset(Lexer *self, TSLength position) {
self->token_start_position = position;
self->current_position = position;
if (self->chunk && (position.bytes < self->chunk_start || position.bytes >= self->chunk_start + self->chunk_size)) {
if (self->chunk && (position.bytes < self->chunk_start ||
position.bytes >= self->chunk_start + self->chunk_size)) {
self->chunk = 0;
self->chunk_start = 0;
self->chunk_size = 0;
}
self->lookahead_size = 0;
self->lookahead = 0;
self->data.lookahead = 0;
}
void ts_lexer_set_input(TSLexer *self, TSInput input) {
void ts_lexer_set_input(Lexer *self, TSInput input) {
self->input = input;
ts_lexer__reset(self, ts_length_zero());
}
void ts_lexer_reset(TSLexer *self, TSLength position) {
void ts_lexer_reset(Lexer *self, TSLength position) {
if (!ts_length_eq(position, self->current_position))
ts_lexer__reset(self, position);
return;
}
void ts_lexer_start(TSLexer *self, TSStateId lex_state) {
void ts_lexer_start(Lexer *self, TSStateId lex_state) {
LOG("start_lex state:%d, pos:%lu", lex_state, self->current_position.chars);
self->token_start_position = self->current_position;
self->result_symbol = 0;
self->data.result_symbol = 0;
if (!self->chunk)
ts_lexer__get_chunk(self);