From a3679fbb1fde3f9f69cf205b5b1dde859c587de4 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 19 May 2016 16:25:44 -0700 Subject: [PATCH] Distinguish separators from main tokens via a property on transitions It was incorrect to store it as a property on the lexical states themselves --- include/tree_sitter/parser.h | 21 ++- .../build_tables/lex_conflict_manager_spec.cc | 6 +- spec/compiler/build_tables/lex_item_spec.cc | 160 ++++++++++-------- src/compiler/build_tables/build_lex_table.cc | 15 +- src/compiler/build_tables/lex_item.cc | 25 +-- src/compiler/build_tables/lex_item.h | 2 +- .../build_tables/lex_item_transitions.cc | 43 +++-- src/compiler/generate_code/c_code.cc | 9 +- src/compiler/lex_table.cc | 7 +- src/compiler/lex_table.h | 3 +- src/compiler/rules/metadata.h | 2 +- src/runtime/lexer.c | 15 +- src/runtime/stack.c | 2 +- 13 files changed, 157 insertions(+), 153 deletions(-) diff --git a/include/tree_sitter/parser.h b/include/tree_sitter/parser.h index 3d644da6..9e1860ba 100644 --- a/include/tree_sitter/parser.h +++ b/include/tree_sitter/parser.h @@ -33,8 +33,7 @@ typedef struct { typedef struct TSLexer { void (*start_fn)(struct TSLexer *, TSStateId); - void (*start_token_fn)(struct TSLexer *); - bool (*advance_fn)(struct TSLexer *, TSStateId); + bool (*advance_fn)(struct TSLexer *, TSStateId, bool); TSTree *(*accept_fn)(struct TSLexer *, TSSymbol, TSSymbolMetadata, const char *, bool fragile); @@ -103,18 +102,22 @@ struct TSLanguage { next_state: \ lookahead = lexer->lookahead; -#define START_TOKEN() lexer->start_token_fn(lexer); - #define GO_TO_STATE(state_value) \ { \ state = state_value; \ goto next_state; \ } -#define ADVANCE(state_value) \ - { \ - lexer->advance_fn(lexer, state_value); \ - GO_TO_STATE(state_value); \ +#define ADVANCE(state_value) \ + { \ + lexer->advance_fn(lexer, state_value, true); \ + GO_TO_STATE(state_value); \ + } + +#define SKIP(state_value) \ + { \ + lexer->advance_fn(lexer, state_value, false); \ + GO_TO_STATE(state_value); \ } #define ACCEPT_FRAGILE_TOKEN(symbol) \ @@ -128,7 +131,7 @@ struct TSLanguage { #define LEX_ERROR() \ if (error_mode) { \ if (state == ts_lex_state_error) \ - lexer->advance_fn(lexer, state); \ + lexer->advance_fn(lexer, state, true); \ GO_TO_STATE(ts_lex_state_error) \ } else { \ return lexer->accept_fn(lexer, ts_builtin_sym_error, (TSSymbolMetadata){}, \ diff --git a/spec/compiler/build_tables/lex_conflict_manager_spec.cc b/spec/compiler/build_tables/lex_conflict_manager_spec.cc index caa99ef6..98f6b3bf 100644 --- a/spec/compiler/build_tables/lex_conflict_manager_spec.cc +++ b/spec/compiler/build_tables/lex_conflict_manager_spec.cc @@ -16,7 +16,7 @@ describe("LexConflictManager::resolve(new_action, old_action)", []() { Symbol sym3(2, true); it("favors advance actions over empty accept token actions", [&]() { - update = conflict_manager.resolve(AdvanceAction(2, {0, 0}), AcceptTokenAction()); + update = conflict_manager.resolve(AdvanceAction(2, {0, 0}, true), AcceptTokenAction()); AssertThat(update, IsTrue()); }); @@ -60,14 +60,14 @@ describe("LexConflictManager::resolve(new_action, old_action)", []() { describe("advance/accept-token conflicts", [&]() { describe("when the token to accept has higher precedence", [&]() { it("prefers the accept-token action", [&]() { - update = conflict_manager.resolve(AdvanceAction(1, { 1, 2 }), AcceptTokenAction(sym3, 3, true)); + update = conflict_manager.resolve(AdvanceAction(1, { 1, 2 }, true), AcceptTokenAction(sym3, 3, true)); AssertThat(update, IsFalse()); }); }); describe("when the token to accept does not have a higher precedence", [&]() { it("favors the advance action", [&]() { - update = conflict_manager.resolve(AdvanceAction(1, { 1, 2 }), AcceptTokenAction(sym3, 2, true)); + update = conflict_manager.resolve(AdvanceAction(1, { 1, 2 }, true), AcceptTokenAction(sym3, 2, true)); AssertThat(update, IsTrue()); }); }); diff --git a/spec/compiler/build_tables/lex_item_spec.cc b/spec/compiler/build_tables/lex_item_spec.cc index b129be1d..4a867915 100644 --- a/spec/compiler/build_tables/lex_item_spec.cc +++ b/spec/compiler/build_tables/lex_item_spec.cc @@ -7,41 +7,11 @@ using namespace rules; using namespace build_tables; +typedef LexItemSet::Transition Transition; START_TEST describe("LexItem", []() { - describe("is_token_start()", [&]() { - Symbol sym(1); - rule_ptr token_start = make_shared(str("a"), map({ - { START_TOKEN, true } - })); - - it("returns true for rules designated as token starts", [&]() { - LexItem item(sym, token_start); - AssertThat(item.is_token_start(), IsTrue()); - }); - - it("returns false for rules not designated as token starts", [&]() { - AssertThat(LexItem(sym, make_shared(str("a"), map({ - { PRECEDENCE, 5 } - }))).is_token_start(), IsFalse()); - AssertThat(LexItem(sym, str("a")).is_token_start(), IsFalse()); - }); - - describe("when given a sequence containing a token start", [&]() { - it("returns true when the rule before the token start may be blank", [&]() { - LexItem item(sym, seq({ repeat(str("a")), token_start })); - AssertThat(item.is_token_start(), IsTrue()); - }); - - it("returns false when the rule before the token start cannot be blank", [&]() { - LexItem item(sym, seq({ str("a"), token_start })); - AssertThat(item.is_token_start(), IsFalse()); - }); - }); - }); - describe("completion_status()", [&]() { it("indicates whether the item is done, its precedence, and whether it is a string", [&]() { LexItem item1(Symbol(0, true), character({ 'a', 'b', 'c' })); @@ -77,11 +47,35 @@ describe("LexItemSet::transitions()", [&]() { Equals(LexItemSet::TransitionMap({ { CharacterSet().include('x'), - { + Transition{ LexItemSet({ LexItem(Symbol(1), blank()), }), - PrecedenceRange() + PrecedenceRange(), + false + } + } + }))); + }); + + it("marks transitions that are within the main token (as opposed to separators)", [&]() { + LexItemSet item_set({ + LexItem(Symbol(1), metadata(character({ 'x' }), { + {MAIN_TOKEN, true} + })), + }); + + AssertThat( + item_set.transitions(), + Equals(LexItemSet::TransitionMap({ + { + CharacterSet().include('x'), + Transition{ + LexItemSet({ + LexItem(Symbol(1), metadata(blank(), { {MAIN_TOKEN, true}})), + }), + PrecedenceRange(), + true } } }))); @@ -102,7 +96,7 @@ describe("LexItemSet::transitions()", [&]() { Equals(LexItemSet::TransitionMap({ { CharacterSet().include('w'), - { + Transition{ LexItemSet({ LexItem(Symbol(1), seq({ character({ 'x' }), @@ -110,7 +104,8 @@ describe("LexItemSet::transitions()", [&]() { character({ 'z' }), })), }), - PrecedenceRange() + PrecedenceRange(), + false } } }))); @@ -136,7 +131,7 @@ describe("LexItemSet::transitions()", [&]() { Equals(LexItemSet::TransitionMap({ { CharacterSet().include('v'), - { + Transition{ // The outer precedence is now 'active', because we are within its // contained rule. LexItemSet({ @@ -151,7 +146,8 @@ describe("LexItemSet::transitions()", [&]() { }), // No precedence is applied upon entering a rule. - PrecedenceRange() + PrecedenceRange(), + false } } }))); @@ -164,7 +160,7 @@ describe("LexItemSet::transitions()", [&]() { Equals(LexItemSet::TransitionMap({ { CharacterSet().include('w'), - { + Transition{ // The inner precedence is now 'active' LexItemSet({ LexItem(Symbol(1), seq({ @@ -176,7 +172,8 @@ describe("LexItemSet::transitions()", [&]() { }), // The outer precedence is applied. - PrecedenceRange(3) + PrecedenceRange(3), + false } } }))); @@ -189,7 +186,7 @@ describe("LexItemSet::transitions()", [&]() { Equals(LexItemSet::TransitionMap({ { CharacterSet().include('x'), - { + Transition{ LexItemSet({ LexItem(Symbol(1), seq({ active_prec(3, character({ 'y' })), @@ -198,7 +195,8 @@ describe("LexItemSet::transitions()", [&]() { }), // The inner precedence is applied. - PrecedenceRange(4) + PrecedenceRange(4), + false } } }))); @@ -211,11 +209,12 @@ describe("LexItemSet::transitions()", [&]() { Equals(LexItemSet::TransitionMap({ { CharacterSet().include('y'), - { + Transition{ LexItemSet({ LexItem(Symbol(1), character({ 'z' })), }), - PrecedenceRange(3) + PrecedenceRange(3), + false } } }))); @@ -238,23 +237,25 @@ describe("LexItemSet::transitions()", [&]() { Equals(LexItemSet::TransitionMap({ { CharacterSet().include('x'), - { + Transition{ LexItemSet({ LexItem(Symbol(1), seq({ character({ 'y' }), character({ 'z' }), })), }), - PrecedenceRange() + PrecedenceRange(), + false } }, { CharacterSet().include('y'), - { + Transition{ LexItemSet({ LexItem(Symbol(1), character({ 'z' })), }), - PrecedenceRange() + PrecedenceRange(), + false } } }))); @@ -282,7 +283,7 @@ describe("LexItemSet::transitions()", [&]() { Equals(LexItemSet::TransitionMap({ { CharacterSet().include('a'), - { + Transition{ LexItemSet({ LexItem(Symbol(1), seq({ character({ 'b' }), @@ -293,17 +294,19 @@ describe("LexItemSet::transitions()", [&]() { })), LexItem(Symbol(1), character({ 'b' })), }), - PrecedenceRange() + PrecedenceRange(), + false } }, { CharacterSet().include('c'), - { + Transition{ LexItemSet({ LexItem(Symbol(2), repeat1(character({ 'c' }))), LexItem(Symbol(2), blank()), }), - PrecedenceRange() + PrecedenceRange(), + false } } }))); @@ -319,12 +322,13 @@ describe("LexItemSet::transitions()", [&]() { Equals(LexItemSet::TransitionMap({ { CharacterSet().include('a'), - { + Transition{ LexItemSet({ LexItem(Symbol(1), active_prec(-1, repeat1(character({ 'a' })))), LexItem(Symbol(1), active_prec(-1, blank())), }), - PrecedenceRange(-1) + PrecedenceRange(-1), + false } } }))); @@ -349,30 +353,33 @@ describe("LexItemSet::transitions()", [&]() { Equals(LexItemSet::TransitionMap({ { CharacterSet().include('a', 'b'), - { + Transition{ LexItemSet({ LexItem(Symbol(1), active_prec(2, character({ 'x' }))), }), - PrecedenceRange(2) + PrecedenceRange(2), + false } }, { CharacterSet().include('c', 'd'), - { + Transition{ LexItemSet({ LexItem(Symbol(1), active_prec(2, character({ 'x' }))), LexItem(Symbol(1), active_prec(3, character({ 'y' }))), }), - PrecedenceRange(2, 3) + PrecedenceRange(2, 3), + false } }, { CharacterSet().include('e', 'f'), - { + Transition{ LexItemSet({ LexItem(Symbol(1), active_prec(3, character({ 'y' }))), }), - PrecedenceRange(3) + PrecedenceRange(3), + false } }, }))); @@ -397,21 +404,23 @@ describe("LexItemSet::transitions()", [&]() { Equals(LexItemSet::TransitionMap({ { CharacterSet().include('a').include('e', 'f'), - { + Transition{ LexItemSet({ LexItem(Symbol(1), character({ 'y' })), }), - PrecedenceRange() + PrecedenceRange(), + false } }, { CharacterSet().include('b', 'd'), - { + Transition{ LexItemSet({ LexItem(Symbol(1), character({ 'x' })), LexItem(Symbol(1), character({ 'y' })), }), - PrecedenceRange() + PrecedenceRange(), + false } }, }))); @@ -436,21 +445,23 @@ describe("LexItemSet::transitions()", [&]() { Equals(LexItemSet::TransitionMap({ { CharacterSet().include_all().exclude('/').exclude('\\'), - { + Transition{ LexItemSet({ LexItem(Symbol(1), character({ '/' })), }), - PrecedenceRange() + PrecedenceRange(), + false } }, { CharacterSet().include('\\'), - { + Transition{ LexItemSet({ LexItem(Symbol(1), character({ '/' })), LexItem(Symbol(1), seq({ character({ '/' }), character({ '/' }) })), }), - PrecedenceRange() + PrecedenceRange(), + false } }, }))); @@ -465,30 +476,33 @@ describe("LexItemSet::transitions()", [&]() { AssertThat(set1.transitions(), Equals(LexItemSet::TransitionMap({ { CharacterSet().include('a', 'd'), - { + Transition{ LexItemSet({ LexItem(Symbol(1), blank()), }), - PrecedenceRange() + PrecedenceRange(), + false } }, { CharacterSet().include('e', 'f'), - { + Transition{ LexItemSet({ LexItem(Symbol(1), blank()), LexItem(Symbol(2), blank()), }), - PrecedenceRange() + PrecedenceRange(), + false } }, { CharacterSet().include('g', 'i'), - { + Transition{ LexItemSet({ LexItem(Symbol(2), blank()), }), - PrecedenceRange() + PrecedenceRange(), + false } }, }))); diff --git a/src/compiler/build_tables/build_lex_table.cc b/src/compiler/build_tables/build_lex_table.cc index 174f8459..ca174e77 100644 --- a/src/compiler/build_tables/build_lex_table.cc +++ b/src/compiler/build_tables/build_lex_table.cc @@ -34,7 +34,7 @@ using rules::Repeat; using rules::Symbol; using rules::Metadata; using rules::Seq; -using rules::START_TOKEN; +using rules::MAIN_TOKEN; using rules::PRECEDENCE; using rules::IS_ACTIVE; @@ -79,7 +79,6 @@ class LexTableBuilder { lex_state_ids[item_set] = state_id; add_accept_token_actions(item_set, state_id); add_advance_actions(item_set, state_id); - add_token_start(item_set, state_id); return state_id; } else { return pair->second; @@ -90,7 +89,7 @@ class LexTableBuilder { for (const auto &pair : item_set.transitions()) { const CharacterSet &characters = pair.first; const LexItemSet::Transition &transition = pair.second; - AdvanceAction action(-1, transition.precedence); + AdvanceAction action(-1, transition.precedence, transition.in_main_token); auto current_action = lex_table.state(state_id).accept_action; if (conflict_manager.resolve(action, current_action)) { @@ -114,12 +113,6 @@ class LexTableBuilder { } } - void add_token_start(const LexItemSet &item_set, LexStateId state_id) { - for (const auto &item : item_set.entries) - if (item.is_token_start()) - lex_table.state(state_id).is_token_start = true; - } - void mark_fragile_tokens() { for (LexState &state : lex_table.states) if (state.accept_action.is_present()) @@ -152,8 +145,8 @@ class LexTableBuilder { symbol, Metadata::build( Seq::build({ - Metadata::build(separator_rule, { { START_TOKEN, true } }), - Metadata::build(rule, { { PRECEDENCE, 0 } }), + separator_rule, + Metadata::build(rule, { { PRECEDENCE, 0 }, { MAIN_TOKEN, 1 } }), }), { { PRECEDENCE, INT_MIN }, { IS_ACTIVE, true } }))); return result; diff --git a/src/compiler/build_tables/lex_item.cc b/src/compiler/build_tables/lex_item.cc index 7f5f3a45..72c3b44d 100644 --- a/src/compiler/build_tables/lex_item.cc +++ b/src/compiler/build_tables/lex_item.cc @@ -26,28 +26,6 @@ bool LexItem::operator==(const LexItem &other) const { return (other.lhs == lhs) && other.rule->operator==(*rule); } -bool LexItem::is_token_start() const { - class IsTokenStart : public rules::RuleFn { - bool apply_to(const rules::Seq *rule) { - return apply(rule->left) || - (rule_can_be_blank(rule->left) && apply(rule->right)); - } - - bool apply_to(const rules::Metadata *rule) { - return (rule->value_for(rules::START_TOKEN).second) || apply(rule->rule); - } - - bool apply_to(const rules::Choice *rule) { - for (const rule_ptr &element : rule->elements) - if (apply(element)) - return true; - return false; - } - }; - - return IsTokenStart().apply(rule); -} - LexItem::CompletionStatus LexItem::completion_status() const { class GetCompletionStatus : public rules::RuleFn { protected: @@ -119,7 +97,8 @@ LexItemSet::TransitionMap LexItemSet::transitions() const { } bool LexItemSet::Transition::operator==(const LexItemSet::Transition &other) const { - return destination == other.destination && precedence == other.precedence; + return destination == other.destination && precedence == other.precedence && + in_main_token == other.in_main_token; } } // namespace build_tables diff --git a/src/compiler/build_tables/lex_item.h b/src/compiler/build_tables/lex_item.h index 7c72849e..80cd1e6b 100644 --- a/src/compiler/build_tables/lex_item.h +++ b/src/compiler/build_tables/lex_item.h @@ -27,7 +27,6 @@ class LexItem { }; bool operator==(const LexItem &other) const; - bool is_token_start() const; CompletionStatus completion_status() const; rules::Symbol lhs; @@ -56,6 +55,7 @@ class LexItemSet { struct LexItemSet::Transition { LexItemSet destination; PrecedenceRange precedence; + bool in_main_token; bool operator==(const LexItemSet::Transition &) const; }; diff --git a/src/compiler/build_tables/lex_item_transitions.cc b/src/compiler/build_tables/lex_item_transitions.cc index d95acc66..f1050ead 100644 --- a/src/compiler/build_tables/lex_item_transitions.cc +++ b/src/compiler/build_tables/lex_item_transitions.cc @@ -32,6 +32,7 @@ using rules::Repeat; using rules::Metadata; using rules::PRECEDENCE; using rules::IS_ACTIVE; +using rules::MAIN_TOKEN; typedef LexItemSet::Transition Transition; typedef LexItemSet::TransitionMap TransitionMap; @@ -39,13 +40,15 @@ class TransitionBuilder : public rules::RuleFn { TransitionMap *transitions; const Symbol &item_lhs; vector *precedence_stack; + bool in_main_token; Transition transform_transition(const Transition &transition, function callback) { LexItemSet destination; for (const LexItem &item : transition.destination.entries) destination.entries.insert(LexItem(item.lhs, callback(item.rule))); - return Transition{ destination, transition.precedence }; + return Transition{ destination, transition.precedence, + transition.in_main_token }; } void add_transition(TransitionMap *transitions, CharacterSet new_characters, @@ -56,8 +59,6 @@ class TransitionBuilder : public rules::RuleFn { while (iter != transitions->end()) { CharacterSet existing_characters = iter->first; Transition &existing_transition = iter->second; - LexItemSet &existing_item_set = existing_transition.destination; - PrecedenceRange &existing_precedence = existing_transition.precedence; CharacterSet intersecting_characters = existing_characters.remove_set(new_characters); @@ -70,17 +71,17 @@ class TransitionBuilder : public rules::RuleFn { if (!existing_characters.is_empty()) new_entries.push_back({ - existing_characters, - Transition{ existing_item_set, existing_precedence }, + existing_characters, existing_transition, }); - existing_item_set.entries.insert( + existing_transition.destination.entries.insert( new_transition.destination.entries.begin(), new_transition.destination.entries.end()); - existing_precedence.add(new_transition.precedence); + existing_transition.precedence.add(new_transition.precedence); + existing_transition.in_main_token |= new_transition.in_main_token; + new_entries.push_back({ - intersecting_characters, - Transition{ existing_item_set, existing_precedence }, + intersecting_characters, existing_transition, }); transitions->erase(iter++); @@ -97,11 +98,11 @@ class TransitionBuilder : public rules::RuleFn { if (!precedence_stack->empty()) precedence.add(precedence_stack->back()); - add_transition( - transitions, *character_set, - Transition{ - LexItemSet({ LexItem(item_lhs, Blank::build()) }), precedence, - }); + add_transition(transitions, *character_set, + Transition{ + LexItemSet({ LexItem(item_lhs, Blank::build()) }), + precedence, in_main_token, + }); } void apply_to(const Choice *choice) { @@ -144,6 +145,9 @@ class TransitionBuilder : public rules::RuleFn { if (has_active_precedence) precedence_stack->push_back(metadata->value_for(PRECEDENCE).first); + if (metadata->value_for(MAIN_TOKEN).second) + in_main_token = true; + auto metadata_value = metadata->value; if (metadata_value.count(PRECEDENCE)) metadata_value.insert({ IS_ACTIVE, true }); @@ -165,20 +169,23 @@ class TransitionBuilder : public rules::RuleFn { public: TransitionBuilder(TransitionMap *transitions, const Symbol &item_lhs, - vector *precedence_stack) + vector *precedence_stack, bool in_main_token) : transitions(transitions), item_lhs(item_lhs), - precedence_stack(precedence_stack) {} + precedence_stack(precedence_stack), + in_main_token(in_main_token) {} TransitionBuilder(TransitionMap *transitions, TransitionBuilder *other) : transitions(transitions), item_lhs(other->item_lhs), - precedence_stack(other->precedence_stack) {} + precedence_stack(other->precedence_stack), + in_main_token(other->in_main_token) {} }; void lex_item_transitions(TransitionMap *transitions, const LexItem &item) { vector precedence_stack; - TransitionBuilder(transitions, item.lhs, &precedence_stack).apply(item.rule); + TransitionBuilder(transitions, item.lhs, &precedence_stack, false) + .apply(item.rule); } } // namespace build_tables diff --git a/src/compiler/generate_code/c_code.cc b/src/compiler/generate_code/c_code.cc index 2fd14e4d..dacc434d 100644 --- a/src/compiler/generate_code/c_code.cc +++ b/src/compiler/generate_code/c_code.cc @@ -328,7 +328,10 @@ class CCodeGenerator { } void add_advance_action(const AdvanceAction &action) { - line("ADVANCE(" + to_string(action.state_index) + ");"); + if (action.in_main_token) + line("ADVANCE(" + to_string(action.state_index) + ");"); + else + line("SKIP(" + to_string(action.state_index) + ");"); } void add_accept_token_action(const AcceptTokenAction &action) { @@ -529,6 +532,10 @@ class CCodeGenerator { } } + string _boolean(bool value) { + return value ? "true" : "false"; + } + bool has_sanitized_name(string name) { for (const auto &pair : sanitized_names) if (pair.second == name) diff --git a/src/compiler/lex_table.cc b/src/compiler/lex_table.cc index 0d7bec4c..191d40dc 100644 --- a/src/compiler/lex_table.cc +++ b/src/compiler/lex_table.cc @@ -15,8 +15,11 @@ using rules::CharacterSet; AdvanceAction::AdvanceAction() : state_index(-1) {} AdvanceAction::AdvanceAction(size_t state_index, - PrecedenceRange precedence_range) - : state_index(state_index), precedence_range(precedence_range) {} + PrecedenceRange precedence_range, + bool in_main_token) + : state_index(state_index), + precedence_range(precedence_range), + in_main_token(in_main_token) {} bool AdvanceAction::operator==(const AdvanceAction &other) const { return (state_index == other.state_index) && diff --git a/src/compiler/lex_table.h b/src/compiler/lex_table.h index 708774d7..7696ab37 100644 --- a/src/compiler/lex_table.h +++ b/src/compiler/lex_table.h @@ -20,12 +20,13 @@ typedef enum { struct AdvanceAction { AdvanceAction(); - AdvanceAction(size_t, PrecedenceRange); + AdvanceAction(size_t, PrecedenceRange, bool); bool operator==(const AdvanceAction &action) const; size_t state_index; PrecedenceRange precedence_range; + bool in_main_token; }; struct AcceptTokenAction { diff --git a/src/compiler/rules/metadata.h b/src/compiler/rules/metadata.h index 333cb697..72cdbed0 100644 --- a/src/compiler/rules/metadata.h +++ b/src/compiler/rules/metadata.h @@ -15,7 +15,7 @@ enum Associativity { }; enum MetadataKey { - START_TOKEN, + MAIN_TOKEN, PRECEDENCE, ASSOCIATIVITY, IS_TOKEN, diff --git a/src/runtime/lexer.c b/src/runtime/lexer.c index 66c7dd66..47331e15 100644 --- a/src/runtime/lexer.c +++ b/src/runtime/lexer.c @@ -52,20 +52,15 @@ static void ts_lexer__start(TSLexer *self, TSStateId lex_state) { LOG_LOOKAHEAD(); self->starting_state = lex_state; + self->token_start_position = self->current_position; if (!self->chunk) ts_lexer__get_chunk(self); if (!self->lookahead_size) ts_lexer__get_lookahead(self); } -static void ts_lexer__start_token(TSLexer *self) { - LOG("start_token chars:%lu, rows:%lu, columns:%lu", - self->current_position.chars, self->current_position.rows, - self->current_position.columns); - self->token_start_position = self->current_position; -} - -static bool ts_lexer__advance(TSLexer *self, TSStateId state) { +static bool ts_lexer__advance(TSLexer *self, TSStateId state, + bool in_main_token) { LOG("advance state:%d", state); if (self->chunk == empty_chunk) @@ -83,6 +78,9 @@ static bool ts_lexer__advance(TSLexer *self, TSStateId state) { } } + if (!in_main_token) + self->token_start_position = self->current_position; + if (self->current_position.bytes >= self->chunk_start + self->chunk_size) ts_lexer__get_chunk(self); @@ -125,7 +123,6 @@ static TSTree *ts_lexer__accept(TSLexer *self, TSSymbol symbol, void ts_lexer_init(TSLexer *self) { *self = (TSLexer){ .start_fn = ts_lexer__start, - .start_token_fn = ts_lexer__start_token, .advance_fn = ts_lexer__advance, .accept_fn = ts_lexer__accept, .chunk = NULL, diff --git a/src/runtime/stack.c b/src/runtime/stack.c index d738bb1b..47d7858b 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -501,7 +501,7 @@ bool ts_stack_print_dot_graph(Stack *self, const char **symbol_names, FILE *f) { fprintf(f, "rankdir=\"RL\";\n"); fprintf(f, "edge [arrowhead=none]\n"); - Array(StackNode *) visited_nodes = array_new(); + Array(StackNode *)visited_nodes = array_new(); array_clear(&self->pop_paths); for (size_t i = 0; i < self->heads.size; i++) {