diff --git a/include/tree_sitter/parser.h b/include/tree_sitter/parser.h index a97adf61..95f29135 100644 --- a/include/tree_sitter/parser.h +++ b/include/tree_sitter/parser.h @@ -34,7 +34,7 @@ typedef struct TSLexer { void (*start_token_fn)(struct TSLexer *); bool (*advance_fn)(struct TSLexer *, TSStateId); TSTree *(*accept_fn)(struct TSLexer *, TSSymbol, TSSymbolMetadata, - const char *); + const char *, bool fragile); const char *chunk; size_t chunk_start; @@ -107,9 +107,13 @@ struct TSLanguage { GO_TO_STATE(state_index); \ } +#define ACCEPT_FRAGILE_TOKEN(symbol) \ + return lexer->accept_fn(lexer, symbol, ts_symbol_metadata[symbol], \ + ts_symbol_names[symbol], true); + #define ACCEPT_TOKEN(symbol) \ return lexer->accept_fn(lexer, symbol, ts_symbol_metadata[symbol], \ - ts_symbol_names[symbol]); + ts_symbol_names[symbol], false); #define LEX_ERROR() \ if (error_mode) { \ diff --git a/spec/compiler/build_tables/lex_conflict_manager_spec.cc b/spec/compiler/build_tables/lex_conflict_manager_spec.cc index cd80b5a8..975912ea 100644 --- a/spec/compiler/build_tables/lex_conflict_manager_spec.cc +++ b/spec/compiler/build_tables/lex_conflict_manager_spec.cc @@ -38,6 +38,11 @@ describe("LexConflictManager::resolve(new_action, old_action)", []() { update = conflict_manager.resolve(LexAction::Accept(sym1, 2, false), LexAction::Accept(sym2, 1, false)); AssertThat(update, IsTrue()); }); + + it("adds the discarded token to the 'fragile tokens' set", [&]() { + update = conflict_manager.resolve(LexAction::Accept(sym2, 1, false), LexAction::Accept(sym1, 2, false)); + AssertThat(conflict_manager.fragile_tokens, Contains(sym2)); + }); }); describe("when one token is string-based and the other is regexp-based", [&]() { diff --git a/spec/fixtures/parsers/c.c b/spec/fixtures/parsers/c.c index 2d385f37..971f6463 100644 --- a/spec/fixtures/parsers/c.c +++ b/spec/fixtures/parsers/c.c @@ -359,7 +359,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { (lookahead == '_') || ('a' <= lookahead && lookahead <= 'z')) ADVANCE(15); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 16: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -369,7 +369,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'u') ADVANCE(17); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 17: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -379,7 +379,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 't') ADVANCE(18); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 18: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -389,7 +389,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'o') ADVANCE(19); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 19: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -406,7 +406,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'o') ADVANCE(21); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 21: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -416,7 +416,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'n') ADVANCE(22); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 22: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -426,7 +426,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 's') ADVANCE(23); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 23: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -436,7 +436,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 't') ADVANCE(24); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 24: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -454,7 +454,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'x') ADVANCE(26); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 26: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -464,7 +464,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 't') ADVANCE(27); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 27: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -474,7 +474,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'e') ADVANCE(28); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 28: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -484,7 +484,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'r') ADVANCE(29); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 29: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -494,7 +494,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'n') ADVANCE(30); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 30: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -511,7 +511,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'o') ADVANCE(32); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 32: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -521,7 +521,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'n') ADVANCE(33); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 33: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -531,7 +531,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'g') ADVANCE(34); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 34: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -548,7 +548,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'e') ADVANCE(36); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 36: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -561,7 +561,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(37); if (lookahead == 's') ADVANCE(43); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 37: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -571,7 +571,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'i') ADVANCE(38); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 38: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -581,7 +581,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 's') ADVANCE(39); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 39: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -591,7 +591,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 't') ADVANCE(40); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 40: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -601,7 +601,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'e') ADVANCE(41); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 41: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -611,7 +611,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'r') ADVANCE(42); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 42: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -628,7 +628,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 't') ADVANCE(44); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 44: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -638,7 +638,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'r') ADVANCE(45); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 45: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -648,7 +648,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'i') ADVANCE(46); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 46: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -659,7 +659,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'c') ADVANCE(47); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 47: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -669,7 +669,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 't') ADVANCE(48); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 48: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -691,7 +691,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(54); if (lookahead == 't') ADVANCE(59); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 50: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -701,7 +701,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'o') ADVANCE(51); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 51: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -711,7 +711,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'r') ADVANCE(52); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 52: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -721,7 +721,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 't') ADVANCE(53); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 53: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -738,7 +738,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'g') ADVANCE(55); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 55: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -748,7 +748,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'n') ADVANCE(56); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 56: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -758,7 +758,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'e') ADVANCE(57); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 57: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -768,7 +768,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'd') ADVANCE(58); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 58: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -787,7 +787,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(60); if (lookahead == 'r') ADVANCE(64); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 60: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -797,7 +797,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 't') ADVANCE(61); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 61: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -807,7 +807,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'i') ADVANCE(62); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 62: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -818,7 +818,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'c') ADVANCE(63); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 63: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -835,7 +835,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'u') ADVANCE(65); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 65: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -846,7 +846,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'c') ADVANCE(66); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 66: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -856,7 +856,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 't') ADVANCE(67); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 67: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -873,7 +873,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'y') ADVANCE(69); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 69: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -883,7 +883,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'p') ADVANCE(70); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 70: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -893,7 +893,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'e') ADVANCE(71); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 71: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -903,7 +903,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'd') ADVANCE(72); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 72: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -913,7 +913,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'e') ADVANCE(73); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 73: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -923,7 +923,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'f') ADVANCE(74); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 74: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -940,7 +940,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'n') ADVANCE(76); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 76: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -950,7 +950,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 's') ADVANCE(77); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 77: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -960,7 +960,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'i') ADVANCE(78); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 78: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -970,7 +970,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'g') ADVANCE(79); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 79: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -980,7 +980,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'n') ADVANCE(80); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 80: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -990,7 +990,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'e') ADVANCE(81); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 81: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1000,7 +1000,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'd') ADVANCE(82); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 82: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1017,7 +1017,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'o') ADVANCE(84); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 84: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1027,7 +1027,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'l') ADVANCE(85); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 85: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1036,7 +1036,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'a') ADVANCE(86); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 86: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1046,7 +1046,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 't') ADVANCE(87); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 87: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1056,7 +1056,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'i') ADVANCE(88); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 88: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1066,7 +1066,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'l') ADVANCE(89); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 89: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1076,7 +1076,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'e') ADVANCE(90); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 90: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1132,7 +1132,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(54); if (lookahead == 't') ADVANCE(94); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 94: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1142,7 +1142,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'r') ADVANCE(64); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 95: START_TOKEN(); if ((lookahead == '\t') || @@ -1259,7 +1259,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(50); if (lookahead == 'i') ADVANCE(54); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 104: START_TOKEN(); if ((lookahead == '\t') || @@ -1604,7 +1604,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'e') ADVANCE(134); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 134: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1614,7 +1614,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 's') ADVANCE(43); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 135: START_TOKEN(); if ((lookahead == '\t') || @@ -1838,7 +1838,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { (lookahead == '/') || (lookahead == '\\'))) ADVANCE(156); - ACCEPT_TOKEN(sym_preproc_arg); + ACCEPT_FRAGILE_TOKEN(sym_preproc_arg); case 146: if (lookahead == '*') ADVANCE(147); @@ -1852,7 +1852,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { (lookahead == '/') || (lookahead == '\\'))) ADVANCE(156); - ACCEPT_TOKEN(sym_preproc_arg); + ACCEPT_FRAGILE_TOKEN(sym_preproc_arg); case 147: if (lookahead == '\n') ADVANCE(11); @@ -1865,7 +1865,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { (lookahead == '*') || (lookahead == '\\'))) ADVANCE(147); - ACCEPT_TOKEN(sym_preproc_arg); + ACCEPT_FRAGILE_TOKEN(sym_preproc_arg); case 148: if (lookahead == '\n') ADVANCE(11); @@ -1878,7 +1878,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { (lookahead == '/') || (lookahead == '\\'))) ADVANCE(147); - ACCEPT_TOKEN(sym_preproc_arg); + ACCEPT_FRAGILE_TOKEN(sym_preproc_arg); case 149: ACCEPT_TOKEN(sym_comment); case 150: @@ -1893,7 +1893,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { (lookahead == '*') || (lookahead == '\\'))) ADVANCE(147); - ACCEPT_TOKEN(sym_preproc_arg); + ACCEPT_FRAGILE_TOKEN(sym_preproc_arg); case 151: if (lookahead == '\n') ADVANCE(11); @@ -1906,7 +1906,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { (lookahead == '*') || (lookahead == '\\'))) ADVANCE(147); - ACCEPT_TOKEN(sym_preproc_arg); + ACCEPT_FRAGILE_TOKEN(sym_preproc_arg); case 152: if (lookahead == '\\') ADVANCE(153); @@ -1932,7 +1932,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { (lookahead == '\n') || (lookahead == '\\'))) ADVANCE(156); - ACCEPT_TOKEN(sym_preproc_arg); + ACCEPT_FRAGILE_TOKEN(sym_preproc_arg); case 155: if (lookahead == '\\') ADVANCE(154); @@ -1940,7 +1940,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { (lookahead == '\n') || (lookahead == '\\'))) ADVANCE(156); - ACCEPT_TOKEN(sym_preproc_arg); + ACCEPT_FRAGILE_TOKEN(sym_preproc_arg); case 156: if (lookahead == '\\') ADVANCE(154); @@ -1948,7 +1948,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { (lookahead == '\n') || (lookahead == '\\'))) ADVANCE(156); - ACCEPT_TOKEN(sym_preproc_arg); + ACCEPT_FRAGILE_TOKEN(sym_preproc_arg); case 157: START_TOKEN(); ACCEPT_TOKEN(anon_sym_LF); @@ -2062,7 +2062,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'o') ADVANCE(164); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 164: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2072,7 +2072,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(15); if (lookahead == 'r') ADVANCE(165); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 165: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || diff --git a/spec/fixtures/parsers/cpp.c b/spec/fixtures/parsers/cpp.c index 6ce41029..44708085 100644 --- a/spec/fixtures/parsers/cpp.c +++ b/spec/fixtures/parsers/cpp.c @@ -322,7 +322,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { (lookahead == '_') || ('a' <= lookahead && lookahead <= 'z')) ADVANCE(5); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 6: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -332,7 +332,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'o') ADVANCE(7); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 7: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -342,7 +342,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'n') ADVANCE(8); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 8: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -352,7 +352,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 's') ADVANCE(9); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 9: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -362,7 +362,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 't') ADVANCE(10); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 10: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -372,7 +372,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'e') ADVANCE(11); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 11: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -383,7 +383,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'x') ADVANCE(12); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 12: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -393,7 +393,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'p') ADVANCE(13); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 13: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -403,7 +403,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'r') ADVANCE(14); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 14: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -421,7 +421,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'x') ADVANCE(16); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 16: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -434,7 +434,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(17); if (lookahead == 't') ADVANCE(23); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 17: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -444,7 +444,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'l') ADVANCE(18); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 18: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -454,7 +454,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'i') ADVANCE(19); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 19: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -465,7 +465,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'c') ADVANCE(20); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 20: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -475,7 +475,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'i') ADVANCE(21); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 21: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -485,7 +485,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 't') ADVANCE(22); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 22: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -502,7 +502,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'e') ADVANCE(24); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 24: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -512,7 +512,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'r') ADVANCE(25); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 25: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -522,7 +522,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'n') ADVANCE(26); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 26: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -539,7 +539,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'r') ADVANCE(28); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 28: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -549,7 +549,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'i') ADVANCE(29); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 29: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -559,7 +559,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'e') ADVANCE(30); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 30: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -569,7 +569,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'n') ADVANCE(31); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 31: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -579,7 +579,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'd') ADVANCE(32); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 32: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -596,7 +596,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'n') ADVANCE(34); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 34: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -606,7 +606,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'l') ADVANCE(35); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 35: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -616,7 +616,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'i') ADVANCE(36); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 36: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -626,7 +626,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'n') ADVANCE(37); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 37: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -636,7 +636,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'e') ADVANCE(38); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 38: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -653,7 +653,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'u') ADVANCE(40); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 40: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -663,7 +663,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 't') ADVANCE(41); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 41: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -672,7 +672,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'a') ADVANCE(42); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 42: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -682,7 +682,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'b') ADVANCE(43); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 43: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -692,7 +692,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'l') ADVANCE(44); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 44: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -702,7 +702,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'e') ADVANCE(45); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 45: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -718,7 +718,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'a') ADVANCE(47); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 47: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -728,7 +728,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'm') ADVANCE(48); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 48: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -738,7 +738,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'e') ADVANCE(49); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 49: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -748,7 +748,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 's') ADVANCE(50); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 50: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -758,7 +758,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'p') ADVANCE(51); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 51: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -767,7 +767,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'a') ADVANCE(52); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 52: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -778,7 +778,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'c') ADVANCE(53); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 53: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -788,7 +788,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'e') ADVANCE(54); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 54: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -805,7 +805,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'e') ADVANCE(56); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 56: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -815,7 +815,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'g') ADVANCE(57); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 57: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -825,7 +825,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'i') ADVANCE(58); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 58: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -835,7 +835,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 's') ADVANCE(59); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 59: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -845,7 +845,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 't') ADVANCE(60); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 60: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -855,7 +855,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'e') ADVANCE(61); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 61: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -865,7 +865,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'r') ADVANCE(62); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 62: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -882,7 +882,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 't') ADVANCE(64); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 64: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -891,7 +891,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'a') ADVANCE(65); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 65: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -901,7 +901,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 't') ADVANCE(66); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 66: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -911,7 +911,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'i') ADVANCE(67); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 67: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -922,7 +922,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'c') ADVANCE(68); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 68: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -942,7 +942,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(70); if (lookahead == 'y') ADVANCE(81); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 70: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -952,7 +952,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'r') ADVANCE(71); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 71: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -962,7 +962,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'e') ADVANCE(72); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 72: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -971,7 +971,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'a') ADVANCE(73); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 73: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -981,7 +981,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'd') ADVANCE(74); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 74: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -989,7 +989,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == '_') ADVANCE(75); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 75: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -999,7 +999,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'l') ADVANCE(76); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 76: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1009,7 +1009,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'o') ADVANCE(77); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 77: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1020,7 +1020,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'c') ADVANCE(78); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 78: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1029,7 +1029,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'a') ADVANCE(79); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 79: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1039,7 +1039,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'l') ADVANCE(80); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 80: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1056,7 +1056,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'p') ADVANCE(82); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 82: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1066,7 +1066,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'e') ADVANCE(83); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 83: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1076,7 +1076,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'd') ADVANCE(84); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 84: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1086,7 +1086,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'e') ADVANCE(85); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 85: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1096,7 +1096,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'f') ADVANCE(86); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 86: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1113,7 +1113,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'i') ADVANCE(88); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 88: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1123,7 +1123,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'r') ADVANCE(89); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 89: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1133,7 +1133,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 't') ADVANCE(90); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 90: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1143,7 +1143,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'u') ADVANCE(91); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 91: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1152,7 +1152,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'a') ADVANCE(92); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 92: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1162,7 +1162,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'l') ADVANCE(93); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 93: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2002,7 +2002,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'o') ADVANCE(169); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 169: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2012,7 +2012,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'n') ADVANCE(170); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 170: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2022,7 +2022,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 's') ADVANCE(171); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 171: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2032,7 +2032,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 't') ADVANCE(172); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 172: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2049,7 +2049,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'o') ADVANCE(174); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 174: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2059,7 +2059,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'l') ADVANCE(175); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 175: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2068,7 +2068,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'a') ADVANCE(176); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 176: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2078,7 +2078,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 't') ADVANCE(177); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 177: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2088,7 +2088,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'i') ADVANCE(178); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 178: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2098,7 +2098,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'l') ADVANCE(179); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 179: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2108,7 +2108,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'e') ADVANCE(180); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 180: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2369,7 +2369,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'e') ADVANCE(194); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 194: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2382,7 +2382,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(195); if (lookahead == 'l') ADVANCE(200); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 195: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2391,7 +2391,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'a') ADVANCE(196); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 196: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2401,7 +2401,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'u') ADVANCE(197); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 197: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2411,7 +2411,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'l') ADVANCE(198); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 198: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2421,7 +2421,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 't') ADVANCE(199); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 199: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2438,7 +2438,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'e') ADVANCE(201); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 201: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2448,7 +2448,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 't') ADVANCE(202); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 202: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2458,7 +2458,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'e') ADVANCE(203); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 203: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2960,7 +2960,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'o') ADVANCE(240); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 240: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2970,7 +2970,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'n') ADVANCE(241); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 241: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2980,7 +2980,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 's') ADVANCE(242); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 242: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2990,7 +2990,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 't') ADVANCE(243); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 243: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -3010,7 +3010,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'n') ADVANCE(245); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 245: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -3024,7 +3024,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(246); if (lookahead == 'l') ADVANCE(35); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 246: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -3034,7 +3034,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 't') ADVANCE(247); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 247: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -3044,7 +3044,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'i') ADVANCE(248); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 248: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -3053,7 +3053,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'a') ADVANCE(249); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 249: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -3063,7 +3063,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'l') ADVANCE(250); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 250: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -3073,7 +3073,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'i') ADVANCE(251); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 251: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -3082,7 +3082,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'z') ADVANCE(252); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 252: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -3092,7 +3092,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'e') ADVANCE(253); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 253: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -3102,7 +3102,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'r') ADVANCE(254); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 254: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -3110,7 +3110,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == '_') ADVANCE(255); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 255: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -3120,7 +3120,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'l') ADVANCE(256); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 256: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -3130,7 +3130,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 'i') ADVANCE(257); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 257: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -3140,7 +3140,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 's') ADVANCE(258); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 258: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -3150,7 +3150,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(5); if (lookahead == 't') ADVANCE(259); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 259: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -3170,7 +3170,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(88); if (lookahead == 'o') ADVANCE(174); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case ts_lex_state_error: START_TOKEN(); if (lookahead == 0) diff --git a/spec/fixtures/parsers/golang.c b/spec/fixtures/parsers/golang.c index 74d33e1c..94b62bb8 100644 --- a/spec/fixtures/parsers/golang.c +++ b/spec/fixtures/parsers/golang.c @@ -431,7 +431,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { (lookahead == '_') || ('a' <= lookahead && lookahead <= 'z')) ADVANCE(33); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 34: START_TOKEN(); if (lookahead == 0) @@ -590,7 +590,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'n') ADVANCE(51); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 51: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -600,7 +600,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 't') ADVANCE(52); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 52: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -610,7 +610,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'e') ADVANCE(53); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 53: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -620,7 +620,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'r') ADVANCE(54); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 54: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -630,7 +630,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'f') ADVANCE(55); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 55: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -639,7 +639,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'a') ADVANCE(56); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 56: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -650,7 +650,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'c') ADVANCE(57); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 57: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -660,7 +660,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'e') ADVANCE(58); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 58: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -676,7 +676,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'a') ADVANCE(60); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 60: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -686,7 +686,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'p') ADVANCE(61); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 61: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -703,7 +703,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 't') ADVANCE(63); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 63: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -713,7 +713,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'r') ADVANCE(64); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 64: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -723,7 +723,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'u') ADVANCE(65); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 65: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -734,7 +734,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'c') ADVANCE(66); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 66: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -744,7 +744,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 't') ADVANCE(67); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 67: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1003,7 +1003,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'o') ADVANCE(90); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 90: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1013,7 +1013,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'r') ADVANCE(91); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 91: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1030,7 +1030,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'f') ADVANCE(93); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 93: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1047,7 +1047,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'e') ADVANCE(95); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 95: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1057,7 +1057,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 't') ADVANCE(96); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 96: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1067,7 +1067,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'u') ADVANCE(97); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 97: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1077,7 +1077,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'r') ADVANCE(98); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 98: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1087,7 +1087,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'n') ADVANCE(99); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 99: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1103,7 +1103,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'a') ADVANCE(101); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 101: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1113,7 +1113,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'r') ADVANCE(102); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 102: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1354,7 +1354,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'l') ADVANCE(124); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 124: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1364,7 +1364,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 's') ADVANCE(125); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 125: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -1374,7 +1374,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'e') ADVANCE(126); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 126: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2000,7 +2000,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(90); if (lookahead == 'u') ADVANCE(155); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 155: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2010,7 +2010,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'n') ADVANCE(156); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 156: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2021,7 +2021,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'c') ADVANCE(157); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 157: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2043,7 +2043,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(159); if (lookahead == 'n') ADVANCE(51); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 159: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2053,7 +2053,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'p') ADVANCE(160); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 160: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2063,7 +2063,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'o') ADVANCE(161); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 161: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2073,7 +2073,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'r') ADVANCE(162); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 162: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2083,7 +2083,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 't') ADVANCE(163); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 163: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2099,7 +2099,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'a') ADVANCE(165); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 165: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2110,7 +2110,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'c') ADVANCE(166); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 166: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2120,7 +2120,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'k') ADVANCE(167); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 167: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2129,7 +2129,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'a') ADVANCE(168); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 168: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2139,7 +2139,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'g') ADVANCE(169); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 169: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2149,7 +2149,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'e') ADVANCE(170); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 170: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2168,7 +2168,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(172); if (lookahead == 'e') ADVANCE(95); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 172: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2178,7 +2178,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'n') ADVANCE(173); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 173: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2188,7 +2188,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'g') ADVANCE(174); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 174: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2198,7 +2198,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'e') ADVANCE(175); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 175: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2215,7 +2215,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'y') ADVANCE(177); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 177: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2225,7 +2225,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'p') ADVANCE(178); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 178: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || @@ -2235,7 +2235,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(33); if (lookahead == 'e') ADVANCE(179); - ACCEPT_TOKEN(sym__identifier); + ACCEPT_FRAGILE_TOKEN(sym__identifier); case 179: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'Z') || diff --git a/spec/fixtures/parsers/javascript.c b/spec/fixtures/parsers/javascript.c index 5c710376..1239ff7e 100644 --- a/spec/fixtures/parsers/javascript.c +++ b/spec/fixtures/parsers/javascript.c @@ -437,7 +437,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { (lookahead == '_') || ('a' <= lookahead && lookahead <= 'z')) ADVANCE(9); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 10: if (lookahead == '\'') ADVANCE(6); @@ -509,9 +509,9 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { case 17: if (lookahead == 'g') ADVANCE(18); - ACCEPT_TOKEN(sym_comment); + ACCEPT_FRAGILE_TOKEN(sym_comment); case 18: - ACCEPT_TOKEN(sym_regex); + ACCEPT_FRAGILE_TOKEN(sym_regex); case 19: if (lookahead == '*') ADVANCE(16); @@ -540,7 +540,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { (lookahead == '\\') || (lookahead == 'g'))) ADVANCE(15); - ACCEPT_TOKEN(sym_regex); + ACCEPT_FRAGILE_TOKEN(sym_regex); case 21: if (lookahead == '*') ADVANCE(22); @@ -550,7 +550,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { (lookahead == '*') || (lookahead == 'g'))) ADVANCE(24); - ACCEPT_TOKEN(sym_regex); + ACCEPT_FRAGILE_TOKEN(sym_regex); case 22: if (lookahead == '/') ADVANCE(23); @@ -559,7 +559,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(24); LEX_ERROR(); case 23: - ACCEPT_TOKEN(sym_comment); + ACCEPT_FRAGILE_TOKEN(sym_comment); case 24: if (lookahead == '*') ADVANCE(22); @@ -573,7 +573,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { if (!((lookahead == 0) || (lookahead == '*'))) ADVANCE(24); - ACCEPT_TOKEN(sym_regex); + ACCEPT_FRAGILE_TOKEN(sym_regex); case 26: if (lookahead == '*') ADVANCE(16); @@ -586,7 +586,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { (lookahead == '/') || (lookahead == '\\'))) ADVANCE(15); - ACCEPT_TOKEN(sym_regex); + ACCEPT_FRAGILE_TOKEN(sym_regex); case 27: if (lookahead == 'g') ADVANCE(28); @@ -594,17 +594,17 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { (lookahead == '\n') || (lookahead == 'g'))) ADVANCE(29); - ACCEPT_TOKEN(sym_regex); + ACCEPT_FRAGILE_TOKEN(sym_regex); case 28: if (!((lookahead == 0) || (lookahead == '\n'))) ADVANCE(29); - ACCEPT_TOKEN(sym_regex); + ACCEPT_FRAGILE_TOKEN(sym_regex); case 29: if (!((lookahead == 0) || (lookahead == '\n'))) ADVANCE(29); - ACCEPT_TOKEN(sym_comment); + ACCEPT_FRAGILE_TOKEN(sym_comment); case 30: if (lookahead == '/') ADVANCE(31); @@ -627,11 +627,11 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { (lookahead == '\\') || (lookahead == 'g'))) ADVANCE(34); - ACCEPT_TOKEN(sym_regex); + ACCEPT_FRAGILE_TOKEN(sym_regex); case 32: if (lookahead == 'g') ADVANCE(18); - ACCEPT_TOKEN(sym_regex); + ACCEPT_FRAGILE_TOKEN(sym_regex); case 33: if (lookahead == '/') ADVANCE(32); @@ -641,7 +641,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { (lookahead == '/') || (lookahead == '\\'))) ADVANCE(34); - ACCEPT_TOKEN(sym_regex); + ACCEPT_FRAGILE_TOKEN(sym_regex); case 34: if (lookahead == '/') ADVANCE(32); @@ -678,7 +678,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'r') ADVANCE(40); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 40: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -689,7 +689,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'e') ADVANCE(41); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 41: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -699,7 +699,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'a') ADVANCE(42); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 42: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -710,7 +710,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'k') ADVANCE(43); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 43: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -729,7 +729,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'e') ADVANCE(45); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 45: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -740,7 +740,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'l') ADVANCE(46); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 46: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -751,7 +751,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'e') ADVANCE(47); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 47: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -762,7 +762,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 't') ADVANCE(48); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 48: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -773,7 +773,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'e') ADVANCE(49); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 49: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -797,7 +797,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(55); if (lookahead == 'u') ADVANCE(57); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 51: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -808,7 +808,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'l') ADVANCE(52); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 52: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -819,7 +819,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 's') ADVANCE(53); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 53: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -830,7 +830,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'e') ADVANCE(54); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 54: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -849,7 +849,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'r') ADVANCE(56); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 56: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -868,7 +868,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'n') ADVANCE(58); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 58: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -880,7 +880,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'c') ADVANCE(59); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 59: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -891,7 +891,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 't') ADVANCE(60); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 60: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -902,7 +902,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'i') ADVANCE(61); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 61: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -913,7 +913,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'o') ADVANCE(62); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 62: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -924,7 +924,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'n') ADVANCE(63); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 63: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -943,7 +943,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'f') ADVANCE(65); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 65: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -965,7 +965,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(67); if (lookahead == 'u') ADVANCE(69); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 67: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -976,7 +976,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'w') ADVANCE(68); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 68: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -995,7 +995,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'l') ADVANCE(70); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 70: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1006,7 +1006,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'l') ADVANCE(71); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 71: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1025,7 +1025,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'e') ADVANCE(73); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 73: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1036,7 +1036,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 't') ADVANCE(74); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 74: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1047,7 +1047,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'u') ADVANCE(75); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 75: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1058,7 +1058,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'r') ADVANCE(76); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 76: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1069,7 +1069,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'n') ADVANCE(77); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 77: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1088,7 +1088,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'w') ADVANCE(79); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 79: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1099,7 +1099,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'i') ADVANCE(80); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 80: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1110,7 +1110,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 't') ADVANCE(81); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 81: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1122,7 +1122,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'c') ADVANCE(82); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 82: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1133,7 +1133,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'h') ADVANCE(83); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 83: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1158,7 +1158,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(89); if (lookahead == 'y') ADVANCE(93); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 85: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1169,7 +1169,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'r') ADVANCE(86); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 86: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1180,7 +1180,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'o') ADVANCE(87); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 87: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1191,7 +1191,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'w') ADVANCE(88); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 88: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1213,7 +1213,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(90); if (lookahead == 'y') ADVANCE(92); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 90: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1224,7 +1224,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'e') ADVANCE(91); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 91: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1251,7 +1251,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'p') ADVANCE(94); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 94: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1262,7 +1262,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'e') ADVANCE(95); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 95: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1273,7 +1273,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'o') ADVANCE(96); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 96: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1284,7 +1284,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'f') ADVANCE(97); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 97: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1303,7 +1303,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'n') ADVANCE(99); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 99: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1314,7 +1314,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'd') ADVANCE(100); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 100: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1325,7 +1325,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'e') ADVANCE(101); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 101: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1336,7 +1336,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'f') ADVANCE(102); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 102: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1347,7 +1347,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'i') ADVANCE(103); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 103: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1358,7 +1358,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'n') ADVANCE(104); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 104: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1369,7 +1369,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'e') ADVANCE(105); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 105: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1380,7 +1380,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'd') ADVANCE(106); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 106: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1398,7 +1398,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'a') ADVANCE(108); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 108: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1409,7 +1409,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'r') ADVANCE(109); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 109: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1428,7 +1428,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'h') ADVANCE(111); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 111: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1439,7 +1439,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'i') ADVANCE(112); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 112: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1450,7 +1450,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'l') ADVANCE(113); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 113: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1461,7 +1461,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'e') ADVANCE(114); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 114: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1806,7 +1806,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(51); if (lookahead == 'u') ADVANCE(57); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 161: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1820,7 +1820,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(162); if (lookahead == 'y') ADVANCE(93); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 162: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -1831,7 +1831,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'u') ADVANCE(90); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 163: START_TOKEN(); if ((lookahead == '\t') || @@ -2589,7 +2589,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(65); if (lookahead == 'n') ADVANCE(195); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 195: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -2611,7 +2611,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 't') ADVANCE(197); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 197: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -2621,7 +2621,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'a') ADVANCE(198); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 198: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -2632,7 +2632,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'n') ADVANCE(199); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 199: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -2644,7 +2644,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'c') ADVANCE(200); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 200: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -2655,7 +2655,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'e') ADVANCE(201); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 201: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -2666,7 +2666,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'o') ADVANCE(202); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 202: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -2677,7 +2677,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'f') ADVANCE(203); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 203: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -2820,7 +2820,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'a') ADVANCE(207); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 207: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -2831,7 +2831,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 't') ADVANCE(208); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 208: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -2843,7 +2843,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'c') ADVANCE(209); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 209: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -2854,7 +2854,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'h') ADVANCE(210); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 210: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -2881,7 +2881,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(55); if (lookahead == 'u') ADVANCE(57); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 212: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -2892,7 +2892,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'n') ADVANCE(213); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 213: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -2902,7 +2902,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'a') ADVANCE(214); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 214: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -2913,7 +2913,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'l') ADVANCE(215); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 215: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -2924,7 +2924,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'l') ADVANCE(216); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 216: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -2935,7 +2935,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'y') ADVANCE(217); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 217: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -3212,7 +3212,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'n') ADVANCE(228); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 228: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -3454,7 +3454,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'a') ADVANCE(246); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 246: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -3465,7 +3465,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 's') ADVANCE(247); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 247: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -3476,7 +3476,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'e') ADVANCE(248); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 248: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -3495,7 +3495,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'e') ADVANCE(250); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 250: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -3509,7 +3509,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(251); if (lookahead == 'l') ADVANCE(46); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 251: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -3519,7 +3519,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'a') ADVANCE(252); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 252: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -3530,7 +3530,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'u') ADVANCE(253); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 253: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -3541,7 +3541,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'l') ADVANCE(254); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 254: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -3552,7 +3552,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 't') ADVANCE(255); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 255: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -3780,7 +3780,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'a') ADVANCE(260); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 260: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -3793,7 +3793,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(247); if (lookahead == 't') ADVANCE(208); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 261: START_TOKEN(); if ((lookahead == '\t') || @@ -3929,7 +3929,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'l') ADVANCE(264); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 264: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -3940,7 +3940,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 's') ADVANCE(265); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 265: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || @@ -3951,7 +3951,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(9); if (lookahead == 'e') ADVANCE(266); - ACCEPT_TOKEN(sym_identifier); + ACCEPT_FRAGILE_TOKEN(sym_identifier); case 266: if ((lookahead == '$') || ('0' <= lookahead && lookahead <= '9') || diff --git a/src/compiler/build_tables/build_lex_table.cc b/src/compiler/build_tables/build_lex_table.cc index 26681c2f..79a05522 100644 --- a/src/compiler/build_tables/build_lex_table.cc +++ b/src/compiler/build_tables/build_lex_table.cc @@ -32,7 +32,7 @@ using rules::Symbol; class LexTableBuilder { const LexicalGrammar lex_grammar; - const LexConflictManager conflict_manager; + LexConflictManager conflict_manager; ParseTable *parse_table; unordered_map lex_state_ids; LexTable lex_table; @@ -59,6 +59,8 @@ class LexTableBuilder { build_lex_item_set(parse_table->all_symbols(), true); populate_lex_state(error_item_set, LexTable::ERROR_STATE_ID); + mark_fragile_tokens(); + return lex_table; } @@ -153,6 +155,17 @@ class LexTableBuilder { if (item.is_token_start()) lex_table.state(state_id).is_token_start = true; } + + void mark_fragile_tokens() { + for (LexState &state : lex_table.states) + if (state.default_action.type == LexActionTypeAccept) + if (has_fragile_token(state.default_action.symbol)) + state.default_action.type = LexActionTypeAcceptFragile; + } + + bool has_fragile_token(const Symbol &symbol) { + return conflict_manager.fragile_tokens.find(symbol) != conflict_manager.fragile_tokens.end(); + } }; LexTable build_lex_table(ParseTable *table, const LexicalGrammar &grammar) { diff --git a/src/compiler/build_tables/lex_conflict_manager.cc b/src/compiler/build_tables/lex_conflict_manager.cc index e1702c79..dbc8b42a 100644 --- a/src/compiler/build_tables/lex_conflict_manager.cc +++ b/src/compiler/build_tables/lex_conflict_manager.cc @@ -10,7 +10,7 @@ LexConflictManager::LexConflictManager(const LexicalGrammar &grammar) : grammar(grammar) {} bool LexConflictManager::resolve(const LexAction &new_action, - const LexAction &old_action) const { + const LexAction &old_action) { if (new_action.type < old_action.type) return !resolve(old_action, new_action); @@ -24,16 +24,27 @@ bool LexConflictManager::resolve(const LexAction &new_action, switch (new_action.type) { case LexActionTypeAccept: { int new_precedence = new_action.precedence_range.min; + + bool result; if (new_precedence > old_precedence) - return true; + result = true; else if (new_precedence < old_precedence) - return false; + result = false; else if (new_action.is_string && !old_action.is_string) - return true; + result = true; else if (old_action.is_string && !new_action.is_string) - return false; + result = false; + else if (new_action.symbol.index < old_action.symbol.index) + result = true; else - return new_action.symbol.index < old_action.symbol.index; + result = false; + + if (result) + fragile_tokens.insert(old_action.symbol); + else + fragile_tokens.insert(new_action.symbol); + + return result; } case LexActionTypeAdvance: diff --git a/src/compiler/build_tables/lex_conflict_manager.h b/src/compiler/build_tables/lex_conflict_manager.h index 50ab3f70..a73048b4 100644 --- a/src/compiler/build_tables/lex_conflict_manager.h +++ b/src/compiler/build_tables/lex_conflict_manager.h @@ -1,8 +1,10 @@ #ifndef COMPILER_BUILD_TABLES_LEX_CONFLICT_MANAGER_H_ #define COMPILER_BUILD_TABLES_LEX_CONFLICT_MANAGER_H_ +#include #include "tree_sitter/compiler.h" #include "compiler/lexical_grammar.h" +#include "compiler/rules/symbol.h" namespace tree_sitter { @@ -15,7 +17,9 @@ class LexConflictManager { public: explicit LexConflictManager(const LexicalGrammar &); - bool resolve(const LexAction &, const LexAction &) const; + bool resolve(const LexAction &, const LexAction &); + + std::set fragile_tokens; }; } // namespace build_tables diff --git a/src/compiler/generate_code/c_code.cc b/src/compiler/generate_code/c_code.cc index e81e63a0..3c4e4638 100644 --- a/src/compiler/generate_code/c_code.cc +++ b/src/compiler/generate_code/c_code.cc @@ -302,6 +302,9 @@ class CCodeGenerator { case LexActionTypeAccept: line("ACCEPT_TOKEN(" + symbol_id(action.symbol) + ");"); break; + case LexActionTypeAcceptFragile: + line("ACCEPT_FRAGILE_TOKEN(" + symbol_id(action.symbol) + ");"); + break; case LexActionTypeError: line("LEX_ERROR();"); break; diff --git a/src/compiler/lex_table.h b/src/compiler/lex_table.h index b6288a07..e96be64b 100644 --- a/src/compiler/lex_table.h +++ b/src/compiler/lex_table.h @@ -14,6 +14,7 @@ namespace tree_sitter { typedef enum { LexActionTypeError, LexActionTypeAccept, + LexActionTypeAcceptFragile, LexActionTypeAdvance } LexActionType; diff --git a/src/runtime/lexer.c b/src/runtime/lexer.c index ec9e1d50..d2b69fc0 100644 --- a/src/runtime/lexer.c +++ b/src/runtime/lexer.c @@ -85,20 +85,24 @@ static bool ts_lexer__advance(TSLexer *self, TSStateId state) { static TSTree *ts_lexer__accept(TSLexer *self, TSSymbol symbol, TSSymbolMetadata metadata, - const char *symbol_name) { + const char *symbol_name, bool fragile) { TSLength size = ts_length_sub(self->current_position, self->token_start_position); TSLength padding = ts_length_sub(self->token_start_position, self->token_end_position); self->token_end_position = self->current_position; + TSTree *result; if (symbol == ts_builtin_sym_error) { LOG("error_char"); - return ts_tree_make_error(size, padding, self->lookahead); + result = ts_tree_make_error(size, padding, self->lookahead); } else { LOG("accept_token sym:%s", symbol_name); - return ts_tree_make_leaf(symbol, padding, size, metadata); + result = ts_tree_make_leaf(symbol, padding, size, metadata); } + + result->options.fragile_left = fragile; + return result; } /*