diff --git a/test/fixtures/test_grammars/external_and_internal_anonymous_tokens/scanner.c b/test/fixtures/test_grammars/external_and_internal_anonymous_tokens/scanner.c index eb999d15..644bbd0a 100644 --- a/test/fixtures/test_grammars/external_and_internal_anonymous_tokens/scanner.c +++ b/test/fixtures/test_grammars/external_and_internal_anonymous_tokens/scanner.c @@ -1,4 +1,9 @@ -#include "../external_and_internal_tokens/scanner.c" +#include + +enum { + STRING, + LINE_BREAK +}; void *tree_sitter_external_and_internal_anonymous_tokens_external_scanner_create() { return NULL; @@ -28,9 +33,38 @@ bool tree_sitter_external_and_internal_anonymous_tokens_external_scanner_scan( TSLexer *lexer, const bool *whitelist ) { - return tree_sitter_external_and_internal_tokens_external_scanner_scan( - payload, - lexer, - whitelist - ); + // If a line-break is a valid lookahead token, only skip spaces. + if (whitelist[LINE_BREAK]) { + while (lexer->lookahead == ' ' || lexer->lookahead == '\r') { + lexer->advance(lexer, true); + } + + if (lexer->lookahead == '\n') { + lexer->advance(lexer, false); + lexer->result_symbol = LINE_BREAK; + return true; + } + } + + // If a line-break is not a valid lookahead token, skip line breaks as well + // as spaces. + if (whitelist[STRING]) { + while (lexer->lookahead == ' ' || lexer->lookahead == '\r' || lexer->lookahead == '\n') { + lexer->advance(lexer, true); + } + + if (lexer->lookahead == '\'') { + lexer->advance(lexer, false); + + while (lexer->lookahead != '\'') { + lexer->advance(lexer, false); + } + + lexer->advance(lexer, false); + lexer->result_symbol = STRING; + return true; + } + } + + return false; }