Fixed some test failures due to CRLF line endings

This commit is contained in:
Andrew Helwer 2021-09-24 22:42:38 -04:00
parent 2bee7c9b75
commit 0dc1bd806f
4 changed files with 19 additions and 7 deletions

View file

@ -35,7 +35,7 @@ bool tree_sitter_external_and_internal_tokens_external_scanner_scan(
) {
// If a line-break is a valid lookahead token, only skip spaces.
if (whitelist[LINE_BREAK]) {
while (lexer->lookahead == ' ') {
while (lexer->lookahead == ' ' || lexer->lookahead == '\r') {
lexer->advance(lexer, true);
}
@ -49,7 +49,7 @@ bool tree_sitter_external_and_internal_tokens_external_scanner_scan(
// If a line-break is not a valid lookahead token, skip line breaks as well
// as spaces.
if (whitelist[STRING]) {
while (lexer->lookahead == ' ' || lexer->lookahead == '\n') {
while (lexer->lookahead == ' ' || lexer->lookahead == '\r' || lexer->lookahead == '\n') {
lexer->advance(lexer, true);
}

View file

@ -46,7 +46,8 @@ bool tree_sitter_external_tokens_external_scanner_scan(
if (whitelist[percent_string]) {
while (lexer->lookahead == ' ' ||
lexer->lookahead == '\t' ||
lexer->lookahead == '\n') {
lexer->lookahead == '\n' ||
lexer->lookahead == '\r') {
lexer->advance(lexer, true);
}

View file

@ -23,7 +23,7 @@ void tree_sitter_inverted_external_token_external_scanner_deserialize(
bool tree_sitter_inverted_external_token_external_scanner_scan(
void *payload, TSLexer *lexer, const bool *whitelist) {
while (lexer->lookahead == ' ') {
while (lexer->lookahead == ' ' || lexer->lookahead == '\r') {
lexer->advance(lexer, true);
}
@ -34,7 +34,7 @@ bool tree_sitter_inverted_external_token_external_scanner_scan(
lexer->mark_end(lexer);
// Skip whitespace *after* having marked the end.
while (lexer->lookahead == ' ' || lexer->lookahead == '\n') {
while (lexer->lookahead == ' ' || lexer->lookahead == '\n' || lexer->lookahead == '\r') {
lexer->advance(lexer, true);
}