Merge pull request #1412 from ahelwer/windows-test-failures

Fixed some test failures due to CRLF line endings

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ env:
   CARGO_INCREMENTAL: 0
 
 jobs:
-  tests:
+  unix-tests:
     name: Unix tests
     runs-on: ${{ matrix.os }}
     strategy:
@@ -70,7 +70,9 @@ jobs:
         run: script/build-wasm
 
       - name: Build CLI
-        run: RUSTFLAGS="-D warnings" cargo build --release
+        run: |
+          RUSTFLAGS="-D warnings"
+          cargo build --release
 
       - name: Set up fixture parsers
         run: |
@@ -105,3 +107,43 @@ jobs:
             lib/binding_web/tree-sitter.wasm
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  windows-tests:
+    name: Windows tests
+    runs-on: windows-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v2
+
+      - name: Cache artifacts
+        id: cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }}
+
+      - name: Install rust
+        if: steps.cache.outputs.cache-hit != 'true'
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          profile: minimal
+
+      - name: Check Rust code formatting
+        run: cargo fmt -- --check
+
+      - name: Build CLI
+        run: |
+          $env:RUSTFLAGS="-D warnings"
+          cargo build --release
+
+      - name: Set up fixture parsers
+        run: |
+          script/fetch-fixtures.cmd
+          script/generate-fixtures.cmd
+
+      - name: Run main tests
+        run: script/test

cli/src/tests/corpus_test.rs

@@ -229,12 +229,15 @@ fn test_feature_corpus_files() {
 
             eprintln!("test language: {:?}", language_name);
 
-            let expected_message = fs::read_to_string(&error_message_path).unwrap();
+            let expected_message = fs::read_to_string(&error_message_path)
+                .unwrap()
+                .replace("\r\n", "\n");
             if let Err(e) = generate_result {
-                if e.to_string() != expected_message {
+                let actual_message = e.to_string().replace("\r\n", "\n");
+                if expected_message != actual_message {
                     eprintln!(
                         "Unexpected error message.\n\nExpected:\n\n{}\nActual:\n\n{}\n",
-                        expected_message, e
+                        expected_message, actual_message
                     );
                     failure_count += 1;
                 }

cli/src/util.rs

@@ -1,10 +1,12 @@
-use anyhow::{anyhow, Context, Result};
+use anyhow::Result;
 use std::io;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 use std::thread;
 use tree_sitter::Parser;
 
+#[cfg(unix)]
+use anyhow::{anyhow, Context};
 #[cfg(unix)]
 use std::path::PathBuf;
 #[cfg(unix)]

test/fixtures/test_grammars/external_and_internal_tokens/scanner.c

@@ -35,7 +35,7 @@ bool tree_sitter_external_and_internal_tokens_external_scanner_scan(
 ) {
   // If a line-break is a valid lookahead token, only skip spaces.
   if (whitelist[LINE_BREAK]) {
-    while (lexer->lookahead == ' ') {
+    while (lexer->lookahead == ' ' || lexer->lookahead == '\r') {
       lexer->advance(lexer, true);
     }
 
@@ -49,7 +49,7 @@ bool tree_sitter_external_and_internal_tokens_external_scanner_scan(
   // If a line-break is not a valid lookahead token, skip line breaks as well
   // as spaces.
   if (whitelist[STRING]) {
-    while (lexer->lookahead == ' ' || lexer->lookahead == '\n') {
+    while (lexer->lookahead == ' ' || lexer->lookahead == '\r' || lexer->lookahead == '\n') {
       lexer->advance(lexer, true);
     }
 

test/fixtures/test_grammars/external_tokens/scanner.c

@@ -46,7 +46,8 @@ bool tree_sitter_external_tokens_external_scanner_scan(
   if (whitelist[percent_string]) {
     while (lexer->lookahead == ' ' ||
            lexer->lookahead == '\t' ||
-           lexer->lookahead == '\n') {
+           lexer->lookahead == '\n' ||
+           lexer->lookahead == '\r') {
       lexer->advance(lexer, true);
     }
 

test/fixtures/test_grammars/inverted_external_token/scanner.c

@@ -23,7 +23,7 @@ void tree_sitter_inverted_external_token_external_scanner_deserialize(
 
 bool tree_sitter_inverted_external_token_external_scanner_scan(
   void *payload, TSLexer *lexer, const bool *whitelist) {
-  while (lexer->lookahead == ' ') {
+  while (lexer->lookahead == ' ' || lexer->lookahead == '\r') {
     lexer->advance(lexer, true);
   }
 
@@ -34,7 +34,7 @@ bool tree_sitter_inverted_external_token_external_scanner_scan(
     lexer->mark_end(lexer);
 
     // Skip whitespace *after* having marked the end.
-    while (lexer->lookahead == ' ' || lexer->lookahead == '\n') {
+    while (lexer->lookahead == ' ' || lexer->lookahead == '\n' || lexer->lookahead == '\r') {
       lexer->advance(lexer, true);
     }