From 0dc1bd806f51cdfce4e7a9ddaa2c0c174fd8c2ad Mon Sep 17 00:00:00 2001 From: Andrew Helwer Date: Fri, 24 Sep 2021 22:42:38 -0400 Subject: [PATCH 01/11] Fixed some test failures due to CRLF line endings --- cli/src/tests/corpus_test.rs | 15 +++++++++++++-- .../external_and_internal_tokens/scanner.c | 4 ++-- .../test_grammars/external_tokens/scanner.c | 3 ++- .../inverted_external_token/scanner.c | 4 ++-- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/cli/src/tests/corpus_test.rs b/cli/src/tests/corpus_test.rs index bfb2b52c..87df8b28 100644 --- a/cli/src/tests/corpus_test.rs +++ b/cli/src/tests/corpus_test.rs @@ -231,10 +231,21 @@ fn test_feature_corpus_files() { let expected_message = fs::read_to_string(&error_message_path).unwrap(); if let Err(e) = generate_result { - if e.to_string() != expected_message { + let actual_message = e.to_string(); + let first_diff = expected_message + .lines() + .zip(actual_message.lines()) + .enumerate() + .filter(|(_, (expected, actual))| expected != actual) + .next(); + if let Some((line_number,(expected_line, actual_line))) = first_diff { eprintln!( "Unexpected error message.\n\nExpected:\n\n{}\nActual:\n\n{}\n", - expected_message, e + expected_message, actual_message + ); + eprintln!( + "First difference on line {}; expected:\n{}\nActual:\n{}\n", + line_number, expected_line, actual_line ); failure_count += 1; } diff --git a/test/fixtures/test_grammars/external_and_internal_tokens/scanner.c b/test/fixtures/test_grammars/external_and_internal_tokens/scanner.c index 837a1d1a..43a4bc9d 100644 --- a/test/fixtures/test_grammars/external_and_internal_tokens/scanner.c +++ b/test/fixtures/test_grammars/external_and_internal_tokens/scanner.c @@ -35,7 +35,7 @@ bool tree_sitter_external_and_internal_tokens_external_scanner_scan( ) { // If a line-break is a valid lookahead token, only skip spaces. if (whitelist[LINE_BREAK]) { - while (lexer->lookahead == ' ') { + while (lexer->lookahead == ' ' || lexer->lookahead == '\r') { lexer->advance(lexer, true); } @@ -49,7 +49,7 @@ bool tree_sitter_external_and_internal_tokens_external_scanner_scan( // If a line-break is not a valid lookahead token, skip line breaks as well // as spaces. if (whitelist[STRING]) { - while (lexer->lookahead == ' ' || lexer->lookahead == '\n') { + while (lexer->lookahead == ' ' || lexer->lookahead == '\r' || lexer->lookahead == '\n') { lexer->advance(lexer, true); } diff --git a/test/fixtures/test_grammars/external_tokens/scanner.c b/test/fixtures/test_grammars/external_tokens/scanner.c index 3fea84b3..c187f41b 100644 --- a/test/fixtures/test_grammars/external_tokens/scanner.c +++ b/test/fixtures/test_grammars/external_tokens/scanner.c @@ -46,7 +46,8 @@ bool tree_sitter_external_tokens_external_scanner_scan( if (whitelist[percent_string]) { while (lexer->lookahead == ' ' || lexer->lookahead == '\t' || - lexer->lookahead == '\n') { + lexer->lookahead == '\n' || + lexer->lookahead == '\r') { lexer->advance(lexer, true); } diff --git a/test/fixtures/test_grammars/inverted_external_token/scanner.c b/test/fixtures/test_grammars/inverted_external_token/scanner.c index c2cd6ce9..260994c5 100644 --- a/test/fixtures/test_grammars/inverted_external_token/scanner.c +++ b/test/fixtures/test_grammars/inverted_external_token/scanner.c @@ -23,7 +23,7 @@ void tree_sitter_inverted_external_token_external_scanner_deserialize( bool tree_sitter_inverted_external_token_external_scanner_scan( void *payload, TSLexer *lexer, const bool *whitelist) { - while (lexer->lookahead == ' ') { + while (lexer->lookahead == ' ' || lexer->lookahead == '\r') { lexer->advance(lexer, true); } @@ -34,7 +34,7 @@ bool tree_sitter_inverted_external_token_external_scanner_scan( lexer->mark_end(lexer); // Skip whitespace *after* having marked the end. - while (lexer->lookahead == ' ' || lexer->lookahead == '\n') { + while (lexer->lookahead == ' ' || lexer->lookahead == '\n' || lexer->lookahead == '\r') { lexer->advance(lexer, true); } From 0c3fb6e6a26fdc0b4fa8529edb25e34214643c33 Mon Sep 17 00:00:00 2001 From: Andrew Helwer Date: Fri, 24 Sep 2021 22:49:25 -0400 Subject: [PATCH 02/11] Fixed cargo fmt issues --- cli/src/tests/corpus_test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/tests/corpus_test.rs b/cli/src/tests/corpus_test.rs index 87df8b28..4692382d 100644 --- a/cli/src/tests/corpus_test.rs +++ b/cli/src/tests/corpus_test.rs @@ -238,7 +238,7 @@ fn test_feature_corpus_files() { .enumerate() .filter(|(_, (expected, actual))| expected != actual) .next(); - if let Some((line_number,(expected_line, actual_line))) = first_diff { + if let Some((line_number, (expected_line, actual_line))) = first_diff { eprintln!( "Unexpected error message.\n\nExpected:\n\n{}\nActual:\n\n{}\n", expected_message, actual_message From 3d1f760e31ffc9a825e562a10a5be817109fb2f2 Mon Sep 17 00:00:00 2001 From: Andrew Helwer Date: Tue, 28 Sep 2021 18:27:25 -0400 Subject: [PATCH 03/11] Simplified corpus test output comparisons --- cli/src/tests/corpus_test.rs | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/cli/src/tests/corpus_test.rs b/cli/src/tests/corpus_test.rs index 4692382d..7533601f 100644 --- a/cli/src/tests/corpus_test.rs +++ b/cli/src/tests/corpus_test.rs @@ -229,24 +229,14 @@ fn test_feature_corpus_files() { eprintln!("test language: {:?}", language_name); - let expected_message = fs::read_to_string(&error_message_path).unwrap(); + let expected_message = fs::read_to_string(&error_message_path).unwrap().replace("\r\n", "\n"); if let Err(e) = generate_result { - let actual_message = e.to_string(); - let first_diff = expected_message - .lines() - .zip(actual_message.lines()) - .enumerate() - .filter(|(_, (expected, actual))| expected != actual) - .next(); - if let Some((line_number, (expected_line, actual_line))) = first_diff { + let actual_message = e.to_string().replace("\r\n", "\n"); + if expected_message != actual_message { eprintln!( "Unexpected error message.\n\nExpected:\n\n{}\nActual:\n\n{}\n", expected_message, actual_message ); - eprintln!( - "First difference on line {}; expected:\n{}\nActual:\n{}\n", - line_number, expected_line, actual_line - ); failure_count += 1; } } else { From 02253866d72c71ebc0f4e98db5a47b7c6aa97130 Mon Sep 17 00:00:00 2001 From: Andrew Helwer Date: Tue, 28 Sep 2021 18:28:56 -0400 Subject: [PATCH 04/11] Added windows to github CI workflow --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bfcd9f8c..44d194ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,7 @@ jobs: os: - macos-latest - ubuntu-latest + - windows-latest steps: - name: Checkout repo uses: actions/checkout@v2 From e8e4dafa92ace2421a41b902b5e9d9e9ddf894e9 Mon Sep 17 00:00:00 2001 From: Andrew Helwer Date: Tue, 28 Sep 2021 18:31:04 -0400 Subject: [PATCH 05/11] Fixed cargo fmt suggestions --- cli/src/tests/corpus_test.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cli/src/tests/corpus_test.rs b/cli/src/tests/corpus_test.rs index 7533601f..3a861b5c 100644 --- a/cli/src/tests/corpus_test.rs +++ b/cli/src/tests/corpus_test.rs @@ -229,7 +229,9 @@ fn test_feature_corpus_files() { eprintln!("test language: {:?}", language_name); - let expected_message = fs::read_to_string(&error_message_path).unwrap().replace("\r\n", "\n"); + let expected_message = fs::read_to_string(&error_message_path) + .unwrap() + .replace("\r\n", "\n"); if let Err(e) = generate_result { let actual_message = e.to_string().replace("\r\n", "\n"); if expected_message != actual_message { From eeff48d79ffaf447c23b8b7621cf379f6e8ce0b9 Mon Sep 17 00:00:00 2001 From: Andrew Helwer Date: Tue, 28 Sep 2021 18:35:40 -0400 Subject: [PATCH 06/11] Specify bash shell --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44d194ff..ff757164 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,7 @@ jobs: echo PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH" >> $GITHUB_ENV - name: Read Emscripten version + shell: bash run: | printf 'EMSCRIPTEN_VERSION=%s\n' "$(cat cli/emscripten-version)" >> $GITHUB_ENV From 652235a6b88070c9782942c4c9e712a043118daa Mon Sep 17 00:00:00 2001 From: Andrew Helwer Date: Tue, 28 Sep 2021 18:45:27 -0400 Subject: [PATCH 07/11] Moved windows build out into separate job --- .github/workflows/ci.yml | 42 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff757164..70840371 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ env: CARGO_INCREMENTAL: 0 jobs: - tests: + unix-tests: name: Unix tests runs-on: ${{ matrix.os }} strategy: @@ -24,7 +24,6 @@ jobs: os: - macos-latest - ubuntu-latest - - windows-latest steps: - name: Checkout repo uses: actions/checkout@v2 @@ -36,7 +35,6 @@ jobs: echo PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH" >> $GITHUB_ENV - name: Read Emscripten version - shell: bash run: | printf 'EMSCRIPTEN_VERSION=%s\n' "$(cat cli/emscripten-version)" >> $GITHUB_ENV @@ -107,3 +105,41 @@ jobs: lib/binding_web/tree-sitter.wasm env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + windows-tests: + name: Windows tests + runs-on: windows-latest + steps: + - name: Checkout repo + uses: actions/checkout@v2 + + - name: Cache artifacts + id: cache + uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }} + + - name: Install rust + if: steps.cache.outputs.cache-hit != 'true' + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + profile: minimal + + - name: Check Rust code formatting + run: cargo fmt -- --check + + - name: Build CLI + run: RUSTFLAGS="-D warnings" cargo build --release + + - name: Set up fixture parsers + run: | + script/fetch-fixtures.cmd + script/generate-fixtures.cmd + + - name: Run main tests + run: script/test From 8eef790a610313ec41ef5c38f41aca0da19ac806 Mon Sep 17 00:00:00 2001 From: Andrew Helwer Date: Tue, 28 Sep 2021 18:51:09 -0400 Subject: [PATCH 08/11] Build rust with warnings=errors --- .github/workflows/ci.yml | 8 ++++++-- cli/src/util.rs | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 70840371..28e5979d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,7 +70,9 @@ jobs: run: script/build-wasm - name: Build CLI - run: RUSTFLAGS="-D warnings" cargo build --release + run: | + RUSTFLAGS="-D warnings" + cargo build --release - name: Set up fixture parsers run: | @@ -134,7 +136,9 @@ jobs: run: cargo fmt -- --check - name: Build CLI - run: RUSTFLAGS="-D warnings" cargo build --release + run: | + $env:RUSTFLAGS="-D warnings" + cargo build --release - name: Set up fixture parsers run: | diff --git a/cli/src/util.rs b/cli/src/util.rs index 396702c8..db69bbd6 100644 --- a/cli/src/util.rs +++ b/cli/src/util.rs @@ -1,4 +1,4 @@ -use anyhow::{anyhow, Context, Result}; +use anyhow::{Result}; use std::io; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; From 815c281c488c32d3ce70a949b1b78ebc7e3eaa39 Mon Sep 17 00:00:00 2001 From: Andrew Helwer Date: Tue, 28 Sep 2021 18:52:12 -0400 Subject: [PATCH 09/11] Fix ci.yml format --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28e5979d..ac588053 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -137,8 +137,8 @@ jobs: - name: Build CLI run: | - $env:RUSTFLAGS="-D warnings" - cargo build --release + $env:RUSTFLAGS="-D warnings" + cargo build --release - name: Set up fixture parsers run: | From 3b5f9cb64288f12d66b1ef8a41548dedeef5fcbd Mon Sep 17 00:00:00 2001 From: Andrew Helwer Date: Tue, 28 Sep 2021 18:53:24 -0400 Subject: [PATCH 10/11] Fixed cargo fmt --- cli/src/util.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/util.rs b/cli/src/util.rs index db69bbd6..82341492 100644 --- a/cli/src/util.rs +++ b/cli/src/util.rs @@ -1,4 +1,4 @@ -use anyhow::{Result}; +use anyhow::Result; use std::io; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; From 993e93eeee6682babb3e8860dd7db68020d9a91f Mon Sep 17 00:00:00 2001 From: Andrew Helwer Date: Tue, 28 Sep 2021 19:06:05 -0400 Subject: [PATCH 11/11] Fix linux build --- cli/src/util.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cli/src/util.rs b/cli/src/util.rs index 82341492..acd8acbf 100644 --- a/cli/src/util.rs +++ b/cli/src/util.rs @@ -5,6 +5,8 @@ use std::sync::Arc; use std::thread; use tree_sitter::Parser; +#[cfg(unix)] +use anyhow::{anyhow, Context}; #[cfg(unix)] use std::path::PathBuf; #[cfg(unix)]