diff --git a/cli/benches/benchmark.rs b/cli/benches/benchmark.rs index 51246f03..1dfc8ece 100644 --- a/cli/benches/benchmark.rs +++ b/cli/benches/benchmark.rs @@ -209,7 +209,7 @@ fn parse(path: &Path, max_path_length: usize, mut action: impl FnMut(&[u8])) -> fn get_language(path: &Path) -> Language { let src_dir = GRAMMARS_DIR.join(path).join("src"); TEST_LOADER - .load_language_at_path(&src_dir, &src_dir) + .load_language_at_path(&src_dir, &[&src_dir]) .with_context(|| format!("Failed to load language at path {:?}", src_dir)) .unwrap() } diff --git a/cli/loader/src/lib.rs b/cli/loader/src/lib.rs index 8febf222..53af492e 100644 --- a/cli/loader/src/lib.rs +++ b/cli/loader/src/lib.rs @@ -322,12 +322,16 @@ impl Loader { language .get_or_try_init(|| { let src_path = path.join("src"); - self.load_language_at_path(&src_path, &src_path) + self.load_language_at_path(&src_path, &[&src_path]) }) .cloned() } - pub fn load_language_at_path(&self, src_path: &Path, header_path: &Path) -> Result { + pub fn load_language_at_path( + &self, + src_path: &Path, + header_paths: &[&Path], + ) -> Result { let grammar_path = src_path.join("grammar.json"); #[derive(Deserialize)] @@ -339,13 +343,13 @@ impl Loader { let grammar_json: GrammarJSON = serde_json::from_reader(BufReader::new(&mut grammar_file)) .with_context(|| "Failed to parse grammar.json")?; - self.load_language_at_path_with_name(src_path, &header_path, &grammar_json.name) + self.load_language_at_path_with_name(src_path, header_paths, &grammar_json.name) } pub fn load_language_at_path_with_name( &self, src_path: &Path, - header_path: &Path, + header_paths: &[&Path], name: &str, ) -> Result { let mut lib_name = name.to_string(); @@ -391,7 +395,7 @@ impl Loader { { if recompile { self.compile_parser_to_dylib( - header_path, + header_paths, &parser_path, &scanner_path, &library_path, @@ -413,7 +417,7 @@ impl Loader { fn compile_parser_to_dylib( &self, - header_path: &Path, + header_paths: &[&Path], parser_path: &Path, scanner_path: &Option, library_path: &PathBuf, @@ -433,7 +437,10 @@ impl Loader { } if compiler.is_like_msvc() { - command.args(&["/nologo", "/LD", "/I"]).arg(header_path); + command.args(&["/nologo", "/LD"]); + header_paths.iter().for_each(|path| { + command.arg(format!("/I{}", path.to_string_lossy())); + }); if self.debug_build { command.arg("/Od"); } else { @@ -451,11 +458,13 @@ impl Loader { .arg("-shared") .arg("-fno-exceptions") .arg("-g") - .arg("-I") - .arg(header_path) .arg("-o") .arg(&library_path); + header_paths.iter().for_each(|path| { + command.arg(format!("-I{}", path.to_string_lossy())); + }); + if !cfg!(windows) { command.arg("-fPIC"); } diff --git a/cli/src/tests/helpers/fixtures.rs b/cli/src/tests/helpers/fixtures.rs index 7cb981ae..da4f305b 100644 --- a/cli/src/tests/helpers/fixtures.rs +++ b/cli/src/tests/helpers/fixtures.rs @@ -1,3 +1,4 @@ +use anyhow::Context; use lazy_static::lazy_static; use std::path::{Path, PathBuf}; use std::{env, fs}; @@ -28,7 +29,10 @@ pub fn fixtures_dir<'a>() -> &'static Path { pub fn get_language(name: &str) -> Language { TEST_LOADER - .load_language_at_path(&GRAMMARS_DIR.join(name).join("src"), &HEADER_DIR) + .load_language_at_path( + &GRAMMARS_DIR.join(name).join("src"), + &[&HEADER_DIR, &GRAMMARS_DIR.join(name).join("src")], + ) .unwrap() } @@ -93,7 +97,18 @@ pub fn get_test_language(name: &str, parser_code: &str, path: Option<&Path>) -> } } + let header_path = src_dir.join("tree_sitter"); + fs::create_dir_all(&header_path).unwrap(); + fs::write(&header_path.join("parser.h"), tree_sitter::PARSER_HEADER) + .with_context(|| { + format!( + "Failed to write {:?}", + header_path.join("parser.h").file_name().unwrap() + ) + }) + .unwrap(); + TEST_LOADER - .load_language_at_path_with_name(&src_dir, &HEADER_DIR, name) + .load_language_at_path_with_name(&src_dir, &[&HEADER_DIR], name) .unwrap() } diff --git a/cli/src/tests/highlight_test.rs b/cli/src/tests/highlight_test.rs index 12c120ab..b43bb0c6 100644 --- a/cli/src/tests/highlight_test.rs +++ b/cli/src/tests/highlight_test.rs @@ -329,10 +329,11 @@ fn test_highlighting_empty_lines() { fn test_highlighting_carriage_returns() { let source = "a = \"a\rb\"\r\nb\r"; + // FIXME(amaanq): figure why this changed w/ JS's grammar changes assert_eq!( &to_html(source, &JS_HIGHLIGHT).unwrap(), &[ - "a = "ab"\n", + "a = "ab"\n", "b\n", ], ); diff --git a/cli/src/tests/parser_test.rs b/cli/src/tests/parser_test.rs index a223a453..511ef02a 100644 --- a/cli/src/tests/parser_test.rs +++ b/cli/src/tests/parser_test.rs @@ -432,7 +432,7 @@ fn test_parsing_after_editing_tree_that_depends_on_column_values() { let dir = fixtures_dir() .join("test_grammars") .join("uses_current_column"); - let grammar = fs::read_to_string(&dir.join("grammar.json")).unwrap(); + let grammar = fs::read_to_string(dir.join("grammar.json")).unwrap(); let (grammar_name, parser_code) = generate_parser_for_grammar(&grammar).unwrap(); let mut parser = Parser::new(); @@ -858,16 +858,16 @@ fn test_parsing_with_multiple_included_ranges() { let template_string_node = js_tree .root_node() .descendant_for_byte_range( - source_code.find("
").unwrap(), - source_code.find("Hello").unwrap(), + source_code.find("`<").unwrap(), + source_code.find(">`").unwrap(), ) .unwrap(); assert_eq!(template_string_node.kind(), "template_string"); let open_quote_node = template_string_node.child(0).unwrap(); - let interpolation_node1 = template_string_node.child(1).unwrap(); - let interpolation_node2 = template_string_node.child(2).unwrap(); - let close_quote_node = template_string_node.child(3).unwrap(); + let interpolation_node1 = template_string_node.child(2).unwrap(); + let interpolation_node2 = template_string_node.child(4).unwrap(); + let close_quote_node = template_string_node.child(6).unwrap(); parser.set_language(&get_language("html")).unwrap(); let html_ranges = &[ diff --git a/cli/src/tests/query_test.rs b/cli/src/tests/query_test.rs index 4b26a5d9..c6cbbe0a 100644 --- a/cli/src/tests/query_test.rs +++ b/cli/src/tests/query_test.rs @@ -377,7 +377,7 @@ fn test_query_errors_on_impossible_patterns() { Query::new( &js_lang, "[ - (function (identifier)) + (function_expression (identifier)) (function_declaration (identifier)) (generator_function_declaration (identifier)) ]", @@ -387,7 +387,7 @@ fn test_query_errors_on_impossible_patterns() { Query::new( &js_lang, "[ - (function (identifier)) + (function_expression (identifier)) (function_declaration (object)) (generator_function_declaration (identifier)) ]", @@ -395,7 +395,7 @@ fn test_query_errors_on_impossible_patterns() { Err(QueryError { kind: QueryErrorKind::Structure, row: 2, - offset: 88, + offset: 99, column: 42, message: [ " (function_declaration (object))", // @@ -589,7 +589,7 @@ fn test_query_matches_with_multiple_patterns_same_root() { " (pair key: (property_identifier) @method-def - value: (function)) + value: (function_expression)) (pair key: (property_identifier) @method-def @@ -1499,7 +1499,7 @@ fn test_query_matches_with_simple_alternatives() { " (pair key: [(property_identifier) (string)] @key - value: [(function) @val1 (arrow_function) @val2]) + value: [(function_expression) @val1 (arrow_function) @val2]) ", ) .unwrap(); @@ -2835,12 +2835,12 @@ fn test_query_captures_basic() { r#" (pair key: _ @method.def - (function + (function_expression name: (identifier) @method.alias)) (variable_declarator name: _ @function.def - value: (function + value: (function_expression name: (identifier) @function.alias)) ":" @delimiter @@ -3078,7 +3078,7 @@ fn test_query_captures_with_duplicates() { r#" (variable_declarator name: (identifier) @function - value: (function)) + value: (function_expression)) (identifier) @variable "#, @@ -4498,7 +4498,7 @@ fn test_capture_quantifiers() { language: get_language("javascript"), pattern: r#"[ (function_declaration name:(identifier) @name) - (function) + (function_expression) ] @fun"#, capture_quantifiers: &[ (0, "fun", CaptureQuantifier::One), diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 0db1f554..30e5b405 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -14,12 +14,12 @@ rust-version.workspace = true build = "binding_rust/build.rs" include = [ - "/binding_rust/*", - "/Cargo.toml", - "/include/*", - "/src/*.h", - "/src/*.c", - "/src/unicode/*", + "/binding_rust/*", + "/Cargo.toml", + "/src/*.h", + "/src/*.c", + "/src/unicode/*", + "/include/tree_sitter/api.h", ] [features] diff --git a/lib/binding_rust/lib.rs b/lib/binding_rust/lib.rs index 32d61b45..30579e35 100644 --- a/lib/binding_rust/lib.rs +++ b/lib/binding_rust/lib.rs @@ -42,7 +42,7 @@ pub const LANGUAGE_VERSION: usize = ffi::TREE_SITTER_LANGUAGE_VERSION as usize; pub const MIN_COMPATIBLE_LANGUAGE_VERSION: usize = ffi::TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION as usize; -pub const PARSER_HEADER: &'static str = include_str!("../include/tree_sitter/parser.h"); +pub const PARSER_HEADER: &str = include_str!("../src/parser.h"); /// An opaque object that defines how to parse a particular language. The code for each /// `Language` is generated by the Tree-sitter CLI. diff --git a/lib/binding_web/exports.json b/lib/binding_web/exports.json index dfc9ff54..3cf8565d 100644 --- a/lib/binding_web/exports.json +++ b/lib/binding_web/exports.json @@ -5,7 +5,6 @@ "_realloc", "__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE6__initEPKcm", - "__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE9__grow_byEmmmmmm", "__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE9push_backEc", "__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE9push_backEw", "__ZNKSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE4copyEPcmm", diff --git a/lib/binding_web/test/language-test.js b/lib/binding_web/test/language-test.js index 394e32b4..467b56c5 100644 --- a/lib/binding_web/test/language-test.js +++ b/lib/binding_web/test/language-test.js @@ -64,7 +64,7 @@ describe("Lookahead iterator", () => { lookahead.delete(); }); - const expected = ["identifier", "comment", "(", "*", "formal_parameters"]; + const expected = ["identifier", "comment", "html_comment", "(", "*", "formal_parameters", "ERROR"]; it("should iterate over valid symbols in the state", () => { const symbols = Array.from(lookahead); assert.includeMembers(symbols, expected); diff --git a/lib/binding_web/test/query-test.js b/lib/binding_web/test/query-test.js index 34fd5cd1..f53b8d15 100644 --- a/lib/binding_web/test/query-test.js +++ b/lib/binding_web/test/query-test.js @@ -103,8 +103,8 @@ describe("Query", () => { const matches = query.matches(tree.rootNode); assert.deepEqual(formatMatches(matches), [ - { pattern: 0, captures: [{name: "name", text: "giraffe" }] }, - { pattern: 0, captures: [{name: "name", text: "gross" }] }, + { pattern: 0, captures: [{ name: "name", text: "giraffe" }] }, + { pattern: 0, captures: [{ name: "name", text: "gross" }] }, ]); }); }); @@ -124,12 +124,12 @@ describe("Query", () => { query = JavaScript.query(` (pair key: _ @method.def - (function + (function_expression name: (identifier) @method.alias)) (variable_declarator name: _ @function.def - value: (function + value: (function_expression name: (identifier) @function.alias)) ":" @delimiter @@ -256,7 +256,7 @@ describe("Query", () => { (array (identifier) @pre (identifier) @post) `); - const captures = query.captures(tree.rootNode, null, null, {matchLimit: 32}); + const captures = query.captures(tree.rootNode, null, null, { matchLimit: 32 }); assert.ok(query.didExceedMatchLimit()); }); diff --git a/lib/src/language.h b/lib/src/language.h index 55b5d89b..c89dc739 100644 --- a/lib/src/language.h +++ b/lib/src/language.h @@ -6,7 +6,7 @@ extern "C" { #endif #include "./subtree.h" -#include "tree_sitter/parser.h" +#include "./parser.h" #define ts_builtin_sym_error_repeat (ts_builtin_sym_error - 1) diff --git a/lib/src/lexer.h b/lib/src/lexer.h index c1a5bfdb..445c4fdc 100644 --- a/lib/src/lexer.h +++ b/lib/src/lexer.h @@ -8,7 +8,7 @@ extern "C" { #include "./length.h" #include "./subtree.h" #include "tree_sitter/api.h" -#include "tree_sitter/parser.h" +#include "./parser.h" typedef struct { TSLexer data; diff --git a/lib/include/tree_sitter/parser.h b/lib/src/parser.h similarity index 100% rename from lib/include/tree_sitter/parser.h rename to lib/src/parser.h diff --git a/lib/src/subtree.h b/lib/src/subtree.h index 76faff5e..8f97afa1 100644 --- a/lib/src/subtree.h +++ b/lib/src/subtree.h @@ -13,7 +13,7 @@ extern "C" { #include "./error_costs.h" #include "./host.h" #include "tree_sitter/api.h" -#include "tree_sitter/parser.h" +#include "./parser.h" #define TS_TREE_STATE_NONE USHRT_MAX #define NULL_SUBTREE ((Subtree) {.ptr = NULL}) diff --git a/lib/src/wasm.c b/lib/src/wasm.c index 379bd8c0..4fa2a6cb 100644 --- a/lib/src/wasm.c +++ b/lib/src/wasm.c @@ -1,5 +1,5 @@ #include "tree_sitter/api.h" -#include "tree_sitter/parser.h" +#include "./parser.h" #include #ifdef TREE_SITTER_FEATURE_WASM @@ -12,7 +12,6 @@ #include "./atomic.h" #include "./lexer.h" #include "./wasm.h" -#include "./lexer.h" #include "./wasm/wasm-stdlib.h" // The following symbols from the C and C++ standard libraries are available diff --git a/lib/src/wasm.h b/lib/src/wasm.h index 849a3c1c..c3324cc8 100644 --- a/lib/src/wasm.h +++ b/lib/src/wasm.h @@ -6,7 +6,7 @@ extern "C" { #endif #include "tree_sitter/api.h" -#include "tree_sitter/parser.h" +#include "./parser.h" bool ts_wasm_store_start(TSWasmStore *, TSLexer *, const TSLanguage *); void ts_wasm_store_stop(TSWasmStore *); diff --git a/test/fixtures/error_corpus/javascript_errors.txt b/test/fixtures/error_corpus/javascript_errors.txt index e2f21176..9b92e10c 100644 --- a/test/fixtures/error_corpus/javascript_errors.txt +++ b/test/fixtures/error_corpus/javascript_errors.txt @@ -80,10 +80,10 @@ if ({a: 'b'} {c: 'd'}) { (expression_statement (assignment_expression (identifier) - (function (formal_parameters (identifier)) (statement_block (expression_statement (identifier))))) + (function_expression (formal_parameters (identifier)) (statement_block (expression_statement (identifier))))) (MISSING ";")) (expression_statement - (function (formal_parameters (identifier)) (statement_block (expression_statement (identifier)))))))) + (function_expression (formal_parameters (identifier)) (statement_block (expression_statement (identifier)))))))) =================================================== Extra tokens at the end of the file @@ -148,14 +148,14 @@ const h = `i ${j(k} l` (lexical_declaration (variable_declarator (identifier) - (template_string (template_substitution - (augmented_assignment_expression (identifier) (MISSING identifier)))))) + (template_string (string_fragment) (template_substitution + (augmented_assignment_expression (identifier) (MISSING identifier))) (string_fragment)))) (lexical_declaration (variable_declarator (identifier) - (template_string (template_substitution (call_expression + (template_string (string_fragment) (template_substitution (call_expression (identifier) - (arguments (identifier) (MISSING ")")))))))) + (arguments (identifier) (MISSING ")")))) (string_fragment))))) ========================================================= Long sequences of invalid tokens diff --git a/test/fixtures/test_grammars/epsilon_external_tokens/scanner.c b/test/fixtures/test_grammars/epsilon_external_tokens/scanner.c index 85bc7c60..d25c1ff3 100644 --- a/test/fixtures/test_grammars/epsilon_external_tokens/scanner.c +++ b/test/fixtures/test_grammars/epsilon_external_tokens/scanner.c @@ -1,4 +1,4 @@ -#include +#include "tree_sitter/parser.h" enum TokenType { ZERO_WIDTH_TOKEN diff --git a/test/fixtures/test_grammars/external_and_internal_anonymous_tokens/scanner.c b/test/fixtures/test_grammars/external_and_internal_anonymous_tokens/scanner.c index 644bbd0a..187c675e 100644 --- a/test/fixtures/test_grammars/external_and_internal_anonymous_tokens/scanner.c +++ b/test/fixtures/test_grammars/external_and_internal_anonymous_tokens/scanner.c @@ -1,4 +1,4 @@ -#include +#include "tree_sitter/parser.h" enum { STRING, diff --git a/test/fixtures/test_grammars/external_and_internal_tokens/scanner.c b/test/fixtures/test_grammars/external_and_internal_tokens/scanner.c index 43a4bc9d..55454f29 100644 --- a/test/fixtures/test_grammars/external_and_internal_tokens/scanner.c +++ b/test/fixtures/test_grammars/external_and_internal_tokens/scanner.c @@ -1,4 +1,4 @@ -#include +#include "tree_sitter/parser.h" enum { STRING, diff --git a/test/fixtures/test_grammars/external_extra_tokens/scanner.c b/test/fixtures/test_grammars/external_extra_tokens/scanner.c index ac6d8407..95d58426 100644 --- a/test/fixtures/test_grammars/external_extra_tokens/scanner.c +++ b/test/fixtures/test_grammars/external_extra_tokens/scanner.c @@ -1,4 +1,4 @@ -#include +#include "tree_sitter/parser.h" enum { COMMENT, diff --git a/test/fixtures/test_grammars/external_tokens/scanner.c b/test/fixtures/test_grammars/external_tokens/scanner.c index 490100d4..163ae51b 100644 --- a/test/fixtures/test_grammars/external_tokens/scanner.c +++ b/test/fixtures/test_grammars/external_tokens/scanner.c @@ -1,4 +1,4 @@ -#include +#include "tree_sitter/parser.h" enum { percent_string, diff --git a/test/fixtures/test_grammars/external_unicode_column_alignment/scanner.c b/test/fixtures/test_grammars/external_unicode_column_alignment/scanner.c index a9e98735..3380bba2 100644 --- a/test/fixtures/test_grammars/external_unicode_column_alignment/scanner.c +++ b/test/fixtures/test_grammars/external_unicode_column_alignment/scanner.c @@ -1,4 +1,5 @@ -#include +#include "tree_sitter/parser.h" + #include #include diff --git a/test/fixtures/test_grammars/get_col_should_hang_not_crash/scanner.c b/test/fixtures/test_grammars/get_col_should_hang_not_crash/scanner.c index d21ec6d4..000647af 100644 --- a/test/fixtures/test_grammars/get_col_should_hang_not_crash/scanner.c +++ b/test/fixtures/test_grammars/get_col_should_hang_not_crash/scanner.c @@ -1,4 +1,4 @@ -#include +#include "tree_sitter/parser.h" unsigned tree_sitter_get_col_should_hang_not_crash_external_scanner_serialize() { return 0; } diff --git a/test/fixtures/test_grammars/inverted_external_token/scanner.c b/test/fixtures/test_grammars/inverted_external_token/scanner.c index 260994c5..33fde6d1 100644 --- a/test/fixtures/test_grammars/inverted_external_token/scanner.c +++ b/test/fixtures/test_grammars/inverted_external_token/scanner.c @@ -1,4 +1,4 @@ -#include +#include "tree_sitter/parser.h" enum { LINE_BREAK diff --git a/test/fixtures/test_grammars/uses_current_column/scanner.c b/test/fixtures/test_grammars/uses_current_column/scanner.c index 62b16392..b2b59281 100644 --- a/test/fixtures/test_grammars/uses_current_column/scanner.c +++ b/test/fixtures/test_grammars/uses_current_column/scanner.c @@ -1,6 +1,7 @@ +#include "tree_sitter/parser.h" + #include #include -#include enum TokenType { INDENT,