traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #2894 from amaanq/parser-h

Browse source 
fix: do not install `parser.h`
This commit is contained in:
Amaan Qureshi 2024-02-02 12:06:36 -05:00 • committed by GitHub
commit ff89b539f2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
27 changed files with 88 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

2
cli/benches/benchmark.rs
View file

@ -209,7 +209,7 @@ fn parse(path: &Path, max_path_length: usize, mut action: impl FnMut(&[u8])) ->
fn get_language(path: &Path) -> Language {
let src_dir = GRAMMARS_DIR.join(path).join("src");
TEST_LOADER
.load_language_at_path(&src_dir, &src_dir)
.load_language_at_path(&src_dir, &[&src_dir])
.with_context(|| format!("Failed to load language at path {:?}", src_dir))
.unwrap()
}

27
cli/loader/src/lib.rs
View file

@ -322,12 +322,16 @@ impl Loader {
language
.get_or_try_init(|| {
let src_path = path.join("src");
self.load_language_at_path(&src_path, &src_path)
self.load_language_at_path(&src_path, &[&src_path])
})
.cloned()
}
pub fn load_language_at_path(&self, src_path: &Path, header_path: &Path) -> Result<Language> {
pub fn load_language_at_path(
&self,
src_path: &Path,
header_paths: &[&Path],
) -> Result<Language> {
let grammar_path = src_path.join("grammar.json");
#[derive(Deserialize)]
@ -339,13 +343,13 @@ impl Loader {
let grammar_json: GrammarJSON = serde_json::from_reader(BufReader::new(&mut grammar_file))
.with_context(|| "Failed to parse grammar.json")?;
self.load_language_at_path_with_name(src_path, &header_path, &grammar_json.name)
self.load_language_at_path_with_name(src_path, header_paths, &grammar_json.name)
}
pub fn load_language_at_path_with_name(
&self,
src_path: &Path,
header_path: &Path,
header_paths: &[&Path],
name: &str,
) -> Result<Language> {
let mut lib_name = name.to_string();
@ -391,7 +395,7 @@ impl Loader {
{
if recompile {
self.compile_parser_to_dylib(
header_path,
header_paths,
&parser_path,
&scanner_path,
&library_path,
@ -413,7 +417,7 @@ impl Loader {
fn compile_parser_to_dylib(
&self,
header_path: &Path,
header_paths: &[&Path],
parser_path: &Path,
scanner_path: &Option<PathBuf>,
library_path: &PathBuf,
@ -433,7 +437,10 @@ impl Loader {
}
if compiler.is_like_msvc() {
command.args(&["/nologo", "/LD", "/I"]).arg(header_path);
command.args(&["/nologo", "/LD"]);
header_paths.iter().for_each(|path| {
command.arg(format!("/I{}", path.to_string_lossy()));
});
if self.debug_build {
command.arg("/Od");
} else {
@ -451,11 +458,13 @@ impl Loader {
.arg("-shared")
.arg("-fno-exceptions")
.arg("-g")
.arg("-I")
.arg(header_path)
.arg("-o")
.arg(&library_path);
header_paths.iter().for_each(|path| {
command.arg(format!("-I{}", path.to_string_lossy()));
});
if !cfg!(windows) {
command.arg("-fPIC");
}

19
cli/src/tests/helpers/fixtures.rs
View file

@ -1,3 +1,4 @@
use anyhow::Context;
use lazy_static::lazy_static;
use std::path::{Path, PathBuf};
use std::{env, fs};
@ -28,7 +29,10 @@ pub fn fixtures_dir<'a>() -> &'static Path {
pub fn get_language(name: &str) -> Language {
TEST_LOADER
.load_language_at_path(&GRAMMARS_DIR.join(name).join("src"), &HEADER_DIR)
.load_language_at_path(
&GRAMMARS_DIR.join(name).join("src"),
&[&HEADER_DIR, &GRAMMARS_DIR.join(name).join("src")],
)
.unwrap()
}
@ -93,7 +97,18 @@ pub fn get_test_language(name: &str, parser_code: &str, path: Option<&Path>) ->
}
}
let header_path = src_dir.join("tree_sitter");
fs::create_dir_all(&header_path).unwrap();
fs::write(&header_path.join("parser.h"), tree_sitter::PARSER_HEADER)
.with_context(|| {
format!(
"Failed to write {:?}",
header_path.join("parser.h").file_name().unwrap()
)
})
.unwrap();
TEST_LOADER
.load_language_at_path_with_name(&src_dir, &HEADER_DIR, name)
.load_language_at_path_with_name(&src_dir, &[&HEADER_DIR], name)
.unwrap()
}

3
cli/src/tests/highlight_test.rs
View file

@ -329,10 +329,11 @@ fn test_highlighting_empty_lines() {
fn test_highlighting_carriage_returns() {
let source = "a = \"a\rb\"\r\nb\r";
// FIXME(amaanq): figure why this changed w/ JS's grammar changes
assert_eq!(
&to_html(source, &JS_HIGHLIGHT).unwrap(),
&[
"<span class=variable>a</span> <span class=operator>=</span> <span class=string>&quot;a<span class=carriage-return></span>b&quot;</span>\n",
"<span class=variable>a</span> <span class=operator>=</span> <span class=string>&quot;a<span class=variable>b</span>&quot;</span>\n",
"<span class=variable>b</span>\n",
],
);

12
cli/src/tests/parser_test.rs
View file

@ -432,7 +432,7 @@ fn test_parsing_after_editing_tree_that_depends_on_column_values() {
let dir = fixtures_dir()
.join("test_grammars")
.join("uses_current_column");
let grammar = fs::read_to_string(&dir.join("grammar.json")).unwrap();
let grammar = fs::read_to_string(dir.join("grammar.json")).unwrap();
let (grammar_name, parser_code) = generate_parser_for_grammar(&grammar).unwrap();
let mut parser = Parser::new();
@ -858,16 +858,16 @@ fn test_parsing_with_multiple_included_ranges() {
let template_string_node = js_tree
.root_node()
.descendant_for_byte_range(
source_code.find("<div>").unwrap(),
source_code.find("Hello").unwrap(),
source_code.find("`<").unwrap(),
source_code.find(">`").unwrap(),
)
.unwrap();
assert_eq!(template_string_node.kind(), "template_string");
let open_quote_node = template_string_node.child(0).unwrap();
let interpolation_node1 = template_string_node.child(1).unwrap();
let interpolation_node2 = template_string_node.child(2).unwrap();
let close_quote_node = template_string_node.child(3).unwrap();
let interpolation_node1 = template_string_node.child(2).unwrap();
let interpolation_node2 = template_string_node.child(4).unwrap();
let close_quote_node = template_string_node.child(6).unwrap();
parser.set_language(&get_language("html")).unwrap();
let html_ranges = &[

18
cli/src/tests/query_test.rs
View file

@ -377,7 +377,7 @@ fn test_query_errors_on_impossible_patterns() {
Query::new(
&js_lang,
"[
(function (identifier))
(function_expression (identifier))
(function_declaration (identifier))
(generator_function_declaration (identifier))
]",
@ -387,7 +387,7 @@ fn test_query_errors_on_impossible_patterns() {
Query::new(
&js_lang,
"[
(function (identifier))
(function_expression (identifier))
(function_declaration (object))
(generator_function_declaration (identifier))
]",
@ -395,7 +395,7 @@ fn test_query_errors_on_impossible_patterns() {
Err(QueryError {
kind: QueryErrorKind::Structure,
row: 2,
offset: 88,
offset: 99,
column: 42,
message: [
" (function_declaration (object))", //
@ -589,7 +589,7 @@ fn test_query_matches_with_multiple_patterns_same_root() {
"
(pair
key: (property_identifier) @method-def
value: (function))
value: (function_expression))
(pair
key: (property_identifier) @method-def
@ -1499,7 +1499,7 @@ fn test_query_matches_with_simple_alternatives() {
"
(pair
key: [(property_identifier) (string)] @key
value: [(function) @val1 (arrow_function) @val2])
value: [(function_expression) @val1 (arrow_function) @val2])
",
)
.unwrap();
@ -2835,12 +2835,12 @@ fn test_query_captures_basic() {
r#"
(pair
key: _ @method.def
(function
(function_expression
name: (identifier) @method.alias))
(variable_declarator
name: _ @function.def
value: (function
value: (function_expression
name: (identifier) @function.alias))
":" @delimiter
@ -3078,7 +3078,7 @@ fn test_query_captures_with_duplicates() {
r#"
(variable_declarator
name: (identifier) @function
value: (function))
value: (function_expression))
(identifier) @variable
"#,
@ -4498,7 +4498,7 @@ fn test_capture_quantifiers() {
language: get_language("javascript"),
pattern: r#"[
(function_declaration name:(identifier) @name)
(function)
(function_expression)
] @fun"#,
capture_quantifiers: &[
(0, "fun", CaptureQuantifier::One),

12
lib/Cargo.toml
View file

@ -14,12 +14,12 @@ rust-version.workspace = true
build = "binding_rust/build.rs"
include = [
"/binding_rust/*",
"/Cargo.toml",
"/include/*",
"/src/*.h",
"/src/*.c",
"/src/unicode/*",
"/binding_rust/*",
"/Cargo.toml",
"/src/*.h",
"/src/*.c",
"/src/unicode/*",
"/include/tree_sitter/api.h",
]
[features]

2
lib/binding_rust/lib.rs
View file

@ -42,7 +42,7 @@ pub const LANGUAGE_VERSION: usize = ffi::TREE_SITTER_LANGUAGE_VERSION as usize;
pub const MIN_COMPATIBLE_LANGUAGE_VERSION: usize =
ffi::TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION as usize;
pub const PARSER_HEADER: &'static str = include_str!("../include/tree_sitter/parser.h");
pub const PARSER_HEADER: &str = include_str!("../src/parser.h");
/// An opaque object that defines how to parse a particular language. The code for each
/// `Language` is generated by the Tree-sitter CLI.

1
lib/binding_web/exports.json
View file

@ -5,7 +5,6 @@
"_realloc",
"__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE6__initEPKcm",
"__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE9__grow_byEmmmmmm",
"__ZNSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE9push_backEc",
"__ZNSt3__212basic_stringIwNS_11char_traitsIwEENS_9allocatorIwEEE9push_backEw",
"__ZNKSt3__212basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEE4copyEPcmm",

2
lib/binding_web/test/language-test.js
View file

@ -64,7 +64,7 @@ describe("Lookahead iterator", () => {
lookahead.delete();
});
const expected = ["identifier", "comment", "(", "*", "formal_parameters"];
const expected = ["identifier", "comment", "html_comment", "(", "*", "formal_parameters", "ERROR"];
it("should iterate over valid symbols in the state", () => {
const symbols = Array.from(lookahead);
assert.includeMembers(symbols, expected);

10
lib/binding_web/test/query-test.js
View file

@ -103,8 +103,8 @@ describe("Query", () => {
const matches = query.matches(tree.rootNode);
assert.deepEqual(formatMatches(matches), [
{ pattern: 0, captures: [{name: "name", text: "giraffe" }] },
{ pattern: 0, captures: [{name: "name", text: "gross" }] },
{ pattern: 0, captures: [{ name: "name", text: "giraffe" }] },
{ pattern: 0, captures: [{ name: "name", text: "gross" }] },
]);
});
});
@ -124,12 +124,12 @@ describe("Query", () => {
query = JavaScript.query(`
(pair
key: _ @method.def
(function
(function_expression
name: (identifier) @method.alias))
(variable_declarator
name: _ @function.def
value: (function
value: (function_expression
name: (identifier) @function.alias))
":" @delimiter
@ -256,7 +256,7 @@ describe("Query", () => {
(array (identifier) @pre (identifier) @post)
`);
const captures = query.captures(tree.rootNode, null, null, {matchLimit: 32});
const captures = query.captures(tree.rootNode, null, null, { matchLimit: 32 });
assert.ok(query.didExceedMatchLimit());
});

2
lib/src/language.h
View file

@ -6,7 +6,7 @@ extern "C" {
#endif
#include "./subtree.h"
#include "tree_sitter/parser.h"
#include "./parser.h"
#define ts_builtin_sym_error_repeat (ts_builtin_sym_error - 1)

2
lib/src/lexer.h
View file

@ -8,7 +8,7 @@ extern "C" {
#include "./length.h"
#include "./subtree.h"
#include "tree_sitter/api.h"
#include "tree_sitter/parser.h"
#include "./parser.h"
typedef struct {
TSLexer data;

0
lib/include/tree_sitter/parser.h → lib/src/parser.h
View file

2
lib/src/subtree.h
View file

@ -13,7 +13,7 @@ extern "C" {
#include "./error_costs.h"
#include "./host.h"
#include "tree_sitter/api.h"
#include "tree_sitter/parser.h"
#include "./parser.h"
#define TS_TREE_STATE_NONE USHRT_MAX
#define NULL_SUBTREE ((Subtree) {.ptr = NULL})

3
lib/src/wasm.c
View file

@ -1,5 +1,5 @@
#include "tree_sitter/api.h"
#include "tree_sitter/parser.h"
#include "./parser.h"
#include <stdint.h>
#ifdef TREE_SITTER_FEATURE_WASM
@ -12,7 +12,6 @@
#include "./atomic.h"
#include "./lexer.h"
#include "./wasm.h"
#include "./lexer.h"
#include "./wasm/wasm-stdlib.h"
// The following symbols from the C and C++ standard libraries are available

2
lib/src/wasm.h
View file

@ -6,7 +6,7 @@ extern "C" {
#endif
#include "tree_sitter/api.h"
#include "tree_sitter/parser.h"
#include "./parser.h"
bool ts_wasm_store_start(TSWasmStore *, TSLexer *, const TSLanguage *);
void ts_wasm_store_stop(TSWasmStore *);

12
test/fixtures/error_corpus/javascript_errors.txt vendored
View file

@ -80,10 +80,10 @@ if ({a: 'b'} {c: 'd'}) {
(expression_statement
(assignment_expression
(identifier)
(function (formal_parameters (identifier)) (statement_block (expression_statement (identifier)))))
(function_expression (formal_parameters (identifier)) (statement_block (expression_statement (identifier)))))
(MISSING ";"))
(expression_statement
(function (formal_parameters (identifier)) (statement_block (expression_statement (identifier))))))))
(function_expression (formal_parameters (identifier)) (statement_block (expression_statement (identifier))))))))
===================================================
Extra tokens at the end of the file
@ -148,14 +148,14 @@ const h = `i ${j(k} l`
(lexical_declaration
(variable_declarator
(identifier)
(template_string (template_substitution
(augmented_assignment_expression (identifier) (MISSING identifier))))))
(template_string (string_fragment) (template_substitution
(augmented_assignment_expression (identifier) (MISSING identifier))) (string_fragment))))
(lexical_declaration
(variable_declarator
(identifier)
(template_string (template_substitution (call_expression
(template_string (string_fragment) (template_substitution (call_expression
(identifier)
(arguments (identifier) (MISSING ")"))))))))
(arguments (identifier) (MISSING ")")))) (string_fragment)))))
=========================================================
Long sequences of invalid tokens

2
test/fixtures/test_grammars/epsilon_external_tokens/scanner.c vendored
View file

@ -1,4 +1,4 @@
#include <tree_sitter/parser.h>
#include "tree_sitter/parser.h"
enum TokenType {
ZERO_WIDTH_TOKEN

2
test/fixtures/test_grammars/external_and_internal_anonymous_tokens/scanner.c vendored
View file

@ -1,4 +1,4 @@
#include <tree_sitter/parser.h>
#include "tree_sitter/parser.h"
enum {
STRING,

2
test/fixtures/test_grammars/external_and_internal_tokens/scanner.c vendored
View file

@ -1,4 +1,4 @@
#include <tree_sitter/parser.h>
#include "tree_sitter/parser.h"
enum {
STRING,

2
test/fixtures/test_grammars/external_extra_tokens/scanner.c vendored
View file

@ -1,4 +1,4 @@
#include <tree_sitter/parser.h>
#include "tree_sitter/parser.h"
enum {
COMMENT,

2
test/fixtures/test_grammars/external_tokens/scanner.c vendored
View file

@ -1,4 +1,4 @@
#include <tree_sitter/parser.h>
#include "tree_sitter/parser.h"
enum {
percent_string,

3
test/fixtures/test_grammars/external_unicode_column_alignment/scanner.c vendored
View file

@ -1,4 +1,5 @@
#include <tree_sitter/parser.h>
#include "tree_sitter/parser.h"
#include <wctype.h>
#include <string.h>

2
test/fixtures/test_grammars/get_col_should_hang_not_crash/scanner.c vendored
View file

@ -1,4 +1,4 @@
#include <tree_sitter/parser.h>
#include "tree_sitter/parser.h"
unsigned tree_sitter_get_col_should_hang_not_crash_external_scanner_serialize() { return 0; }

2
test/fixtures/test_grammars/inverted_external_token/scanner.c vendored
View file

@ -1,4 +1,4 @@
#include <tree_sitter/parser.h>
#include "tree_sitter/parser.h"
enum {
LINE_BREAK

3
test/fixtures/test_grammars/uses_current_column/scanner.c vendored
View file

@ -1,6 +1,7 @@
#include "tree_sitter/parser.h"
#include <stdlib.h>
#include <wctype.h>
#include <tree_sitter/parser.h>
enum TokenType {
INDENT,