traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into simpler-large-char-set-code

Browse source
This commit is contained in:
Max Brunsfeld 2024-04-12 10:03:46 -07:00
commit 3498498449
81 changed files with 918 additions and 734 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Cargo.lock generated
View file

@ -1447,6 +1447,7 @@ dependencies = [
"regex",
"serde",
"serde_json",
"tempfile",
"tree-sitter",
"tree-sitter-highlight",
"tree-sitter-tags",

13
cli/benches/benchmark.rs
View file

@ -1,9 +1,14 @@
use std::{
collections::BTreeMap,
env, fs,
path::{Path, PathBuf},
str,
time::Instant,
usize,
};
use anyhow::Context;
use lazy_static::lazy_static;
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::time::Instant;
use std::{env, fs, str, usize};
use tree_sitter::{Language, Parser, Query};
use tree_sitter_loader::{CompileConfig, Loader};

8
cli/config/src/lib.rs
View file

@ -1,10 +1,10 @@
#![doc = include_str!("../README.md")]
use std::{env, fs, path::PathBuf};
use anyhow::{anyhow, Context, Result};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::path::PathBuf;
use std::{env, fs};
/// Holds the contents of tree-sitter's configuration file.
///
@ -62,8 +62,8 @@ impl Config {
///
/// - Location specified by the path parameter if provided
/// - `$TREE_SITTER_DIR/config.json`, if the `TREE_SITTER_DIR` environment variable is set
/// - `tree-sitter/config.json` in your default user configuration directory, as determined
/// by [`dirs::config_dir`](https://docs.rs/dirs/*/dirs/fn.config_dir.html)
/// - `tree-sitter/config.json` in your default user configuration directory, as determined by
/// [`dirs::config_dir`](https://docs.rs/dirs/*/dirs/fn.config_dir.html)
/// - `$HOME/.tree-sitter/config.json` as a fallback from where tree-sitter _used_ to store
/// its configuration
pub fn load(path: Option<PathBuf>) -> Result<Self> {

1
cli/loader/Cargo.toml
View file

@ -26,6 +26,7 @@ once_cell.workspace = true
regex.workspace = true
serde.workspace = true
serde_json.workspace = true
tempfile.workspace = true
tree-sitter.workspace = true
tree-sitter-highlight.workspace = true

2
cli/loader/emscripten-version
View file

@ -1 +1 @@
3.1.37
3.1.55

53
cli/loader/src/lib.rs
View file

@ -1,14 +1,18 @@
#![doc = include_str!("../README.md")]
use std::collections::HashMap;
use std::ffi::{OsStr, OsString};
use std::io::{BufRead, BufReader};
use std::ops::Range;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::sync::Mutex;
use std::time::SystemTime;
use std::{env, fs, mem};
use std::{
collections::HashMap,
env,
ffi::{OsStr, OsString},
fs,
io::{BufRead, BufReader},
mem,
ops::Range,
path::{Path, PathBuf},
process::Command,
sync::Mutex,
time::SystemTime,
};
use anyhow::{anyhow, Context, Error, Result};
use fs4::FileExt;
@ -126,11 +130,12 @@ pub struct CompileConfig<'a> {
}
impl<'a> CompileConfig<'a> {
#[must_use]
pub fn new(
src_path: &'a Path,
externals: Option<&'a [PathBuf]>,
output_path: Option<PathBuf>,
) -> CompileConfig<'a> {
) -> Self {
Self {
src_path,
header_paths: vec![src_path],
@ -445,7 +450,7 @@ impl Loader {
let parser_path = config.src_path.join("parser.c");
config.scanner_path = self.get_scanner_path(config.src_path);
let mut paths_to_check = vec![parser_path.clone()];
let mut paths_to_check = vec![parser_path];
if let Some(scanner_path) = config.scanner_path.as_ref() {
paths_to_check.push(scanner_path.clone());
@ -484,7 +489,9 @@ impl Loader {
}
let lock_path = if env::var("CROSS_RUNNER").is_ok() {
PathBuf::from("/tmp")
tempfile::tempdir()
.unwrap()
.path()
.join("tree-sitter")
.join("lock")
.join(format!("{}.lock", config.name))
@ -499,7 +506,8 @@ impl Loader {
if let Ok(lock_file) = fs::OpenOptions::new().write(true).open(&lock_path) {
recompile = false;
if lock_file.try_lock_exclusive().is_err() {
// if we can't acquire the lock, another process is compiling the parser, wait for it and don't recompile
// if we can't acquire the lock, another process is compiling the parser, wait for
// it and don't recompile
lock_file.lock_exclusive()?;
recompile = false;
} else {
@ -1016,7 +1024,7 @@ impl Loader {
language_name: grammar_json.name.clone(),
scope: config_json.scope,
language_id,
file_types: config_json.file_types.unwrap_or(Vec::new()),
file_types: config_json.file_types.unwrap_or_default(),
content_regex: Self::regex(config_json.content_regex.as_deref()),
first_line_regex: Self::regex(config_json.first_line_regex.as_deref()),
injection_regex: Self::regex(config_json.injection_regex.as_deref()),
@ -1043,8 +1051,11 @@ impl Loader {
.push(self.language_configurations.len());
}
self.language_configurations
.push(unsafe { mem::transmute(configuration) });
self.language_configurations.push(unsafe {
mem::transmute::<LanguageConfiguration<'_>, LanguageConfiguration<'static>>(
configuration,
)
});
if set_current_path_config
&& self.language_configuration_in_current_path.is_none()
@ -1083,8 +1094,11 @@ impl Loader {
highlight_names: &self.highlight_names,
use_all_highlight_names: self.use_all_highlight_names,
};
self.language_configurations
.push(unsafe { mem::transmute(configuration) });
self.language_configurations.push(unsafe {
mem::transmute::<LanguageConfiguration<'_>, LanguageConfiguration<'static>>(
configuration,
)
});
self.languages_by_id
.push((parser_path.to_owned(), OnceCell::new(), None));
}
@ -1322,8 +1336,7 @@ impl<'a> LanguageConfiguration<'a> {
.unwrap_or_else(|| ranges.last().unwrap());
error.offset = offset_within_section - range.start;
error.row = source[range.start..offset_within_section]
.chars()
.filter(|c| *c == '\n')
.matches(|c| c == '\n')
.count();
Error::from(error).context(format!("Error in query file {path:?}"))
}

4
cli/npm/install.js
View file

@ -40,7 +40,7 @@ const matrix = {
// Determine the URL of the file.
const platform = matrix.platform[process.platform];
const arch = platform && platform.arch[process.arch];
const arch = platform?.arch[process.arch];
if (!platform || !platform.name || !arch || !arch.name) {
console.error(
@ -91,7 +91,7 @@ function get(url, callback) {
}
};
const proxyEnv = process.env['HTTPS_PROXY'] || process.env['https_proxy'];
const proxyEnv = process.env.HTTPS_PROXY || process.env.https_proxy;
if (!proxyEnv) {
https.get(url, processResponse);
return;

12
cli/src/generate/build_tables/build_lex_table.rs
View file

@ -1,14 +1,18 @@
use std::{
collections::{hash_map::Entry, HashMap, VecDeque},
mem,
};
use log::info;
use super::{coincident_tokens::CoincidentTokenIndex, token_conflicts::TokenConflictMap};
use crate::generate::{
build_tables::{coincident_tokens::CoincidentTokenIndex, token_conflicts::TokenConflictMap},
dedup::split_state_id_groups,
grammars::{LexicalGrammar, SyntaxGrammar},
nfa::{CharacterSet, NfaCursor},
rules::{Symbol, TokenSet},
tables::{AdvanceAction, LexState, LexTable, ParseStateId, ParseTable},
};
use log::info;
use std::collections::{hash_map::Entry, HashMap, VecDeque};
use std::mem;
pub const LARGE_CHARACTER_RANGE_COUNT: usize = 8;

51
cli/src/generate/build_tables/build_parse_table.rs
View file

@ -1,25 +1,30 @@
use super::item::{ParseItem, ParseItemSet, ParseItemSetCore};
use super::item_set_builder::ParseItemSetBuilder;
use crate::generate::grammars::PrecedenceEntry;
use crate::generate::grammars::{
InlinedProductionMap, LexicalGrammar, SyntaxGrammar, VariableType,
use std::{
cmp::Ordering,
collections::{BTreeMap, HashMap, HashSet, VecDeque},
fmt::Write,
hash::BuildHasherDefault,
};
use crate::generate::node_types::VariableInfo;
use crate::generate::rules::{Associativity, Precedence, Symbol, SymbolType, TokenSet};
use crate::generate::tables::{
FieldLocation, GotoAction, ParseAction, ParseState, ParseStateId, ParseTable, ParseTableEntry,
ProductionInfo, ProductionInfoId,
};
use anyhow::{anyhow, Result};
use std::cmp::Ordering;
use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
use std::fmt::Write;
use std::hash::BuildHasherDefault;
use std::u32;
use anyhow::{anyhow, Result};
use indexmap::{map::Entry, IndexMap};
use rustc_hash::FxHasher;
use super::{
item::{ParseItem, ParseItemSet, ParseItemSetCore},
item_set_builder::ParseItemSetBuilder,
};
use crate::generate::{
grammars::{
InlinedProductionMap, LexicalGrammar, PrecedenceEntry, SyntaxGrammar, VariableType,
},
node_types::VariableInfo,
rules::{Associativity, Precedence, Symbol, SymbolType, TokenSet},
tables::{
FieldLocation, GotoAction, ParseAction, ParseState, ParseStateId, ParseTable,
ParseTableEntry, ProductionInfo, ProductionInfoId,
},
};
// For conflict reporting, each parse state is associated with an example
// sequence of symbols that could lead to that parse state.
type SymbolSequence = Vec<Symbol>;
@ -293,7 +298,7 @@ impl<'a> ParseTableBuilder<'a> {
}
}
reduction_info.precedence = precedence.clone();
reduction_info.precedence.clone_from(precedence);
if let Err(i) = reduction_info.symbols.binary_search(&symbol) {
reduction_info.symbols.insert(i, symbol);
}
@ -599,13 +604,13 @@ impl<'a> ParseTableBuilder<'a> {
write!(&mut msg, " {}", self.symbol_name(symbol)).unwrap();
}
write!(
writeln!(
&mut msg,
" • {} …\n\n",
" • {} …\n",
self.symbol_name(&conflicting_lookahead)
)
.unwrap();
write!(&mut msg, "Possible interpretations:\n\n").unwrap();
writeln!(&mut msg, "Possible interpretations:\n").unwrap();
let mut interpretations = conflicting_items
.iter()
@ -680,7 +685,7 @@ impl<'a> ParseTableBuilder<'a> {
}
let mut resolution_count = 0;
write!(&mut msg, "\nPossible resolutions:\n\n").unwrap();
writeln!(&mut msg, "\nPossible resolutions:\n").unwrap();
let mut shift_items = Vec::new();
let mut reduce_items = Vec::new();
for item in conflicting_items {
@ -956,7 +961,7 @@ fn populate_following_tokens(
for entry in result.iter_mut() {
entry.insert(*extra);
}
result[extra.index] = all_tokens.clone();
result[extra.index].clone_from(&all_tokens);
}
}
}

9
cli/src/generate/build_tables/coincident_tokens.rs
View file

@ -1,8 +1,11 @@
use crate::generate::grammars::LexicalGrammar;
use crate::generate::rules::Symbol;
use crate::generate::tables::{ParseStateId, ParseTable};
use std::fmt;
use crate::generate::{
grammars::LexicalGrammar,
rules::Symbol,
tables::{ParseStateId, ParseTable},
};
pub struct CoincidentTokenIndex<'a> {
entries: Vec<Vec<ParseStateId>>,
grammar: &'a LexicalGrammar,

19
cli/src/generate/build_tables/item.rs
View file

@ -1,10 +1,15 @@
use crate::generate::grammars::{LexicalGrammar, Production, ProductionStep, SyntaxGrammar};
use crate::generate::rules::{Associativity, Precedence, Symbol, SymbolType, TokenSet};
use std::{
cmp::Ordering,
fmt,
hash::{Hash, Hasher},
};
use lazy_static::lazy_static;
use std::cmp::Ordering;
use std::fmt;
use std::hash::{Hash, Hasher};
use std::u32;
use crate::generate::{
grammars::{LexicalGrammar, Production, ProductionStep, SyntaxGrammar},
rules::{Associativity, Precedence, Symbol, SymbolType, TokenSet},
};
lazy_static! {
static ref START_PRODUCTION: Production = Production {
@ -128,7 +133,7 @@ impl<'a> ParseItem<'a> {
/// Create an item like this one, but advanced by one step.
#[must_use]
pub const fn successor(&self) -> ParseItem<'a> {
pub const fn successor(&self) -> Self {
ParseItem {
variable_index: self.variable_index,
production: self.production,

27
cli/src/generate/build_tables/item_set_builder.rs
View file

@ -1,8 +1,13 @@
use std::{
collections::{HashMap, HashSet},
fmt,
};
use super::item::{ParseItem, ParseItemDisplay, ParseItemSet, TokenSetDisplay};
use crate::generate::grammars::{InlinedProductionMap, LexicalGrammar, SyntaxGrammar};
use crate::generate::rules::{Symbol, SymbolType, TokenSet};
use std::collections::{HashMap, HashSet};
use std::fmt;
use crate::generate::{
grammars::{InlinedProductionMap, LexicalGrammar, SyntaxGrammar},
rules::{Symbol, SymbolType, TokenSet},
};
#[derive(Clone, Debug, PartialEq, Eq)]
struct TransitiveClosureAddition<'a> {
@ -70,8 +75,8 @@ impl<'a> ParseItemSetBuilder<'a> {
// The FIRST set of a non-terminal `i` is the union of the following sets:
// * the set of all terminals that appear at the beginnings of i's productions
// * the FIRST sets of all the non-terminals that appear at the beginnings
// of i's productions
// * the FIRST sets of all the non-terminals that appear at the beginnings of i's
// productions
//
// Rather than computing these sets using recursion, we use an explicit stack
// called `symbols_to_process`.
@ -130,11 +135,11 @@ impl<'a> ParseItemSetBuilder<'a> {
// item set when `i` occurs as the next symbol in one if its core items. The
// structure of an *addition* is as follows:
// * `item` - the new item that must be added as part of the expansion of `i`
// * `lookaheads` - lookahead tokens that can always come after that item in
// the expansion of `i`
// * `propagates_lookaheads` - a boolean indicating whether or not `item` can
// occur at the *end* of the expansion of `i`, so that i's own current
// lookahead tokens can occur after `item`.
// * `lookaheads` - lookahead tokens that can always come after that item in the expansion
// of `i`
// * `propagates_lookaheads` - a boolean indicating whether or not `item` can occur at the
// *end* of the expansion of `i`, so that i's own current lookahead tokens can occur
// after `item`.
//
// Again, rather than computing these additions recursively, we use an explicit
// stack called `entries_to_process`.

20
cli/src/generate/build_tables/minimize_parse_table.rs
View file

@ -1,13 +1,17 @@
use super::token_conflicts::TokenConflictMap;
use crate::generate::dedup::split_state_id_groups;
use crate::generate::grammars::{LexicalGrammar, SyntaxGrammar, VariableType};
use crate::generate::rules::{AliasMap, Symbol, TokenSet};
use crate::generate::tables::{
GotoAction, ParseAction, ParseState, ParseStateId, ParseTable, ParseTableEntry,
use std::{
collections::{HashMap, HashSet},
mem,
};
use log::info;
use std::collections::{HashMap, HashSet};
use std::mem;
use super::token_conflicts::TokenConflictMap;
use crate::generate::{
dedup::split_state_id_groups,
grammars::{LexicalGrammar, SyntaxGrammar, VariableType},
rules::{AliasMap, Symbol, TokenSet},
tables::{GotoAction, ParseAction, ParseState, ParseStateId, ParseTable, ParseTableEntry},
};
pub fn minimize_parse_table(
parse_table: &mut ParseTable,

8
cli/src/generate/build_tables/mod.rs
View file

@ -6,6 +6,11 @@ mod item_set_builder;
mod minimize_parse_table;
mod token_conflicts;
use std::collections::{BTreeSet, HashMap};
use anyhow::Result;
use log::info;
use self::{
build_lex_table::build_lex_table,
build_parse_table::{build_parse_table, ParseStateInfo},
@ -20,9 +25,6 @@ use crate::generate::{
rules::{AliasMap, Symbol, SymbolType, TokenSet},
tables::{LexTable, ParseAction, ParseTable, ParseTableEntry},
};
use anyhow::Result;
use log::info;
use std::collections::{BTreeSet, HashMap};
pub use build_lex_table::LARGE_CHARACTER_RANGE_COUNT;

23
cli/src/generate/build_tables/token_conflicts.rs
View file

@ -1,10 +1,11 @@
use crate::generate::build_tables::item::TokenSetDisplay;
use crate::generate::grammars::{LexicalGrammar, SyntaxGrammar};
use crate::generate::nfa::{CharacterSet, NfaCursor, NfaTransition};
use crate::generate::rules::TokenSet;
use std::cmp::Ordering;
use std::collections::HashSet;
use std::fmt;
use std::{cmp::Ordering, collections::HashSet, fmt};
use crate::generate::{
build_tables::item::TokenSetDisplay,
grammars::{LexicalGrammar, SyntaxGrammar},
nfa::{CharacterSet, NfaCursor, NfaTransition},
rules::TokenSet,
};
#[derive(Clone, Debug, Default, PartialEq, Eq)]
struct TokenConflictStatus {
@ -372,9 +373,11 @@ fn compute_conflict_status(
#[cfg(test)]
mod tests {
use super::*;
use crate::generate::grammars::{Variable, VariableType};
use crate::generate::prepare_grammar::{expand_tokens, ExtractedLexicalGrammar};
use crate::generate::rules::{Precedence, Rule, Symbol};
use crate::generate::{
grammars::{Variable, VariableType},
prepare_grammar::{expand_tokens, ExtractedLexicalGrammar},
rules::{Precedence, Rule, Symbol},
};
#[test]
fn test_starting_characters() {

24
cli/src/generate/dsl.js
View file

@ -23,7 +23,7 @@ function alias(rule, value) {
}
}
throw new Error('Invalid alias value ' + value);
throw new Error(`Invalid alias value ${value}`);
}
function blank() {
@ -35,7 +35,7 @@ function blank() {
function field(name, rule) {
return {
type: "FIELD",
name: name,
name,
content: normalize(rule)
}
}
@ -156,7 +156,7 @@ function seq(...elements) {
function sym(name) {
return {
type: "SYMBOL",
name: name
name
};
}
@ -201,17 +201,17 @@ function normalize(value) {
if (typeof value.type === 'string') {
return value;
} else {
throw new TypeError("Invalid rule: " + value.toString());
throw new TypeError(`Invalid rule: ${value}`);
}
}
}
function RuleBuilder(ruleMap) {
return new Proxy({}, {
get(target, propertyName) {
get(_, propertyName) {
const symbol = sym(propertyName);
if (!ruleMap || ruleMap.hasOwnProperty(propertyName)) {
if (!ruleMap || Object.prototype.hasOwnProperty.call(ruleMap, propertyName)) {
return symbol;
} else {
const error = new ReferenceError(`Undefined symbol '${propertyName}'`);
@ -256,10 +256,10 @@ function grammar(baseGrammar, options) {
}
const ruleMap = {};
for (const key in options.rules) {
for (const key of Object.keys(options.rules)) {
ruleMap[key] = true;
}
for (const key in baseGrammar.rules) {
for (const key of Object.keys(baseGrammar.rules)) {
ruleMap[key] = true;
}
for (const external of externals) {
@ -279,16 +279,16 @@ function grammar(baseGrammar, options) {
throw new Error("Grammar's 'name' property must not start with a digit and cannot contain non-word characters.");
}
let rules = Object.assign({}, baseGrammar.rules);
const rules = Object.assign({}, baseGrammar.rules);
if (options.rules) {
if (typeof options.rules !== "object") {
throw new Error("Grammar's 'rules' property must be an object.");
}
for (const ruleName in options.rules) {
for (const ruleName of Object.keys(options.rules)) {
const ruleFn = options.rules[ruleName];
if (typeof ruleFn !== "function") {
throw new Error("Grammar rules must all be functions. '" + ruleName + "' rule is not.");
throw new Error(`Grammar rules must all be functions. '${ruleName}' rule is not.`);
}
rules[ruleName] = normalize(ruleFn.call(ruleBuilder, ruleBuilder, baseGrammar.rules[ruleName]));
}
@ -403,7 +403,7 @@ function grammar(baseGrammar, options) {
});
}
if (Object.keys(rules).length == 0) {
if (Object.keys(rules).length === 0) {
throw new Error("Grammar must have at least one rule.");
}

17
cli/src/generate/grammar_files.rs
View file

@ -1,13 +1,18 @@
use super::write_file;
use std::{
fs,
fs::File,
io::BufReader,
path::{Path, PathBuf},
str,
};
use anyhow::{anyhow, Context, Result};
use heck::{ToKebabCase, ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase};
use indoc::indoc;
use serde::Deserialize;
use serde_json::{json, Map, Value};
use std::fs::File;
use std::io::BufReader;
use std::path::{Path, PathBuf};
use std::{fs, str};
use super::write_file;
const CLI_VERSION: &str = env!("CARGO_PKG_VERSION");
const CLI_VERSION_PLACEHOLDER: &str = "CLI_VERSION";
@ -281,7 +286,7 @@ pub fn generate_grammar_files(
|path| {
let build_rs =
fs::read_to_string(path).with_context(|| "Failed to read build.rs")?;
if !build_rs.contains("/utf-8") {
if !build_rs.contains("-utf-8") {
let index = build_rs
.find(" let parser_path = src_dir.join(\"parser.c\")")
.ok_or_else(|| anyhow!(indoc!{

10
cli/src/generate/grammars.rs
View file

@ -1,7 +1,9 @@
use super::nfa::Nfa;
use super::rules::{Alias, Associativity, Precedence, Rule, Symbol};
use std::collections::HashMap;
use std::fmt;
use std::{collections::HashMap, fmt};
use super::{
nfa::Nfa,
rules::{Alias, Associativity, Precedence, Rule, Symbol},
};
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub enum VariableType {

13
cli/src/generate/mod.rs
View file

@ -1,17 +1,20 @@
use self::grammars::InputGrammar;
use std::{
env, fs,
io::Write,
path::{Path, PathBuf},
process::{Command, Stdio},
};
use anyhow::{anyhow, Context, Result};
use build_tables::build_tables;
use grammar_files::path_in_ignore;
use grammars::InputGrammar;
use lazy_static::lazy_static;
use parse_grammar::parse_grammar;
use prepare_grammar::prepare_grammar;
use regex::{Regex, RegexBuilder};
use render::render_c_code;
use semver::Version;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::{env, fs};
mod build_tables;
mod dedup;

48
cli/src/generate/node_types.rs
View file

@ -1,9 +1,15 @@
use super::grammars::{LexicalGrammar, SyntaxGrammar, VariableType};
use super::rules::{Alias, AliasMap, Symbol, SymbolType};
use std::{
cmp::Ordering,
collections::{BTreeMap, HashMap, HashSet},
};
use anyhow::{anyhow, Result};
use serde::Serialize;
use std::cmp::Ordering;
use std::collections::{BTreeMap, HashMap, HashSet};
use super::{
grammars::{LexicalGrammar, SyntaxGrammar, VariableType},
rules::{Alias, AliasMap, Symbol, SymbolType},
};
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum ChildType {
@ -134,18 +140,17 @@ impl ChildQuantity {
/// * `types` - The types of visible children the field can contain.
/// * `optional` - Do `N` nodes always have this field?
/// * `multiple` - Can `N` nodes have multiple children for this field?
/// 3. `children_without_fields` - The *other* named children of `N` that are
/// not associated with fields. Data regarding these children:
/// 3. `children_without_fields` - The *other* named children of `N` that are not associated with
/// fields. Data regarding these children:
/// * `types` - The types of named children with no field.
/// * `optional` - Do `N` nodes always have at least one named child with no field?
/// * `multiple` - Can `N` nodes have multiple named children with no field?
///
/// Each summary must account for some indirect factors:
/// 1. hidden nodes. When a parent node `N` has a hidden child `C`, the visible
/// children of `C` *appear* to be direct children of `N`.
/// 2. aliases. If a parent node type `M` is aliased as some other type `N`,
/// then nodes which *appear* to have type `N` may have internal structure based
/// on `M`.
/// 1. hidden nodes. When a parent node `N` has a hidden child `C`, the visible children of `C`
/// *appear* to be direct children of `N`.
/// 2. aliases. If a parent node type `M` is aliased as some other type `N`, then nodes which
/// *appear* to have type `N` may have internal structure based on `M`.
pub fn get_variable_info(
syntax_grammar: &SyntaxGrammar,
lexical_grammar: &LexicalGrammar,
@ -218,7 +223,8 @@ pub fn get_variable_info(
.entry(field_name)
.or_insert_with(ChildQuantity::zero);
// Inherit the types and quantities of hidden children associated with fields.
// Inherit the types and quantities of hidden children associated with
// fields.
if child_is_hidden && child_symbol.is_non_terminal() {
let child_variable_info = &result[child_symbol.index];
did_change |= extend_sorted(
@ -523,8 +529,8 @@ pub fn generate_node_types_json(
let fields_json = node_type_json.fields.as_mut().unwrap();
for (new_field, field_info) in &info.fields {
let field_json = fields_json.entry(new_field.clone()).or_insert_with(|| {
// If another rule is aliased with the same name, and does *not* have this field,
// then this field cannot be required.
// If another rule is aliased with the same name, and does *not* have this
// field, then this field cannot be required.
let mut field_json = FieldInfoJSON::default();
if node_type_existed {
field_json.required = false;
@ -534,8 +540,8 @@ pub fn generate_node_types_json(
populate_field_info_json(field_json, field_info);
}
// If another rule is aliased with the same name, any fields that aren't present in this
// cannot be required.
// If another rule is aliased with the same name, any fields that aren't present in
// this cannot be required.
for (existing_field, field_json) in fields_json.iter_mut() {
if !info.fields.contains_key(existing_field) {
field_json.required = false;
@ -715,11 +721,13 @@ where
#[cfg(test)]
mod tests {
use super::*;
use crate::generate::grammars::{
InputGrammar, LexicalVariable, Production, ProductionStep, SyntaxVariable, Variable,
use crate::generate::{
grammars::{
InputGrammar, LexicalVariable, Production, ProductionStep, SyntaxVariable, Variable,
},
prepare_grammar::prepare_grammar,
rules::Rule,
};
use crate::generate::prepare_grammar::prepare_grammar;
use crate::generate::rules::Rule;
#[test]
fn test_node_types_simple() {

30
cli/src/generate/parse_grammar.rs
View file

@ -1,9 +1,12 @@
use super::grammars::{InputGrammar, PrecedenceEntry, Variable, VariableType};
use super::rules::{Precedence, Rule};
use anyhow::{anyhow, Result};
use serde::Deserialize;
use serde_json::{Map, Value};
use super::{
grammars::{InputGrammar, PrecedenceEntry, Variable, VariableType},
rules::{Precedence, Rule},
};
#[derive(Deserialize)]
#[serde(tag = "type")]
#[allow(non_camel_case_types)]
@ -163,19 +166,18 @@ fn parse_rule(json: RuleJSON) -> Rule {
RuleJSON::PATTERN { value, flags } => Rule::Pattern(
value,
flags.map_or(String::new(), |f| {
f.chars()
.filter(|c| {
if *c == 'i' {
true
} else {
// silently ignore unicode flags
if *c != 'u' && *c != 'v' {
eprintln!("Warning: unsupported flag {c}");
}
false
f.matches(|c| {
if c == 'i' {
true
} else {
// silently ignore unicode flags
if c != 'u' && c != 'v' {
eprintln!("Warning: unsupported flag {c}");
}
})
.collect()
false
}
})
.collect()
}),
),
RuleJSON::SYMBOL { name } => Rule::NamedSymbol(name),

10
cli/src/generate/prepare_grammar/expand_repeats.rs
View file

@ -1,8 +1,10 @@
use std::{collections::HashMap, mem};
use super::ExtractedSyntaxGrammar;
use crate::generate::grammars::{Variable, VariableType};
use crate::generate::rules::{Rule, Symbol};
use std::collections::HashMap;
use std::mem;
use crate::generate::{
grammars::{Variable, VariableType},
rules::{Rule, Symbol},
};
struct Expander {
variable_name: String,

21
cli/src/generate/prepare_grammar/expand_tokens.rs
View file

@ -1,15 +1,18 @@
use super::ExtractedLexicalGrammar;
use crate::generate::grammars::{LexicalGrammar, LexicalVariable};
use crate::generate::nfa::{CharacterSet, Nfa, NfaState};
use crate::generate::rules::{Precedence, Rule};
use std::collections::HashMap;
use anyhow::{anyhow, Context, Result};
use lazy_static::lazy_static;
use regex_syntax::ast::{
parse, Ast, ClassPerlKind, ClassSet, ClassSetBinaryOpKind, ClassSetItem, ClassUnicodeKind,
RepetitionKind, RepetitionRange,
};
use std::collections::HashMap;
use std::i32;
use super::ExtractedLexicalGrammar;
use crate::generate::{
grammars::{LexicalGrammar, LexicalVariable},
nfa::{CharacterSet, Nfa, NfaState},
rules::{Precedence, Rule},
};
lazy_static! {
static ref UNICODE_CATEGORIES: HashMap<&'static str, Vec<u32>> =
@ -539,8 +542,10 @@ impl NfaBuilder {
#[cfg(test)]
mod tests {
use super::*;
use crate::generate::grammars::Variable;
use crate::generate::nfa::{NfaCursor, NfaTransition};
use crate::generate::{
grammars::Variable,
nfa::{NfaCursor, NfaTransition},
};
fn simulate_nfa<'a>(grammar: &'a LexicalGrammar, s: &'a str) -> Option<(usize, &'a str)> {
let start_states = grammar.variables.iter().map(|v| v.start_state).collect();

16
cli/src/generate/prepare_grammar/extract_default_aliases.rs
View file

@ -1,5 +1,7 @@
use crate::generate::grammars::{LexicalGrammar, SyntaxGrammar};
use crate::generate::rules::{Alias, AliasMap, Symbol, SymbolType};
use crate::generate::{
grammars::{LexicalGrammar, SyntaxGrammar},
rules::{Alias, AliasMap, Symbol, SymbolType},
};
#[derive(Clone, Default)]
struct SymbolStatus {
@ -14,8 +16,8 @@ struct SymbolStatus {
// This has two benefits:
// * It reduces the overhead of storing production-specific alias info in the parse table.
// * Within an `ERROR` node, no context-specific aliases will be applied. This transformation
// ensures that the children of an `ERROR` node have symbols that are consistent with the
// way that they would appear in a valid syntax tree.
// ensures that the children of an `ERROR` node have symbols that are consistent with the way that
// they would appear in a valid syntax tree.
pub(super) fn extract_default_aliases(
syntax_grammar: &mut SyntaxGrammar,
lexical_grammar: &LexicalGrammar,
@ -162,10 +164,10 @@ pub(super) fn extract_default_aliases(
#[cfg(test)]
mod tests {
use super::*;
use crate::generate::grammars::{
LexicalVariable, Production, ProductionStep, SyntaxVariable, VariableType,
use crate::generate::{
grammars::{LexicalVariable, Production, ProductionStep, SyntaxVariable, VariableType},
nfa::Nfa,
};
use crate::generate::nfa::Nfa;
#[test]
fn test_extract_simple_aliases() {

13
cli/src/generate/prepare_grammar/extract_tokens.rs
View file

@ -1,9 +1,12 @@
use super::{ExtractedLexicalGrammar, ExtractedSyntaxGrammar, InternedGrammar};
use crate::generate::grammars::{ExternalToken, Variable, VariableType};
use crate::generate::rules::{MetadataParams, Rule, Symbol, SymbolType};
use std::{collections::HashMap, mem};
use anyhow::{anyhow, Result};
use std::collections::HashMap;
use std::mem;
use super::{ExtractedLexicalGrammar, ExtractedSyntaxGrammar, InternedGrammar};
use crate::generate::{
grammars::{ExternalToken, Variable, VariableType},
rules::{MetadataParams, Rule, Symbol, SymbolType},
};
pub(super) fn extract_tokens(
mut grammar: InternedGrammar,

11
cli/src/generate/prepare_grammar/flatten_grammar.rs
View file

@ -1,10 +1,11 @@
use super::ExtractedSyntaxGrammar;
use crate::generate::grammars::{
Production, ProductionStep, SyntaxGrammar, SyntaxVariable, Variable,
};
use crate::generate::rules::{Alias, Associativity, Precedence, Rule, Symbol};
use anyhow::{anyhow, Result};
use super::ExtractedSyntaxGrammar;
use crate::generate::{
grammars::{Production, ProductionStep, SyntaxGrammar, SyntaxVariable, Variable},
rules::{Alias, Associativity, Precedence, Rule, Symbol},
};
struct RuleFlattener {
production: Production,
precedence_stack: Vec<Precedence>,

9
cli/src/generate/prepare_grammar/intern_symbols.rs
View file

@ -1,8 +1,11 @@
use super::InternedGrammar;
use crate::generate::grammars::{InputGrammar, Variable, VariableType};
use crate::generate::rules::{Rule, Symbol};
use anyhow::{anyhow, Result};
use super::InternedGrammar;
use crate::generate::{
grammars::{InputGrammar, Variable, VariableType},
rules::{Rule, Symbol},
};
pub(super) fn intern_symbols(grammar: &InputGrammar) -> Result<InternedGrammar> {
let interner = Interner { grammar };

30
cli/src/generate/prepare_grammar/mod.rs
View file

@ -6,26 +6,28 @@ mod flatten_grammar;
mod intern_symbols;
mod process_inlines;
pub use self::expand_tokens::expand_tokens;
use self::expand_repeats::expand_repeats;
use self::extract_default_aliases::extract_default_aliases;
use self::extract_tokens::extract_tokens;
use self::flatten_grammar::flatten_grammar;
use self::intern_symbols::intern_symbols;
use self::process_inlines::process_inlines;
use super::grammars::{
ExternalToken, InlinedProductionMap, InputGrammar, LexicalGrammar, PrecedenceEntry,
SyntaxGrammar, Variable,
};
use super::rules::{AliasMap, Precedence, Rule, Symbol};
use anyhow::{anyhow, Result};
use std::{
cmp::Ordering,
collections::{hash_map, HashMap, HashSet},
mem,
};
use anyhow::{anyhow, Result};
pub use self::expand_tokens::expand_tokens;
use self::{
expand_repeats::expand_repeats, extract_default_aliases::extract_default_aliases,
extract_tokens::extract_tokens, flatten_grammar::flatten_grammar,
intern_symbols::intern_symbols, process_inlines::process_inlines,
};
use super::{
grammars::{
ExternalToken, InlinedProductionMap, InputGrammar, LexicalGrammar, PrecedenceEntry,
SyntaxGrammar, Variable,
},
rules::{AliasMap, Precedence, Rule, Symbol},
};
pub struct IntermediateGrammar<T, U> {
variables: Vec<Variable>,
extra_symbols: Vec<T>,

14
cli/src/generate/prepare_grammar/process_inlines.rs
View file

@ -1,9 +1,11 @@
use std::collections::HashMap;
use anyhow::{anyhow, Result};
use crate::generate::{
grammars::{InlinedProductionMap, LexicalGrammar, Production, ProductionStep, SyntaxGrammar},
rules::SymbolType,
};
use anyhow::{anyhow, Result};
use std::collections::HashMap;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
struct ProductionStepId {
@ -152,7 +154,7 @@ impl InlinedProductionMapBuilder {
self.productions
.iter()
.position(|p| *p == production)
.unwrap_or({
.unwrap_or_else(|| {
self.productions.push(production);
self.productions.len() - 1
})
@ -223,8 +225,10 @@ pub(super) fn process_inlines(
#[cfg(test)]
mod tests {
use super::*;
use crate::generate::grammars::{LexicalVariable, SyntaxVariable, VariableType};
use crate::generate::rules::{Associativity, Precedence, Symbol};
use crate::generate::{
grammars::{LexicalVariable, SyntaxVariable, VariableType},
rules::{Associativity, Precedence, Symbol},
};
#[test]
fn test_basic_inlining() {

52
cli/src/generate/render.rs
View file

@ -1,3 +1,10 @@
use std::{
cmp,
collections::{HashMap, HashSet},
fmt::Write,
mem::swap,
};
use super::{
build_tables::Tables,
grammars::{ExternalToken, LexicalGrammar, SyntaxGrammar, VariableType},
@ -8,12 +15,6 @@ use super::{
ParseTableEntry,
},
};
use std::{
cmp,
collections::{HashMap, HashSet},
fmt::Write,
mem::swap,
};
const SMALL_STATE_THRESHOLD: usize = 64;
const ABI_VERSION_MIN: usize = 13;
@ -27,7 +28,7 @@ macro_rules! add {
}
macro_rules! add_whitespace {
($this: tt) => {{
($this:tt) => {{
for _ in 0..$this.indent_level {
write!(&mut $this.buffer, " ").unwrap();
}
@ -43,13 +44,13 @@ macro_rules! add_line {
}
macro_rules! indent {
($this: tt) => {
($this:tt) => {
$this.indent_level += 1;
};
}
macro_rules! dedent {
($this: tt) => {
($this:tt) => {
assert_ne!($this.indent_level, 0);
$this.indent_level -= 1;
};
@ -172,8 +173,8 @@ impl Generator {
}
// Two anonymous tokens with different flags but the same string value
// should be represented with the same symbol in the public API. Examples:
// * "<" and token(prec(1, "<"))
// * "(" and token.immediate("(")
// * "<" and token(prec(1, "<"))
// * "(" and token.immediate("(")
else if symbol.is_terminal() {
let metadata = self.metadata_for_symbol(*symbol);
for other_symbol in &self.parse_table.symbols {
@ -215,22 +216,22 @@ impl Generator {
});
// Some aliases match an existing symbol in the grammar.
let alias_id;
if let Some(existing_symbol) = existing_symbol {
alias_id = self.symbol_ids[&self.symbol_map[&existing_symbol]].clone();
let alias_id = if let Some(existing_symbol) = existing_symbol {
self.symbol_ids[&self.symbol_map[&existing_symbol]].clone()
}
// Other aliases don't match any existing symbol, and need their own identifiers.
// Other aliases don't match any existing symbol, and need their own
// identifiers.
else {
if let Err(i) = self.unique_aliases.binary_search(alias) {
self.unique_aliases.insert(i, alias.clone());
}
alias_id = if alias.is_named {
if alias.is_named {
format!("alias_sym_{}", self.sanitize_identifier(&alias.value))
} else {
format!("anon_alias_sym_{}", self.sanitize_identifier(&alias.value))
};
}
}
};
self.alias_ids.entry(alias.clone()).or_insert(alias_id);
}
@ -1612,16 +1613,15 @@ impl Generator {
/// * `parse_table` - The generated parse table for the language
/// * `main_lex_table` - The generated lexing table for the language
/// * `keyword_lex_table` - The generated keyword lexing table for the language
/// * `keyword_capture_token` - A symbol indicating which token is used
/// for keyword capture, if any.
/// * `keyword_capture_token` - A symbol indicating which token is used for keyword capture, if any.
/// * `syntax_grammar` - The syntax grammar extracted from the language's grammar
/// * `lexical_grammar` - The lexical grammar extracted from the language's grammar
/// * `default_aliases` - A map describing the global rename rules that should apply.
/// the keys are symbols that are *always* aliased in the same way, and the values
/// are the aliases that are applied to those symbols.
/// * `abi_version` - The language ABI version that should be generated. Usually
/// you want Tree-sitter's current version, but right after making an ABI
/// change, it may be useful to generate code with the previous ABI.
/// * `default_aliases` - A map describing the global rename rules that should apply. the keys are
/// symbols that are *always* aliased in the same way, and the values are the aliases that are
/// applied to those symbols.
/// * `abi_version` - The language ABI version that should be generated. Usually you want
/// Tree-sitter's current version, but right after making an ABI change, it may be useful to
/// generate code with the previous ABI.
#[allow(clippy::too_many_arguments)]
pub fn render_c_code(
name: &str,

6
cli/src/generate/rules.rs
View file

@ -1,7 +1,9 @@
use super::grammars::VariableType;
use smallbitvec::SmallBitVec;
use std::{collections::HashMap, fmt};
use smallbitvec::SmallBitVec;
use super::grammars::VariableType;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub enum SymbolType {
External,

7
cli/src/generate/tables.rs
View file

@ -1,6 +1,9 @@
use super::nfa::CharacterSet;
use super::rules::{Alias, Symbol, TokenSet};
use std::collections::BTreeMap;
use super::{
nfa::CharacterSet,
rules::{Alias, Symbol, TokenSet},
};
pub type ProductionInfoId = usize;
pub type ParseStateId = usize;
pub type LexStateId = usize;

15
cli/src/highlight.rs
View file

@ -1,14 +1,12 @@
use std::{
collections::HashMap, fmt::Write, fs, io, path, str, sync::atomic::AtomicUsize, time::Instant,
};
use ansi_term::Color;
use anyhow::Result;
use lazy_static::lazy_static;
use serde::ser::SerializeMap;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde::{ser::SerializeMap, Deserialize, Deserializer, Serialize, Serializer};
use serde_json::{json, Value};
use std::collections::HashMap;
use std::fmt::Write;
use std::sync::atomic::AtomicUsize;
use std::time::Instant;
use std::{fs, io, path, str, usize};
use tree_sitter_highlight::{HighlightConfiguration, HighlightEvent, Highlighter, HtmlRenderer};
use tree_sitter_loader::Loader;
@ -417,9 +415,10 @@ pub fn html(
#[cfg(test)]
mod tests {
use super::*;
use std::env;
use super::*;
const JUNGLE_GREEN: &str = "#26A69A";
const DARK_CYAN: &str = "#00AF87";

17
cli/src/main.rs
View file

@ -1,17 +1,21 @@
use std::{
collections::HashSet,
env, fs,
path::{Path, PathBuf},
};
use anstyle::{AnsiColor, Color, Style};
use anyhow::{anyhow, Context, Result};
use clap::{crate_authors, Args, Command, FromArgMatches as _, Subcommand};
use glob::glob;
use regex::Regex;
use std::collections::HashSet;
use std::path::{Path, PathBuf};
use std::{env, fs, u64};
use tree_sitter::{ffi, Parser, Point};
use tree_sitter_cli::test::TestOptions;
use tree_sitter_cli::{
generate, highlight, logger,
parse::{self, ParseFileOptions, ParseOutput},
playground, query, tags, test, test_highlight, test_tags, util, wasm,
playground, query, tags, test,
test::TestOptions,
test_highlight, test_tags, util, wasm,
};
use tree_sitter_config::Config;
use tree_sitter_highlight::Highlighter;
@ -866,8 +870,7 @@ fn run() -> Result<()> {
let open_in_browser = !playground_options.quiet;
let grammar_path = playground_options
.grammar_path
.map(PathBuf::from)
.unwrap_or(current_dir);
.map_or(current_dir, PathBuf::from);
playground::serve(&grammar_path, open_in_browser)?;
}

16
cli/src/parse.rs
View file

@ -1,12 +1,16 @@
use super::util;
use std::{
fmt, fs,
io::{self, Write},
path::Path,
sync::atomic::AtomicUsize,
time::{Duration, Instant},
};
use anyhow::{anyhow, Context, Result};
use std::io::{self, Write};
use std::path::Path;
use std::sync::atomic::AtomicUsize;
use std::time::{Duration, Instant};
use std::{fmt, fs, usize};
use tree_sitter::{ffi, InputEdit, Language, LogType, Parser, Point, Tree};
use super::util;
#[derive(Debug)]
pub struct Edit {
pub position: usize,

8
cli/src/playground.rs
View file

@ -1,5 +1,3 @@
use super::wasm;
use anyhow::{anyhow, Context, Result};
use std::{
borrow::Cow,
env, fs,
@ -7,10 +5,14 @@ use std::{
path::{Path, PathBuf},
str::{self, FromStr as _},
};
use anyhow::{anyhow, Context, Result};
use tiny_http::{Header, Response, Server};
use super::wasm;
macro_rules! optional_resource {
($name: tt, $path: tt) => {
($name:tt, $path:tt) => {
#[cfg(TREE_SITTER_EMBED_WASM_BINDING)]
fn $name(tree_sitter_dir: Option<&Path>) -> Cow<'static, [u8]> {
if let Some(tree_sitter_dir) = tree_sitter_dir {

6
cli/src/query.rs
View file

@ -1,5 +1,3 @@
use crate::query_testing;
use anyhow::{Context, Result};
use std::{
fs,
io::{self, Write},
@ -7,8 +5,12 @@ use std::{
path::Path,
time::Instant,
};
use anyhow::{Context, Result};
use tree_sitter::{Language, Parser, Point, Query, QueryCursor};
use crate::query_testing;
#[allow(clippy::too_many_arguments)]
pub fn query_files_at_paths(
language: &Language,

3
cli/src/query_testing.rs
View file

@ -1,7 +1,8 @@
use std::fs;
use anyhow::{anyhow, Result};
use lazy_static::lazy_static;
use regex::Regex;
use std::fs;
use tree_sitter::{Language, Parser, Point};
lazy_static! {

15
cli/src/tags.rs
View file

@ -1,12 +1,17 @@
use super::util;
use std::{
fs,
io::{self, Write},
path::Path,
str,
time::Instant,
};
use anyhow::{anyhow, Result};
use std::io::{self, Write};
use std::path::Path;
use std::time::Instant;
use std::{fs, str};
use tree_sitter_loader::{Config, Loader};
use tree_sitter_tags::TagsContext;
use super::util;
pub fn generate_tags(
loader: &Loader,
loader_config: &Config,

38
cli/src/test.rs
View file

@ -1,20 +1,26 @@
use super::util;
use std::{
collections::BTreeMap,
ffi::OsStr,
fs,
io::{self, Write},
path::{Path, PathBuf},
str,
};
use ansi_term::Colour;
use anyhow::{anyhow, Context, Result};
use difference::{Changeset, Difference};
use indoc::indoc;
use lazy_static::lazy_static;
use regex::bytes::{Regex as ByteRegex, RegexBuilder as ByteRegexBuilder};
use regex::Regex;
use std::collections::BTreeMap;
use std::ffi::OsStr;
use std::fs;
use std::io::{self, Write};
use std::path::{Path, PathBuf};
use std::str;
use regex::{
bytes::{Regex as ByteRegex, RegexBuilder as ByteRegexBuilder},
Regex,
};
use tree_sitter::{format_sexp, Language, LogType, Parser, Query};
use walkdir::WalkDir;
use super::util;
lazy_static! {
static ref HEADER_REGEX: ByteRegex = ByteRegexBuilder::new(
r"^(?x)
@ -300,13 +306,15 @@ fn run_tests(
let expected_output = format_sexp(&output, 0);
let actual_output = format_sexp(&actual, 0);
// Only bail early before updating if the actual is not the output, sometimes
// users want to test cases that are intended to have errors, hence why this
// Only bail early before updating if the actual is not the output,
// sometimes users want to test cases that
// are intended to have errors, hence why this
// check isn't shown above
if actual.contains("ERROR") || actual.contains("MISSING") {
*has_parse_errors = true;
// keep the original `expected` output if the actual output has an error
// keep the original `expected` output if the actual output has an
// error
corrected_entries.push((
name.clone(),
input,
@ -424,9 +432,9 @@ fn write_tests_to_buffer(
if i > 0 {
writeln!(buffer)?;
}
write!(
writeln!(
buffer,
"{}\n{name}\n{}\n{input}\n{}\n\n{}\n",
"{}\n{name}\n{}\n{input}\n{}\n\n{}",
"=".repeat(*header_delim_len),
"=".repeat(*header_delim_len),
"-".repeat(*divider_delim_len),
@ -654,7 +662,7 @@ fn parse_test_content(name: String, content: &str, file_path: Option<PathBuf>) -
}
}
prev_attributes = attributes;
prev_name = test_name.unwrap_or(String::new());
prev_name = test_name.unwrap_or_default();
prev_header_len = header_delim_len;
prev_header_end = header_range.end;
}

3
cli/src/test_highlight.rs
View file

@ -1,5 +1,4 @@
use std::fs;
use std::path::Path;
use std::{fs, path::Path};
use ansi_term::Colour;
use anyhow::{anyhow, Result};

3
cli/src/test_tags.rs
View file

@ -1,5 +1,4 @@
use std::fs;
use std::path::Path;
use std::{fs, path::Path};
use ansi_term::Colour;
use anyhow::{anyhow, Result};

14
cli/src/tests/async_context_test.rs
View file

@ -1,10 +1,14 @@
use super::helpers::fixtures::get_language;
use std::future::Future;
use std::pin::{pin, Pin};
use std::ptr;
use std::task::{self, Context, Poll, RawWaker, RawWakerVTable, Waker};
use std::{
future::Future,
pin::{pin, Pin},
ptr,
task::{self, Context, Poll, RawWaker, RawWakerVTable, Waker},
};
use tree_sitter::Parser;
use super::helpers::fixtures::get_language;
#[test]
fn test_node_in_fut() {
let (ret, pended) = tokio_like_spawn(async {

10
cli/src/tests/corpus_test.rs
View file

@ -1,3 +1,8 @@
use std::{collections::HashMap, env, fs};
use tree_sitter::{LogType, Node, Parser, Point, Range, Tree};
use tree_sitter_proc_macro::test_with_seed;
use super::helpers::{
allocations,
edits::{get_random_edit, invert_edit},
@ -14,9 +19,6 @@ use crate::{
test::{parse_tests, print_diff, print_diff_key, strip_sexp_fields, TestEntry},
util,
};
use std::{collections::HashMap, env, fs};
use tree_sitter::{LogType, Node, Parser, Point, Range, Tree};
use tree_sitter_proc_macro::test_with_seed;
#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
fn test_corpus_for_bash(seed: usize) {
@ -215,7 +217,7 @@ fn test_language_corpus(
// Perform a random series of edits and reparse.
let mut undo_stack = Vec::new();
for _ in 0..1 + rand.unsigned(*EDIT_COUNT) {
for _ in 0..=rand.unsigned(*EDIT_COUNT) {
let edit = get_random_edit(&mut rand, &input);
undo_stack.push(invert_edit(&input, &edit));
perform_edit(&mut tree, &mut input, &edit).unwrap();

28
cli/src/tests/detect_language.rs
View file

@ -1,8 +1,9 @@
use crate::tests::helpers::fixtures::scratch_dir;
use std::{fs, path::Path};
use std::path::Path;
use tree_sitter_loader::Loader;
use crate::tests::helpers::fixtures::scratch_dir;
#[test]
fn detect_language_by_first_line_regex() {
let strace_dir = tree_sitter_dir(
@ -32,25 +33,25 @@ fn detect_language_by_first_line_regex() {
assert_eq!(config[0].scope.as_ref().unwrap(), "source.strace");
let file_name = strace_dir.path().join("strace.log");
std::fs::write(&file_name, "execve\nworld").unwrap();
fs::write(&file_name, "execve\nworld").unwrap();
assert_eq!(
get_lang_scope(&loader, &file_name),
Some("source.strace".into())
);
let file_name = strace_dir.path().join("strace.log");
std::fs::write(&file_name, "447845 execve\nworld").unwrap();
fs::write(&file_name, "447845 execve\nworld").unwrap();
assert_eq!(
get_lang_scope(&loader, &file_name),
Some("source.strace".into())
);
let file_name = strace_dir.path().join("strace.log");
std::fs::write(&file_name, "hello\nexecve").unwrap();
fs::write(&file_name, "hello\nexecve").unwrap();
assert!(get_lang_scope(&loader, &file_name).is_none());
let file_name = strace_dir.path().join("strace.log");
std::fs::write(&file_name, "").unwrap();
fs::write(&file_name, "").unwrap();
assert!(get_lang_scope(&loader, &file_name).is_none());
let dummy_dir = tree_sitter_dir(
@ -75,7 +76,7 @@ fn detect_language_by_first_line_regex() {
.find_language_configurations_at_path(dummy_dir.path(), false)
.unwrap();
let file_name = dummy_dir.path().join("strace.dummy");
std::fs::write(&file_name, "execve").unwrap();
fs::write(&file_name, "execve").unwrap();
assert_eq!(
get_lang_scope(&loader, &file_name),
Some("source.dummy".into())
@ -84,15 +85,14 @@ fn detect_language_by_first_line_regex() {
fn tree_sitter_dir(package_json: &str, name: &str) -> tempfile::TempDir {
let temp_dir = tempfile::tempdir().unwrap();
std::fs::write(temp_dir.path().join("package.json"), package_json).unwrap();
std::fs::create_dir(temp_dir.path().join("src")).unwrap();
std::fs::create_dir(temp_dir.path().join("src/tree_sitter")).unwrap();
std::fs::write(
fs::write(temp_dir.path().join("package.json"), package_json).unwrap();
fs::create_dir_all(temp_dir.path().join("src/tree_sitter")).unwrap();
fs::write(
temp_dir.path().join("src/grammar.json"),
format!(r#"{{"name":"{name}"}}"#),
)
.unwrap();
std::fs::write(
fs::write(
temp_dir.path().join("src/parser.c"),
format!(
r##"
@ -107,7 +107,7 @@ fn tree_sitter_dir(package_json: &str, name: &str) -> tempfile::TempDir {
),
)
.unwrap();
std::fs::write(
fs::write(
temp_dir.path().join("src/tree_sitter/parser.h"),
include_str!("../../../lib/src/parser.h"),
)
@ -115,7 +115,7 @@ fn tree_sitter_dir(package_json: &str, name: &str) -> tempfile::TempDir {
temp_dir
}
// if we manage to get the language scope, it means we correctly detected the file-type
// If we manage to get the language scope, it means we correctly detected the file-type
fn get_lang_scope(loader: &Loader, file_name: &Path) -> Option<String> {
loader
.language_configuration_for_file_name(file_name)

6
cli/src/tests/helpers/edits.rs
View file

@ -1,7 +1,7 @@
use std::{ops::Range, str};
use super::random::Rand;
use crate::parse::Edit;
use std::ops::Range;
use std::str;
#[derive(Debug)]
pub struct ReadRecorder<'a> {
@ -31,7 +31,7 @@ impl<'a> ReadRecorder<'a> {
pub fn strings_read(&self) -> Vec<&'a str> {
let mut result = Vec::new();
let mut last_range: Option<Range<usize>> = None;
let mut last_range = Option::<Range<usize>>::None;
for index in &self.indices_read {
if let Some(ref mut range) = &mut last_range {
if range.end == *index {

7
cli/src/tests/helpers/fixtures.rs
View file

@ -1,7 +1,10 @@
use std::{
env, fs,
path::{Path, PathBuf},
};
use anyhow::Context;
use lazy_static::lazy_static;
use std::path::{Path, PathBuf};
use std::{env, fs};
use tree_sitter::Language;
use tree_sitter_highlight::HighlightConfiguration;
use tree_sitter_loader::{CompileConfig, Loader};

3
cli/src/tests/helpers/mod.rs
View file

@ -5,9 +5,10 @@ pub(super) mod query_helpers;
pub(super) mod random;
pub(super) mod scope_sequence;
use std::env;
use lazy_static::lazy_static;
use rand::Rng;
use std::env;
lazy_static! {
pub static ref LOG_ENABLED: bool = env::var("TREE_SITTER_LOG").is_ok();

3
cli/src/tests/helpers/query_helpers.rs
View file

@ -1,5 +1,6 @@
use rand::prelude::Rng;
use std::{cmp::Ordering, fmt::Write, ops::Range};
use rand::prelude::Rng;
use tree_sitter::{
Language, Node, Parser, Point, Query, QueryCapture, QueryCursor, QueryMatch, Tree, TreeCursor,
};

2
cli/src/tests/helpers/random.rs
View file

@ -15,7 +15,7 @@ impl Rand {
}
pub fn unsigned(&mut self, max: usize) -> usize {
self.0.gen_range(0..max + 1)
self.0.gen_range(0..=max)
}
pub fn words(&mut self, max_count: usize) -> Vec<u8> {

18
cli/src/tests/highlight_test.rs
View file

@ -1,13 +1,18 @@
use super::helpers::fixtures::{get_highlight_config, get_language, get_language_queries_path};
use std::{
ffi::CString,
fs,
os::raw::c_char,
ptr, slice, str,
sync::atomic::{AtomicUsize, Ordering},
};
use lazy_static::lazy_static;
use std::ffi::CString;
use std::os::raw::c_char;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::{fs, ptr, slice, str};
use tree_sitter_highlight::{
c, Error, Highlight, HighlightConfiguration, HighlightEvent, Highlighter, HtmlRenderer,
};
use super::helpers::fixtures::{get_highlight_config, get_language, get_language_queries_path};
lazy_static! {
static ref JS_HIGHLIGHT: HighlightConfiguration =
get_highlight_config("javascript", Some("injections.scm"), &HIGHLIGHT_NAMES);
@ -748,8 +753,7 @@ fn to_token_vector<'a>(
for (i, l) in s.split('\n').enumerate() {
let l = l.trim_end_matches('\r');
if i > 0 {
lines.push(line);
line = Vec::new();
lines.push(std::mem::take(&mut line));
}
if !l.is_empty() {
line.push((l, highlights.clone()));

3
cli/src/tests/language_test.rs
View file

@ -1,6 +1,7 @@
use super::helpers::fixtures::get_language;
use tree_sitter::Parser;
use super::helpers::fixtures::get_language;
#[test]
fn test_lookahead_iterator() {
let mut parser = Parser::new();

28
cli/src/tests/node_test.rs
View file

@ -1,11 +1,14 @@
use super::helpers::edits::get_random_edit;
use super::helpers::fixtures::{fixtures_dir, get_language, get_test_language};
use super::helpers::random::Rand;
use crate::generate::generate_parser_for_grammar;
use crate::parse::perform_edit;
use std::fs;
use tree_sitter::{Node, Parser, Point, Tree};
use super::helpers::{
edits::get_random_edit,
fixtures::{fixtures_dir, get_language, get_test_language},
random::Rand,
};
use crate::{generate::generate_parser_for_grammar, parse::perform_edit};
const JSON_EXAMPLE: &str = r#"
[
@ -847,10 +850,11 @@ fn test_node_numeric_symbols_respect_simple_aliases() {
parser.set_language(&get_language("python")).unwrap();
// Example 1:
// Python argument lists can contain "splat" arguments, which are not allowed within
// other expressions. This includes `parenthesized_list_splat` nodes like `(*b)`. These
// `parenthesized_list_splat` nodes are aliased as `parenthesized_expression`. Their numeric
// `symbol`, aka `kind_id` should match that of a normal `parenthesized_expression`.
// Python argument lists can contain "splat" arguments, which are not allowed
// within other expressions. This includes `parenthesized_list_splat` nodes
// like `(*b)`. These `parenthesized_list_splat` nodes are aliased as
// `parenthesized_expression`. Their numeric `symbol`, aka `kind_id` should
// match that of a normal `parenthesized_expression`.
let tree = parser.parse("(a((*b)))", None).unwrap();
let root = tree.root_node();
assert_eq!(
@ -872,9 +876,9 @@ fn test_node_numeric_symbols_respect_simple_aliases() {
assert_eq!(inner_expr_node.kind_id(), outer_expr_node.kind_id());
// Example 2:
// Ruby handles the unary (negative) and binary (minus) `-` operators using two different
// tokens. One or more of these is an external token that's aliased as `-`. Their numeric
// kind ids should match.
// Ruby handles the unary (negative) and binary (minus) `-` operators using two
// different tokens. One or more of these is an external token that's
// aliased as `-`. Their numeric kind ids should match.
parser.set_language(&get_language("ruby")).unwrap();
let tree = parser.parse("-a - b", None).unwrap();
let root = tree.root_node();

107
cli/src/tests/parser_hang_test.rs
View file

@ -1,15 +1,17 @@
// For some reasons `Command::spawn` doesn't work in CI env for many exotic arches.
#![cfg(all(any(target_arch = "x86_64", target_arch = "x86"), not(sanitizing)))]
use std::{
env::VarError,
process::{Command, Stdio},
};
use tree_sitter::Parser;
use crate::{
generate::{generate_parser_for_grammar, load_grammar_file},
tests::helpers::fixtures::{fixtures_dir, get_test_language},
};
use std::{
env::VarError,
process::{Command, Stdio},
};
use tree_sitter::Parser;
// The `sanitizing` cfg is required to don't run tests under specific sunitizer
// because they don't work well with subprocesses _(it's an assumption)_.
@ -35,7 +37,7 @@ fn test_grammar_that_should_hang_and_not_segfault() {
let tests_exec_path = std::env::args()
.next()
.expect("Failed get get tests executable path");
.expect("Failed to get tests executable path");
match std::env::var(test_var) {
Ok(v) if v == test_name => {
@ -45,60 +47,59 @@ fn test_grammar_that_should_hang_and_not_segfault() {
Err(VarError::NotPresent) => {
eprintln!(" parent process id {}", std::process::id());
if true {
let mut command = Command::new(tests_exec_path);
command.arg(test_name).env(test_var, test_name);
if std::env::args().any(|x| x == "--nocapture") {
command.arg("--nocapture");
} else {
command.stdout(Stdio::null()).stderr(Stdio::null());
}
match command.spawn() {
Ok(mut child) => {
std::thread::sleep(std::time::Duration::from_millis(parent_sleep_millis));
match child.try_wait() {
Ok(Some(status)) if status.success() => {
panic!("Child wasn't hang and exited successfully")
}
Ok(Some(status)) => panic!(
"Child wasn't hang and exited with status code: {:?}",
status.code()
),
_ => (),
}
if let Err(e) = child.kill() {
eprintln!(
"Failed to kill hang test sub process id: {}, error: {e}",
child.id()
);
let mut command = Command::new(tests_exec_path);
command.arg(test_name).env(test_var, test_name);
if std::env::args().any(|x| x == "--nocapture") {
command.arg("--nocapture");
} else {
command.stdout(Stdio::null()).stderr(Stdio::null());
}
match command.spawn() {
Ok(mut child) => {
std::thread::sleep(std::time::Duration::from_millis(parent_sleep_millis));
match child.try_wait() {
Ok(Some(status)) if status.success() => {
panic!("Child didn't hang and exited successfully")
}
Ok(Some(status)) => panic!(
"Child didn't hang and exited with status code: {:?}",
status.code()
),
_ => (),
}
if let Err(e) = child.kill() {
eprintln!(
"Failed to kill hang test's process id: {}, error: {e}",
child.id()
);
}
Err(e) => panic!("{e}"),
}
Err(e) => panic!("{e}"),
}
}
Err(e) => panic!("Env var error: {e}"),
_ => unreachable!(),
}
fn hang_test() {
let test_grammar_dir = fixtures_dir()
.join("test_grammars")
.join("get_col_should_hang_not_crash");
let grammar_json = load_grammar_file(&test_grammar_dir.join("grammar.js"), None).unwrap();
let (parser_name, parser_code) =
generate_parser_for_grammar(grammar_json.as_str()).unwrap();
let language =
get_test_language(&parser_name, &parser_code, Some(test_grammar_dir.as_path()));
let mut parser = Parser::new();
parser.set_language(&language).unwrap();
let code_that_should_hang = "\nHello";
parser.parse(code_that_should_hang, None).unwrap();
}
}
fn hang_test() {
let test_grammar_dir = fixtures_dir()
.join("test_grammars")
.join("get_col_should_hang_not_crash");
let grammar_json = load_grammar_file(&test_grammar_dir.join("grammar.js"), None).unwrap();
let (parser_name, parser_code) = generate_parser_for_grammar(grammar_json.as_str()).unwrap();
let language = get_test_language(&parser_name, &parser_code, Some(test_grammar_dir.as_path()));
let mut parser = Parser::new();
parser.set_language(&language).unwrap();
let code_that_should_hang = "\nHello";
parser.parse(code_that_should_hang, None).unwrap();
}

18
cli/src/tests/parser_test.rs
View file

@ -1,3 +1,12 @@
use std::{
fs,
sync::atomic::{AtomicUsize, Ordering},
thread, time,
};
use tree_sitter::{IncludedRangesError, InputEdit, LogType, Parser, Point, Range};
use tree_sitter_proc_macro::retry;
use super::helpers::{
allocations,
edits::{invert_edit, ReadRecorder},
@ -8,13 +17,6 @@ use crate::{
parse::{perform_edit, Edit},
tests::helpers::fixtures::fixtures_dir,
};
use std::{
fs,
sync::atomic::{AtomicUsize, Ordering},
thread, time,
};
use tree_sitter::{IncludedRangesError, InputEdit, LogType, Parser, Point, Range};
use tree_sitter_proc_macro::retry;
#[test]
fn test_parsing_simple_string() {
@ -97,7 +99,7 @@ fn test_parsing_with_debug_graph_enabled() {
parser.print_dot_graphs(&debug_graph_file);
parser.parse("const zero = 0", None).unwrap();
debug_graph_file.seek(std::io::SeekFrom::Start(0)).unwrap();
debug_graph_file.rewind().unwrap();
let log_reader = BufReader::new(debug_graph_file)
.lines()
.map(|l| l.expect("Failed to read line from graph log"));

3
cli/src/tests/pathological_test.rs
View file

@ -1,6 +1,7 @@
use super::helpers::{allocations, fixtures::get_language};
use tree_sitter::Parser;
use super::helpers::{allocations, fixtures::get_language};
#[test]
fn test_pathological_example_1() {
let language = "cpp";

8
cli/src/tests/proc_macro/src/lib.rs
View file

@ -10,8 +10,8 @@ use syn::{
pub fn retry(args: TokenStream, input: TokenStream) -> TokenStream {
let count = parse_macro_input!(args as LitInt);
let input = parse_macro_input!(input as ItemFn);
let attrs = input.attrs.clone();
let name = input.sig.ident.clone();
let attrs = &input.attrs;
let name = &input.sig.ident;
TokenStream::from(quote! {
#(#attrs),*
@ -98,8 +98,8 @@ pub fn test_with_seed(args: TokenStream, input: TokenStream) -> TokenStream {
let seed_fn = seed_fn.iter();
let func = parse_macro_input!(input as ItemFn);
let attrs = func.attrs.clone();
let name = func.sig.ident.clone();
let attrs = &func.attrs;
let name = &func.sig.ident;
TokenStream::from(quote! {
#[test]

35
cli/src/tests/query_test.rs
View file

@ -1,3 +1,14 @@
use std::{env, fmt::Write};
use indoc::indoc;
use lazy_static::lazy_static;
use rand::{prelude::StdRng, SeedableRng};
use tree_sitter::{
CaptureQuantifier, Language, Node, Parser, Point, Query, QueryCursor, QueryError,
QueryErrorKind, QueryPredicate, QueryPredicateArg, QueryProperty,
};
use unindent::Unindent;
use super::helpers::{
allocations,
fixtures::{get_language, get_test_language},
@ -8,15 +19,6 @@ use crate::{
generate::generate_parser_for_grammar,
tests::helpers::query_helpers::{collect_captures, collect_matches},
};
use indoc::indoc;
use lazy_static::lazy_static;
use rand::{prelude::StdRng, SeedableRng};
use std::{env, fmt::Write};
use tree_sitter::{
CaptureQuantifier, Language, Node, Parser, Point, Query, QueryCursor, QueryError,
QueryErrorKind, QueryPredicate, QueryPredicateArg, QueryProperty,
};
use unindent::Unindent;
lazy_static! {
static ref EXAMPLE_FILTER: Option<String> = env::var("TREE_SITTER_TEST_EXAMPLE_FILTER").ok();
@ -889,12 +891,12 @@ fn test_query_matches_with_immediate_siblings() {
let language = get_language("python");
// The immediate child operator '.' can be used in three similar ways:
// 1. Before the first child node in a pattern, it means that there cannot be any
// named siblings before that child node.
// 1. Before the first child node in a pattern, it means that there cannot be any named
// siblings before that child node.
// 2. After the last child node in a pattern, it means that there cannot be any named
// sibling after that child node.
// 2. Between two child nodes in a pattern, it specifies that there cannot be any
// named siblings between those two child snodes.
// 2. Between two child nodes in a pattern, it specifies that there cannot be any named
// siblings between those two child snodes.
let query = Query::new(
&language,
"
@ -1423,7 +1425,8 @@ fn test_query_matches_with_nested_optional_nodes() {
allocations::record(|| {
let language = get_language("javascript");
// A function call, optionally containing a function call, which optionally contains a number
// A function call, optionally containing a function call, which optionally contains a
// number
let query = Query::new(
&language,
"
@ -3267,8 +3270,8 @@ fn test_query_captures_with_too_many_nested_results() {
// appearance.
// 2. This pattern captures the root `call_expression`.
// 3. This pattern's result also depends on the final child (the template string).
// 4. In between the `call_expression` and the possible `template_string`, there can
// be an arbitrarily deep subtree.
// 4. In between the `call_expression` and the possible `template_string`, there can be an
// arbitrarily deep subtree.
//
// This means that, if any patterns match *after* the initial `call_expression` is
// captured, but before the final `template_string` is found, those matches must

10
cli/src/tests/tags_test.rs
View file

@ -1,14 +1,16 @@
use super::helpers::{
allocations,
fixtures::{get_language, get_language_queries_path},
};
use std::{
ffi::{CStr, CString},
fs, ptr, slice, str,
};
use tree_sitter::Point;
use tree_sitter_tags::{c_lib as c, Error, TagsConfiguration, TagsContext};
use super::helpers::{
allocations,
fixtures::{get_language, get_language_queries_path},
};
const PYTHON_TAG_QUERY: &str = r#"
(
(function_definition

9
cli/src/tests/test_highlight_test.rs
View file

@ -1,9 +1,12 @@
use super::helpers::fixtures::{get_highlight_config, get_language, test_loader};
use crate::query_testing::{parse_position_comments, Assertion};
use crate::test_highlight::get_highlight_positions;
use tree_sitter::{Parser, Point};
use tree_sitter_highlight::{Highlight, Highlighter};
use super::helpers::fixtures::{get_highlight_config, get_language, test_loader};
use crate::{
query_testing::{parse_position_comments, Assertion},
test_highlight::get_highlight_positions,
};
#[test]
fn test_highlight_test_with_basic_test() {
let language = get_language("javascript");

9
cli/src/tests/test_tags_test.rs
View file

@ -1,9 +1,12 @@
use super::helpers::fixtures::{get_language, get_tags_config};
use crate::query_testing::{parse_position_comments, Assertion};
use crate::test_tags::get_tag_positions;
use tree_sitter::{Parser, Point};
use tree_sitter_tags::TagsContext;
use super::helpers::fixtures::{get_language, get_tags_config};
use crate::{
query_testing::{parse_position_comments, Assertion},
test_tags::get_tag_positions,
};
#[test]
fn test_tags_test_with_basic_test() {
let language = get_language("python");

3
cli/src/tests/text_provider_test.rs
View file

@ -1,8 +1,9 @@
use std::{iter, sync::Arc};
use crate::tests::helpers::fixtures::get_language;
use tree_sitter::{Language, Node, Parser, Point, Query, QueryCursor, TextProvider, Tree};
use crate::tests::helpers::fixtures::get_language;
fn parse_text(text: impl AsRef<[u8]>) -> (Tree, Language) {
let language = get_language("c");
let mut parser = Parser::new();

7
cli/src/tests/tree_test.rs
View file

@ -1,9 +1,10 @@
use super::helpers::edits::invert_edit;
use super::helpers::fixtures::get_language;
use crate::parse::{perform_edit, Edit};
use std::str;
use tree_sitter::{InputEdit, Parser, Point, Range, Tree};
use super::helpers::{edits::invert_edit, fixtures::get_language};
use crate::parse::{perform_edit, Edit};
#[test]
fn test_tree_edit() {
let mut parser = Parser::new();

6
cli/src/tests/wasm_language_test.rs
View file

@ -1,10 +1,12 @@
use crate::tests::helpers::{allocations, fixtures::WASM_DIR};
use lazy_static::lazy_static;
use std::fs;
use lazy_static::lazy_static;
use tree_sitter::{
wasmtime::Engine, Parser, Query, QueryCursor, WasmError, WasmErrorKind, WasmStore,
};
use crate::tests::helpers::{allocations, fixtures::WASM_DIR};
lazy_static! {
static ref ENGINE: Engine = Engine::default();
}

6
cli/src/wasm.rs
View file

@ -1,13 +1,15 @@
use super::generate::parse_grammar::GrammarJSON;
use anyhow::{anyhow, Context, Result};
use std::{
fs,
path::{Path, PathBuf},
};
use anyhow::{anyhow, Context, Result};
use tree_sitter::wasm_stdlib_symbols;
use tree_sitter_loader::Loader;
use wasmparser::Parser;
use super::generate::parse_grammar::GrammarJSON;
pub fn load_language_wasm_file(language_dir: &Path) -> Result<(String, Vec<u8>)> {
let grammar_name = get_grammar_name(language_dir)
.with_context(|| "Failed to get wasm filename")

22
docs/assets/js/playground.js
View file

@ -1,7 +1,7 @@
let tree;
(async () => {
const CAPTURE_REGEX = /@\s*([\w\._-]+)/g;
const CAPTURE_REGEX = /@\s*([\w._-]+)/g;
const COLORS_BY_INDEX = [
'blue',
'chocolate',
@ -18,8 +18,6 @@ let tree;
'sienna',
];
const scriptURL = document.currentScript.getAttribute('src');
const codeInput = document.getElementById('code-input');
const languageSelect = document.getElementById('language-select');
const loggingCheckbox = document.getElementById('logging-checkbox');
@ -102,8 +100,8 @@ let tree;
handleQueryChange();
}
async function handleCodeChange(editor, changes) {
const newText = codeEditor.getValue() + '\n';
async function handleCodeChange(_editor, changes) {
const newText = `${codeEditor.getValue()}\n`;
const edits = tree && changes && changes.map(treeEditForEditorChange);
const start = performance.now();
@ -128,9 +126,9 @@ let tree;
isRendering++;
const cursor = tree.walk();
let currentRenderCount = parseCount;
const currentRenderCount = parseCount;
let row = '';
let rows = [];
const rows = [];
let finishedRow = false;
let visitedChildren = false;
let indentLevel = 0;
@ -319,7 +317,7 @@ let tree;
start.column > end.column
)
) {
let swap = end;
const swap = end;
end = start;
start = swap;
}
@ -445,14 +443,14 @@ let tree;
}
function debounce(func, wait, immediate) {
var timeout;
let timeout;
return function() {
var context = this, args = arguments;
var later = function() {
const context = this, args = arguments;
const later = function() {
timeout = null;
if (!immediate) func.apply(context, args);
};
var callNow = immediate && !timeout;
const callNow = immediate && !timeout;
clearTimeout(timeout);
timeout = setTimeout(later, wait);
if (callNow) func.apply(context, args);

33
highlight/src/c_lib.rs
View file

@ -1,13 +1,13 @@
use super::{Error, Highlight, HighlightConfiguration, Highlighter, HtmlRenderer};
use std::{
collections::HashMap, ffi::CStr, fmt, os::raw::c_char, process::abort, slice, str,
sync::atomic::AtomicUsize,
};
use regex::Regex;
use std::collections::HashMap;
use std::ffi::CStr;
use std::os::raw::c_char;
use std::process::abort;
use std::sync::atomic::AtomicUsize;
use std::{fmt, slice, str};
use tree_sitter::Language;
use super::{Error, Highlight, HighlightConfiguration, Highlighter, HtmlRenderer};
pub struct TSHighlighter {
languages: HashMap<String, (Option<Regex>, HighlightConfiguration)>,
attribute_strings: Vec<&'static [u8]>,
@ -72,8 +72,8 @@ pub unsafe extern "C" fn ts_highlighter_new(
/// `this` must be non-null and must be a valid pointer to a [`TSHighlighter`] instance
/// created by [`ts_highlighter_new`].
///
/// The caller must ensure that any `*const c_char` (C-style string) parameters are valid for the lifetime of
/// the [`TSHighlighter`] instance, and are non-null.
/// The caller must ensure that any `*const c_char` (C-style string) parameters are valid for the
/// lifetime of the [`TSHighlighter`] instance, and are non-null.
#[no_mangle]
pub unsafe extern "C" fn ts_highlighter_add_language(
this: *mut TSHighlighter,
@ -188,8 +188,8 @@ pub unsafe extern "C" fn ts_highlight_buffer_delete(this: *mut TSHighlightBuffer
/// `this` must be non-null and must be a valid pointer to a [`TSHighlightBuffer`] instance
/// created by [`ts_highlight_buffer_new`].
///
/// The returned pointer, a C-style string, must not outlive the [`TSHighlightBuffer`] instance, else the
/// data will point to garbage.
/// The returned pointer, a C-style string, must not outlive the [`TSHighlightBuffer`] instance,
/// else the data will point to garbage.
///
/// To get the length of the HTML content, use [`ts_highlight_buffer_len`].
#[no_mangle]
@ -205,8 +205,8 @@ pub unsafe extern "C" fn ts_highlight_buffer_content(this: *const TSHighlightBuf
/// `this` must be non-null and must be a valid pointer to a [`TSHighlightBuffer`] instance
/// created by [`ts_highlight_buffer_new`].
///
/// The returned pointer, a C-style array of [`u32`]s, must not outlive the [`TSHighlightBuffer`] instance, else the
/// data will point to garbage.
/// The returned pointer, a C-style array of [`u32`]s, must not outlive the [`TSHighlightBuffer`]
/// instance, else the data will point to garbage.
///
/// To get the length of the array, use [`ts_highlight_buffer_line_count`].
#[no_mangle]
@ -245,10 +245,11 @@ pub unsafe extern "C" fn ts_highlight_buffer_line_count(this: *const TSHighlight
///
/// # Safety
///
/// The caller must ensure that `scope_name`, `source_code`, `output`, and `cancellation_flag` are valid for
/// the lifetime of the [`TSHighlighter`] instance, and are non-null.
/// The caller must ensure that `scope_name`, `source_code`, `output`, and `cancellation_flag` are
/// valid for the lifetime of the [`TSHighlighter`] instance, and are non-null.
///
/// `this` must be a non-null pointer to a [`TSHighlighter`] instance created by [`ts_highlighter_new`]
/// `this` must be a non-null pointer to a [`TSHighlighter`] instance created by
/// [`ts_highlighter_new`]
#[no_mangle]
pub unsafe extern "C" fn ts_highlighter_highlight(
this: *const TSHighlighter,

34
highlight/src/lib.rs
View file

@ -1,12 +1,14 @@
#![doc = include_str!("../README.md")]
pub mod c_lib;
pub use c_lib as c;
use std::{
collections::HashSet,
iter, mem, ops, str,
sync::atomic::{AtomicUsize, Ordering},
};
pub use c_lib as c;
use lazy_static::lazy_static;
use std::collections::HashSet;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::{iter, mem, ops, str, usize};
use thiserror::Error;
use tree_sitter::{
Language, LossyUtf8, Node, Parser, Point, Query, QueryCaptures, QueryCursor, QueryError,
@ -246,10 +248,10 @@ impl HighlightConfiguration {
/// * `language` - The Tree-sitter `Language` that should be used for parsing.
/// * `highlights_query` - A string containing tree patterns for syntax highlighting. This
/// should be non-empty, otherwise no syntax highlights will be added.
/// * `injections_query` - A string containing tree patterns for injecting other languages
/// into the document. This can be empty if no injections are desired.
/// * `locals_query` - A string containing tree patterns for tracking local variable
/// definitions and references. This can be empty if local variable tracking is not needed.
/// * `injections_query` - A string containing tree patterns for injecting other languages into
/// the document. This can be empty if no injections are desired.
/// * `locals_query` - A string containing tree patterns for tracking local variable definitions
/// and references. This can be empty if local variable tracking is not needed.
///
/// Returns a `HighlightConfiguration` that can then be used with the `highlight` method.
pub fn new(
@ -399,8 +401,8 @@ impl HighlightConfiguration {
}
// Return the list of this configuration's capture names that are neither present in the
// list of predefined 'canonical' names nor start with an underscore (denoting 'private' captures
// used as part of capture internals).
// list of predefined 'canonical' names nor start with an underscore (denoting 'private'
// captures used as part of capture internals).
#[must_use]
pub fn nonconformant_capture_names(&self, capture_names: &HashSet<&str>) -> Vec<&str> {
let capture_names = if capture_names.is_empty() {
@ -532,12 +534,12 @@ impl<'a> HighlightIterLayer<'a> {
// Compute the ranges that should be included when parsing an injection.
// This takes into account three things:
// * `parent_ranges` - The ranges must all fall within the *current* layer's ranges.
// * `nodes` - Every injection takes place within a set of nodes. The injection ranges
// are the ranges of those nodes.
// * `includes_children` - For some injections, the content nodes' children should be
// excluded from the nested document, so that only the content nodes' *own* content
// is reparsed. For other injections, the content nodes' entire ranges should be
// reparsed, including the ranges of their children.
// * `nodes` - Every injection takes place within a set of nodes. The injection ranges are the
// ranges of those nodes.
// * `includes_children` - For some injections, the content nodes' children should be excluded
// from the nested document, so that only the content nodes' *own* content is reparsed. For
// other injections, the content nodes' entire ranges should be reparsed, including the ranges
// of their children.
fn intersect_ranges(
parent_ranges: &[Range],
nodes: &[Node],

6
lib/binding_rust/build.rs
View file

@ -1,5 +1,7 @@
use std::path::{Path, PathBuf};
use std::{env, fs};
use std::{
env, fs,
path::{Path, PathBuf},
};
fn main() {
let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());

7
lib/binding_rust/ffi.rs
View file

@ -18,10 +18,11 @@ extern "C" {
pub(crate) fn _ts_dup(handle: *mut std::os::raw::c_void) -> std::os::raw::c_int;
}
use std::{marker::PhantomData, mem::ManuallyDrop, ptr::NonNull, str};
use crate::{
Language, LookaheadIterator, Node, Parser, Query, QueryCursor, QueryError, Tree, TreeCursor,
};
use std::{marker::PhantomData, mem::ManuallyDrop, ptr::NonNull, str};
impl Language {
/// Reconstructs a [`Language`] from a raw pointer.
@ -90,7 +91,7 @@ impl<'tree> Node<'tree> {
///
/// `ptr` must be non-null.
#[must_use]
pub const unsafe fn from_raw(raw: TSNode) -> Node<'tree> {
pub const unsafe fn from_raw(raw: TSNode) -> Self {
Self(raw, PhantomData)
}
@ -108,7 +109,7 @@ impl<'a> TreeCursor<'a> {
///
/// `ptr` must be non-null.
#[must_use]
pub const unsafe fn from_raw(raw: TSTreeCursor) -> TreeCursor<'a> {
pub const unsafe fn from_raw(raw: TSTreeCursor) -> Self {
Self(raw, PhantomData)
}

306
lib/binding_rust/lib.rs
View file

@ -7,7 +7,6 @@ mod util;
use std::os::unix::io::AsRawFd;
#[cfg(windows)]
use std::os::windows::io::AsRawHandle;
use std::{
char, error,
ffi::CStr,
@ -21,7 +20,6 @@ use std::{
ptr::{self, NonNull},
slice, str,
sync::atomic::AtomicUsize,
u16,
};
#[cfg(feature = "wasm")]
@ -48,8 +46,8 @@ pub const MIN_COMPATIBLE_LANGUAGE_VERSION: usize =
pub const ARRAY_HEADER: &str = include_str!("../src/array.h");
pub const PARSER_HEADER: &str = include_str!("../src/parser.h");
/// An opaque object that defines how to parse a particular language. The code for each
/// `Language` is generated by the Tree-sitter CLI.
/// An opaque object that defines how to parse a particular language. The code
/// for each `Language` is generated by the Tree-sitter CLI.
#[doc(alias = "TSLanguage")]
#[derive(Debug, PartialEq, Eq, Hash)]
#[repr(transparent)]
@ -70,8 +68,8 @@ pub struct Point {
pub column: usize,
}
/// A range of positions in a multi-line text document, both in terms of bytes and of
/// rows and columns.
/// A range of positions in a multi-line text document, both in terms of bytes
/// and of rows and columns.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct Range {
pub start_byte: usize,
@ -97,11 +95,13 @@ pub struct InputEdit {
#[repr(transparent)]
pub struct Node<'tree>(ffi::TSNode, PhantomData<&'tree ()>);
/// A stateful object that this is used to produce a [`Tree`] based on some source code.
/// A stateful object that this is used to produce a [`Tree`] based on some
/// source code.
#[doc(alias = "TSParser")]
pub struct Parser(NonNull<ffi::TSParser>);
/// A stateful object that is used to look up symbols valid in a specific parse state
/// A stateful object that is used to look up symbols valid in a specific parse
/// state
#[doc(alias = "TSLookaheadIterator")]
pub struct LookaheadIterator(NonNull<ffi::TSLookaheadIterator>);
struct LookaheadNamesIterator<'a>(&'a mut LookaheadIterator);
@ -222,7 +222,8 @@ where
fn text(&mut self, node: Node) -> Self::I;
}
/// A particular [`Node`] that has been captured with a particular name within a [`Query`].
/// A particular [`Node`] that has been captured with a particular name within a
/// [`Query`].
#[derive(Clone, Copy, Debug)]
#[repr(C)]
pub struct QueryCapture<'tree> {
@ -230,7 +231,8 @@ pub struct QueryCapture<'tree> {
pub index: u32,
}
/// An error that occurred when trying to assign an incompatible [`Language`] to a [`Parser`].
/// An error that occurred when trying to assign an incompatible [`Language`] to
/// a [`Parser`].
#[derive(Debug, PartialEq, Eq)]
pub struct LanguageError {
version: usize,
@ -282,8 +284,8 @@ pub struct LossyUtf8<'a> {
}
impl Language {
/// Get the ABI version number that indicates which version of the Tree-sitter CLI
/// that was used to generate this [`Language`].
/// Get the ABI version number that indicates which version of the
/// Tree-sitter CLI that was used to generate this [`Language`].
#[doc(alias = "ts_language_version")]
#[must_use]
pub fn version(&self) -> usize {
@ -444,11 +446,13 @@ impl Parser {
/// Set the language that the parser should use for parsing.
///
/// Returns a Result indicating whether or not the language was successfully
/// assigned. True means assignment succeeded. False means there was a version
/// mismatch: the language was generated with an incompatible version of the
/// Tree-sitter CLI. Check the language's version using [`Language::version`]
/// and compare it to this library's [`LANGUAGE_VERSION`](LANGUAGE_VERSION) and
/// [`MIN_COMPATIBLE_LANGUAGE_VERSION`](MIN_COMPATIBLE_LANGUAGE_VERSION) constants.
/// assigned. True means assignment succeeded. False means there was a
/// version mismatch: the language was generated with an incompatible
/// version of the Tree-sitter CLI. Check the language's version using
/// [`Language::version`] and compare it to this library's
/// [`LANGUAGE_VERSION`](LANGUAGE_VERSION) and
/// [`MIN_COMPATIBLE_LANGUAGE_VERSION`](MIN_COMPATIBLE_LANGUAGE_VERSION)
/// constants.
#[doc(alias = "ts_parser_set_language")]
pub fn set_language(&mut self, language: &Language) -> Result<(), LanguageError> {
let version = language.version();
@ -523,9 +527,9 @@ impl Parser {
}
/// Set the destination to which the parser should write debugging graphs
/// during parsing. The graphs are formatted in the DOT language. You may want
/// to pipe these graphs directly to a `dot(1)` process in order to generate
/// SVG output.
/// during parsing. The graphs are formatted in the DOT language. You may
/// want to pipe these graphs directly to a `dot(1)` process in order to
/// generate SVG output.
#[doc(alias = "ts_parser_print_dot_graphs")]
pub fn print_dot_graphs(
&mut self,
@ -559,10 +563,9 @@ impl Parser {
///
/// # Arguments:
/// * `text` The UTF8-encoded text to parse.
/// * `old_tree` A previous syntax tree parsed from the same document.
/// If the text of the document has changed since `old_tree` was
/// created, then you must edit `old_tree` to match the new text using
/// [`Tree::edit`].
/// * `old_tree` A previous syntax tree parsed from the same document. If the text of the
/// document has changed since `old_tree` was created, then you must edit `old_tree` to match
/// the new text using [`Tree::edit`].
///
/// Returns a [`Tree`] if parsing succeeded, or `None` if:
/// * The parser has not yet had a language assigned with [`Parser::set_language`]
@ -582,10 +585,9 @@ impl Parser {
///
/// # Arguments:
/// * `text` The UTF16-encoded text to parse.
/// * `old_tree` A previous syntax tree parsed from the same document.
/// If the text of the document has changed since `old_tree` was
/// created, then you must edit `old_tree` to match the new text using
/// [`Tree::edit`].
/// * `old_tree` A previous syntax tree parsed from the same document. If the text of the
/// document has changed since `old_tree` was created, then you must edit `old_tree` to match
/// the new text using [`Tree::edit`].
pub fn parse_utf16(
&mut self,
input: impl AsRef<[u16]>,
@ -602,14 +604,13 @@ impl Parser {
/// Parse UTF8 text provided in chunks by a callback.
///
/// # Arguments:
/// * `callback` A function that takes a byte offset and position and
/// returns a slice of UTF8-encoded text starting at that byte offset
/// and position. The slices can be of any length. If the given position
/// is at the end of the text, the callback should return an empty slice.
/// * `old_tree` A previous syntax tree parsed from the same document.
/// If the text of the document has changed since `old_tree` was
/// created, then you must edit `old_tree` to match the new text using
/// [`Tree::edit`].
/// * `callback` A function that takes a byte offset and position and returns a slice of
/// UTF8-encoded text starting at that byte offset and position. The slices can be of any
/// length. If the given position is at the end of the text, the callback should return an
/// empty slice.
/// * `old_tree` A previous syntax tree parsed from the same document. If the text of the
/// document has changed since `old_tree` was created, then you must edit `old_tree` to match
/// the new text using [`Tree::edit`].
pub fn parse_with<T: AsRef<[u8]>, F: FnMut(usize, Point) -> T>(
&mut self,
callback: &mut F,
@ -618,8 +619,8 @@ impl Parser {
// A pointer to this payload is passed on every call to the `read` C function.
// The payload contains two things:
// 1. A reference to the rust `callback`.
// 2. The text that was returned from the previous call to `callback`.
// This allows the callback to return owned values like vectors.
// 2. The text that was returned from the previous call to `callback`. This allows the
// callback to return owned values like vectors.
let mut payload: (&mut F, Option<T>) = (callback, None);
// This C function is passed to Tree-sitter as the input callback.
@ -652,14 +653,13 @@ impl Parser {
/// Parse UTF16 text provided in chunks by a callback.
///
/// # Arguments:
/// * `callback` A function that takes a code point offset and position and
/// returns a slice of UTF16-encoded text starting at that byte offset
/// and position. The slices can be of any length. If the given position
/// is at the end of the text, the callback should return an empty slice.
/// * `old_tree` A previous syntax tree parsed from the same document.
/// If the text of the document has changed since `old_tree` was
/// created, then you must edit `old_tree` to match the new text using
/// [`Tree::edit`].
/// * `callback` A function that takes a code point offset and position and returns a slice of
/// UTF16-encoded text starting at that byte offset and position. The slices can be of any
/// length. If the given position is at the end of the text, the callback should return an
/// empty slice.
/// * `old_tree` A previous syntax tree parsed from the same document. If the text of the
/// document has changed since `old_tree` was created, then you must edit `old_tree` to match
/// the new text using [`Tree::edit`].
pub fn parse_utf16_with<T: AsRef<[u16]>, F: FnMut(usize, Point) -> T>(
&mut self,
callback: &mut F,
@ -668,8 +668,8 @@ impl Parser {
// A pointer to this payload is passed on every call to the `read` C function.
// The payload contains two things:
// 1. A reference to the rust `callback`.
// 2. The text that was returned from the previous call to `callback`.
// This allows the callback to return owned values like vectors.
// 2. The text that was returned from the previous call to `callback`. This allows the
// callback to return owned values like vectors.
let mut payload: (&mut F, Option<T>) = (callback, None);
// This C function is passed to Tree-sitter as the input callback.
@ -707,9 +707,10 @@ impl Parser {
/// Instruct the parser to start the next parse from the beginning.
///
/// If the parser previously failed because of a timeout or a cancellation, then by default, it
/// will resume where it left off on the next call to [`parse`](Parser::parse) or other parsing
/// functions. If you don't want to resume, and instead intend to use this parser to parse some
/// If the parser previously failed because of a timeout or a cancellation,
/// then by default, it will resume where it left off on the next call
/// to [`parse`](Parser::parse) or other parsing functions. If you don't
/// want to resume, and instead intend to use this parser to parse some
/// other document, you must call `reset` first.
#[doc(alias = "ts_parser_reset")]
pub fn reset(&mut self) {
@ -725,8 +726,8 @@ impl Parser {
unsafe { ffi::ts_parser_timeout_micros(self.0.as_ptr()) }
}
/// Set the maximum duration in microseconds that parsing should be allowed to
/// take before halting.
/// Set the maximum duration in microseconds that parsing should be allowed
/// to take before halting.
///
/// If parsing takes longer than this, it will halt early, returning `None`.
/// See [`parse`](Parser::parse) for more information.
@ -737,20 +738,21 @@ impl Parser {
/// Set the ranges of text that the parser should include when parsing.
///
/// By default, the parser will always include entire documents. This function
/// allows you to parse only a *portion* of a document but still return a syntax
/// tree whose ranges match up with the document as a whole. You can also pass
/// multiple disjoint ranges.
/// By default, the parser will always include entire documents. This
/// function allows you to parse only a *portion* of a document but
/// still return a syntax tree whose ranges match up with the document
/// as a whole. You can also pass multiple disjoint ranges.
///
/// If `ranges` is empty, then the entire document will be parsed. Otherwise,
/// the given ranges must be ordered from earliest to latest in the document,
/// and they must not overlap. That is, the following must hold for all
/// `i` < `length - 1`:
/// If `ranges` is empty, then the entire document will be parsed.
/// Otherwise, the given ranges must be ordered from earliest to latest
/// in the document, and they must not overlap. That is, the following
/// must hold for all `i` < `length - 1`:
/// ```text
/// ranges[i].end_byte <= ranges[i + 1].start_byte
/// ```
/// If this requirement is not satisfied, method will return [`IncludedRangesError`]
/// error with an offset in the passed ranges slice pointing to a first incorrect range.
/// If this requirement is not satisfied, method will return
/// [`IncludedRangesError`] error with an offset in the passed ranges
/// slice pointing to a first incorrect range.
#[doc(alias = "ts_parser_set_included_ranges")]
pub fn set_included_ranges(&mut self, ranges: &[Range]) -> Result<(), IncludedRangesError> {
let ts_ranges = ranges
@ -814,8 +816,9 @@ impl Parser {
/// Set the parser's current cancellation flag pointer.
///
/// If a pointer is assigned, then the parser will periodically read from
/// this pointer during parsing. If it reads a non-zero value, it will halt early,
/// returning `None`. See [`parse`](Parser::parse) for more information.
/// this pointer during parsing. If it reads a non-zero value, it will halt
/// early, returning `None`. See [`parse`](Parser::parse) for more
/// information.
///
/// # Safety
///
@ -891,13 +894,15 @@ impl Tree {
self.root_node().walk()
}
/// Compare this old edited syntax tree to a new syntax tree representing the same
/// document, returning a sequence of ranges whose syntactic structure has changed.
/// Compare this old edited syntax tree to a new syntax tree representing
/// the same document, returning a sequence of ranges whose syntactic
/// structure has changed.
///
/// For this to work correctly, this syntax tree must have been edited such that its
/// ranges match up to the new tree. Generally, you'll want to call this method right
/// after calling one of the [`Parser::parse`] functions. Call it on the old tree that
/// was passed to parse, and pass the new tree that was returned from `parse`.
/// For this to work correctly, this syntax tree must have been edited such
/// that its ranges match up to the new tree. Generally, you'll want to
/// call this method right after calling one of the [`Parser::parse`]
/// functions. Call it on the old tree that was passed to parse, and
/// pass the new tree that was returned from `parse`.
#[doc(alias = "ts_tree_get_changed_ranges")]
#[must_use]
pub fn changed_ranges(&self, other: &Self) -> impl ExactSizeIterator<Item = Range> {
@ -931,8 +936,9 @@ impl Tree {
}
/// Print a graph of the tree to the given file descriptor.
/// The graph is formatted in the DOT language. You may want to pipe this graph
/// directly to a `dot(1)` process in order to generate SVG output.
/// The graph is formatted in the DOT language. You may want to pipe this
/// graph directly to a `dot(1)` process in order to generate SVG
/// output.
#[doc(alias = "ts_tree_print_dot_graph")]
pub fn print_dot_graph(
&self,
@ -1030,8 +1036,8 @@ impl<'tree> Node<'tree> {
/// Check if this node is *named*.
///
/// Named nodes correspond to named rules in the grammar, whereas *anonymous* nodes
/// correspond to string literals in the grammar.
/// Named nodes correspond to named rules in the grammar, whereas
/// *anonymous* nodes correspond to string literals in the grammar.
#[doc(alias = "ts_node_is_named")]
#[must_use]
pub fn is_named(&self) -> bool {
@ -1040,8 +1046,8 @@ impl<'tree> Node<'tree> {
/// Check if this node is *extra*.
///
/// Extra nodes represent things like comments, which are not required the grammar,
/// but can appear anywhere.
/// Extra nodes represent things like comments, which are not required the
/// grammar, but can appear anywhere.
#[doc(alias = "ts_node_is_extra")]
#[must_use]
pub fn is_extra(&self) -> bool {
@ -1055,8 +1061,8 @@ impl<'tree> Node<'tree> {
unsafe { ffi::ts_node_has_changes(self.0) }
}
/// Check if this node represents a syntax error or contains any syntax errors anywhere
/// within it.
/// Check if this node represents a syntax error or contains any syntax
/// errors anywhere within it.
#[doc(alias = "ts_node_has_error")]
#[must_use]
pub fn has_error(&self) -> bool {
@ -1065,8 +1071,8 @@ impl<'tree> Node<'tree> {
/// Check if this node represents a syntax error.
///
/// Syntax errors represent parts of the code that could not be incorporated into a
/// valid syntax tree.
/// Syntax errors represent parts of the code that could not be incorporated
/// into a valid syntax tree.
#[doc(alias = "ts_node_is_error")]
#[must_use]
pub fn is_error(&self) -> bool {
@ -1089,8 +1095,8 @@ impl<'tree> Node<'tree> {
/// Check if this node is *missing*.
///
/// Missing nodes are inserted by the parser in order to recover from certain kinds of
/// syntax errors.
/// Missing nodes are inserted by the parser in order to recover from
/// certain kinds of syntax errors.
#[doc(alias = "ts_node_is_missing")]
#[must_use]
pub fn is_missing(&self) -> bool {
@ -1117,8 +1123,8 @@ impl<'tree> Node<'tree> {
self.start_byte()..self.end_byte()
}
/// Get the range of source code that this node represents, both in terms of raw bytes
/// and of row/column coordinates.
/// Get the range of source code that this node represents, both in terms of
/// raw bytes and of row/column coordinates.
#[must_use]
pub fn range(&self) -> Range {
Range {
@ -1204,8 +1210,8 @@ impl<'tree> Node<'tree> {
/// Get this node's child with the given numerical field id.
///
/// See also [`child_by_field_name`](Node::child_by_field_name). You can convert a field name to
/// an id using [`Language::field_id_for_name`].
/// See also [`child_by_field_name`](Node::child_by_field_name). You can
/// convert a field name to an id using [`Language::field_id_for_name`].
#[doc(alias = "ts_node_child_by_field_id")]
#[must_use]
pub fn child_by_field_id(&self, field_id: u16) -> Option<Self> {
@ -1225,12 +1231,12 @@ impl<'tree> Node<'tree> {
/// Iterate over this node's children.
///
/// A [`TreeCursor`] is used to retrieve the children efficiently. Obtain
/// a [`TreeCursor`] by calling [`Tree::walk`] or [`Node::walk`]. To avoid unnecessary
/// allocations, you should reuse the same cursor for subsequent calls to
/// this method.
/// a [`TreeCursor`] by calling [`Tree::walk`] or [`Node::walk`]. To avoid
/// unnecessary allocations, you should reuse the same cursor for
/// subsequent calls to this method.
///
/// If you're walking the tree recursively, you may want to use the [`TreeCursor`]
/// APIs directly instead.
/// If you're walking the tree recursively, you may want to use the
/// [`TreeCursor`] APIs directly instead.
pub fn children<'cursor>(
&self,
cursor: &'cursor mut TreeCursor<'tree>,
@ -1432,11 +1438,11 @@ impl<'tree> Node<'tree> {
/// Edit this node to keep it in-sync with source code that has been edited.
///
/// This function is only rarely needed. When you edit a syntax tree with the
/// [`Tree::edit`] method, all of the nodes that you retrieve from the tree
/// afterward will already reflect the edit. You only need to use [`Node::edit`]
/// when you have a specific [`Node`] instance that you want to keep and continue
/// to use after an edit.
/// This function is only rarely needed. When you edit a syntax tree with
/// the [`Tree::edit`] method, all of the nodes that you retrieve from
/// the tree afterward will already reflect the edit. You only need to
/// use [`Node::edit`] when you have a specific [`Node`] instance that
/// you want to keep and continue to use after an edit.
#[doc(alias = "ts_node_edit")]
pub fn edit(&mut self, edit: &InputEdit) {
let edit = edit.into();
@ -1480,7 +1486,7 @@ impl fmt::Display for Node<'_> {
if sexp.is_empty() {
write!(f, "")
} else if !f.alternate() {
write!(f, "{}", sexp)
write!(f, "{sexp}")
} else {
write!(f, "{}", format_sexp(&sexp, f.width().unwrap_or(0)))
}
@ -1537,8 +1543,8 @@ impl<'cursor> TreeCursor<'cursor> {
/// Move this cursor to the first child of its current node.
///
/// This returns `true` if the cursor successfully moved, and returns `false`
/// if there were no children.
/// This returns `true` if the cursor successfully moved, and returns
/// `false` if there were no children.
#[doc(alias = "ts_tree_cursor_goto_first_child")]
pub fn goto_first_child(&mut self) -> bool {
unsafe { ffi::ts_tree_cursor_goto_first_child(&mut self.0) }
@ -1559,8 +1565,9 @@ impl<'cursor> TreeCursor<'cursor> {
/// Move this cursor to the parent of its current node.
///
/// This returns `true` if the cursor successfully moved, and returns `false`
/// if there was no parent node (the cursor was already on the root node).
/// This returns `true` if the cursor successfully moved, and returns
/// `false` if there was no parent node (the cursor was already on the
/// root node).
#[doc(alias = "ts_tree_cursor_goto_parent")]
pub fn goto_parent(&mut self) -> bool {
unsafe { ffi::ts_tree_cursor_goto_parent(&mut self.0) }
@ -1568,8 +1575,8 @@ impl<'cursor> TreeCursor<'cursor> {
/// Move this cursor to the next sibling of its current node.
///
/// This returns `true` if the cursor successfully moved, and returns `false`
/// if there was no next sibling node.
/// This returns `true` if the cursor successfully moved, and returns
/// `false` if there was no next sibling node.
#[doc(alias = "ts_tree_cursor_goto_next_sibling")]
pub fn goto_next_sibling(&mut self) -> bool {
unsafe { ffi::ts_tree_cursor_goto_next_sibling(&mut self.0) }
@ -1598,11 +1605,11 @@ impl<'cursor> TreeCursor<'cursor> {
unsafe { ffi::ts_tree_cursor_goto_previous_sibling(&mut self.0) }
}
/// Move this cursor to the first child of its current node that extends beyond
/// the given byte offset.
/// Move this cursor to the first child of its current node that extends
/// beyond the given byte offset.
///
/// This returns the index of the child node if one was found, and returns `None`
/// if no such child was found.
/// This returns the index of the child node if one was found, and returns
/// `None` if no such child was found.
#[doc(alias = "ts_tree_cursor_goto_first_child_for_byte")]
pub fn goto_first_child_for_byte(&mut self, index: usize) -> Option<usize> {
let result =
@ -1610,11 +1617,11 @@ impl<'cursor> TreeCursor<'cursor> {
(result >= 0).then_some(result as usize)
}
/// Move this cursor to the first child of its current node that extends beyond
/// the given byte offset.
/// Move this cursor to the first child of its current node that extends
/// beyond the given byte offset.
///
/// This returns the index of the child node if one was found, and returns `None`
/// if no such child was found.
/// This returns the index of the child node if one was found, and returns
/// `None` if no such child was found.
#[doc(alias = "ts_tree_cursor_goto_first_child_for_point")]
pub fn goto_first_child_for_point(&mut self, point: Point) -> Option<usize> {
let result =
@ -1630,10 +1637,10 @@ impl<'cursor> TreeCursor<'cursor> {
/// Re-initialize a tree cursor to the same position as another cursor.
///
/// Unlike [`reset`](TreeCursor::reset), this will not lose parent information and
/// allows reusing already created cursors.
/// Unlike [`reset`](TreeCursor::reset), this will not lose parent
/// information and allows reusing already created cursors.
#[doc(alias = "ts_tree_cursor_reset_to")]
pub fn reset_to(&mut self, cursor: &TreeCursor<'cursor>) {
pub fn reset_to(&mut self, cursor: &Self) {
unsafe { ffi::ts_tree_cursor_reset_to(&mut self.0, &cursor.0) };
}
}
@ -1692,8 +1699,8 @@ impl LookaheadIterator {
/// Reset the lookahead iterator to another state.
///
/// This returns `true` if the iterator was reset to the given state and `false`
/// otherwise.
/// This returns `true` if the iterator was reset to the given state and
/// `false` otherwise.
#[doc(alias = "ts_lookahead_iterator_reset_state")]
pub fn reset_state(&mut self, state: u16) -> bool {
unsafe { ffi::ts_lookahead_iterator_reset_state(self.0.as_ptr(), state) }
@ -1774,7 +1781,7 @@ impl Query {
let mut line_start = 0;
let mut row = 0;
let mut line_containing_error = None;
for line in source.split('\n') {
for line in source.lines() {
let line_end = line_start + line.len() + 1;
if line_end > offset {
line_containing_error = Some(line);
@ -2109,7 +2116,8 @@ impl Query {
Ok(result)
}
/// Get the byte offset where the given pattern starts in the query's source.
/// Get the byte offset where the given pattern starts in the query's
/// source.
#[doc(alias = "ts_query_start_byte_for_pattern")]
#[must_use]
pub fn start_byte_for_pattern(&self, pattern_index: usize) -> usize {
@ -2181,8 +2189,8 @@ impl Query {
/// Disable a certain capture within a query.
///
/// This prevents the capture from being returned in matches, and also avoids any
/// resource usage associated with recording the capture.
/// This prevents the capture from being returned in matches, and also
/// avoids any resource usage associated with recording the capture.
#[doc(alias = "ts_query_disable_capture")]
pub fn disable_capture(&mut self, name: &str) {
unsafe {
@ -2196,8 +2204,8 @@ impl Query {
/// Disable a certain pattern within a query.
///
/// This prevents the pattern from matching, and also avoids any resource usage
/// associated with the pattern.
/// This prevents the pattern from matching, and also avoids any resource
/// usage associated with the pattern.
#[doc(alias = "ts_query_disable_pattern")]
pub fn disable_pattern(&mut self, index: usize) {
unsafe { ffi::ts_query_disable_pattern(self.ptr.as_ptr(), index as u32) }
@ -2219,8 +2227,8 @@ impl Query {
/// Check if a given step in a query is 'definite'.
///
/// A query step is 'definite' if its parent pattern will be guaranteed to match
/// successfully once it reaches the step.
/// A query step is 'definite' if its parent pattern will be guaranteed to
/// match successfully once it reaches the step.
#[doc(alias = "ts_query_is_pattern_guaranteed_at_step")]
#[must_use]
pub fn is_pattern_guaranteed_at_step(&self, byte_offset: usize) -> bool {
@ -2297,7 +2305,8 @@ impl Default for QueryCursor {
impl QueryCursor {
/// Create a new cursor for executing a given query.
///
/// The cursor stores the state that is needed to iteratively search for matches.
/// The cursor stores the state that is needed to iteratively search for
/// matches.
#[doc(alias = "ts_query_cursor_new")]
#[must_use]
pub fn new() -> Self {
@ -2313,8 +2322,8 @@ impl QueryCursor {
unsafe { ffi::ts_query_cursor_match_limit(self.ptr.as_ptr()) }
}
/// Set the maximum number of in-progress matches for this cursor. The limit must be > 0 and
/// <= 65536.
/// Set the maximum number of in-progress matches for this cursor. The
/// limit must be > 0 and <= 65536.
#[doc(alias = "ts_query_cursor_set_match_limit")]
pub fn set_match_limit(&mut self, limit: u32) {
unsafe {
@ -2322,8 +2331,8 @@ impl QueryCursor {
}
}
/// Check if, on its last execution, this cursor exceeded its maximum number of
/// in-progress matches.
/// Check if, on its last execution, this cursor exceeded its maximum number
/// of in-progress matches.
#[doc(alias = "ts_query_cursor_did_exceed_match_limit")]
#[must_use]
pub fn did_exceed_match_limit(&self) -> bool {
@ -2332,9 +2341,10 @@ impl QueryCursor {
/// Iterate over all of the matches in the order that they were found.
///
/// Each match contains the index of the pattern that matched, and a list of captures.
/// Because multiple patterns can match the same set of nodes, one match may contain
/// captures that appear *before* some of the captures from a previous match.
/// Each match contains the index of the pattern that matched, and a list of
/// captures. Because multiple patterns can match the same set of nodes,
/// one match may contain captures that appear *before* some of the
/// captures from a previous match.
#[doc(alias = "ts_query_cursor_exec")]
pub fn matches<'query, 'cursor: 'query, 'tree, T: TextProvider<I>, I: AsRef<[u8]>>(
&'cursor mut self,
@ -2354,10 +2364,11 @@ impl QueryCursor {
}
}
/// Iterate over all of the individual captures in the order that they appear.
/// Iterate over all of the individual captures in the order that they
/// appear.
///
/// This is useful if you don't care about which pattern matched, and just want a single,
/// ordered sequence of captures.
/// This is useful if you don't care about which pattern matched, and just
/// want a single, ordered sequence of captures.
#[doc(alias = "ts_query_cursor_exec")]
pub fn captures<'query, 'cursor: 'query, 'tree, T: TextProvider<I>, I: AsRef<[u8]>>(
&'cursor mut self,
@ -2377,7 +2388,8 @@ impl QueryCursor {
}
}
/// Set the range in which the query will be executed, in terms of byte offsets.
/// Set the range in which the query will be executed, in terms of byte
/// offsets.
#[doc(alias = "ts_query_cursor_set_byte_range")]
pub fn set_byte_range(&mut self, range: ops::Range<usize>) -> &mut Self {
unsafe {
@ -2390,7 +2402,8 @@ impl QueryCursor {
self
}
/// Set the range in which the query will be executed, in terms of rows and columns.
/// Set the range in which the query will be executed, in terms of rows and
/// columns.
#[doc(alias = "ts_query_cursor_set_point_range")]
pub fn set_point_range(&mut self, range: ops::Range<Point>) -> &mut Self {
unsafe {
@ -2406,13 +2419,15 @@ impl QueryCursor {
/// Set the maximum start depth for a query cursor.
///
/// This prevents cursors from exploring children nodes at a certain depth.
/// Note if a pattern includes many children, then they will still be checked.
/// Note if a pattern includes many children, then they will still be
/// checked.
///
/// The zero max start depth value can be used as a special behavior and
/// it helps to destructure a subtree by staying on a node and using captures
/// for interested parts. Note that the zero max start depth only limit a search
/// depth for a pattern's root node but other nodes that are parts of the pattern
/// may be searched at any depth what defined by the pattern structure.
/// it helps to destructure a subtree by staying on a node and using
/// captures for interested parts. Note that the zero max start depth
/// only limit a search depth for a pattern's root node but other nodes
/// that are parts of the pattern may be searched at any depth what
/// defined by the pattern structure.
///
/// Set to `None` to remove the maximum start depth.
#[doc(alias = "ts_query_cursor_set_max_start_depth")]
@ -2496,7 +2511,7 @@ impl<'tree> QueryMatch<'_, 'tree> {
} else if let Some(ref first_chunk) = self.first_chunk {
first_chunk.as_ref()
} else {
Default::default()
&[]
}
}
}
@ -2808,7 +2823,7 @@ impl<'a> Iterator for LossyUtf8<'a> {
}
match std::str::from_utf8(self.bytes) {
Ok(valid) => {
self.bytes = Default::default();
self.bytes = &[];
Some(valid)
}
Err(error) => {
@ -2886,6 +2901,7 @@ impl fmt::Display for QueryError {
}
#[doc(hidden)]
#[must_use]
pub fn format_sexp(sexp: &str, initial_indent_level: usize) -> String {
let mut indent_level = initial_indent_level;
let mut formatted = String::new();

3
lib/binding_rust/util.rs
View file

@ -1,6 +1,7 @@
use super::FREE_FN;
use std::os::raw::c_void;
use super::FREE_FN;
/// A raw pointer and a length, exposed as an iterator.
pub struct CBufferIter<T> {
ptr: *mut T,

4
lib/binding_rust/wasm_language.rs
View file

@ -1,4 +1,3 @@
use crate::{ffi, Language, LanguageError, Parser, FREE_FN};
use std::{
error,
ffi::{CStr, CString},
@ -6,8 +5,11 @@ use std::{
mem::{self, MaybeUninit},
os::raw::c_char,
};
pub use wasmtime_c_api::wasmtime;
use crate::{ffi, Language, LanguageError, Parser, FREE_FN};
// Force Cargo to include wasmtime-c-api as a dependency of this crate,
// even though it is only used by the C code.
#[allow(unused)]

25
lib/binding_web/binding.js
View file

@ -91,7 +91,7 @@ class ParserImpl {
let rangeCount = 0;
let rangeAddress = 0;
if (options && options.includedRanges) {
if (options?.includedRanges) {
rangeCount = options.includedRanges.length;
rangeAddress = C._calloc(rangeCount, SIZE_OF_RANGE);
let address = rangeAddress;
@ -349,6 +349,7 @@ class Node {
childForFieldName(fieldName) {
const fieldId = this.tree.language.fields.indexOf(fieldName);
if (fieldId !== -1) return this.childForFieldId(fieldId);
return null;
}
fieldNameForChild(index) {
@ -365,6 +366,7 @@ class Node {
childrenForFieldName(fieldName) {
const fieldId = this.tree.language.fields.indexOf(fieldName);
if (fieldId !== -1 && fieldId !== 0) return this.childrenForFieldId(fieldId);
return [];
}
childrenForFieldId(fieldId) {
@ -863,6 +865,7 @@ class Language {
lookaheadIterator(stateId) {
const address = C._ts_lookahead_iterator_new(this[0], stateId);
if (address) return new LookaheadIterable(INTERNAL, address, this);
return null;
}
query(source) {
@ -991,7 +994,7 @@ class Language {
if (steps[2].type === 'capture') {
const captureName1 = steps[1].name;
const captureName2 = steps[2].name;
textPredicates[i].push(function(captures) {
textPredicates[i].push((captures) => {
const nodes1 = [];
const nodes2 = [];
for (const c of captures) {
@ -1012,7 +1015,7 @@ class Language {
const stringValue = steps[2].value;
const matches = (n) => n.text === stringValue;
const doesNotMatch = (n) => n.text !== stringValue;
textPredicates[i].push(function(captures) {
textPredicates[i].push((captures) => {
const nodes = [];
for (const c of captures) {
if (c.name === captureName) nodes.push(c.node);
@ -1048,7 +1051,7 @@ class Language {
captureName = steps[1].name;
const regex = new RegExp(steps[2].value);
matchAll = !operator.startsWith('any-');
textPredicates[i].push(function(captures) {
textPredicates[i].push((captures) => {
const nodes = [];
for (const c of captures) {
if (c.name === captureName) nodes.push(c.node.text);
@ -1119,7 +1122,7 @@ class Language {
}
captureName = steps[1].name;
const values = steps.slice(2).map((s) => s.value);
textPredicates[i].push(function(captures) {
textPredicates[i].push((captures) => {
const nodes = [];
for (const c of captures) {
if (c.name === captureName) nodes.push(c.node.text);
@ -1301,7 +1304,7 @@ class Query {
const startAddress = getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32');
const didExceedMatchLimit = getValue(TRANSFER_BUFFER + 2 * SIZE_OF_INT, 'i32');
const result = new Array(rawCount);
this.exceededMatchLimit = !!didExceedMatchLimit;
this.exceededMatchLimit = Boolean(didExceedMatchLimit);
let filteredCount = 0;
let address = startAddress;
@ -1364,7 +1367,7 @@ class Query {
const startAddress = getValue(TRANSFER_BUFFER + SIZE_OF_INT, 'i32');
const didExceedMatchLimit = getValue(TRANSFER_BUFFER + 2 * SIZE_OF_INT, 'i32');
const result = [];
this.exceededMatchLimit = !!didExceedMatchLimit;
this.exceededMatchLimit = Boolean(didExceedMatchLimit);
const captures = [];
let address = startAddress;
@ -1490,15 +1493,15 @@ function unmarshalNode(tree, address = TRANSFER_BUFFER) {
}
function marshalTreeCursor(cursor, address = TRANSFER_BUFFER) {
setValue(address + 0 * SIZE_OF_INT, cursor[0], 'i32'),
setValue(address + 1 * SIZE_OF_INT, cursor[1], 'i32'),
setValue(address + 0 * SIZE_OF_INT, cursor[0], 'i32');
setValue(address + 1 * SIZE_OF_INT, cursor[1], 'i32');
setValue(address + 2 * SIZE_OF_INT, cursor[2], 'i32');
setValue(address + 3 * SIZE_OF_INT, cursor[3], 'i32');
}
function unmarshalTreeCursor(cursor) {
cursor[0] = getValue(TRANSFER_BUFFER + 0 * SIZE_OF_INT, 'i32'),
cursor[1] = getValue(TRANSFER_BUFFER + 1 * SIZE_OF_INT, 'i32'),
cursor[0] = getValue(TRANSFER_BUFFER + 0 * SIZE_OF_INT, 'i32');
cursor[1] = getValue(TRANSFER_BUFFER + 1 * SIZE_OF_INT, 'i32');
cursor[2] = getValue(TRANSFER_BUFFER + 2 * SIZE_OF_INT, 'i32');
cursor[3] = getValue(TRANSFER_BUFFER + 3 * SIZE_OF_INT, 'i32');
}

6
lib/binding_web/imports.js
View file

@ -1,5 +1,5 @@
mergeInto(LibraryManager.library, {
tree_sitter_parse_callback: function(
tree_sitter_parse_callback(
inputBufferAddress,
index,
row,
@ -7,7 +7,7 @@ mergeInto(LibraryManager.library, {
lengthAddress,
) {
const INPUT_BUFFER_SIZE = 10 * 1024;
const string = currentParseCallback(index, {row: row, column: column});
const string = currentParseCallback(index, {row, column});
if (typeof string === 'string') {
setValue(lengthAddress, string.length, 'i32');
stringToUTF16(string, inputBufferAddress, INPUT_BUFFER_SIZE);
@ -16,7 +16,7 @@ mergeInto(LibraryManager.library, {
}
},
tree_sitter_log_callback: function(isLexMessage, messageAddress) {
tree_sitter_log_callback(isLexMessage, messageAddress) {
if (currentLogCallback) {
const message = UTF8ToString(messageAddress);
currentLogCallback(message, isLexMessage !== 0);

2
lib/binding_web/test/parser-test.js
View file

@ -251,7 +251,7 @@ describe('Parser', () => {
it('handles long input strings', () => {
const repeatCount = 10000;
const inputString = '[' + '0,'.repeat(repeatCount) + ']';
const inputString = `[${Array(repeatCount).fill('0').join(',')}]`;
tree = parser.parse(inputString);
assert.equal(tree.rootNode.type, 'program');

91
lib/binding_web/test/tree-test.js
View file

@ -244,62 +244,47 @@ describe('Tree', () => {
endIndex: 13,
});
{
const copy = tree.walk();
copy.resetTo(cursor);
const copy = tree.walk();
copy.resetTo(cursor);
assert(copy.gotoPreviousSibling());
assertCursorState(copy, {
nodeType: '+',
nodeIsNamed: false,
startPosition: {row: 0, column: 6},
endPosition: {row: 0, column: 7},
startIndex: 6,
endIndex: 7,
});
assert(copy.gotoPreviousSibling());
assertCursorState(copy, {
nodeType: '+',
nodeIsNamed: false,
startPosition: {row: 0, column: 6},
endPosition: {row: 0, column: 7},
startIndex: 6,
endIndex: 7,
});
assert(copy.gotoPreviousSibling());
assertCursorState(copy, {
nodeType: 'binary_expression',
nodeIsNamed: true,
startPosition: {row: 0, column: 0},
endPosition: {row: 0, column: 5},
startIndex: 0,
endIndex: 5,
});
assert(copy.gotoPreviousSibling());
assertCursorState(copy, {
nodeType: 'binary_expression',
nodeIsNamed: true,
startPosition: {row: 0, column: 0},
endPosition: {row: 0, column: 5},
startIndex: 0,
endIndex: 5,
});
assert(copy.gotoLastChild());
assertCursorState(copy, {
nodeType: 'identifier',
nodeIsNamed: true,
startPosition: {row: 0, column: 4},
endPosition: {row: 0, column: 5},
startIndex: 4,
endIndex: 5,
});
assert(copy.gotoLastChild());
assertCursorState(copy, {
nodeType: 'identifier',
nodeIsNamed: true,
startPosition: {row: 0, column: 4},
endPosition: {row: 0, column: 5},
startIndex: 4,
endIndex: 5,
});
assert(copy.gotoParent());
assert(copy.gotoParent());
assert.equal(copy.nodeType, 'binary_expression');
assert(copy.gotoParent());
assert.equal(copy.nodeType, 'expression_statement');
assert(copy.gotoParent());
assert.equal(copy.nodeType, 'program');
assert(!copy.gotoParent());
}
// const childIndex = cursor.gotoFirstChildForIndex(12);
// assertCursorState(cursor, {
// nodeType: 'identifier',
// nodeIsNamed: true,
// startPosition: {row: 0, column: 12},
// endPosition: {row: 0, column: 13},
// startIndex: 12,
// endIndex: 13
// });
// assert.equal(childIndex, 2);
// assert(!cursor.gotoNextSibling());
// assert(cursor.gotoParent());
assert(copy.gotoParent());
assert(copy.gotoParent());
assert.equal(copy.nodeType, 'binary_expression');
assert(copy.gotoParent());
assert.equal(copy.nodeType, 'expression_statement');
assert(copy.gotoParent());
assert.equal(copy.nodeType, 'program');
assert(!copy.gotoParent());
assert(cursor.gotoParent());
assert.equal(cursor.nodeType, 'binary_expression');
@ -414,7 +399,7 @@ function spliceInput(input, startIndex, lengthRemoved, newText) {
function getExtent(text) {
let row = 0;
let index;
for (index = 0; index != -1; index = text.indexOf('\n', index)) {
for (index = 0; index !== -1; index = text.indexOf('\n', index)) {
index++;
row++;
}

6
rustfmt.toml Normal file
View file

@ -0,0 +1,6 @@
comment_width = 100
format_code_in_doc_comments = true
format_macro_matchers = true
imports_granularity = "Crate"
group_imports = "StdExternalCrate"
wrap_comments = true

14
tags/src/c_lib.rs
View file

@ -1,12 +1,12 @@
use super::{Error, TagsConfiguration, TagsContext};
use std::collections::HashMap;
use std::ffi::CStr;
use std::os::raw::c_char;
use std::process::abort;
use std::sync::atomic::AtomicUsize;
use std::{fmt, slice, str};
use std::{
collections::HashMap, ffi::CStr, fmt, os::raw::c_char, process::abort, slice, str,
sync::atomic::AtomicUsize,
};
use tree_sitter::Language;
use super::{Error, TagsConfiguration, TagsContext};
const BUFFER_TAGS_RESERVE_CAPACITY: usize = 100;
const BUFFER_DOCS_RESERVE_CAPACITY: usize = 1024;

41
tags/src/lib.rs
View file

@ -2,14 +2,19 @@
pub mod c_lib;
use std::{
char,
collections::HashMap,
ffi::{CStr, CString},
mem,
ops::Range,
os::raw::c_char,
str,
sync::atomic::{AtomicUsize, Ordering},
};
use memchr::memchr;
use regex::Regex;
use std::collections::HashMap;
use std::ffi::{CStr, CString};
use std::ops::Range;
use std::os::raw::c_char;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::{char, mem, str};
use thiserror::Error;
use tree_sitter::{
Language, LossyUtf8, Parser, Point, Query, QueryCursor, QueryError, QueryPredicateArg, Tree,
@ -281,8 +286,9 @@ impl TagsContext {
unsafe { self.parser.set_cancellation_flag(cancellation_flag) };
let tree = self.parser.parse(source, None).ok_or(Error::Cancelled)?;
// The `matches` iterator borrows the `Tree`, which prevents it from being moved.
// But the tree is really just a pointer, so it's actually ok to move it.
// The `matches` iterator borrows the `Tree`, which prevents it from being
// moved. But the tree is really just a pointer, so it's actually ok to
// move it.
let tree_ref = unsafe { mem::transmute::<_, &'static Tree>(&tree) };
let matches = self
.cursor
@ -456,7 +462,8 @@ where
}
}
// Generate a doc string from all of the doc nodes, applying any strip regexes.
// Generate a doc string from all of the doc nodes, applying any strip
// regexes.
let mut docs = None;
for doc_node in &doc_nodes[docs_start_index..] {
if let Ok(content) = str::from_utf8(&self.source[doc_node.byte_range()])
@ -479,9 +486,9 @@ where
let range = rng.start.min(name_range.start)..rng.end.max(name_range.end);
let span = name_node.start_position()..name_node.end_position();
// Compute tag properties that depend on the text of the containing line. If the
// previous tag occurred on the same line, then reuse results from the previous tag.
let line_range;
// Compute tag properties that depend on the text of the containing line. If
// the previous tag occurred on the same line, then
// reuse results from the previous tag.
let mut prev_utf16_column = 0;
let mut prev_utf8_byte = name_range.start - span.start.column;
let line_info = self.prev_line_info.as_ref().and_then(|info| {
@ -491,20 +498,20 @@ where
None
}
});
if let Some(line_info) = line_info {
line_range = line_info.line_range.clone();
let line_range = if let Some(line_info) = line_info {
if line_info.utf8_position.column <= span.start.column {
prev_utf8_byte = line_info.utf8_byte;
prev_utf16_column = line_info.utf16_column;
}
line_info.line_range.clone()
} else {
line_range = self::line_range(
self::line_range(
self.source,
name_range.start,
span.start,
MAX_LINE_LEN,
);
}
)
};
let utf16_start_column = prev_utf16_column
+ utf16_len(&self.source[prev_utf8_byte..name_range.start]);

3
xtask/src/bump.rs
View file

@ -1,5 +1,4 @@
use std::cmp::Ordering;
use std::path::Path;
use std::{cmp::Ordering, path::Path};
use git2::{DiffOptions, Repository};
use indoc::indoc;