traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Handle is?, is-not?, and set! predicate functions in queries

Browse source
This commit is contained in:
Max Brunsfeld 2019-09-18 17:40:13 -07:00
parent b15e90bd26
commit 27149902f8
4 changed files with 256 additions and 142 deletions
Download patch file Download diff file Expand all files Collapse all files

26
cli/src/tests/query_test.rs
View file

@ -1,7 +1,9 @@
use super::helpers::allocations;
use super::helpers::fixtures::get_language;
use std::fmt::Write;
use tree_sitter::{Node, Parser, Query, QueryCapture, QueryCursor, QueryError, QueryMatch};
use tree_sitter::{
Node, Parser, Query, QueryCapture, QueryCursor, QueryError, QueryMatch, QueryProperty,
};
#[test]
fn test_query_errors_on_invalid_syntax() {
@ -642,20 +644,30 @@ fn test_query_captures_with_set_properties() {
r#"
((call_expression (identifier) @foo)
(set! name something)
(set! age 24))
(set! cool))
(property_identifier) @bar"#,
((property_identifier) @bar
(is? cool)
(is-not? name something))"#,
)
.unwrap();
assert_eq!(
query.pattern_properties(0),
query.property_settings(0),
&[
("name".to_string(), "something".to_string()),
("age".to_string(), "24".to_string()),
QueryProperty::new("name", Some("something"), None),
QueryProperty::new("cool", None, None),
]
);
assert_eq!(query.property_settings(1), &[]);
assert_eq!(query.property_predicates(0), &[]);
assert_eq!(
query.property_predicates(1),
&[
(QueryProperty::new("cool", None, None), true),
(QueryProperty::new("name", Some("something"), None), false),
]
);
assert_eq!(query.pattern_properties(1), &[])
});
}

162
lib/binding_rust/lib.rs
View file

@ -139,18 +139,26 @@ pub struct TreePropertyCursor<'a, P> {
}
#[derive(Debug)]
enum QueryPredicate {
enum TextPredicate {
CaptureEqString(u32, String),
CaptureEqCapture(u32, u32),
CaptureMatchString(u32, regex::bytes::Regex),
}
#[derive(Debug, PartialEq, Eq)]
pub struct QueryProperty {
pub key: Box<str>,
pub value: Option<Box<str>>,
pub capture_id: Option<usize>,
}
#[derive(Debug)]
pub struct Query {
ptr: NonNull<ffi::TSQuery>,
capture_names: Vec<String>,
predicates: Vec<Vec<QueryPredicate>>,
properties: Vec<Box<[(String, String)]>>,
text_predicates: Vec<Box<[TextPredicate]>>,
property_settings: Vec<Box<[QueryProperty]>>,
property_predicates: Vec<Box<[(QueryProperty, bool)]>>,
}
pub struct QueryCursor(NonNull<ffi::TSQueryCursor>);
@ -1002,8 +1010,9 @@ impl Query {
let mut result = Query {
ptr: unsafe { NonNull::new_unchecked(ptr) },
capture_names: Vec::with_capacity(capture_count as usize),
predicates: Vec::with_capacity(pattern_count),
properties: Vec::with_capacity(pattern_count),
text_predicates: Vec::with_capacity(pattern_count),
property_predicates: Vec::with_capacity(pattern_count),
property_settings: Vec::with_capacity(pattern_count),
};
// Build a vector of strings to store the capture names.
@ -1044,8 +1053,9 @@ impl Query {
let type_capture = ffi::TSQueryPredicateStepType_TSQueryPredicateStepTypeCapture;
let type_string = ffi::TSQueryPredicateStepType_TSQueryPredicateStepTypeString;
let mut pattern_properties = Vec::new();
let mut pattern_predicates = Vec::new();
let mut text_predicates = Vec::new();
let mut property_predicates = Vec::new();
let mut property_settings = Vec::new();
for p in predicate_steps.split(|s| s.type_ == type_done) {
if p.is_empty() {
continue;
@ -1075,10 +1085,10 @@ impl Query {
)));
}
pattern_predicates.push(if p[2].type_ == type_capture {
QueryPredicate::CaptureEqCapture(p[1].value_id, p[2].value_id)
text_predicates.push(if p[2].type_ == type_capture {
TextPredicate::CaptureEqCapture(p[1].value_id, p[2].value_id)
} else {
QueryPredicate::CaptureEqString(
TextPredicate::CaptureEqString(
p[1].value_id,
string_values[p[2].value_id as usize].clone(),
)
@ -1106,7 +1116,7 @@ impl Query {
}
let regex = &string_values[p[2].value_id as usize];
pattern_predicates.push(QueryPredicate::CaptureMatchString(
text_predicates.push(TextPredicate::CaptureMatchString(
p[1].value_id,
regex::bytes::Regex::new(regex).map_err(|_| {
QueryError::Predicate(format!("Invalid regex '{}'", regex))
@ -1114,23 +1124,22 @@ impl Query {
));
}
"set!" => {
if p.len() != 3 {
return Err(QueryError::Predicate(format!(
"Wrong number of arguments to set! predicate. Expected 2, got {}.",
p.len() - 1
)));
}
if p[1].type_ != type_string || p[2].type_ != type_string {
return Err(QueryError::Predicate(
"Argument to set! predicate must be strings.".to_string(),
));
}
let key = &string_values[p[1].value_id as usize];
let value = &string_values[p[2].value_id as usize];
"set!" => property_settings.push(Self::parse_property(
"set!",
&result.capture_names,
&string_values,
&p[1..],
)?),
pattern_properties.push((key.to_string(), value.to_string()));
}
"is?" | "is-not?" => property_predicates.push((
Self::parse_property(
&operator_name,
&result.capture_names,
&string_values,
&p[1..],
)?,
operator_name == "is?",
)),
_ => {
return Err(QueryError::Predicate(format!(
@ -1142,20 +1151,24 @@ impl Query {
}
result
.properties
.push(pattern_properties.into_boxed_slice());
result.predicates.push(pattern_predicates);
.text_predicates
.push(text_predicates.into_boxed_slice());
result
.property_predicates
.push(property_predicates.into_boxed_slice());
result
.property_settings
.push(property_settings.into_boxed_slice());
}
Ok(result)
}
pub fn start_byte_for_pattern(&self, pattern_index: usize) -> usize {
if pattern_index >= self.predicates.len() {
if pattern_index >= self.text_predicates.len() {
panic!(
"Pattern index is {} but the pattern count is {}",
pattern_index,
self.predicates.len(),
self.text_predicates.len(),
);
}
unsafe {
@ -1171,8 +1184,73 @@ impl Query {
&self.capture_names
}
pub fn pattern_properties(&self, index: usize) -> &[(String, String)] {
&self.properties[index]
pub fn property_predicates(&self, index: usize) -> &[(QueryProperty, bool)] {
&self.property_predicates[index]
}
pub fn property_settings(&self, index: usize) -> &[QueryProperty] {
&self.property_settings[index]
}
fn parse_property(
function_name: &str,
capture_names: &[String],
string_values: &[String],
args: &[ffi::TSQueryPredicateStep],
) -> Result<QueryProperty, QueryError> {
if args.len() == 0 || args.len() > 3 {
return Err(QueryError::Predicate(format!(
"Wrong number of arguments to {} predicate. Expected 1 to 3, got {}.",
function_name,
args.len(),
)));
}
let mut i = 0;
let mut capture_id = None;
if args[i].type_ == ffi::TSQueryPredicateStepType_TSQueryPredicateStepTypeCapture {
capture_id = Some(args[i].value_id as usize);
i += 1;
if i == args.len() {
return Err(QueryError::Predicate(format!(
"No key specified for {} predicate.",
function_name,
)));
}
if args[i].type_ == ffi::TSQueryPredicateStepType_TSQueryPredicateStepTypeCapture {
return Err(QueryError::Predicate(format!(
"Invalid arguments to {} predicate. Expected string, got @{}",
function_name, capture_names[args[i].value_id as usize]
)));
}
}
let key = &string_values[args[i].value_id as usize];
i += 1;
let mut value = None;
if i < args.len() {
if args[i].type_ == ffi::TSQueryPredicateStepType_TSQueryPredicateStepTypeCapture {
return Err(QueryError::Predicate(format!(
"Invalid arguments to {} predicate. Expected string, got @{}",
function_name, capture_names[args[i].value_id as usize]
)));
}
value = Some(string_values[args[i].value_id as usize].as_str());
}
Ok(QueryProperty::new(key, value, capture_id))
}
}
impl QueryProperty {
pub fn new(key: &str, value: Option<&str>, capture_id: Option<usize>) -> Self {
QueryProperty {
capture_id,
key: key.to_string().into_boxed_str(),
value: value.map(|s| s.to_string().into_boxed_str()),
}
}
}
@ -1196,7 +1274,7 @@ impl QueryCursor {
if ffi::ts_query_cursor_next_match(ptr, m.as_mut_ptr()) {
let m = m.assume_init();
let captures = slice::from_raw_parts(m.captures, m.capture_count as usize);
if Self::captures_match_condition(
if Self::captures_match_text_predicates(
query,
captures,
m.pattern_index as usize,
@ -1234,7 +1312,7 @@ impl QueryCursor {
) {
let m = m.assume_init();
let captures = slice::from_raw_parts(m.captures, m.capture_count as usize);
if Self::captures_match_condition(
if Self::captures_match_text_predicates(
query,
captures,
m.pattern_index as usize,
@ -1256,25 +1334,25 @@ impl QueryCursor {
})
}
fn captures_match_condition<'a>(
fn captures_match_text_predicates<'a>(
query: &'a Query,
captures: &'a [ffi::TSQueryCapture],
pattern_index: usize,
text_callback: &mut impl FnMut(Node<'a>) -> &'a [u8],
) -> bool {
query.predicates[pattern_index]
query.text_predicates[pattern_index]
.iter()
.all(|predicate| match predicate {
QueryPredicate::CaptureEqCapture(i, j) => {
TextPredicate::CaptureEqCapture(i, j) => {
let node1 = Self::capture_for_id(captures, *i).unwrap();
let node2 = Self::capture_for_id(captures, *j).unwrap();
text_callback(node1) == text_callback(node2)
}
QueryPredicate::CaptureEqString(i, s) => {
TextPredicate::CaptureEqString(i, s) => {
let node = Self::capture_for_id(captures, *i).unwrap();
text_callback(node) == s.as_bytes()
}
QueryPredicate::CaptureMatchString(i, r) => {
TextPredicate::CaptureMatchString(i, r) => {
let node = Self::capture_for_id(captures, *i).unwrap();
r.is_match(text_callback(node))
}

198
lib/binding_web/binding.js
View file

@ -719,7 +719,9 @@ class Language {
stringValues[i] = UTF8ToString(valueAddress, nameLength);
}
const properties = new Array(patternCount);
const setProperties = new Array(patternCount);
const assertedProperties = new Array(patternCount);
const refutedProperties = new Array(patternCount);
const predicates = new Array(patternCount);
for (let i = 0; i < patternCount; i++) {
const predicatesAddress = C._ts_query_predicates_for_pattern(
@ -730,7 +732,6 @@ class Language {
const stepCount = getValue(TRANSFER_BUFFER, 'i32');
predicates[i] = [];
properties[i] = null;
const steps = [];
let stepAddress = predicatesAddress;
@ -744,18 +745,96 @@ class Language {
} else if (stepType === PREDICATE_STEP_TYPE_STRING) {
steps.push({type: 'string', value: stringValues[stepValueId]});
} else if (steps.length > 0) {
const predicate = buildQueryPredicate(steps);
if (typeof predicate === 'function') {
predicates[i].push(predicate);
} else {
if (!properties[i]) properties[i] = {};
properties[i][predicate.key] = predicate.value;
if (steps[0].type !== 'string') {
throw new Error('Predicates must begin with a literal value');
}
const operator = steps[0].value;
switch (operator) {
case 'eq?':
if (steps.length !== 3) throw new Error(
`Wrong number of arguments to \`eq?\` predicate. Expected 2, got ${steps.length - 1}`
);
if (steps[1].type !== 'capture') throw new Error(
`First argument of \`eq?\` predicate must be a capture. Got "${steps[1].value}"`
);
if (steps[2].type === 'capture') {
const captureName1 = steps[1].name;
const captureName2 = steps[2].name;
predicates[i].push(function(captures) {
let node1, node2
for (const c of captures) {
if (c.name === captureName1) node1 = c.node;
if (c.name === captureName2) node2 = c.node;
}
return node1.text === node2.text
});
} else {
const captureName = steps[1].name;
const stringValue = steps[2].value;
predicates[i].push(function(captures) {
for (const c of captures) {
if (c.name === captureName) return c.node.text === stringValue;
}
return false;
});
}
break;
case 'match?':
if (steps.length !== 3) throw new Error(
`Wrong number of arguments to \`match?\` predicate. Expected 2, got ${steps.length - 1}.`
);
if (steps[1].type !== 'capture') throw new Error(
`First argument of \`match?\` predicate must be a capture. Got "${steps[1].value}".`
);
if (steps[2].type !== 'string') throw new Error(
`Second argument of \`match?\` predicate must be a string. Got @${steps[2].value}.`
);
const captureName = steps[1].name;
const regex = new RegExp(steps[2].value);
predicates[i].push(function(captures) {
for (const c of captures) {
if (c.name === captureName) return regex.test(c.node.text);
}
return false;
});
break;
case 'set!':
if (steps.length < 2 || steps.length > 3) throw new Error(
`Wrong number of arguments to \`set!\` predicate. Expected 1 or 2. Got ${steps.length - 1}.`
);
if (steps.some(s => s.type !== 'string')) throw new Error(
`Arguments to \`set!\` predicate must be a strings.".`
);
if (!setProperties[i]) setProperties[i] = {};
setProperties[i][steps[1].value] = steps[2] ? steps[2].value : null;
break;
case 'is?':
case 'is-not?':
if (steps.length < 2 || steps.length > 3) throw new Error(
`Wrong number of arguments to \`${operator}\` predicate. Expected 1 or 2. Got ${steps.length - 1}.`
);
if (steps.some(s => s.type !== 'string')) throw new Error(
`Arguments to \`${operator}\` predicate must be a strings.".`
);
const properties = operator === 'is?' ? assertedProperties : refutedProperties;
if (!properties[i]) properties[i] = {};
properties[i][steps[1].value] = steps[2] ? steps[2].value : null;
break;
default:
throw new Error(`Unknown query predicate \`${steps[0].value}\``);
}
steps.length = 0;
}
}
Object.freeze(properties[i]);
Object.freeze(setProperties[i]);
Object.freeze(assertedProperties[i]);
Object.freeze(refutedProperties[i]);
}
C._free(sourceAddress);
@ -764,7 +843,9 @@ class Language {
address,
captureNames,
predicates,
Object.freeze(properties)
Object.freeze(setProperties),
Object.freeze(assertedProperties),
Object.freeze(refutedProperties)
);
}
@ -801,12 +882,17 @@ class Language {
}
class Query {
constructor(internal, address, captureNames, predicates, properties) {
constructor(
internal, address, captureNames, predicates,
setProperties, assertedProperties, refutedProperties
) {
assertInternal(internal);
this[0] = address;
this.captureNames = captureNames;
this.predicates = predicates;
this.patternProperties = properties;
this.setProperties = setProperties;
this.assertedProperties = assertedProperties;
this.refutedProperties = refutedProperties;
}
delete() {
@ -843,10 +929,13 @@ class Query {
address = unmarshalCaptures(this, node.tree, address, captures);
if (this.predicates[pattern].every(p => p(captures))) {
result[i] = {pattern, captures};
const setProperties = this.setProperties[pattern];
if (setProperties) result[i].setProperties = setProperties;
const assertedProperties = this.assertedProperties[pattern];
if (assertedProperties) result[i].assertedProperties = assertedProperties;
const refutedProperties = this.refutedProperties[pattern];
if (refutedProperties) result[i].refutedProperties = refutedProperties;
}
const properties = this.patternProperties[pattern];
if (properties) result[i].properties = properties;
}
C._free(startAddress);
@ -887,8 +976,12 @@ class Query {
if (this.predicates[pattern].every(p => p(captures))) {
const capture = captures[captureIndex];
const properties = this.patternProperties[pattern];
if (properties) capture.properties = properties;
const setProperties = this.setProperties[pattern];
if (setProperties) capture.setProperties = setProperties;
const assertedProperties = this.assertedProperties[pattern];
if (assertedProperties) capture.assertedProperties = assertedProperties;
const refutedProperties = this.refutedProperties[pattern];
if (refutedProperties) capture.refutedProperties = refutedProperties;
result.push(capture);
}
}
@ -898,77 +991,6 @@ class Query {
}
}
function buildQueryPredicate(steps) {
if (steps[0].type !== 'string') {
throw new Error('Predicates must begin with a literal value');
}
switch (steps[0].value) {
case 'eq?':
if (steps.length !== 3) throw new Error(
`Wrong number of arguments to \`eq?\` predicate. Expected 2, got ${steps.length - 1}`
);
if (steps[1].type !== 'capture') throw new Error(
`First argument of \`eq?\` predicate must be a capture. Got "${steps[1].value}"`
);
if (steps[2].type === 'capture') {
const captureName1 = steps[1].name;
const captureName2 = steps[2].name;
return function(captures) {
let node1, node2
for (const c of captures) {
if (c.name === captureName1) node1 = c.node;
if (c.name === captureName2) node2 = c.node;
}
return node1.text === node2.text
}
} else {
const captureName = steps[1].name;
const stringValue = steps[2].value;
return function(captures) {
for (const c of captures) {
if (c.name === captureName) return c.node.text === stringValue;
}
return false;
}
}
case 'match?':
if (steps.length !== 3) throw new Error(
`Wrong number of arguments to \`match?\` predicate. Expected 2, got ${steps.length - 1}.`
);
if (steps[1].type !== 'capture') throw new Error(
`First argument of \`match?\` predicate must be a capture. Got "${steps[1].value}".`
);
if (steps[2].type !== 'string') throw new Error(
`Second argument of \`match?\` predicate must be a string. Got @${steps[2].value}.`
);
const captureName = steps[1].name;
const regex = new RegExp(steps[2].value);
return function(captures) {
for (const c of captures) {
if (c.name === captureName) return regex.test(c.node.text);
}
return false;
}
case 'set!':
if (steps.length !== 3) throw new Error(
`Wrong number of arguments to \`set!\` predicate. Expected 2, got ${steps.length - 1}.`
);
if (steps[1].type !== 'string' || steps[2].type !== 'string') throw new Error(
`Arguments to \`set!\` predicate must be a strings.".`
);
return {
key: steps[1].value,
value: steps[2].value,
};
default:
throw new Error(`Unknown query predicate \`${steps[0].value}\``);
}
}
function unmarshalCaptures(query, tree, address, result) {
for (let i = 0, n = result.length; i < n; i++) {
const captureIndex = getValue(address, 'i32');

12
lib/binding_web/test/query-test.js
View file

@ -202,16 +202,18 @@ describe("Query", () => {
tree = parser.parse(`a(b.c);`);
query = JavaScript.query(`
((call_expression (identifier) @func)
(set! foo bar)
(set! baz quux))
(set! foo)
(set! bar baz))
(property_identifier) @prop
((property_identifier) @prop
(is? foo)
(is-not? bar baz))
`);
const captures = query.captures(tree.rootNode);
assert.deepEqual(formatCaptures(captures), [
{name: 'func', text: 'a', properties: {foo: 'bar', baz: 'quux'}},
{name: 'prop', text: 'c'},
{name: 'func', text: 'a', setProperties: {foo: null, bar: 'baz'}},
{name: 'prop', text: 'c', assertedProperties: {foo: null}, refutedProperties: {bar: 'baz'}},
]);
});
});