feat(xtask): generate JSON schema for cli TestSummary
This commit is contained in:
Will Lillis
parent
d546e28abf
commit
c7b5f89392
9 changed files with 376 additions and 7 deletions
65
Cargo.lock
generated
65
Cargo.lock
generated
|
|
@ -576,6 +576,12 @@ version = "1.0.5"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
|
||||
|
||||
[[package]]
|
||||
name = "dyn-clone"
|
||||
version = "1.0.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.15.0"
|
||||
|
|
@ -1438,6 +1444,26 @@ dependencies = [
|
|||
"getrandom 0.2.16",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ref-cast"
|
||||
version = "1.0.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d"
|
||||
dependencies = [
|
||||
"ref-cast-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ref-cast-impl"
|
||||
version = "1.0.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regalloc2"
|
||||
version = "0.12.2"
|
||||
|
|
@ -1603,6 +1629,31 @@ dependencies = [
|
|||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "schemars"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "82d20c4491bc164fa2f6c5d44565947a52ad80b9505d8e36f8d54c27c739fcd0"
|
||||
dependencies = [
|
||||
"dyn-clone",
|
||||
"ref-cast",
|
||||
"schemars_derive",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "schemars_derive"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "33d020396d1d138dc19f1165df7545479dcd58d93810dc5d646a16e55abefa80"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"serde_derive_internals",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "semver"
|
||||
version = "1.0.27"
|
||||
|
|
@ -1643,6 +1694,17 @@ dependencies = [
|
|||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive_internals"
|
||||
version = "0.29.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.145"
|
||||
|
|
@ -1947,6 +2009,7 @@ dependencies = [
|
|||
"pretty_assertions",
|
||||
"rand",
|
||||
"regex",
|
||||
"schemars",
|
||||
"semver",
|
||||
"serde",
|
||||
"serde_json",
|
||||
|
|
@ -2740,8 +2803,10 @@ dependencies = [
|
|||
"notify",
|
||||
"notify-debouncer-full",
|
||||
"regex",
|
||||
"schemars",
|
||||
"semver",
|
||||
"serde_json",
|
||||
"tree-sitter-cli",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
|
|||
|
|
@ -137,6 +137,7 @@ rand = "0.8.5"
|
|||
regex = "1.11.3"
|
||||
regex-syntax = "0.8.6"
|
||||
rustc-hash = "2.1.1"
|
||||
schemars = "1.0.4"
|
||||
semver = { version = "1.0.27", features = ["serde"] }
|
||||
serde = { version = "1.0.219", features = ["derive"] }
|
||||
serde_json = { version = "1.0.145", features = ["preserve_order"] }
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ log.workspace = true
|
|||
memchr.workspace = true
|
||||
rand.workspace = true
|
||||
regex.workspace = true
|
||||
schemars.workspace = true
|
||||
semver.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ use anstyle::{AnsiColor, Color, RgbColor};
|
|||
use anyhow::{anyhow, Context, Result};
|
||||
use clap::ValueEnum;
|
||||
use log::info;
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tree_sitter::{
|
||||
ffi, InputEdit, Language, LogType, ParseOptions, ParseState, Parser, Point, Range, Tree,
|
||||
|
|
@ -19,7 +20,7 @@ use tree_sitter::{
|
|||
|
||||
use crate::{fuzz::edits::Edit, logger::paint, util};
|
||||
|
||||
#[derive(Debug, Default, Serialize)]
|
||||
#[derive(Debug, Default, Serialize, JsonSchema)]
|
||||
pub struct Stats {
|
||||
pub successful_parses: usize,
|
||||
pub total_parses: usize,
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ use regex::{
|
|||
bytes::{Regex as ByteRegex, RegexBuilder as ByteRegexBuilder},
|
||||
Regex,
|
||||
};
|
||||
use schemars::{JsonSchema, Schema, SchemaGenerator};
|
||||
use serde::Serialize;
|
||||
use similar::{ChangeTag, TextDiff};
|
||||
use tree_sitter::{format_sexp, Language, LogType, Parser, Query, Tree};
|
||||
|
|
@ -139,36 +140,47 @@ pub struct TestOptions<'a> {
|
|||
}
|
||||
|
||||
/// A stateful object used to collect results from running a grammar's test suite
|
||||
#[derive(Debug, Default, Serialize)]
|
||||
#[derive(Debug, Default, Serialize, JsonSchema)]
|
||||
pub struct TestSummary {
|
||||
// Parse test results and associated data
|
||||
#[schemars(schema_with = "schema_as_array")]
|
||||
#[serde(serialize_with = "serialize_as_array")]
|
||||
pub parse_results: TestResultHierarchy,
|
||||
pub parse_failures: Vec<TestFailure>,
|
||||
pub parse_stats: Stats,
|
||||
#[schemars(skip)]
|
||||
#[serde(skip)]
|
||||
pub has_parse_errors: bool,
|
||||
#[schemars(skip)]
|
||||
#[serde(skip)]
|
||||
pub parse_stat_display: TestStats,
|
||||
|
||||
// Other test results
|
||||
#[schemars(schema_with = "schema_as_array")]
|
||||
#[serde(serialize_with = "serialize_as_array")]
|
||||
pub highlight_results: TestResultHierarchy,
|
||||
#[schemars(schema_with = "schema_as_array")]
|
||||
#[serde(serialize_with = "serialize_as_array")]
|
||||
pub tag_results: TestResultHierarchy,
|
||||
#[schemars(schema_with = "schema_as_array")]
|
||||
#[serde(serialize_with = "serialize_as_array")]
|
||||
pub query_results: TestResultHierarchy,
|
||||
|
||||
// Data used during construction
|
||||
#[schemars(skip)]
|
||||
#[serde(skip)]
|
||||
pub test_num: usize,
|
||||
// Options passed in from the CLI which control how the summary is displayed
|
||||
#[schemars(skip)]
|
||||
#[serde(skip)]
|
||||
pub color: bool,
|
||||
#[schemars(skip)]
|
||||
#[serde(skip)]
|
||||
pub overview_only: bool,
|
||||
#[schemars(skip)]
|
||||
#[serde(skip)]
|
||||
pub update: bool,
|
||||
#[schemars(skip)]
|
||||
#[serde(skip)]
|
||||
pub json: bool,
|
||||
}
|
||||
|
|
@ -194,7 +206,7 @@ impl TestSummary {
|
|||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
#[derive(Debug, Default, JsonSchema)]
|
||||
pub struct TestResultHierarchy {
|
||||
root_group: Vec<TestResult>,
|
||||
traversal_idxs: Vec<usize>,
|
||||
|
|
@ -207,6 +219,10 @@ where
|
|||
results.root_group.serialize(serializer)
|
||||
}
|
||||
|
||||
fn schema_as_array(gen: &mut SchemaGenerator) -> Schema {
|
||||
gen.subschema_for::<Vec<TestResult>>()
|
||||
}
|
||||
|
||||
/// Stores arbitrarily nested parent test groups and child cases. Supports creation
|
||||
/// in DFS traversal order
|
||||
impl TestResultHierarchy {
|
||||
|
|
@ -313,14 +329,16 @@ impl<'a> Iterator for TestResultIterWithDepth<'a> {
|
|||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[derive(Debug, Serialize, JsonSchema)]
|
||||
pub struct TestResult {
|
||||
pub name: String,
|
||||
#[schemars(flatten)]
|
||||
#[serde(flatten)]
|
||||
pub info: TestInfo,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[derive(Debug, Serialize, JsonSchema)]
|
||||
#[schemars(untagged)]
|
||||
#[serde(untagged)]
|
||||
pub enum TestInfo {
|
||||
Group {
|
||||
|
|
@ -329,6 +347,7 @@ pub enum TestInfo {
|
|||
ParseTest {
|
||||
outcome: TestOutcome,
|
||||
// True parse rate, adjusted parse rate
|
||||
#[schemars(schema_with = "parse_rate_schema")]
|
||||
#[serde(serialize_with = "serialize_parse_rates")]
|
||||
parse_rate: Option<(f64, f64)>,
|
||||
test_num: usize,
|
||||
|
|
@ -352,7 +371,11 @@ where
|
|||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Eq, PartialEq, Serialize)]
|
||||
fn parse_rate_schema(gen: &mut SchemaGenerator) -> Schema {
|
||||
gen.subschema_for::<Option<f64>>()
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Eq, PartialEq, Serialize, JsonSchema)]
|
||||
pub enum TestOutcome {
|
||||
// Parse outcomes
|
||||
Passed,
|
||||
|
|
@ -726,7 +749,7 @@ impl std::fmt::Display for TestDiff<'_> {
|
|||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[derive(Debug, Serialize, JsonSchema)]
|
||||
pub struct TestFailure {
|
||||
name: String,
|
||||
actual: String,
|
||||
|
|
|
|||
|
|
@ -21,7 +21,9 @@ bindgen = { version = "0.72.0" }
|
|||
clap.workspace = true
|
||||
indoc.workspace = true
|
||||
regex.workspace = true
|
||||
schemars.workspace = true
|
||||
semver.workspace = true
|
||||
serde_json.workspace = true
|
||||
tree-sitter-cli = { path = "../cli/" }
|
||||
notify = "8.2.0"
|
||||
notify-debouncer-full = "0.6.0"
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ mod embed_sources;
|
|||
mod fetch;
|
||||
mod generate;
|
||||
mod test;
|
||||
mod test_schema;
|
||||
mod upgrade_wasmtime;
|
||||
|
||||
use std::{path::Path, process::Command};
|
||||
|
|
@ -40,6 +41,8 @@ enum Commands {
|
|||
GenerateBindings,
|
||||
/// Generates the fixtures for testing tree-sitter.
|
||||
GenerateFixtures(GenerateFixtures),
|
||||
/// Generates the JSON schema for the test runner summary.
|
||||
GenerateTestSchema,
|
||||
/// Generate the list of exports from Tree-sitter Wasm files.
|
||||
GenerateWasmExports,
|
||||
/// Run the test suite
|
||||
|
|
@ -236,6 +239,7 @@ fn run() -> Result<()> {
|
|||
Commands::GenerateFixtures(generate_fixtures_options) => {
|
||||
generate::run_fixtures(&generate_fixtures_options)?;
|
||||
}
|
||||
Commands::GenerateTestSchema => test_schema::run_test_schema()?,
|
||||
Commands::GenerateWasmExports => generate::run_wasm_exports()?,
|
||||
Commands::Test(test_options) => test::run(&test_options)?,
|
||||
Commands::TestWasm => test::run_wasm()?,
|
||||
|
|
|
|||
25
crates/xtask/src/test_schema.rs
Normal file
25
crates/xtask/src/test_schema.rs
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
use std::path::PathBuf;
|
||||
|
||||
use anyhow::Result;
|
||||
use serde_json::to_writer_pretty;
|
||||
|
||||
use tree_sitter_cli::test::TestSummary;
|
||||
|
||||
pub fn run_test_schema() -> Result<()> {
|
||||
let schema = schemars::schema_for!(TestSummary);
|
||||
|
||||
let xtask_path: PathBuf = env!("CARGO_MANIFEST_DIR").into();
|
||||
let schema_path = xtask_path
|
||||
.parent()
|
||||
.unwrap()
|
||||
.parent()
|
||||
.unwrap()
|
||||
.join("docs")
|
||||
.join("src")
|
||||
.join("assets")
|
||||
.join("schemas")
|
||||
.join("test-summary.schema.json");
|
||||
let mut file = std::fs::File::create(schema_path)?;
|
||||
|
||||
Ok(to_writer_pretty(&mut file, &schema)?)
|
||||
}
|
||||
247
docs/src/assets/schemas/test-summary.schema.json
Normal file
247
docs/src/assets/schemas/test-summary.schema.json
Normal file
|
|
@ -0,0 +1,247 @@
|
|||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "TestSummary",
|
||||
"description": "A stateful object used to collect results from running a grammar's test suite",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"parse_results": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/TestResult"
|
||||
}
|
||||
},
|
||||
"parse_failures": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/TestFailure"
|
||||
}
|
||||
},
|
||||
"parse_stats": {
|
||||
"$ref": "#/$defs/Stats"
|
||||
},
|
||||
"highlight_results": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/TestResult"
|
||||
}
|
||||
},
|
||||
"tag_results": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/TestResult"
|
||||
}
|
||||
},
|
||||
"query_results": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/TestResult"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"parse_results",
|
||||
"parse_failures",
|
||||
"parse_stats",
|
||||
"highlight_results",
|
||||
"tag_results",
|
||||
"query_results"
|
||||
],
|
||||
"$defs": {
|
||||
"TestResult": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"name"
|
||||
],
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"children": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/TestResult"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"children"
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"outcome": {
|
||||
"$ref": "#/$defs/TestOutcome"
|
||||
},
|
||||
"parse_rate": {
|
||||
"type": [
|
||||
"number",
|
||||
"null"
|
||||
],
|
||||
"format": "double"
|
||||
},
|
||||
"test_num": {
|
||||
"type": "integer",
|
||||
"format": "uint",
|
||||
"minimum": 0
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"outcome",
|
||||
"parse_rate",
|
||||
"test_num"
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"outcome": {
|
||||
"$ref": "#/$defs/TestOutcome"
|
||||
},
|
||||
"test_num": {
|
||||
"type": "integer",
|
||||
"format": "uint",
|
||||
"minimum": 0
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"outcome",
|
||||
"test_num"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"TestOutcome": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"Passed",
|
||||
"Failed",
|
||||
"Updated",
|
||||
"Skipped",
|
||||
"Platform"
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"AssertionPassed": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"assertion_count": {
|
||||
"type": "integer",
|
||||
"format": "uint",
|
||||
"minimum": 0
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"assertion_count"
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"AssertionPassed"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"AssertionFailed": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"error": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"error"
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"AssertionFailed"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"TestFailure": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"actual": {
|
||||
"type": "string"
|
||||
},
|
||||
"expected": {
|
||||
"type": "string"
|
||||
},
|
||||
"is_cst": {
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"name",
|
||||
"actual",
|
||||
"expected",
|
||||
"is_cst"
|
||||
]
|
||||
},
|
||||
"Stats": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"successful_parses": {
|
||||
"type": "integer",
|
||||
"format": "uint",
|
||||
"minimum": 0
|
||||
},
|
||||
"total_parses": {
|
||||
"type": "integer",
|
||||
"format": "uint",
|
||||
"minimum": 0
|
||||
},
|
||||
"total_bytes": {
|
||||
"type": "integer",
|
||||
"format": "uint",
|
||||
"minimum": 0
|
||||
},
|
||||
"total_duration": {
|
||||
"$ref": "#/$defs/Duration"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"successful_parses",
|
||||
"total_parses",
|
||||
"total_bytes",
|
||||
"total_duration"
|
||||
]
|
||||
},
|
||||
"Duration": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"secs": {
|
||||
"type": "integer",
|
||||
"format": "uint64",
|
||||
"minimum": 0
|
||||
},
|
||||
"nanos": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"secs",
|
||||
"nanos"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue