traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into node-fields

Browse source
This commit is contained in:
Max Brunsfeld 2019-03-05 08:08:05 -08:00
commit f52271352b
26 changed files with 2655 additions and 233 deletions
Download patch file Download diff file Expand all files Collapse all files

69
cli/src/config.rs Normal file
View file

@ -0,0 +1,69 @@
use super::highlight::Theme;
use serde_derive::{Deserialize, Serialize};
use std::path::{Path, PathBuf};
use std::{env, fs, io};
#[derive(Default, Deserialize, Serialize)]
pub struct Config {
#[serde(skip)]
pub binary_directory: PathBuf,
#[serde(default)]
#[serde(rename = "parser-directories")]
pub parser_directories: Vec<PathBuf>,
#[serde(default)]
pub theme: Theme,
}
impl Config {
pub fn get_path(home_dir: &Path) -> PathBuf {
env::var("TREE_SITTER_DIR")
.map(|p| p.into())
.unwrap_or_else(|_| home_dir.join(".tree-sitter"))
}
pub fn load(home_dir: &Path) -> Self {
let tree_sitter_dir = Self::get_path(home_dir);
let config_path = tree_sitter_dir.join("config.json");
let mut result = fs::read_to_string(&config_path)
.map_err(drop)
.and_then(|json| serde_json::from_str(&json).map_err(drop))
.unwrap_or_else(|_| Self::default());
result.init(home_dir, &tree_sitter_dir);
result
}
pub fn save(&self, home_dir: &Path) -> io::Result<()> {
let tree_sitter_dir = Self::get_path(home_dir);
let config_path = tree_sitter_dir.join("config.json");
let json = serde_json::to_string_pretty(self).expect("Failed to serialize config");
fs::write(config_path, json)
}
pub fn new(home_dir: &Path) -> Self {
let tree_sitter_dir = Self::get_path(home_dir);
let mut result = Self::default();
result.init(home_dir, &tree_sitter_dir);
result
}
fn init(&mut self, home_dir: &Path, tree_sitter_dir: &Path) {
if self.parser_directories.is_empty() {
self.parser_directories = vec![
home_dir.join("github"),
home_dir.join("src"),
home_dir.join("source"),
]
}
let binary_path = tree_sitter_dir.join("bin");
self.binary_directory = binary_path;
fs::create_dir_all(&self.binary_directory).unwrap_or_else(|error| {
panic!(
"Could not find or create parser binary directory {:?}. Error: {}",
self.binary_directory, error
)
});
}
}

11
cli/src/error.rs
View file

@ -1,4 +1,5 @@
use std::io;
use tree_sitter_highlight::PropertySheetError;
#[derive(Debug)]
pub struct Error(pub String);
@ -42,3 +43,13 @@ impl From<String> for Error {
Error(error)
}
}
impl From<PropertySheetError> for Error {
fn from(error: PropertySheetError) -> Self {
match error {
PropertySheetError::InvalidFormat(e) => Self::from(e),
PropertySheetError::InvalidRegex(e) => Self::regex(&e.to_string()),
PropertySheetError::InvalidJSON(e) => Self::from(e),
}
}
}

10
cli/src/generate/dsl.js
View file

@ -1,5 +1,3 @@
const UNICODE_ESCAPE_PATTERN = /\\u([0-9a-f]{4})/gi;
function alias(rule, value) {
const result = {
type: "ALIAS",
@ -180,12 +178,8 @@ function normalize(value) {
};
case RegExp:
return {
type: 'PATTERN',
value: value.source
.replace(
UNICODE_ESCAPE_PATTERN,
(match, group) => String.fromCharCode(parseInt(group, 16))
)
type: 'PATTERN',
value: value.source
};
case ReferenceError:
throw value

373
cli/src/highlight.rs Normal file
View file

@ -0,0 +1,373 @@
use crate::error::Result;
use crate::loader::Loader;
use ansi_term::{Color, Style};
use lazy_static::lazy_static;
use serde::ser::SerializeMap;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde_json::{json, Value};
use std::collections::HashMap;
use std::{fmt, fs, io, path};
use tree_sitter::{Language, PropertySheet};
use tree_sitter_highlight::{highlight, highlight_html, HighlightEvent, Properties, Scope};
lazy_static! {
static ref CSS_STYLES_BY_COLOR_ID: Vec<String> =
serde_json::from_str(include_str!("../vendor/xterm-colors.json")).unwrap();
}
pub struct Theme {
ansi_styles: Vec<Option<Style>>,
css_styles: Vec<Option<String>>,
}
impl Theme {
pub fn load(path: &path::Path) -> io::Result<Self> {
let json = fs::read_to_string(path)?;
Ok(serde_json::from_str(&json).unwrap_or_default())
}
fn ansi_style(&self, scope: Scope) -> Option<&Style> {
self.ansi_styles[scope as usize].as_ref()
}
fn css_style(&self, scope: Scope) -> Option<&str> {
self.css_styles[scope as usize].as_ref().map(|s| s.as_str())
}
}
impl<'de> Deserialize<'de> for Theme {
fn deserialize<D>(deserializer: D) -> std::result::Result<Theme, D::Error>
where
D: Deserializer<'de>,
{
let scope_count = Scope::Unknown as usize + 1;
let mut ansi_styles = vec![None; scope_count];
let mut css_styles = vec![None; scope_count];
if let Ok(colors) = HashMap::<Scope, Value>::deserialize(deserializer) {
for (scope, style_value) in colors {
let mut style = Style::default();
parse_style(&mut style, style_value);
ansi_styles[scope as usize] = Some(style);
css_styles[scope as usize] = Some(style_to_css(style));
}
}
Ok(Self {
ansi_styles,
css_styles,
})
}
}
impl Serialize for Theme {
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
where
S: Serializer,
{
let entry_count = self.ansi_styles.iter().filter(|i| i.is_some()).count();
let mut map = serializer.serialize_map(Some(entry_count))?;
for (i, style) in self.ansi_styles.iter().enumerate() {
let scope = Scope::from_usize(i).unwrap();
if scope == Scope::Unknown {
break;
}
if let Some(style) = style {
let color = style.foreground.map(|color| match color {
Color::Black => json!("black"),
Color::Blue => json!("blue"),
Color::Cyan => json!("cyan"),
Color::Green => json!("green"),
Color::Purple => json!("purple"),
Color::Red => json!("red"),
Color::White => json!("white"),
Color::Yellow => json!("yellow"),
Color::RGB(r, g, b) => json!(format!("#{:x?}{:x?}{:x?}", r, g, b)),
Color::Fixed(n) => json!(n),
});
if style.is_bold || style.is_italic || style.is_underline {
let mut entry = HashMap::new();
if let Some(color) = color {
entry.insert("color", color);
}
if style.is_bold {
entry.insert("bold", Value::Bool(true));
}
if style.is_italic {
entry.insert("italic", Value::Bool(true));
}
if style.is_underline {
entry.insert("underline", Value::Bool(true));
}
map.serialize_entry(&scope, &entry)?;
} else if let Some(color) = color {
map.serialize_entry(&scope, &color)?;
} else {
map.serialize_entry(&scope, &Value::Null)?;
}
} else {
map.serialize_entry(&scope, &Value::Null)?;
}
}
map.end()
}
}
impl Default for Theme {
fn default() -> Self {
serde_json::from_str(
r#"
{
"attribute": {"color": 124, "italic": true},
"comment": {"color": 245, "italic": true},
"constant.builtin": {"color": 94, "bold": true},
"constant": 94,
"constructor": 136,
"embedded": null,
"function.builtin": {"color": 26, "bold": true},
"function": 26,
"keyword": 56,
"number": {"color": 94, "bold": true},
"property": 124,
"operator": {"color": 239, "bold": true},
"punctuation.bracket": 239,
"punctuation.delimiter": 239,
"string.special": 30,
"string": 28,
"tag": 18,
"type": 23,
"type.builtin": {"color": 23, "bold": true},
"variable.builtin": {"bold": true}
}
"#,
)
.unwrap()
}
}
impl fmt::Debug for Theme {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{{")?;
let mut first = true;
for (i, style) in self.ansi_styles.iter().enumerate() {
if let Some(style) = style {
let scope = Scope::from_usize(i).unwrap();
if !first {
write!(f, ", ")?;
}
write!(f, "{:?}: {:?}", scope, style)?;
first = false;
}
}
write!(f, "}}")?;
Ok(())
}
}
fn parse_style(style: &mut Style, json: Value) {
if let Value::Object(entries) = json {
for (property_name, value) in entries {
match property_name.as_str() {
"bold" => *style = style.bold(),
"italic" => *style = style.italic(),
"underline" => *style = style.underline(),
"color" => {
if let Some(color) = parse_color(value) {
*style = style.fg(color);
}
}
_ => {}
}
}
} else if let Some(color) = parse_color(json) {
*style = style.fg(color);
}
}
fn parse_color(json: Value) -> Option<Color> {
match json {
Value::Number(n) => match n.as_u64() {
Some(n) => Some(Color::Fixed(n as u8)),
_ => None,
},
Value::String(s) => match s.to_lowercase().as_str() {
"black" => Some(Color::Black),
"blue" => Some(Color::Blue),
"cyan" => Some(Color::Cyan),
"green" => Some(Color::Green),
"purple" => Some(Color::Purple),
"red" => Some(Color::Red),
"white" => Some(Color::White),
"yellow" => Some(Color::Yellow),
s => {
if s.starts_with("#") && s.len() >= 7 {
if let (Ok(red), Ok(green), Ok(blue)) = (
u8::from_str_radix(&s[1..3], 16),
u8::from_str_radix(&s[3..5], 16),
u8::from_str_radix(&s[5..7], 16),
) {
Some(Color::RGB(red, green, blue))
} else {
None
}
} else {
None
}
}
},
_ => None,
}
}
fn style_to_css(style: Style) -> String {
use std::fmt::Write;
let mut result = "style='".to_string();
if style.is_bold {
write!(&mut result, "font-weight: bold;").unwrap();
}
if style.is_italic {
write!(&mut result, "font-style: italic;").unwrap();
}
if let Some(color) = style.foreground {
write!(&mut result, "color: {};", color_to_css(color)).unwrap();
}
result.push('\'');
result
}
fn color_to_css(color: Color) -> &'static str {
match color {
Color::Black => "black",
Color::Blue => "blue",
Color::Red => "red",
Color::Green => "green",
Color::Yellow => "yellow",
Color::Cyan => "cyan",
Color::Purple => "purple",
Color::White => "white",
Color::Fixed(n) => CSS_STYLES_BY_COLOR_ID[n as usize].as_str(),
_ => panic!("Unsupported color type"),
}
}
pub fn ansi(
loader: &Loader,
theme: &Theme,
source: &[u8],
language: Language,
property_sheet: &PropertySheet<Properties>,
) -> Result<()> {
use std::io::Write;
let stdout = io::stdout();
let mut stdout = stdout.lock();
let mut scope_stack = Vec::new();
for event in highlight(source, language, property_sheet, |s| {
language_for_injection_string(loader, s)
})? {
match event {
HighlightEvent::Source(s) => {
if let Some(style) = scope_stack.last().and_then(|s| theme.ansi_style(*s)) {
write!(&mut stdout, "{}", style.paint(s))?;
} else {
write!(&mut stdout, "{}", s)?;
}
}
HighlightEvent::ScopeStart(s) => {
scope_stack.push(s);
}
HighlightEvent::ScopeEnd => {
scope_stack.pop();
}
}
}
Ok(())
}
pub const HTML_HEADER: &'static str = "
<!doctype HTML>
<head>
<title>Tree-sitter Highlighting</title>
<style>
body {
font-family: monospace
}
.line-number {
user-select: none;
text-align: right;
color: rgba(27,31,35,.3);
padding: 0 10px;
}
.line {
white-space: pre;
}
</style>
</head>
<body>
";
pub const HTML_FOOTER: &'static str = "
</body>
";
pub fn html(
loader: &Loader,
theme: &Theme,
source: &[u8],
language: Language,
property_sheet: &PropertySheet<Properties>,
) -> Result<()> {
use std::io::Write;
let stdout = io::stdout();
let mut stdout = stdout.lock();
write!(&mut stdout, "<table>\n")?;
let lines = highlight_html(
source,
language,
property_sheet,
|s| language_for_injection_string(loader, s),
|scope| {
if let Some(css_style) = theme.css_style(scope) {
css_style
} else {
""
}
},
)?;
for (i, line) in lines.into_iter().enumerate() {
write!(
&mut stdout,
"<tr><td class=line-number>{}</td><td class=line>{}</td></tr>\n",
i + 1,
line
)?;
}
write!(&mut stdout, "</table>\n")?;
Ok(())
}
fn language_for_injection_string<'a>(
loader: &'a Loader,
string: &str,
) -> Option<(Language, &'a PropertySheet<Properties>)> {
match loader.language_configuration_for_injection_string(string) {
Err(message) => {
eprintln!(
"Failed to load language for injection string '{}': {}",
string, message.0
);
None
}
Ok(None) => None,
Ok(Some((language, configuration))) => {
match configuration.highlight_property_sheet(language) {
Err(message) => {
eprintln!(
"Failed to load property sheet for injection string '{}': {}",
string, message.0
);
None
}
Ok(None) => None,
Ok(Some(sheet)) => Some((language, sheet)),
}
}
}
}

2
cli/src/lib.rs
View file

@ -1,5 +1,7 @@
pub mod config;
pub mod error;
pub mod generate;
pub mod highlight;
pub mod loader;
pub mod logger;
pub mod parse;

104
cli/src/loader.rs
View file

@ -1,5 +1,6 @@
use super::error::{Error, Result};
use libloading::{Library, Symbol};
use once_cell::unsync::OnceCell;
use regex::{Regex, RegexBuilder};
use serde_derive::Deserialize;
use std::collections::HashMap;
@ -9,6 +10,7 @@ use std::process::Command;
use std::time::SystemTime;
use std::{fs, mem};
use tree_sitter::{Language, PropertySheet};
use tree_sitter_highlight::{load_property_sheet, Properties};
#[cfg(unix)]
const DYLIB_EXTENSION: &'static str = "so";
@ -20,16 +22,18 @@ const BUILD_TARGET: &'static str = env!("BUILD_TARGET");
struct LanguageRepo {
path: PathBuf,
language: Option<Language>,
language: OnceCell<Language>,
configurations: Vec<LanguageConfiguration>,
}
pub struct LanguageConfiguration {
_name: String,
scope: Option<String>,
_content_regex: Option<Regex>,
_first_line_regex: Option<Regex>,
injection_regex: Option<Regex>,
file_types: Vec<String>,
_highlight_property_sheet: Option<std::result::Result<PropertySheet, PathBuf>>,
highlight_property_sheet_path: Option<PathBuf>,
highlight_property_sheet: OnceCell<Option<PropertySheet<Properties>>>,
}
pub struct Loader {
@ -75,8 +79,23 @@ impl Loader {
}
}
pub fn language_configuration_for_scope(
&self,
scope: &str,
) -> Result<Option<(Language, &LanguageConfiguration)>> {
for (i, repo) in self.language_repos.iter().enumerate() {
for configuration in &repo.configurations {
if configuration.scope.as_ref().map_or(false, |s| s == scope) {
let (language, _) = self.language_configuration_for_id(i)?;
return Ok(Some((language, &configuration)));
}
}
}
Ok(None)
}
pub fn language_configuration_for_file_name(
&mut self,
&self,
path: &Path,
) -> Result<Option<(Language, &LanguageConfiguration)>> {
let ids = path
@ -100,20 +119,43 @@ impl Loader {
Ok(None)
}
pub fn language_configuration_for_injection_string(
&self,
string: &str,
) -> Result<Option<(Language, &LanguageConfiguration)>> {
let mut best_match_length = 0;
let mut best_match_position = None;
for (i, repo) in self.language_repos.iter().enumerate() {
for (j, configuration) in repo.configurations.iter().enumerate() {
if let Some(injection_regex) = &configuration.injection_regex {
if let Some(mat) = injection_regex.find(string) {
let length = mat.end() - mat.start();
if length > best_match_length {
best_match_position = Some((i, j));
best_match_length = length;
}
}
}
}
}
if let Some((i, j)) = best_match_position {
let (language, configurations) = self.language_configuration_for_id(i)?;
Ok(Some((language, &configurations[j])))
} else {
Ok(None)
}
}
fn language_configuration_for_id(
&mut self,
&self,
id: usize,
) -> Result<(Language, &Vec<LanguageConfiguration>)> {
let repo = &self.language_repos[id];
let language = if let Some(language) = repo.language {
language
} else {
let language = repo.language.get_or_try_init(|| {
let src_path = repo.path.join("src");
let language = self.load_language_at_path(&src_path, &src_path)?;
self.language_repos[id].language = Some(language);
language
};
Ok((language, &self.language_repos[id].configurations))
self.load_language_at_path(&src_path, &src_path)
})?;
Ok((*language, &self.language_repos[id].configurations))
}
pub fn load_language_at_path(&self, src_path: &Path, header_path: &Path) -> Result<Language> {
@ -191,7 +233,8 @@ impl Loader {
.arg("-I")
.arg(header_path)
.arg("-o")
.arg(&library_path);
.arg(&library_path)
.arg("-O2");
if let Some(scanner_path) = scanner_path.as_ref() {
if scanner_path.extension() == Some("c".as_ref()) {
command.arg("-xc").arg("-std=c99").arg(scanner_path);
@ -231,13 +274,15 @@ impl Loader {
fn find_language_at_path<'a>(&'a mut self, parser_path: &Path) -> Result<usize> {
#[derive(Deserialize)]
struct LanguageConfigurationJSON {
name: String,
scope: Option<String>,
#[serde(rename = "file-types")]
file_types: Option<Vec<String>>,
#[serde(rename = "content-regex")]
content_regex: Option<String>,
#[serde(rename = "first-line-regex")]
first_line_regex: Option<String>,
#[serde(rename = "injection-regex")]
injection_regex: Option<String>,
highlights: Option<String>,
}
@ -255,7 +300,7 @@ impl Loader {
configurations
.into_iter()
.map(|conf| LanguageConfiguration {
_name: conf.name,
scope: conf.scope,
file_types: conf.file_types.unwrap_or(Vec::new()),
_content_regex: conf
.content_regex
@ -263,7 +308,11 @@ impl Loader {
_first_line_regex: conf
.first_line_regex
.and_then(|r| RegexBuilder::new(&r).multi_line(true).build().ok()),
_highlight_property_sheet: conf.highlights.map(|d| Err(d.into())),
injection_regex: conf
.injection_regex
.and_then(|r| RegexBuilder::new(&r).multi_line(true).build().ok()),
highlight_property_sheet_path: conf.highlights.map(|h| parser_path.join(h)),
highlight_property_sheet: OnceCell::new(),
})
.collect()
});
@ -279,7 +328,7 @@ impl Loader {
self.language_repos.push(LanguageRepo {
path: parser_path.to_owned(),
language: None,
language: OnceCell::new(),
configurations,
});
@ -287,6 +336,25 @@ impl Loader {
}
}
impl LanguageConfiguration {
pub fn highlight_property_sheet(
&self,
language: Language,
) -> Result<Option<&PropertySheet<Properties>>> {
self.highlight_property_sheet
.get_or_try_init(|| {
if let Some(path) = &self.highlight_property_sheet_path {
let sheet_json = fs::read_to_string(path)?;
let sheet = load_property_sheet(language, &sheet_json)?;
Ok(Some(sheet))
} else {
Ok(None)
}
})
.map(Option::as_ref)
}
}
fn needs_recompile(
lib_path: &Path,
parser_c_path: &Path,

85
cli/src/main.rs
View file

@ -4,8 +4,9 @@ use std::fs;
use std::path::Path;
use std::process::exit;
use std::usize;
use tree_sitter_cli::loader::Loader;
use tree_sitter_cli::{error, generate, logger, parse, properties, test};
use tree_sitter_cli::{
config, error, generate, highlight, loader, logger, parse, properties, test,
};
fn main() {
if let Err(e) = run() {
@ -25,6 +26,7 @@ fn run() -> error::Result<()> {
.setting(AppSettings::SubcommandRequiredElseHelp)
.author("Max Brunsfeld <maxbrunsfeld@gmail.com>")
.about("Generates and tests parsers")
.subcommand(SubCommand::with_name("init-config").about("Generate a default config file"))
.subcommand(
SubCommand::with_name("generate")
.about("Generate a parser")
@ -64,16 +66,29 @@ fn run() -> error::Result<()> {
.arg(Arg::with_name("debug").long("debug").short("d"))
.arg(Arg::with_name("debug-graph").long("debug-graph").short("D")),
)
.subcommand(
SubCommand::with_name("highlight")
.about("Highlight a file")
.arg(
Arg::with_name("path")
.index(1)
.multiple(true)
.required(true),
)
.arg(Arg::with_name("scope").long("scope").takes_value(true))
.arg(Arg::with_name("html").long("html").short("h")),
)
.get_matches();
let home_dir = dirs::home_dir().unwrap();
let home_dir = dirs::home_dir().expect("Failed to read home directory");
let current_dir = env::current_dir().unwrap();
let config_dir = home_dir.join(".tree-sitter");
let config = config::Config::load(&home_dir);
let mut loader = loader::Loader::new(config.binary_directory.clone());
fs::create_dir_all(&config_dir).unwrap();
let mut loader = Loader::new(config_dir);
if let Some(matches) = matches.subcommand_matches("generate") {
if matches.subcommand_matches("init-config").is_some() {
let config = config::Config::new(&home_dir);
config.save(&home_dir)?;
} else if let Some(matches) = matches.subcommand_matches("generate") {
if matches.is_present("log") {
logger::init();
}
@ -81,12 +96,14 @@ fn run() -> error::Result<()> {
let grammar_path = matches.value_of("grammar-path");
let minimize = !matches.is_present("no-minimize");
let properties_only = matches.is_present("properties-only");
let parser_only = grammar_path.is_some();
let state_ids_to_log = matches
.values_of("state-ids-to-log")
.map_or(Vec::new(), |ids| {
ids.filter_map(|id| usize::from_str_radix(id, 10).ok())
.collect()
});
if !properties_only {
generate::generate_parser_in_directory(
&current_dir,
@ -95,7 +112,10 @@ fn run() -> error::Result<()> {
state_ids_to_log,
)?;
}
properties::generate_property_sheets_in_directory(&current_dir)?;
if !parser_only {
properties::generate_property_sheets_in_directory(&current_dir)?;
}
} else if let Some(matches) = matches.subcommand_matches("test") {
let debug = matches.is_present("debug");
let debug_graph = matches.is_present("debug-graph");
@ -111,7 +131,7 @@ fn run() -> error::Result<()> {
let debug_graph = matches.is_present("debug-graph");
let quiet = matches.is_present("quiet");
let time = matches.is_present("time");
loader.find_all_languages(&vec![home_dir.join("github")])?;
loader.find_all_languages(&config.parser_directories)?;
let paths = matches
.values_of("path")
.unwrap()
@ -144,6 +164,51 @@ fn run() -> error::Result<()> {
if has_error {
return Err(error::Error(String::new()));
}
} else if let Some(matches) = matches.subcommand_matches("highlight") {
let paths = matches.values_of("path").unwrap().into_iter();
let html_mode = matches.is_present("html");
loader.find_all_languages(&config.parser_directories)?;
if html_mode {
println!("{}", highlight::HTML_HEADER);
}
let language_config;
if let Some(scope) = matches.value_of("scope") {
language_config = loader.language_configuration_for_scope(scope)?;
if language_config.is_none() {
return Err(error::Error(format!("Unknown scope '{}'", scope)));
}
} else {
language_config = None;
}
for path in paths {
let path = Path::new(path);
let (language, language_config) = match language_config {
Some(v) => v,
None => match loader.language_configuration_for_file_name(path)? {
Some(v) => v,
None => {
eprintln!("No language found for path {:?}", path);
continue;
}
},
};
if let Some(sheet) = language_config.highlight_property_sheet(language)? {
let source = fs::read(path)?;
if html_mode {
highlight::html(&loader, &config.theme, &source, language, sheet)?;
} else {
highlight::ansi(&loader, &config.theme, &source, language, sheet)?;
}
} else {
return Err(error::Error(format!(
"No syntax highlighting property sheet specified"
)));
}
}
}
Ok(())

5
cli/src/parse.rs
View file

@ -1,4 +1,4 @@
use super::error::Result;
use super::error::{Error, Result};
use super::util;
use std::fs;
use std::io::{self, Write};
@ -18,7 +18,8 @@ pub fn parse_file_at_path(
let mut _log_session = None;
let mut parser = Parser::new();
parser.set_language(language)?;
let source_code = fs::read(path)?;
let source_code = fs::read(path)
.map_err(|e| Error(format!("Error reading source file {:?}: {}", path, e)))?;
if debug_graph {
_log_session = Some(util::log_graphs(&mut parser, "log.html")?);

326
cli/src/properties.rs
View file

@ -5,7 +5,7 @@ use rsass::sass::Value;
use rsass::selectors::SelectorPart;
use serde_derive::Serialize;
use std::collections::hash_map::Entry;
use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
use std::collections::{btree_map, BTreeMap, HashMap, VecDeque};
use std::fmt::{self, Write};
use std::fs::{self, File};
use std::io::BufWriter;
@ -15,12 +15,13 @@ use tree_sitter::{self, PropertyStateJSON, PropertyTransitionJSON};
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
#[serde(untagged)]
enum PropertyValue {
Number(isize),
String(String),
Object(PropertySet),
Array(Vec<PropertyValue>),
}
type PropertySet = HashMap<String, PropertyValue>;
type PropertySet = BTreeMap<String, PropertyValue>;
type PropertySheetJSON = tree_sitter::PropertySheetJSON<PropertySet>;
type StateId = usize;
type PropertySetId = usize;
@ -160,7 +161,7 @@ impl Builder {
}
fn populate_state(&mut self, item_set: ItemSet, state_id: StateId) {
let mut transition_map: HashSet<(PropertyTransitionJSON, u32)> = HashSet::new();
let mut transitions: HashMap<PropertyTransitionJSON, u32> = HashMap::new();
let mut selector_matches = Vec::new();
// First, compute all of the possible state transition predicates for
@ -173,18 +174,21 @@ impl Builder {
// If this item has more elements remaining in its selector, then
// add a state transition based on the next step.
if let Some(step) = next_step {
transition_map.insert((
PropertyTransitionJSON {
transitions
.entry(PropertyTransitionJSON {
kind: step.kind.clone(),
field: step.field.clone(),
named: step.is_named,
index: step.child_index,
text: step.text_pattern.clone(),
state_id: 0,
},
// Include the rule id so that it can be used when sorting transitions.
item.rule_id,
));
})
.and_modify(|rule_id| {
if item.rule_id > *rule_id {
*rule_id = item.rule_id;
}
})
.or_insert(item.rule_id);
}
// If the item has matched its entire selector, then the item's
// properties are applicable to this state.
@ -196,46 +200,11 @@ impl Builder {
}
}
// For eacy possible state transition, compute the set of items in that transition's
// destination state.
let mut transition_list: Vec<(PropertyTransitionJSON, u32)> = transition_map
.into_iter()
.map(|(mut transition, rule_id)| {
let mut next_item_set = ItemSet::new();
for item in &item_set {
let rule = &self.rules[item.rule_id as usize];
let selector = &rule.selectors[item.selector_id as usize];
let next_step = selector.0.get(item.step_id as usize);
if let Some(step) = next_step {
// If the next step of the item's selector satisfies this transition,
// advance the item to the next part of its selector and add the
// resulting item to this transition's destination state.
if step_matches_transition(step, &transition) {
next_item_set.insert(Item {
rule_id: item.rule_id,
selector_id: item.selector_id,
step_id: item.step_id + 1,
});
}
// If the next step of the item is not an immediate child, then
// include this item in this transition's destination state, because
// the next step of the item might match a descendant node.
if !step.is_immediate {
next_item_set.insert(*item);
}
}
}
transition.state_id = self.add_state(next_item_set);
(transition, rule_id)
})
.collect();
// Ensure that for a given node type, more specific transitions are tried
// first, and in the event of a tie, transitions corresponding to later rules
// in the cascade are tried first.
let mut transition_list: Vec<(PropertyTransitionJSON, u32)> =
transitions.into_iter().collect();
transition_list.sort_by(|a, b| {
(transition_specificity(&b.0).cmp(&transition_specificity(&a.0)))
.then_with(|| b.1.cmp(&a.1))
@ -244,6 +213,39 @@ impl Builder {
.then_with(|| a.0.field.cmp(&b.0.field))
});
// For eacy possible state transition, compute the set of items in that transition's
// destination state.
for (transition, _) in transition_list.iter_mut() {
let mut next_item_set = ItemSet::new();
for item in &item_set {
let rule = &self.rules[item.rule_id as usize];
let selector = &rule.selectors[item.selector_id as usize];
let next_step = selector.0.get(item.step_id as usize);
if let Some(step) = next_step {
// If the next step of the item's selector satisfies this transition,
// advance the item to the next part of its selector and add the
// resulting item to this transition's destination state.
if step_matches_transition(step, &transition) {
next_item_set.insert(Item {
rule_id: item.rule_id,
selector_id: item.selector_id,
step_id: item.step_id + 1,
});
}
// If the next step of the item is not an immediate child, then
// include this item in this transition's destination state, because
// the next step of the item might match a descendant node.
if !step.is_immediate {
next_item_set.insert(*item);
}
}
}
transition.state_id = self.add_state(next_item_set);
}
// Compute the merged properties that apply in the current state.
// Sort the matching property sets by ascending specificity and by
// their order in the sheet. This way, more specific selectors and later
@ -475,38 +477,9 @@ fn generate_property_sheet(path: impl AsRef<Path>, css: &str) -> Result<Property
}
fn parse_property_sheet(path: &Path, css: &str) -> Result<Vec<Rule>> {
let mut i = 0;
let mut schema_paths = Vec::new();
let mut items = rsass::parse_scss_data(css.as_bytes())?;
while i < items.len() {
match &items[i] {
rsass::Item::Import(arg) => {
if let Some(s) = get_sass_string(arg) {
let import_path = resolve_path(path, s)?;
let imported_items = rsass::parse_scss_file(&import_path)?;
items.splice(i..(i + 1), imported_items);
continue;
} else {
return Err(Error("@import arguments must be strings".to_string()));
}
}
rsass::Item::AtRule { name, args, .. } => match name.as_str() {
"schema" => {
if let Some(s) = get_sass_string(args) {
// TODO - use schema
let _schema_path = resolve_path(path, s)?;
items.remove(i);
continue;
} else {
return Err(Error("@schema arguments must be strings".to_string()));
}
}
_ => return Err(Error(format!("Unsupported at-rule '{}'", name))),
},
_ => {}
}
i += 1;
}
process_at_rules(&mut items, &mut schema_paths, path)?;
let mut result = Vec::new();
let selector_prefixes = vec![Vec::new()];
parse_sass_items(items, &selector_prefixes, &mut result)?;
@ -525,10 +498,10 @@ fn parse_sass_items(
rsass::Item::Property(name, value) => {
let value = parse_sass_value(&value)?;
match properties.entry(name.to_string()) {
Entry::Vacant(v) => {
btree_map::Entry::Vacant(v) => {
v.insert(value);
}
Entry::Occupied(mut o) => {
btree_map::Entry::Occupied(mut o) => {
let existing_value = o.get_mut();
if let PropertyValue::Array(items) = existing_value {
items.push(value);
@ -693,6 +666,45 @@ fn parse_sass_items(
Ok(())
}
fn process_at_rules(
items: &mut Vec<rsass::Item>,
schema_paths: &mut Vec<PathBuf>,
path: &Path,
) -> Result<()> {
let mut i = 0;
while i < items.len() {
match &items[i] {
rsass::Item::Import(arg) => {
if let Some(s) = get_sass_string(arg) {
let import_path = resolve_path(path, s)?;
let mut imported_items = rsass::parse_scss_file(&import_path)?;
process_at_rules(&mut imported_items, schema_paths, &import_path)?;
items.splice(i..(i + 1), imported_items);
continue;
} else {
return Err(Error("@import arguments must be strings".to_string()));
}
}
rsass::Item::AtRule { name, args, .. } => match name.as_str() {
"schema" => {
if let Some(s) = get_sass_string(args) {
let schema_path = resolve_path(path, s)?;
schema_paths.push(schema_path);
items.remove(i);
continue;
} else {
return Err(Error("@schema arguments must be strings".to_string()));
}
}
_ => return Err(Error(format!("Unsupported at-rule '{}'", name))),
},
_ => {}
}
i += 1;
}
Ok(())
}
fn parse_sass_value(value: &Value) -> Result<PropertyValue> {
match value {
Value::Literal(s) => {
@ -724,7 +736,7 @@ fn parse_sass_value(value: &Value) -> Result<PropertyValue> {
Ok(PropertyValue::Array(result))
}
Value::Color(_, Some(name)) => Ok(PropertyValue::String(name.clone())),
Value::Numeric(n, _) => Ok(PropertyValue::String(format!("{}", n))),
Value::Numeric(n, _) => Ok(PropertyValue::Number(n.to_integer())),
Value::True => Ok(PropertyValue::String("true".to_string())),
Value::False => Ok(PropertyValue::String("false".to_string())),
_ => Err(Error(format!(
@ -744,23 +756,22 @@ fn get_sass_string(value: &Value) -> Option<&str> {
fn resolve_path(base: &Path, p: &str) -> Result<PathBuf> {
let path = Path::new(p);
let mut result = base.to_owned();
result.pop();
let mut base = base.to_owned();
base.pop();
if path.starts_with(".") {
result.push(path);
if result.exists() {
return Ok(result);
base.push(path);
if base.exists() {
return Ok(base);
}
} else {
loop {
let mut result = base.clone();
result.push("node_modules");
result.push(path);
if result.exists() {
return Ok(result);
}
result.pop();
result.pop();
if !result.pop() {
if !base.pop() {
break;
}
}
@ -795,9 +806,10 @@ fn interpolation_error() -> Error {
mod tests {
use super::*;
use regex::Regex;
use tempfile::TempDir;
#[test]
fn test_properties_immediate_child_and_descendant_selectors() {
fn test_property_sheet_with_immediate_child_and_descendant_selectors() {
let sheet = generate_property_sheet(
"foo.css",
"
@ -829,71 +841,71 @@ mod tests {
// f1 single-element selector
assert_eq!(
*query_simple(&sheet, vec!["f1"]),
props(&[("color", "red")])
props(&[("color", string("red"))])
);
assert_eq!(
*query_simple(&sheet, vec!["f2", "f1"]),
props(&[("color", "red")])
props(&[("color", string("red"))])
);
assert_eq!(
*query_simple(&sheet, vec!["f2", "f3", "f1"]),
props(&[("color", "red")])
props(&[("color", string("red"))])
);
// f2 single-element selector
assert_eq!(
*query_simple(&sheet, vec!["f2"]),
props(&[("color", "indigo"), ("height", "2")])
props(&[("color", string("indigo")), ("height", num(2))])
);
assert_eq!(
*query_simple(&sheet, vec!["f2", "f2"]),
props(&[("color", "indigo"), ("height", "2")])
props(&[("color", string("indigo")), ("height", num(2))])
);
assert_eq!(
*query_simple(&sheet, vec!["f1", "f3", "f2"]),
props(&[("color", "indigo"), ("height", "2")])
props(&[("color", string("indigo")), ("height", num(2))])
);
assert_eq!(
*query_simple(&sheet, vec!["f1", "f6", "f2"]),
props(&[("color", "indigo"), ("height", "2")])
props(&[("color", string("indigo")), ("height", num(2))])
);
// f3 single-element selector
assert_eq!(
*query_simple(&sheet, vec!["f3"]),
props(&[("color", "violet"), ("height", "3")])
props(&[("color", string("violet")), ("height", num(3))])
);
assert_eq!(
*query_simple(&sheet, vec!["f2", "f3"]),
props(&[("color", "violet"), ("height", "3")])
props(&[("color", string("violet")), ("height", num(3))])
);
// f2 child selector
assert_eq!(
*query_simple(&sheet, vec!["f1", "f2"]),
props(&[("color", "green"), ("height", "2")])
props(&[("color", string("green")), ("height", num(2))])
);
assert_eq!(
*query_simple(&sheet, vec!["f2", "f1", "f2"]),
props(&[("color", "green"), ("height", "2")])
props(&[("color", string("green")), ("height", num(2))])
);
assert_eq!(
*query_simple(&sheet, vec!["f3", "f1", "f2"]),
props(&[("color", "green"), ("height", "2")])
props(&[("color", string("green")), ("height", num(2))])
);
// f3 descendant selector
assert_eq!(
*query_simple(&sheet, vec!["f1", "f3"]),
props(&[("color", "blue"), ("height", "3")])
props(&[("color", string("blue")), ("height", num(3))])
);
assert_eq!(
*query_simple(&sheet, vec!["f1", "f2", "f3"]),
props(&[("color", "blue"), ("height", "3")])
props(&[("color", string("blue")), ("height", num(3))])
);
assert_eq!(
*query_simple(&sheet, vec!["f1", "f6", "f7", "f8", "f3"]),
props(&[("color", "blue"), ("height", "3")])
props(&[("color", string("blue")), ("height", num(3))])
);
// no match
@ -902,7 +914,7 @@ mod tests {
}
#[test]
fn test_properties_text_attribute() {
fn test_property_sheet_with_text_attribute() {
let sheet = generate_property_sheet(
"foo.css",
"
@ -927,15 +939,15 @@ mod tests {
assert_eq!(
*query(&sheet, vec![("f1", None, true, 0)], "abc"),
props(&[("color", "red")])
props(&[("color", string("red"))])
);
assert_eq!(
*query(&sheet, vec![("f1", None, true, 0)], "Abc"),
props(&[("color", "green")])
props(&[("color", string("green"))])
);
assert_eq!(
*query(&sheet, vec![("f1", None, true, 0)], "AB_CD"),
props(&[("color", "blue")])
props(&[("color", string("blue"))])
);
assert_eq!(
*query(&sheet, vec![("f2", None, true, 0)], "Abc"),
@ -943,12 +955,12 @@ mod tests {
);
assert_eq!(
*query(&sheet, vec![("f2", None, true, 0)], "ABC"),
props(&[("color", "purple")])
props(&[("color", string("purple"))])
);
}
#[test]
fn test_properties_with_fields() {
fn test_property_sheet_with_fields() {
let sheet = generate_property_sheet(
"foo.css",
"
@ -971,11 +983,11 @@ mod tests {
assert_eq!(
*query(&sheet, vec![("a", None, true, 0)], ""),
props(&[("color", "red")])
props(&[("color", string("red"))])
);
assert_eq!(
*query(&sheet, vec![("a", Some("x"), true, 0)], ""),
props(&[("color", "green")])
props(&[("color", string("green"))])
);
assert_eq!(
*query(
@ -983,7 +995,7 @@ mod tests {
vec![("a", Some("x"), true, 0), ("b", None, true, 0)],
""
),
props(&[("color", "blue")])
props(&[("color", string("blue"))])
);
assert_eq!(
*query(
@ -991,15 +1003,15 @@ mod tests {
vec![("a", Some("x"), true, 0), ("b", Some("y"), true, 0)],
""
),
props(&[("color", "yellow")])
props(&[("color", string("yellow"))])
);
assert_eq!(
*query(&sheet, vec![("b", Some("x"), true, 0)], ""),
props(&[("color", "violet")])
props(&[("color", string("violet"))])
);
assert_eq!(
*query(&sheet, vec![("a", None, true, 0), ("b", None, true, 0)], ""),
props(&[("color", "orange")])
props(&[("color", string("orange"))])
);
assert_eq!(
*query(
@ -1007,12 +1019,12 @@ mod tests {
vec![("a", None, true, 0), ("b", Some("y"), true, 0)],
""
),
props(&[("color", "indigo")])
props(&[("color", string("indigo"))])
);
}
#[test]
fn test_properties_cascade_ordering_as_tie_breaker() {
fn test_property_sheet_with_cascade_ordering_as_tie_breaker() {
let sheet = generate_property_sheet(
"foo.css",
"
@ -1038,7 +1050,7 @@ mod tests {
vec![("f1", None, true, 0), ("f2", None, true, 1)],
"x"
),
props(&[("color", "red")])
props(&[("color", string("red"))])
);
assert_eq!(
*query(
@ -1046,7 +1058,7 @@ mod tests {
vec![("f1", None, true, 1), ("f2", None, true, 1)],
"x"
),
props(&[("color", "green")])
props(&[("color", string("green"))])
);
assert_eq!(
*query(
@ -1054,7 +1066,7 @@ mod tests {
vec![("f1", None, true, 1), ("f2", None, true, 1)],
"a"
),
props(&[("color", "blue")])
props(&[("color", string("blue"))])
);
assert_eq!(
*query(
@ -1062,12 +1074,12 @@ mod tests {
vec![("f1", None, true, 1), ("f2", None, true, 1)],
"ab"
),
props(&[("color", "violet")])
props(&[("color", string("violet"))])
);
}
#[test]
fn test_properties_css_function_calls() {
fn test_property_sheet_with_css_function_calls() {
let sheet = generate_property_sheet(
"foo.css",
"
@ -1096,7 +1108,7 @@ mod tests {
object(&[("name", string("g")), ("args", array(vec![string("h"),]))]),
string("i"),
string("j"),
string("10"),
num(10),
])
),
])
@ -1104,7 +1116,7 @@ mod tests {
}
#[test]
fn test_properties_array_by_declaring_property_multiple_times() {
fn test_property_sheet_with_array_by_declaring_property_multiple_times() {
let sheet = generate_property_sheet(
"foo.css",
"
@ -1144,6 +1156,62 @@ mod tests {
);
}
#[test]
fn test_property_sheet_with_imports() {
let repo_dir = TempDir::new().unwrap();
let properties_dir = repo_dir.path().join("properties");
let dependency_properties_dir = repo_dir
.path()
.join("node_modules")
.join("the-dependency")
.join("properties");
fs::create_dir_all(&properties_dir).unwrap();
fs::create_dir_all(&dependency_properties_dir).unwrap();
let sheet_path1 = properties_dir.join("sheet1.css");
let sheet_path2 = properties_dir.join("sheet2.css");
let dependency_sheet_path1 = dependency_properties_dir.join("dependency-sheet1.css");
let dependency_sheet_path2 = dependency_properties_dir.join("dependency-sheet2.css");
fs::write(
sheet_path2,
r#"
a { x: '1'; }
"#,
)
.unwrap();
fs::write(
dependency_sheet_path1,
r#"
@import "./dependency-sheet2.css";
a { y: '2'; }
"#,
)
.unwrap();
fs::write(
dependency_sheet_path2,
r#"
b { x: '3'; }
"#,
)
.unwrap();
let sheet = generate_property_sheet(
sheet_path1,
r#"
@import "./sheet2.css";
@import "the-dependency/properties/dependency-sheet1.css";
b { y: '4'; }
"#,
)
.unwrap();
let a = query_simple(&sheet, vec!["a"]);
assert_eq!(a["x"], string("1"),);
assert_eq!(a["y"], string("2"),);
let b = query_simple(&sheet, vec!["b"]);
assert_eq!(b["x"], string("3"),);
assert_eq!(b["y"], string("4"),);
}
fn query_simple<'a>(
sheet: &'a PropertySheetJSON,
node_stack: Vec<&'static str>,
@ -1197,9 +1265,13 @@ mod tests {
PropertyValue::String(s.to_string())
}
fn props<'a>(s: &'a [(&'a str, &'a str)]) -> PropertySet {
fn num(n: isize) -> PropertyValue {
PropertyValue::Number(n)
}
fn props<'a>(s: &'a [(&'a str, PropertyValue)]) -> PropertySet {
s.into_iter()
.map(|(a, b)| (a.to_string(), PropertyValue::String(b.to_string())))
.map(|(a, b)| (a.to_string(), b.clone()))
.collect()
}
}

13
cli/src/tests/helpers/fixtures.rs
View file

@ -2,7 +2,8 @@ use crate::loader::Loader;
use lazy_static::lazy_static;
use std::fs;
use std::path::{Path, PathBuf};
use tree_sitter::Language;
use tree_sitter::{Language, PropertySheet};
use tree_sitter_highlight::{load_property_sheet, Properties};
include!("./dirs.rs");
@ -20,6 +21,16 @@ pub fn get_language(name: &str) -> Language {
.unwrap()
}
pub fn get_property_sheet(language_name: &str, sheet_name: &str) -> PropertySheet<Properties> {
let path = GRAMMARS_DIR
.join(language_name)
.join("src")
.join(sheet_name);
let json = fs::read_to_string(path).unwrap();
let language = get_language(language_name);
load_property_sheet(language, &json).unwrap()
}
pub fn get_test_language(name: &str, parser_code: &str, path: Option<&Path>) -> Language {
let parser_c_path = SCRATCH_DIR.join(&format!("{}-parser.c", name));
if !fs::read_to_string(&parser_c_path)

214
cli/src/tests/highlight_test.rs Normal file
View file

@ -0,0 +1,214 @@
use super::helpers::fixtures::{get_language, get_property_sheet};
use lazy_static::lazy_static;
use tree_sitter::{Language, PropertySheet};
use tree_sitter_highlight::{highlight, highlight_html, HighlightEvent, Properties, Scope};
lazy_static! {
static ref JS_SHEET: PropertySheet<Properties> =
get_property_sheet("javascript", "highlights.json");
static ref HTML_SHEET: PropertySheet<Properties> =
get_property_sheet("html", "highlights.json");
static ref SCOPE_CLASS_STRINGS: Vec<String> = {
let mut result = Vec::new();
let mut i = 0;
while let Some(scope) = Scope::from_usize(i) {
result.push(format!("class={:?}", scope));
i += 1;
}
result
};
}
#[test]
fn test_highlighting_injected_html_in_javascript() {
let source = vec!["const s = html `<div>${a < b}</div>`;"].join("\n");
assert_eq!(
&to_token_vector(&source, get_language("javascript"), &JS_SHEET).unwrap(),
&[vec![
("const", vec![Scope::Keyword]),
(" ", vec![]),
("s", vec![Scope::Variable]),
(" ", vec![]),
("=", vec![Scope::Operator]),
(" ", vec![]),
("html", vec![Scope::Function]),
(" ", vec![]),
("`<", vec![Scope::String]),
("div", vec![Scope::String, Scope::Tag]),
(">", vec![Scope::String]),
(
"${",
vec![Scope::String, Scope::Embedded, Scope::PunctuationSpecial]
),
("a", vec![Scope::String, Scope::Embedded, Scope::Variable]),
(" ", vec![Scope::String, Scope::Embedded]),
("<", vec![Scope::String, Scope::Embedded, Scope::Operator]),
(" ", vec![Scope::String, Scope::Embedded]),
("b", vec![Scope::String, Scope::Embedded, Scope::Variable]),
(
"}",
vec![Scope::String, Scope::Embedded, Scope::PunctuationSpecial]
),
("</", vec![Scope::String]),
("div", vec![Scope::String, Scope::Tag]),
(">`", vec![Scope::String]),
(";", vec![Scope::PunctuationDelimiter]),
]]
);
}
#[test]
fn test_highlighting_injected_javascript_in_html() {
let source = vec![
"<body>",
" <script>",
" const x = new Thing();",
" </script>",
"</body>",
]
.join("\n");
assert_eq!(
&to_token_vector(&source, get_language("html"), &HTML_SHEET).unwrap(),
&[
vec![("<", vec![]), ("body", vec![Scope::Tag]), (">", vec![]),],
vec![(" <", vec![]), ("script", vec![Scope::Tag]), (">", vec![]),],
vec![
(" ", vec![]),
("const", vec![Scope::Keyword]),
(" ", vec![]),
("x", vec![Scope::Variable]),
(" ", vec![]),
("=", vec![Scope::Operator]),
(" ", vec![]),
("new", vec![Scope::Keyword]),
(" ", vec![]),
("Thing", vec![Scope::Constructor]),
("(", vec![Scope::PunctuationBracket]),
(")", vec![Scope::PunctuationBracket]),
(";", vec![Scope::PunctuationDelimiter]),
],
vec![
(" </", vec![]),
("script", vec![Scope::Tag]),
(">", vec![]),
],
vec![("</", vec![]), ("body", vec![Scope::Tag]), (">", vec![]),],
]
);
}
#[test]
fn test_highlighting_multiline_scopes_to_html() {
let source = vec![
"const SOMETHING = `",
" one ${",
" two()",
" } three",
"`",
]
.join("\n");
assert_eq!(
&to_html(&source, get_language("javascript"), &JS_SHEET,).unwrap(),
&[
"<span class=Keyword>const</span> <span class=Constant>SOMETHING</span> <span class=Operator>=</span> <span class=String>`</span>\n".to_string(),
"<span class=String> one <span class=Embedded><span class=PunctuationSpecial>${</span></span></span>\n".to_string(),
"<span class=String><span class=Embedded> <span class=Function>two</span><span class=PunctuationBracket>(</span><span class=PunctuationBracket>)</span></span></span>\n".to_string(),
"<span class=String><span class=Embedded> <span class=PunctuationSpecial>}</span></span> three</span>\n".to_string(),
"<span class=String>`</span>\n".to_string(),
]
);
}
#[test]
fn test_highlighting_empty_lines() {
let source = vec![
"class A {",
"",
" b(c) {",
"",
" d(e)",
"",
" }",
"",
"}",
]
.join("\n");
assert_eq!(
&to_html(&source, get_language("javascript"), &JS_SHEET,).unwrap(),
&[
"<span class=Keyword>class</span> <span class=Constructor>A</span> <span class=PunctuationBracket>{</span>\n".to_string(),
"\n".to_string(),
" <span class=Function>b</span><span class=PunctuationBracket>(</span><span class=Variable>c</span><span class=PunctuationBracket>)</span> <span class=PunctuationBracket>{</span>\n".to_string(),
"\n".to_string(),
" <span class=Function>d</span><span class=PunctuationBracket>(</span><span class=Variable>e</span><span class=PunctuationBracket>)</span>\n".to_string(),
"\n".to_string(),
" <span class=PunctuationBracket>}</span>\n".to_string(),
"\n".to_string(),
"<span class=PunctuationBracket>}</span>\n".to_string(),
]
);
}
fn test_language_for_injection_string<'a>(
string: &str,
) -> Option<(Language, &'a PropertySheet<Properties>)> {
match string {
"javascript" => Some((get_language("javascript"), &JS_SHEET)),
"html" => Some((get_language("html"), &HTML_SHEET)),
_ => None,
}
}
fn to_html<'a>(
src: &'a str,
language: Language,
property_sheet: &'a PropertySheet<Properties>,
) -> Result<Vec<String>, String> {
highlight_html(
src.as_bytes(),
language,
property_sheet,
&test_language_for_injection_string,
&|scope| SCOPE_CLASS_STRINGS[scope as usize].as_str(),
)
}
fn to_token_vector<'a>(
src: &'a str,
language: Language,
property_sheet: &'a PropertySheet<Properties>,
) -> Result<Vec<Vec<(&'a str, Vec<Scope>)>>, String> {
let mut lines = Vec::new();
let mut scopes = Vec::new();
let mut line = Vec::new();
for event in highlight(
src.as_bytes(),
language,
property_sheet,
&test_language_for_injection_string,
)? {
match event {
HighlightEvent::ScopeStart(s) => scopes.push(s),
HighlightEvent::ScopeEnd => {
scopes.pop();
}
HighlightEvent::Source(s) => {
for (i, l) in s.lines().enumerate() {
if i > 0 {
lines.push(line);
line = Vec::new();
}
if l.len() > 0 {
line.push((l, scopes.clone()));
}
}
}
}
}
lines.push(line);
Ok(lines)
}

1
cli/src/tests/mod.rs
View file

@ -1,5 +1,6 @@
mod corpus_test;
mod helpers;
mod highlight_test;
mod node_test;
mod parser_test;
mod properties_test;