From f487b163bf9e3d8c7aef0c40c658b77975b52753 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Mon, 11 Feb 2019 12:31:40 +0100 Subject: [PATCH 1/4] binding: Add overflow checks when slicing in `parse` Because of the way TreeSitter parsers are generated, the parse callback can ask for a byte offset at the exact end of the byte slice we're processing, and slicing at the end of the slice in Rust causes a panic, so we need to explicitly ward against this. --- lib/binding/lib.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/binding/lib.rs b/lib/binding/lib.rs index 66444a55..1c7a6caf 100644 --- a/lib/binding/lib.rs +++ b/lib/binding/lib.rs @@ -229,7 +229,8 @@ impl Parser { pub fn parse(&mut self, input: impl AsRef<[u8]>, old_tree: Option<&Tree>) -> Option { let bytes = input.as_ref(); - self.parse_with(&mut |i, _| &bytes[i..], old_tree) + let len = bytes.len(); + self.parse_with(&mut |i, _| if i < len { &bytes[i..] } else { &[] }, old_tree) } pub fn parse_utf16( @@ -238,7 +239,8 @@ impl Parser { old_tree: Option<&Tree>, ) -> Option { let code_points = input.as_ref(); - self.parse_utf16_with(&mut |i, _| &code_points[i..], old_tree) + let len = code_points.len(); + self.parse_utf16_with(&mut |i, _| if i < len { &code_points[i..] } else { &[] }, old_tree) } pub fn parse_with<'a, T: FnMut(usize, Point) -> &'a [u8]>( From 2b685658dbcc79329422c02b01a6d10e144a6cf6 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 11 Feb 2019 09:46:26 -0800 Subject: [PATCH 2/4] lib: 0.3.8 --- Cargo.lock | 4 ++-- lib/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c614545a..62d25d7a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -561,7 +561,7 @@ dependencies = [ [[package]] name = "tree-sitter" -version = "0.3.7" +version = "0.3.8" dependencies = [ "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", "regex 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -592,7 +592,7 @@ dependencies = [ "serde_json 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", "smallbitvec 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "spin 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tree-sitter 0.3.7", + "tree-sitter 0.3.8", ] [[package]] diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 1d5121e3..f3c3efb3 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "tree-sitter" description = "Rust bindings to the Tree-sitter parsing library" -version = "0.3.7" +version = "0.3.8" authors = ["Max Brunsfeld "] license = "MIT" readme = "binding/README.md" From 57c528b6c5ddb4b3f9be89c270a2773a02b4a24b Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 13 Feb 2019 19:30:59 -0800 Subject: [PATCH 3/4] CLI: Determine language symbol from grammar, not package.json Fixes #272 Fixes #277 --- cli/benches/benchmark.rs | 2 +- cli/src/loader.rs | 53 ++++++++++++++++--------------- cli/src/tests/helpers/fixtures.rs | 2 +- 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/cli/benches/benchmark.rs b/cli/benches/benchmark.rs index fa3a5e73..7983bde5 100644 --- a/cli/benches/benchmark.rs +++ b/cli/benches/benchmark.rs @@ -167,6 +167,6 @@ fn parse(parser: &mut Parser, example_path: &Path, max_path_length: usize) -> us fn get_language(name: &str) -> Language { TEST_LOADER - .load_language_at_path(name, &GRAMMARS_DIR.join(name).join("src"), &HEADER_DIR) + .load_language_at_path(&GRAMMARS_DIR.join(name).join("src"), &HEADER_DIR) .unwrap() } diff --git a/cli/src/loader.rs b/cli/src/loader.rs index 678fbb1b..5c2a19a7 100644 --- a/cli/src/loader.rs +++ b/cli/src/loader.rs @@ -3,6 +3,7 @@ use libloading::{Library, Symbol}; use regex::{Regex, RegexBuilder}; use serde_derive::Deserialize; use std::collections::HashMap; +use std::io::BufReader; use std::path::{Path, PathBuf}; use std::process::Command; use std::time::SystemTime; @@ -18,7 +19,6 @@ const DYLIB_EXTENSION: &'static str = "dll"; const BUILD_TARGET: &'static str = env!("BUILD_TARGET"); struct LanguageRepo { - name: String, path: PathBuf, language: Option, configurations: Vec, @@ -109,35 +109,42 @@ impl Loader { language } else { let src_path = repo.path.join("src"); - let language = self.load_language_at_path(&repo.name, &src_path, &src_path)?; + let language = self.load_language_at_path(&src_path, &src_path)?; self.language_repos[id].language = Some(language); language }; Ok((language, &self.language_repos[id].configurations)) } - pub fn load_language_at_path( - &self, - name: &str, - src_path: &Path, - header_path: &Path, - ) -> Result { + pub fn load_language_at_path(&self, src_path: &Path, header_path: &Path) -> Result { + let grammar_path = src_path.join("grammar.json"); let parser_path = src_path.join("parser.c"); + let mut scanner_path = src_path.join("scanner.c"); - let scanner_path; - let scanner_c_path = src_path.join("scanner.c"); - if scanner_c_path.exists() { - scanner_path = Some(scanner_c_path); - } else { - let scanner_cc_path = src_path.join("scanner.cc"); - if scanner_cc_path.exists() { - scanner_path = Some(scanner_cc_path); - } else { - scanner_path = None; - } + #[derive(Deserialize)] + struct GrammarJSON { + name: String, } + let mut grammar_file = fs::File::open(grammar_path)?; + let grammar_json: GrammarJSON = serde_json::from_reader(BufReader::new(&mut grammar_file))?; - self.load_language_from_sources(name, &header_path, &parser_path, &scanner_path) + let scanner_path = if scanner_path.exists() { + Some(scanner_path) + } else { + scanner_path.set_extension("cc"); + if scanner_path.exists() { + Some(scanner_path) + } else { + None + } + }; + + self.load_language_from_sources( + &grammar_json.name, + &header_path, + &parser_path, + &scanner_path, + ) } pub fn load_language_from_sources( @@ -236,7 +243,6 @@ impl Loader { #[derive(Deserialize)] struct PackageJSON { - name: String, #[serde(rename = "tree-sitter")] tree_sitter: Option>, } @@ -272,11 +278,6 @@ impl Loader { } self.language_repos.push(LanguageRepo { - name: package_json - .name - .split_at("tree-sitter-".len()) - .1 - .to_string(), path: parser_path.to_owned(), language: None, configurations, diff --git a/cli/src/tests/helpers/fixtures.rs b/cli/src/tests/helpers/fixtures.rs index a5ea9ed0..8fc00038 100644 --- a/cli/src/tests/helpers/fixtures.rs +++ b/cli/src/tests/helpers/fixtures.rs @@ -16,7 +16,7 @@ pub fn fixtures_dir<'a>() -> &'static Path { pub fn get_language(name: &str) -> Language { TEST_LOADER - .load_language_at_path(name, &GRAMMARS_DIR.join(name).join("src"), &HEADER_DIR) + .load_language_at_path(&GRAMMARS_DIR.join(name).join("src"), &HEADER_DIR) .unwrap() } From b08f8dc855a0198577dd516261c083298621f8fe Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 13 Feb 2019 19:32:39 -0800 Subject: [PATCH 4/4] 0.14.4 --- Cargo.lock | 2 +- cli/Cargo.toml | 2 +- cli/npm/package-lock.json | 2 +- cli/npm/package.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 62d25d7a..9ca3a70a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -572,7 +572,7 @@ dependencies = [ [[package]] name = "tree-sitter-cli" -version = "0.14.3" +version = "0.14.4" dependencies = [ "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index a013190e..54d0eb5e 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "tree-sitter-cli" description = "CLI tool for developing, testing, and using Tree-sitter parsers" -version = "0.14.3" +version = "0.14.4" authors = ["Max Brunsfeld "] edition = "2018" license = "MIT" diff --git a/cli/npm/package-lock.json b/cli/npm/package-lock.json index fae84c3c..65076390 100644 --- a/cli/npm/package-lock.json +++ b/cli/npm/package-lock.json @@ -1,5 +1,5 @@ { "name": "tree-sitter-cli", - "version": "0.14.3", + "version": "0.14.4", "lockfileVersion": 1 } diff --git a/cli/npm/package.json b/cli/npm/package.json index 90a0fe37..e463d6f4 100644 --- a/cli/npm/package.json +++ b/cli/npm/package.json @@ -1,6 +1,6 @@ { "name": "tree-sitter-cli", - "version": "0.14.3", + "version": "0.14.4", "author": "Max Brunsfeld", "license": "MIT", "repository": {