Merge branch 'master' into node-fields

Cargo.lock

@@ -561,7 +561,7 @@ dependencies = [
 
 [[package]]
 name = "tree-sitter"
-version = "0.3.7"
+version = "0.3.8"
 dependencies = [
  "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)",
  "regex 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -572,7 +572,7 @@ dependencies = [
 
 [[package]]
 name = "tree-sitter-cli"
-version = "0.14.3"
+version = "0.14.4"
 dependencies = [
  "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -592,7 +592,7 @@ dependencies = [
  "serde_json 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)",
  "smallbitvec 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "spin 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "tree-sitter 0.3.7",
+ "tree-sitter 0.3.8",
 ]
 
 [[package]]

cli/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "tree-sitter-cli"
 description = "CLI tool for developing, testing, and using Tree-sitter parsers"
-version = "0.14.3"
+version = "0.14.4"
 authors = ["Max Brunsfeld <maxbrunsfeld@gmail.com>"]
 edition = "2018"
 license = "MIT"

cli/benches/benchmark.rs

@@ -168,6 +168,6 @@ fn parse(parser: &mut Parser, example_path: &Path, max_path_length: usize) -> us
 fn get_language(name: &str) -> Language {
     let src_dir = GRAMMARS_DIR.join(name).join("src");
     TEST_LOADER
-        .load_language_at_path(name, &src_dir, &src_dir)
+        .load_language_at_path(&src_dir, &src_dir)
         .unwrap()
 }

cli/npm/package-lock.json

@@ -1,5 +1,5 @@
 {
   "name": "tree-sitter-cli",
-  "version": "0.14.3",
+  "version": "0.14.4",
   "lockfileVersion": 1
 }

cli/npm/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tree-sitter-cli",
-  "version": "0.14.3",
+  "version": "0.14.4",
   "author": "Max Brunsfeld",
   "license": "MIT",
   "repository": {

cli/src/loader.rs

@@ -3,6 +3,7 @@ use libloading::{Library, Symbol};
 use regex::{Regex, RegexBuilder};
 use serde_derive::Deserialize;
 use std::collections::HashMap;
+use std::io::BufReader;
 use std::path::{Path, PathBuf};
 use std::process::Command;
 use std::time::SystemTime;
@@ -18,7 +19,6 @@ const DYLIB_EXTENSION: &'static str = "dll";
 const BUILD_TARGET: &'static str = env!("BUILD_TARGET");
 
 struct LanguageRepo {
-    name: String,
     path: PathBuf,
     language: Option<Language>,
     configurations: Vec<LanguageConfiguration>,
@@ -109,35 +109,42 @@ impl Loader {
             language
         } else {
             let src_path = repo.path.join("src");
-            let language = self.load_language_at_path(&repo.name, &src_path, &src_path)?;
+            let language = self.load_language_at_path(&src_path, &src_path)?;
             self.language_repos[id].language = Some(language);
             language
         };
         Ok((language, &self.language_repos[id].configurations))
     }
 
-    pub fn load_language_at_path(
-        &self,
-        name: &str,
-        src_path: &Path,
-        header_path: &Path,
-    ) -> Result<Language> {
+    pub fn load_language_at_path(&self, src_path: &Path, header_path: &Path) -> Result<Language> {
+        let grammar_path = src_path.join("grammar.json");
         let parser_path = src_path.join("parser.c");
+        let mut scanner_path = src_path.join("scanner.c");
 
-        let scanner_path;
-        let scanner_c_path = src_path.join("scanner.c");
-        if scanner_c_path.exists() {
-            scanner_path = Some(scanner_c_path);
-        } else {
-            let scanner_cc_path = src_path.join("scanner.cc");
-            if scanner_cc_path.exists() {
-                scanner_path = Some(scanner_cc_path);
-            } else {
-                scanner_path = None;
-            }
+        #[derive(Deserialize)]
+        struct GrammarJSON {
+            name: String,
         }
+        let mut grammar_file = fs::File::open(grammar_path)?;
+        let grammar_json: GrammarJSON = serde_json::from_reader(BufReader::new(&mut grammar_file))?;
 
-        self.load_language_from_sources(name, &header_path, &parser_path, &scanner_path)
+        let scanner_path = if scanner_path.exists() {
+            Some(scanner_path)
+        } else {
+            scanner_path.set_extension("cc");
+            if scanner_path.exists() {
+                Some(scanner_path)
+            } else {
+                None
+            }
+        };
+
+        self.load_language_from_sources(
+            &grammar_json.name,
+            &header_path,
+            &parser_path,
+            &scanner_path,
+        )
     }
 
     pub fn load_language_from_sources(
@@ -236,7 +243,6 @@ impl Loader {
 
         #[derive(Deserialize)]
         struct PackageJSON {
-            name: String,
             #[serde(rename = "tree-sitter")]
             tree_sitter: Option<Vec<LanguageConfigurationJSON>>,
         }
@@ -272,11 +278,6 @@ impl Loader {
         }
 
         self.language_repos.push(LanguageRepo {
-            name: package_json
-                .name
-                .split_at("tree-sitter-".len())
-                .1
-                .to_string(),
             path: parser_path.to_owned(),
             language: None,
             configurations,

cli/src/tests/helpers/fixtures.rs

@@ -16,7 +16,7 @@ pub fn fixtures_dir<'a>() -> &'static Path {
 
 pub fn get_language(name: &str) -> Language {
     TEST_LOADER
-        .load_language_at_path(name, &GRAMMARS_DIR.join(name).join("src"), &HEADER_DIR)
+        .load_language_at_path(&GRAMMARS_DIR.join(name).join("src"), &HEADER_DIR)
         .unwrap()
 }
 

lib/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "tree-sitter"
 description = "Rust bindings to the Tree-sitter parsing library"
-version = "0.3.7"
+version = "0.3.8"
 authors = ["Max Brunsfeld <maxbrunsfeld@gmail.com>"]
 license = "MIT"
 readme = "binding/README.md"

lib/binding/lib.rs

@@ -259,7 +259,8 @@ impl Parser {
 
     pub fn parse(&mut self, input: impl AsRef<[u8]>, old_tree: Option<&Tree>) -> Option<Tree> {
         let bytes = input.as_ref();
-        self.parse_with(&mut |i, _| &bytes[i..], old_tree)
+        let len = bytes.len();
+        self.parse_with(&mut |i, _| if i < len { &bytes[i..] } else { &[] }, old_tree)
     }
 
     pub fn parse_utf16(
@@ -268,7 +269,8 @@ impl Parser {
         old_tree: Option<&Tree>,
     ) -> Option<Tree> {
         let code_points = input.as_ref();
-        self.parse_utf16_with(&mut |i, _| &code_points[i..], old_tree)
+        let len = code_points.len();
+        self.parse_utf16_with(&mut |i, _| if i < len { &code_points[i..] } else { &[] }, old_tree)
     }
 
     pub fn parse_with<'a, T: FnMut(usize, Point) -> &'a [u8]>(