traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

generate: Avoid duplicate string tokens in unique symbol map

Browse source
This commit is contained in:
Max Brunsfeld 2020-03-20 11:35:11 -07:00
parent ae1c51051a
commit a003e5f6bd
2 changed files with 52 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

17
cli/src/generate/render.rs
View file

@ -325,12 +325,13 @@ impl Generator {
add_line!(self, "static TSSymbol ts_symbol_map[] = {{");
indent!(self);
for symbol in &self.parse_table.symbols {
let mut mapping = symbol;
// There can be multiple symbols in the grammar that have the same name and kind,
// due to simple aliases. When that happens, ensure that they map to the same
// public-facing symbol. If one of the symbols is not aliased, choose that one
// to be the public-facing symbol. Otherwise, pick the symbol with the lowest
// numeric value.
let mut mapping = symbol;
if let Some(alias) = self.simple_aliases.get(symbol) {
let kind = alias.kind();
for other_symbol in &self.parse_table.symbols {
@ -344,6 +345,20 @@ impl Generator {
}
}
}
// Two anonymous tokens with different flags but the same string value
// should be represented with the same symbol in the public API. Examples:
// * "<" and token(prec(1, "<"))
// * "(" and token.immediate("(")
else if symbol.is_terminal() {
let metadata = self.metadata_for_symbol(*symbol);
for other_symbol in &self.parse_table.symbols {
let other_metadata = self.metadata_for_symbol(*other_symbol);
if other_metadata == metadata {
mapping = other_symbol;
break;
}
}
}
add_line!(
self,