Make precedence more useful within tokens

Choose accept-token actions over advance actions if their rule has a higher precedence.

src/compiler/build_tables/build_lex_table.cc

@@ -135,7 +135,7 @@ class LexTableBuilder {
       LexItem::CompletionStatus completion_status = item.completion_status();
       if (completion_status.is_done) {
         auto current_action = lex_table.state(state_id).default_action;
-        auto action = LexAction::Accept(item.lhs, completion_status.precedence);
+        auto action = LexAction::Accept(item.lhs, completion_status.precedence, completion_status.is_string);
         if (conflict_manager.resolve(action, current_action))
           lex_table.state(state_id).default_action = action;
       }

src/compiler/build_tables/lex_conflict_manager.cc

@@ -28,12 +28,19 @@ bool LexConflictManager::resolve(const LexAction &new_action,
             return true;
           else if (new_precedence < old_precedence)
             return false;
+          else if (new_action.is_string && !old_action.is_string)
+            return true;
+          else if (old_action.is_string && !new_action.is_string)
+            return false;
           else
             return new_action.symbol.index < old_action.symbol.index;
         }
 
         case LexActionTypeAdvance:
-          return true;
+          if (old_precedence > new_action.precedence_range.max)
+            return false;
+          else
+            return true;
 
         default:
           return false;

src/compiler/lex_table.cc

@@ -17,24 +17,25 @@ LexAction::LexAction()
       precedence_range({ 0, 0 }) {}
 
 LexAction::LexAction(LexActionType type, size_t state_index, Symbol symbol,
-                     PrecedenceRange precedence_range)
+                     PrecedenceRange precedence_range, bool is_string)
     : type(type),
       symbol(symbol),
       state_index(state_index),
-      precedence_range(precedence_range) {}
+      precedence_range(precedence_range),
+      is_string(is_string) {}
 
 LexAction LexAction::Error() {
-  return LexAction(LexActionTypeError, -1, Symbol(-1), { 0, 0 });
+  return LexAction(LexActionTypeError, -1, Symbol(-1), { 0, 0 }, false);
 }
 
 LexAction LexAction::Advance(size_t state_index,
                              PrecedenceRange precedence_range) {
   return LexAction(LexActionTypeAdvance, state_index, Symbol(-1),
-                   precedence_range);
+                   precedence_range, false);
 }
 
-LexAction LexAction::Accept(Symbol symbol, int precedence) {
-  return LexAction(LexActionTypeAccept, -1, symbol, { precedence, precedence });
+LexAction LexAction::Accept(Symbol symbol, int precedence, bool is_string) {
+  return LexAction(LexActionTypeAccept, -1, symbol, { precedence, precedence }, is_string);
 }
 
 bool LexAction::operator==(const LexAction &other) const {

src/compiler/lex_table.h

@@ -19,11 +19,11 @@ typedef enum {
 
 class LexAction {
   LexAction(LexActionType type, size_t state_index, rules::Symbol symbol,
-            PrecedenceRange precedence_range);
+            PrecedenceRange precedence_range, bool is_string);
 
  public:
   LexAction();
-  static LexAction Accept(rules::Symbol symbol, int precedence);
+  static LexAction Accept(rules::Symbol symbol, int precedence, bool is_string);
   static LexAction Error();
   static LexAction Advance(size_t state_index, PrecedenceRange precedence_range);
   bool operator==(const LexAction &action) const;
@@ -32,6 +32,7 @@ class LexAction {
   rules::Symbol symbol;
   size_t state_index;
   PrecedenceRange precedence_range;
+  bool is_string;
 };
 
 }  // namespace tree_sitter

src/compiler/prepare_grammar/expand_tokens.cc

@@ -47,7 +47,7 @@ class ExpandTokens : public rules::IdentityRuleFn {
     return make_shared<rules::Metadata>(
       rules::Seq::build(elements),
       std::map<rules::MetadataKey, int>({
-        { rules::IS_TOKEN, 1 }, { rules::PRECEDENCE, 1 },
+        { rules::IS_TOKEN, 1 }, { rules::IS_STRING, 1 },
       }));
   }