Merge branch 'master' into node-fields

lib/src/alloc.h

@@ -15,7 +15,7 @@ void *ts_record_malloc(size_t);
 void *ts_record_calloc(size_t, size_t);
 void *ts_record_realloc(void *, size_t);
 void ts_record_free(void *);
-bool ts_record_allocations_toggle(bool);
+bool ts_toggle_allocation_recording(bool);
 
 static inline void *ts_malloc(size_t size) {
   return ts_record_malloc(size);
@@ -33,10 +33,6 @@ static inline void ts_free(void *buffer) {
   ts_record_free(buffer);
 }
 
-static inline bool ts_toggle_allocation_recording(bool value) {
-  return ts_record_allocations_toggle(value);
-}
-
 #else
 
 #include <stdlib.h>

lib/src/atomic.h

@@ -7,6 +7,10 @@
 
 #include <windows.h>
 
+static inline size_t atomic_load(const volatile size_t *p) {
+  return *p;
+}
+
 static inline uint32_t atomic_inc(volatile uint32_t *p) {
   return InterlockedIncrement(p);
 }
@@ -17,6 +21,10 @@ static inline uint32_t atomic_dec(volatile uint32_t *p) {
 
 #else
 
+static inline size_t atomic_load(const volatile size_t *p) {
+  return __atomic_load_n(p, __ATOMIC_RELAXED);
+}
+
 static inline uint32_t atomic_inc(volatile uint32_t *p) {
   return __sync_add_and_fetch(p, 1u);
 }

lib/src/clock.h

@@ -0,0 +1,141 @@
+#ifndef TREE_SITTER_CLOCK_H_
+#define TREE_SITTER_CLOCK_H_
+
+#include <stdint.h>
+
+typedef uint64_t TSDuration;
+
+#ifdef _WIN32
+
+// Windows:
+// * Represent a time as a performance counter value.
+// * Represent a duration as a number of performance counter ticks.
+
+#include <windows.h>
+typedef uint64_t TSClock;
+
+static inline TSDuration duration_from_micros(uint64_t micros) {
+  LARGE_INTEGER frequency;
+  QueryPerformanceFrequency(&frequency);
+  return micros * (uint64_t)frequency.QuadPart / 1000000;
+}
+
+static inline uint64_t duration_to_micros(TSDuration self) {
+  LARGE_INTEGER frequency;
+  QueryPerformanceFrequency(&frequency);
+  return self * 1000000 / (uint64_t)frequency.QuadPart;
+}
+
+static inline TSClock clock_null(void) {
+  return 0;
+}
+
+static inline TSClock clock_now(void) {
+  LARGE_INTEGER result;
+  QueryPerformanceCounter(&result);
+  return (uint64_t)result.QuadPart;
+}
+
+static inline TSClock clock_after(TSClock base, TSDuration duration) {
+  return base + duration;
+}
+
+static inline bool clock_is_null(TSClock self) {
+  return !self;
+}
+
+static inline bool clock_is_gt(TSClock self, TSClock other) {
+  return self > other;
+}
+
+#elif defined(CLOCK_MONOTONIC)
+
+// POSIX with monotonic clock support (Linux, macOS >= 10.12)
+// * Represent a time as a monotonic (seconds, nanoseconds) pair.
+// * Represent a duration as a number of microseconds.
+//
+// On these platforms, parse timeouts will correspond accurately to
+// real time, regardless of what other processes are running.
+
+#include <time.h>
+typedef struct timespec TSClock;
+
+static inline TSDuration duration_from_micros(uint64_t micros) {
+  return micros;
+}
+
+static inline uint64_t duration_to_micros(TSDuration self) {
+  return self;
+}
+
+static inline TSClock clock_now(void) {
+  TSClock result;
+  clock_gettime(CLOCK_MONOTONIC, &result);
+  return result;
+}
+
+static inline TSClock clock_null(void) {
+  return (TSClock) {0, 0};
+}
+
+static inline TSClock clock_after(TSClock base, TSDuration duration) {
+  TSClock result = base;
+  result.tv_sec += duration / 1000000;
+  result.tv_nsec += (duration % 1000000) * 1000;
+  return result;
+}
+
+static inline bool clock_is_null(TSClock self) {
+  return !self.tv_sec;
+}
+
+static inline bool clock_is_gt(TSClock self, TSClock other) {
+  if (self.tv_sec > other.tv_sec) return true;
+  if (self.tv_sec < other.tv_sec) return false;
+  return self.tv_nsec > other.tv_nsec;
+}
+
+#else
+
+// POSIX without monotonic clock support
+// * Represent a time as a process clock value.
+// * Represent a duration as a number of process clock ticks.
+//
+// On these platforms, parse timeouts may be affected by other processes,
+// which is not ideal, but is better than using a non-monotonic time API
+// like `gettimeofday`.
+
+#include <time.h>
+typedef uint64_t TSClock;
+
+static inline TSDuration duration_from_micros(uint64_t micros) {
+  return micros * (uint64_t)CLOCKS_PER_SEC / 1000000;
+}
+
+static inline uint64_t duration_to_micros(TSDuration self) {
+  return self * 1000000 / (uint64_t)CLOCKS_PER_SEC;
+}
+
+static inline TSClock clock_null(void) {
+  return 0;
+}
+
+static inline TSClock clock_now(void) {
+  return (uint64_t)clock();
+}
+
+static inline TSClock clock_after(TSClock base, TSDuration duration) {
+  return base + duration;
+}
+
+static inline bool clock_is_null(TSClock self) {
+  return !self;
+}
+
+static inline bool clock_is_gt(TSClock self, TSClock other) {
+  return self > other;
+}
+
+#endif
+
+#endif  // TREE_SITTER_CLOCK_H_

lib/src/get_changed_ranges.c

@@ -326,13 +326,13 @@ static inline void iterator_print_state(Iterator *self) {
   TreeCursorEntry entry = *array_back(&self->cursor.stack);
   TSPoint start = iterator_start_position(self).extent;
   TSPoint end = iterator_end_position(self).extent;
-  const char *name = ts_language_symbol_name(self->language, entry.subtree->symbol);
+  const char *name = ts_language_symbol_name(self->language, ts_subtree_symbol(*entry.subtree));
   printf(
     "(%-25s %s\t depth:%u [%u, %u] - [%u, %u])",
     name, self->in_padding ? "(p)" : "   ",
     self->visible_depth,
-    start.row, start.column,
-    end.row, end.column
+    start.row + 1, start.column,
+    end.row + 1, end.column
   );
 }
 #endif
@@ -361,7 +361,7 @@ unsigned ts_subtree_get_changed_ranges(const Subtree *old_tree, const Subtree *n
 
   do {
     #ifdef DEBUG_GET_CHANGED_RANGES
-    printf("At [%-2u, %-2u] Compare ", position.extent.row, position.extent.column);
+    printf("At [%-2u, %-2u] Compare ", position.extent.row + 1, position.extent.column);
     iterator_print_state(&old_iter);
     printf("\tvs\t");
     iterator_print_state(&new_iter);
@@ -443,8 +443,8 @@ unsigned ts_subtree_get_changed_ranges(const Subtree *old_tree, const Subtree *n
       #ifdef DEBUG_GET_CHANGED_RANGES
       printf(
         "  change: [[%u, %u] - [%u, %u]]\n",
-        position.extent.row, position.extent.column,
-        next_position.extent.row, next_position.extent.column
+        position.extent.row + 1, position.extent.column,
+        next_position.extent.row + 1, next_position.extent.column
       );
       #endif
 

lib/src/length.h

@@ -36,7 +36,7 @@ static inline Length length_sub(Length len1, Length len2) {
   return result;
 }
 
-static inline Length length_zero() {
+static inline Length length_zero(void) {
   Length result = {0, {0, 0}};
   return result;
 }

lib/src/lib.c

@@ -4,7 +4,7 @@
 //   - include
 //   - utf8proc
 
-#define _POSIX_SOURCE
+#define _POSIX_C_SOURCE 200112L
 #define UTF8PROC_STATIC
 
 #include "./get_changed_ranges.c"

lib/src/node.c

@@ -22,7 +22,7 @@ TSNode ts_node_new(const TSTree *tree, const Subtree *subtree, Length position,
   };
 }
 
-static inline TSNode ts_node__null() {
+static inline TSNode ts_node__null(void) {
   return ts_node_new(NULL, NULL, length_zero(), 0);
 }
 

lib/src/parser.c

@@ -1,19 +1,22 @@
+#include <time.h>
 #include <assert.h>
 #include <stdio.h>
 #include <limits.h>
 #include <stdbool.h>
 #include "tree_sitter/api.h"
-#include "./subtree.h"
-#include "./lexer.h"
-#include "./length.h"
-#include "./array.h"
-#include "./language.h"
 #include "./alloc.h"
-#include "./stack.h"
-#include "./reusable_node.h"
-#include "./reduce_action.h"
+#include "./array.h"
+#include "./atomic.h"
+#include "./clock.h"
 #include "./error_costs.h"
 #include "./get_changed_ranges.h"
+#include "./language.h"
+#include "./length.h"
+#include "./lexer.h"
+#include "./reduce_action.h"
+#include "./reusable_node.h"
+#include "./stack.h"
+#include "./subtree.h"
 #include "./tree.h"
 
 #define LOG(...)                                                                            \
@@ -42,6 +45,7 @@ static const unsigned MAX_VERSION_COUNT = 6;
 static const unsigned MAX_VERSION_COUNT_OVERFLOW = 4;
 static const unsigned MAX_SUMMARY_DEPTH = 16;
 static const unsigned MAX_COST_DIFFERENCE = 16 * ERROR_COST_PER_SKIPPED_TREE;
+static const unsigned OP_COUNT_PER_TIMEOUT_CHECK = 100;
 
 typedef struct {
   Subtree token;
@@ -62,9 +66,11 @@ struct TSParser {
   ReusableNode reusable_node;
   void *external_scanner_payload;
   FILE *dot_graph_file;
+  TSClock end_clock;
+  TSDuration timeout_duration;
   unsigned accept_count;
-  size_t operation_limit;
-  volatile bool enabled;
+  unsigned operation_count;
+  const volatile size_t *cancellation_flag;
   bool halt_on_error;
   Subtree old_tree;
   TSRangeArray included_range_differences;
@@ -327,7 +333,7 @@ static Subtree ts_parser__lex(TSParser *self, StackVersion version, TSStateId pa
       LOG(
         "lex_external state:%d, row:%u, column:%u",
         lex_mode.external_lex_state,
-        current_position.extent.row,
+        current_position.extent.row + 1,
         current_position.extent.column
       );
       ts_lexer_start(&self->lexer);
@@ -365,7 +371,7 @@ static Subtree ts_parser__lex(TSParser *self, StackVersion version, TSStateId pa
     LOG(
       "lex_internal state:%d, row:%u, column:%u",
       lex_mode.lex_state,
-      current_position.extent.row,
+      current_position.extent.row + 1,
       current_position.extent.column
     );
     ts_lexer_start(&self->lexer);
@@ -1242,7 +1248,11 @@ static void ts_parser__recover(TSParser *self, StackVersion version, Subtree loo
   }
 }
 
-static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_node_reuse) {
+static bool ts_parser__advance(
+  TSParser *self,
+  StackVersion version,
+  bool allow_node_reuse
+) {
   TSStateId state = ts_stack_state(self->stack, version);
   uint32_t position = ts_stack_position(self->stack, version).bytes;
   Subtree last_external_token = ts_stack_last_external_token(self->stack, version);
@@ -1274,6 +1284,17 @@ static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_
   }
 
   for (;;) {
+    if (++self->operation_count == OP_COUNT_PER_TIMEOUT_CHECK) {
+      self->operation_count = 0;
+      if (
+        (self->cancellation_flag && atomic_load(self->cancellation_flag)) ||
+        (!clock_is_null(self->end_clock) && clock_is_gt(clock_now(), self->end_clock))
+      ) {
+        ts_subtree_release(&self->tree_pool, lookahead);
+        return false;
+      }
+    }
+
     StackVersion last_reduction_version = STACK_VERSION_NONE;
 
     for (uint32_t i = 0; i < table_entry.action_count; i++) {
@@ -1302,7 +1323,7 @@ static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_
 
           ts_parser__shift(self, version, next_state, lookahead, action.params.extra);
           if (did_reuse) reusable_node_advance(&self->reusable_node);
-          return;
+          return true;
         }
 
         case TSParseActionTypeReduce: {
@@ -1322,7 +1343,7 @@ static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_
         case TSParseActionTypeAccept: {
           LOG("accept");
           ts_parser__accept(self, version, lookahead);
-          return;
+          return true;
         }
 
         case TSParseActionTypeRecover: {
@@ -1332,7 +1353,7 @@ static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_
 
           ts_parser__recover(self, version, lookahead);
           if (did_reuse) reusable_node_advance(&self->reusable_node);
-          return;
+          return true;
         }
       }
     }
@@ -1371,7 +1392,7 @@ static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_
 
     if (state == ERROR_STATE) {
       ts_parser__recover(self, version, lookahead);
-      return;
+      return true;
     }
 
     if (ts_parser__breakdown_top_of_stack(self, version)) {
@@ -1381,7 +1402,7 @@ static void ts_parser__advance(TSParser *self, StackVersion version, bool allow_
     LOG("detect_error");
     ts_stack_pause(self->stack, version, ts_subtree_leaf_symbol(lookahead));
     ts_subtree_release(&self->tree_pool, lookahead);
-    return;
+    return true;
   }
 }
 
@@ -1480,7 +1501,7 @@ static bool ts_parser_has_outstanding_parse(TSParser *self) {
 
 // Parser - Public
 
-TSParser *ts_parser_new() {
+TSParser *ts_parser_new(void) {
   TSParser *self = ts_calloc(1, sizeof(TSParser));
   ts_lexer_init(&self->lexer);
   array_init(&self->reduce_actions);
@@ -1491,8 +1512,10 @@ TSParser *ts_parser_new() {
   self->reusable_node = reusable_node_new();
   self->dot_graph_file = NULL;
   self->halt_on_error = false;
-  self->enabled = true;
-  self->operation_limit = SIZE_MAX;
+  self->cancellation_flag = NULL;
+  self->timeout_duration = 0;
+  self->end_clock = clock_null();
+  self->operation_count = 0;
   self->old_tree = NULL_SUBTREE;
   self->scratch_tree.ptr = &self->scratch_tree_data;
   self->included_range_differences = (TSRangeArray) array_new();
@@ -1569,20 +1592,20 @@ void ts_parser_halt_on_error(TSParser *self, bool should_halt_on_error) {
   self->halt_on_error = should_halt_on_error;
 }
 
-bool ts_parser_enabled(const TSParser *self) {
-  return self->enabled;
+const size_t *ts_parser_cancellation_flag(const TSParser *self) {
+  return (const size_t *)self->cancellation_flag;
 }
 
-void ts_parser_set_enabled(TSParser *self, bool enabled) {
-  self->enabled = enabled;
+void ts_parser_set_cancellation_flag(TSParser *self, const size_t *flag) {
+  self->cancellation_flag = (const volatile size_t *)flag;
 }
 
-size_t ts_parser_operation_limit(const TSParser *self) {
-  return self->operation_limit;
+uint64_t ts_parser_timeout_micros(const TSParser *self) {
+  return duration_to_micros(self->timeout_duration);
 }
 
-void ts_parser_set_operation_limit(TSParser *self, size_t limit) {
-  self->operation_limit = limit;
+void ts_parser_set_timeout_micros(TSParser *self, uint64_t timeout_micros) {
+  self->timeout_duration = duration_from_micros(timeout_micros);
 }
 
 void ts_parser_set_included_ranges(TSParser *self, const TSRange *ranges, uint32_t count) {
@@ -1645,24 +1668,26 @@ TSTree *ts_parser_parse(TSParser *self, const TSTree *old_tree, TSInput input) {
   }
 
   uint32_t position = 0, last_position = 0, version_count = 0;
-  size_t operation_count = 0;
+  self->operation_count = 0;
+  if (self->timeout_duration) {
+    self->end_clock = clock_after(clock_now(), self->timeout_duration);
+  } else {
+    self->end_clock = clock_null();
+  }
 
   do {
     for (StackVersion version = 0;
          version_count = ts_stack_version_count(self->stack), version < version_count;
          version++) {
-      if (operation_count > self->operation_limit || !self->enabled) return NULL;
-      operation_count++;
-
       bool allow_node_reuse = version_count == 1;
       while (ts_stack_is_active(self->stack, version)) {
         LOG("process version:%d, version_count:%u, state:%d, row:%u, col:%u",
             version, ts_stack_version_count(self->stack),
             ts_stack_state(self->stack, version),
-            ts_stack_position(self->stack, version).extent.row,
+            ts_stack_position(self->stack, version).extent.row + 1,
             ts_stack_position(self->stack, version).extent.column);
 
-        ts_parser__advance(self, version, allow_node_reuse);
+        if (!ts_parser__advance(self, version, allow_node_reuse)) return NULL;
         LOG_STACK();
 
         position = ts_stack_position(self->stack, version).bytes;

lib/src/reusable_node.h

@@ -11,7 +11,7 @@ typedef struct {
   Subtree last_external_token;
 } ReusableNode;
 
-static inline ReusableNode reusable_node_new() {
+static inline ReusableNode reusable_node_new(void) {
   return (ReusableNode) {array_new(), NULL_SUBTREE};
 }
 

lib/src/stack.c

@@ -712,9 +712,9 @@ void ts_stack_clear(Stack *self) {
 }
 
 bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f) {
+  array_reserve(&self->iterators, 32);
   bool was_recording_allocations = ts_toggle_allocation_recording(false);
-  if (!f)
-    f = stderr;
+  if (!f) f = stderr;
 
   fprintf(f, "digraph stack {\n");
   fprintf(f, "rankdir=\"RL\";\n");
@@ -785,7 +785,7 @@ bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f)
       fprintf(
         f,
         " tooltip=\"position: %u,%u\nnode_count:%u\nerror_cost: %u\ndynamic_precedence: %d\"];\n",
-        node->position.extent.row,
+        node->position.extent.row + 1,
         node->position.extent.column,
         node->node_count,
         node->error_cost,

lib/src/stack.h

@@ -92,11 +92,11 @@ StackSummary *ts_stack_get_summary(Stack *, StackVersion);
 // Get the total cost of all errors on the given version of the stack.
 unsigned ts_stack_error_cost(const Stack *, StackVersion version);
 
-// Determine whether the given two stack versions can be merged.
-bool ts_stack_merge(Stack *, StackVersion, StackVersion);
-
 // Merge the given two stack versions if possible, returning true
 // if they were successfully merged and false otherwise.
+bool ts_stack_merge(Stack *, StackVersion, StackVersion);
+
+// Determine whether the given two stack versions can be merged.
 bool ts_stack_can_merge(Stack *, StackVersion, StackVersion);
 
 TSSymbol ts_stack_resume(Stack *, StackVersion);