Merge pull request #29 from tree-sitter/multiple-node-types

Expose nodes' wrapper symbols

include/tree_sitter/runtime.h

@@ -8,6 +8,10 @@ extern "C" {
 #include <stdlib.h>
 #include <stdbool.h>
 
+typedef unsigned short TSSymbol;
+typedef struct TSLanguage TSLanguage;
+typedef struct TSDocument TSDocument;
+
 typedef enum {
   TSInputEncodingUTF8,
   TSInputEncodingUTF16,
@@ -46,9 +50,11 @@ typedef struct {
   size_t offset[3];
 } TSNode;
 
-typedef unsigned short TSSymbol;
-typedef struct TSLanguage TSLanguage;
-typedef struct TSDocument TSDocument;
+typedef struct {
+  TSSymbol value;
+  bool done;
+  void *data;
+} TSSymbolIterator;
 
 size_t ts_node_start_char(TSNode);
 size_t ts_node_start_byte(TSNode);
@@ -57,6 +63,8 @@ size_t ts_node_end_char(TSNode);
 size_t ts_node_end_byte(TSNode);
 TSPoint ts_node_end_point(TSNode);
 TSSymbol ts_node_symbol(TSNode);
+TSSymbolIterator ts_node_symbols(TSNode);
+void ts_symbol_iterator_next(TSSymbolIterator *);
 const char *ts_node_name(TSNode, const TSDocument *);
 char *ts_node_string(TSNode, const TSDocument *);
 bool ts_node_eq(TSNode, TSNode);
@@ -90,7 +98,7 @@ TSNode ts_document_root_node(const TSDocument *);
 size_t ts_document_parse_count(const TSDocument *);
 
 size_t ts_language_symbol_count(const TSLanguage *);
-const char *ts_language_symbol_name(TSLanguage *, TSSymbol);
+const char *ts_language_symbol_name(const TSLanguage *, TSSymbol);
 
 #define ts_builtin_sym_error 0
 #define ts_builtin_sym_end 1

spec/helpers/tree_helpers.cc

@@ -1,4 +1,6 @@
 #include "helpers/tree_helpers.h"
+#include "runtime/document.h"
+#include "runtime/node.h"
 #include <ostream>
 
 using std::string;
@@ -12,6 +14,7 @@ static const char *symbol_names[24] = {
   "twenty-two", "twenty-three"
 };
 
+
 TSTree ** tree_array(std::vector<TSTree *> trees) {
   TSTree ** result = (TSTree **)calloc(trees.size(), sizeof(TSTree *));
   for (size_t i = 0; i < trees.size(); i++)
@@ -20,7 +23,13 @@ TSTree ** tree_array(std::vector<TSTree *> trees) {
 }
 
 ostream &operator<<(std::ostream &stream, const TSTree *tree) {
-  return stream << string(ts_tree_string(tree, symbol_names, true));;
+  static TSLanguage DUMMY_LANGUAGE = {};
+  static TSDocument DUMMY_DOCUMENT = {};
+  DUMMY_DOCUMENT.parser.language = &DUMMY_LANGUAGE;
+  DUMMY_LANGUAGE.symbol_names = symbol_names;
+  TSNode node;
+  node.data = tree;
+  return stream << string(ts_node_string(node, &DUMMY_DOCUMENT));
 }
 
 ostream &operator<<(ostream &stream, const TSNode &node) {

spec/runtime/node_spec.cc

@@ -133,6 +133,43 @@ describe("Node", []() {
     });
   });
 
+  describe("symbols()", [&]() {
+    it("returns an iterator that yields each of the node's symbols", [&]() {
+      const TSLanguage *language = ts_document_language(document);
+
+      TSSymbolIterator iterator = ts_node_symbols(array_node);
+      AssertThat(iterator.done, Equals(false));
+      AssertThat(ts_language_symbol_name(language, iterator.value), Equals("array"));
+
+      ts_symbol_iterator_next(&iterator);
+      AssertThat(iterator.done, Equals(false));
+      AssertThat(ts_language_symbol_name(language, iterator.value), Equals("_value"));
+
+      ts_symbol_iterator_next(&iterator);
+      AssertThat(iterator.done, Equals(true));
+
+      TSNode false_node = ts_node_descendant_for_range(array_node, false_index, false_index + 1);
+      iterator = ts_node_symbols(false_node);
+      AssertThat(iterator.done, Equals(false));
+      AssertThat(ts_language_symbol_name(language, iterator.value), Equals("false"));
+
+      ts_symbol_iterator_next(&iterator);
+      AssertThat(iterator.done, Equals(false));
+      AssertThat(ts_language_symbol_name(language, iterator.value), Equals("_value"));
+
+      ts_symbol_iterator_next(&iterator);
+      AssertThat(iterator.done, Equals(true));
+
+      TSNode comma_node = ts_node_descendant_for_range(array_node, number_end_index, number_end_index);
+      iterator = ts_node_symbols(comma_node);
+      AssertThat(iterator.done, Equals(false));
+      AssertThat(ts_language_symbol_name(language, iterator.value), Equals(","));
+
+      ts_symbol_iterator_next(&iterator);
+      AssertThat(iterator.done, Equals(true));
+    });
+  });
+
   describe("child_count(), child(i)", [&]() {
     it("returns the child node at the given index, including anonymous nodes", [&]() {
       AssertThat(ts_node_child_count(array_node), Equals<size_t>(7));

spec/runtime/tree_spec.cc

@@ -345,42 +345,6 @@ describe("Tree", []() {
       ts_tree_release(different_parent);
     });
   });
-
-  describe("serialization", [&]() {
-    it("returns a readable string", [&]() {
-      char *string1 = ts_tree_string(tree1, names, true);
-      AssertThat(string(string1), Equals("(cat)"));
-      free(string1);
-
-      char *string2 = ts_tree_string(parent1, names, true);
-      AssertThat(string(string2), Equals("(dog (cat) (cat))"));
-      free(string2);
-    });
-
-    it("hides invisible nodes", [&]() {
-      tree2->visible = false;
-
-      char *string1 = ts_tree_string(parent1, names, true);
-      AssertThat(string(string1), Equals("(dog (cat))"));
-      free(string1);
-    });
-
-    describe("when the root node is not visible", [&]() {
-      it("still serializes it", [&]() {
-        parent1->visible = false;
-
-        char *string1 = ts_tree_string(parent1, names, true);
-        AssertThat(string(string1), Equals("(dog (cat) (cat))"));
-        free(string1);
-
-        tree1->visible = false;
-
-        char *string2 = ts_tree_string(tree1, names, true);
-        AssertThat(string(string2), Equals("(cat)"));
-        free(string2);
-      });
-    });
-  });
 });
 
 END_TEST

src/runtime/language.c

@@ -22,6 +22,6 @@ size_t ts_language_symbol_count(const TSLanguage *language) {
   return language->symbol_count;
 }
 
-const char *ts_language_symbol_name(TSLanguage *language, TSSymbol symbol) {
+const char *ts_language_symbol_name(const TSLanguage *language, TSSymbol symbol) {
   return language->symbol_names[symbol];
 }

src/runtime/node.c

@@ -1,4 +1,6 @@
 #include <stdbool.h>
+#include <string.h>
+#include <stdio.h>
 #include "runtime/node.h"
 #include "runtime/tree.h"
 #include "runtime/document.h"
@@ -202,13 +204,86 @@ TSSymbol ts_node_symbol(TSNode self) {
   return ts_node__tree(self)->symbol;
 }
 
+TSSymbolIterator ts_node_symbols(TSNode self) {
+  const TSTree *tree = ts_node__tree(self);
+  return (TSSymbolIterator){
+    .value = tree->symbol,
+    .done = false,
+    .data = (void *)tree,
+  };
+}
+
+void ts_symbol_iterator_next(TSSymbolIterator *self) {
+  const TSTree *tree = (const TSTree *)self->data;
+  const TSTree *parent = tree->context.parent;
+  if (!self->done && parent) {
+    if (parent->child_count == 1 && !parent->visible) {
+      self->value = parent->symbol;
+      self->data = (void *)parent;
+      return;
+    }
+  }
+  self->done = true;
+}
+
 const char *ts_node_name(TSNode self, const TSDocument *document) {
   return document->parser.language->symbol_names[ts_node__tree(self)->symbol];
 }
 
+static size_t write_lookahead_to_string(char *string, size_t limit,
+                                        char lookahead) {
+  switch (lookahead) {
+    case '\0':
+      return snprintf(string, limit, "<EOF>");
+    default:
+      return snprintf(string, limit, "'%c'", lookahead);
+  }
+}
+
+static size_t ts_tree__write_to_string(const TSTree *self,
+                                       const char **symbol_names, char *string,
+                                       size_t limit, bool is_root,
+                                       bool include_anonymous) {
+  if (!self)
+    return snprintf(string, limit, "(NULL)");
+
+  char *cursor = string;
+  char **writer = (limit > 0) ? &cursor : &string;
+  bool visible =
+    is_root || (self->visible && (include_anonymous || self->named));
+
+  if (visible && !is_root)
+    cursor += snprintf(*writer, limit, " ");
+
+  if (visible) {
+    if (self->symbol == ts_builtin_sym_error && self->child_count == 0) {
+      cursor += snprintf(*writer, limit, "(UNEXPECTED ");
+      cursor += write_lookahead_to_string(*writer, limit, self->lookahead_char);
+    } else {
+      cursor += snprintf(*writer, limit, "(%s", symbol_names[self->symbol]);
+    }
+  }
+
+  for (size_t i = 0; i < self->child_count; i++) {
+    TSTree *child = self->children[i];
+    cursor += ts_tree__write_to_string(child, symbol_names, *writer, limit,
+                                       false, include_anonymous);
+  }
+
+  if (visible)
+    cursor += snprintf(*writer, limit, ")");
+
+  return cursor - string;
+}
+
 char *ts_node_string(TSNode self, const TSDocument *document) {
-  return ts_tree_string(ts_node__tree(self),
-                        document->parser.language->symbol_names, false);
+  static char SCRATCH[1];
+  const TSTree *tree = ts_node__tree(self);
+  const char **symbol_names = document->parser.language->symbol_names;
+  size_t size = ts_tree__write_to_string(tree, symbol_names, SCRATCH, 0, true, false) + 1;
+  char *result = ts_malloc(size * sizeof(char));
+  ts_tree__write_to_string(tree, symbol_names, result, size, true, false);
+  return result;
 }
 
 bool ts_node_eq(TSNode self, TSNode other) {

src/runtime/tree.c

@@ -1,8 +1,6 @@
 #include <assert.h>
 #include <limits.h>
-#include <string.h>
 #include <stdbool.h>
-#include <stdio.h>
 #include "tree_sitter/parser.h"
 #include "runtime/alloc.h"
 #include "runtime/tree.h"
@@ -235,63 +233,6 @@ int ts_tree_compare(const TSTree *left, const TSTree *right) {
   return 0;
 }
 
-static size_t write_lookahead_to_string(char *string, size_t limit,
-                                        char lookahead) {
-  switch (lookahead) {
-    case '\0':
-      return snprintf(string, limit, "<EOF>");
-    default:
-      return snprintf(string, limit, "'%c'", lookahead);
-  }
-}
-
-static size_t ts_tree__write_to_string(const TSTree *self,
-                                       const char **symbol_names, char *string,
-                                       size_t limit, bool is_root,
-                                       bool include_anonymous) {
-  if (!self)
-    return snprintf(string, limit, "(NULL)");
-
-  char *cursor = string;
-  char **writer = (limit > 0) ? &cursor : &string;
-  bool visible =
-    is_root || (self->visible && (include_anonymous || self->named));
-
-  if (visible && !is_root)
-    cursor += snprintf(*writer, limit, " ");
-
-  if (visible) {
-    if (self->symbol == ts_builtin_sym_error && self->child_count == 0) {
-      cursor += snprintf(*writer, limit, "(UNEXPECTED ");
-      cursor += write_lookahead_to_string(*writer, limit, self->lookahead_char);
-    } else {
-      cursor += snprintf(*writer, limit, "(%s", symbol_names[self->symbol]);
-    }
-  }
-
-  for (size_t i = 0; i < self->child_count; i++) {
-    TSTree *child = self->children[i];
-    cursor += ts_tree__write_to_string(child, symbol_names, *writer, limit,
-                                       false, include_anonymous);
-  }
-
-  if (visible)
-    cursor += snprintf(*writer, limit, ")");
-
-  return cursor - string;
-}
-
-char *ts_tree_string(const TSTree *self, const char **symbol_names,
-                     bool include_anonymous) {
-  static char SCRATCH[1];
-  size_t size = 1 + ts_tree__write_to_string(self, symbol_names, SCRATCH, 0,
-                                             true, include_anonymous);
-  char *result = ts_malloc(size * sizeof(char));
-  ts_tree__write_to_string(self, symbol_names, result, size, true,
-                           include_anonymous);
-  return result;
-}
-
 static inline long min(long a, long b) {
   return a <= b ? a : b;
 }

src/runtime/tree.h

@@ -50,8 +50,6 @@ void ts_tree_retain(TSTree *tree);
 void ts_tree_release(TSTree *tree);
 bool ts_tree_eq(const TSTree *tree1, const TSTree *tree2);
 int ts_tree_compare(const TSTree *tree1, const TSTree *tree2);
-char *ts_tree_string(const TSTree *tree, const char **names,
-                     bool include_anonymous);
 
 size_t ts_tree_start_column(const TSTree *self);
 size_t ts_tree_end_column(const TSTree *self);