initial commit

src/scanner.c

@@ -0,0 +1,430 @@
+#include "tree_sitter/parser.h"
+#include "tree_sitter/alloc.h"
+#include "tree_sitter/array.h"
+
+#define ERROR_SIZE 64
+#define STACK_COUNT 3
+#define NAME_SEP ' '
+#define STACK_SEP '\n'
+
+enum TokenType {
+  /* push name on stack to match with later */
+  PopBlock,     // [ name ] %}
+  PopPartial,   // [ name ] %}
+  PopVerbatim,  // [ name ] %}
+  /* pop name when done with it */
+  PushBlock,    // name %}
+  PushPartial,  // name [ inline ] %}
+  PushVerbatim, // [ name ] %}
+  /* indicates that an error occurred */
+  MatcherError,
+};
+
+typedef Array(int32_t) Name;
+typedef Array(Name) Stack;
+
+struct Scanner {
+  bool has_error;
+  union {
+    Stack stacks [STACK_COUNT];
+    char error [ERROR_SIZE];
+  };
+};
+
+static Stack* get_stack_for_token(struct Scanner *scanner, enum TokenType token) {
+  if (token >= MatcherError) {
+    return NULL;
+  }
+  return &scanner->stacks[token % STACK_COUNT];
+}
+
+static void reset_scanner(struct Scanner *const scanner) {
+  if (scanner->has_error) {
+    scanner->error[0] = '\0';
+    scanner->has_error = false;
+  } else {
+    for (unsigned i = 0; i < STACK_COUNT; ++i) {
+      Stack *stack = &scanner->stacks[i];
+      for (unsigned i = 0; i < stack->size; ++i) {
+        array_delete(array_get(stack, i));
+      }
+      array_delete(stack);
+    }
+  }
+}
+
+#define scanner_error(scanner, string) do {\
+  struct Scanner *_scanner = scanner;\
+  reset_scanner(_scanner);\
+  _scanner->has_error = true;\
+  _scanner->error[0] = '\0';\
+  /*strncat(_scanner->error, string, ERROR_SIZE - 1);*/\
+} while(0)
+
+#define min(a, b)\
+  ({\
+    typeof(a) _a = (a);\
+    typeof(b) _b = (b);\
+    _a > _b ? _a : _b;\
+  })
+
+void * tree_sitter_django_external_scanner_create() {
+  return ts_calloc(1, sizeof(struct Scanner));
+}
+
+void tree_sitter_django_external_scanner_destroy(void *payload) {
+  struct Scanner *scanner = (struct Scanner*) payload;
+  reset_scanner(scanner);
+  ts_free(payload);
+}
+
+static unsigned write_code(int32_t code, char *out) {
+  if (code <= 0x7F) {
+      // 1 byte: 0xxxxxxx
+      out[0] = code;
+      return 1;
+  } 
+  else if (code <= 0x7FF) {
+      // 2 bytes: 110xxxxx 10xxxxxx
+      out[0] = (code >> 6)  | 0xC0;
+      out[1] = (code & 0x3F) | 0x80;
+      return 2;
+  } 
+  else if (code <= 0xFFFF) {
+      // 3 bytes: 1110xxxx 10xxxxxx 10xxxxxx
+      // Note: Code points U+D800 to U+DFFF are reserved for UTF-16 surrogates and are invalid
+      if (code >= 0xD800 && code <= 0xDFFF) {
+        return 0;
+      }
+
+      out[0] = (code >> 12) | 0xE0;
+      out[1] = ((code >> 6) & 0x3F) | 0x80;
+      out[2] = (code & 0x3F) | 0x80;
+      return 3;
+  } 
+  else if (code <= 0x10FFFF) {
+      // 4 bytes: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+      out[0] = (code >> 18) | 0xF0;
+      out[1] = ((code >> 12) & 0x3F) | 0x80;
+      out[2] = ((code >> 6) & 0x3F) | 0x80;
+      out[3] = (code & 0x3F) | 0x80;
+      return 4;
+  }
+  return 0; // Out of Unicode range
+}
+
+unsigned tree_sitter_django_external_scanner_serialize(
+  void *payload,
+  char *const buffer
+) {
+  struct Scanner *scanner = (struct Scanner*) payload;
+
+write_serialization_error:;
+  char *iter = buffer;
+
+  iter += write_code(scanner->has_error + '0', iter);
+  if (scanner->has_error) {
+    for (int i = 0; i < ERROR_SIZE; ++i) {
+      char value = scanner->error[i];
+      *(iter++) = value;
+      if (value == '\0') {
+        break;
+      }
+    }
+  } else {
+    // Ok scanner requires copying its stack
+    for (unsigned i = 0; i < STACK_COUNT; ++i) {
+      Stack *stack = &scanner->stacks[i];
+      for (unsigned i = 0; i < stack->size; ++i) {
+        Name *name = array_get(stack, i);
+        for (unsigned j = 0; j < name->size; ++j) {
+          int32_t code = *array_get(name, j);
+          unsigned write_amt = write_code(code, iter);
+          if (write_amt == 0) {
+            scanner_error(scanner, "bad code from name");
+            goto write_serialization_error;
+          }
+          iter += write_amt;
+        }
+        iter += write_code(NAME_SEP, iter);
+      }
+      iter += write_code(STACK_SEP, iter);
+    }
+  }
+  return iter - buffer;
+}
+
+static unsigned read_code(const char *iter, int32_t *result) {
+  unsigned char byte0 = iter[0];
+  if (byte0 <= 0x7F) {
+    // 1 byte: 0xxxxxxx
+    *result = byte0;
+    return 1;
+  } else if ((byte0 & 0xE0) == 0xC0) {
+    // 2 bytes: 110xxxxx 10xxxxxx
+    *result = ((byte0 & 0x1F) << 6) | (iter[1] & 0x3F);
+    return 2;
+  } else if ((byte0 & 0xF0) == 0xE0) {
+    // 3 bytes: 1110xxxx 10xxxxxx 10xxxxxx
+    *result = ((byte0 & 0x0F) << 12) | ((iter[1] & 0x3F) << 6) | (iter[2] & 0x3F);
+    return 3;
+  } else if ((byte0 & 0xF8) == 0xF0) {
+    // 4 bytes: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+    *result = ((byte0 & 0x07) << 18) | ((iter[1] & 0x3F) << 12) | ((iter[2] & 0x3F) << 6) | (iter[3] & 0x3F);
+    return 4;
+  }
+  return 0; // invalid UTF-8 leading byte
+}
+
+static bool check_name_start_char(const int32_t letter) {
+  return 'a' <= letter && letter <= 'z' || 'A' <= letter && letter <= 'Z';
+}
+
+static bool check_name_char(const int32_t letter) {
+  return check_name_start_char(letter) || '0' <= letter && letter <= '9' || letter == '_';
+}
+
+void tree_sitter_django_external_scanner_deserialize(
+  void *payload,
+  const char *buffer,
+  unsigned length
+) {
+  struct Scanner *scanner = (struct Scanner*) payload;
+  reset_scanner(scanner);
+  if (length == 0) {
+    return;
+  }
+  int32_t code;
+  unsigned bytes_read = read_code(buffer, &code);
+  if (!bytes_read) {
+    scanner_error(scanner, "received invalid utf8 byte");
+    return;
+  }
+  if (bytes_read > length) {
+    scanner_error(scanner, "status code truncated by length");
+    return;
+  }
+  buffer += bytes_read;
+  switch (code) {
+    default:
+      scanner_error(scanner, "received unrecognized scanner status");
+      return;
+    case '1': {
+      scanner->has_error = true;
+      for (unsigned i = 0; i < length - bytes_read; ++i) {
+        char value = *(buffer++);
+        scanner->error[i] = value;
+        if (value == '\0') {
+          break;
+        }
+      }
+      return;
+    }
+    case '0': {
+      scanner->has_error = false;
+      const char *end = buffer + length - bytes_read;
+      Stack *stack = scanner->stacks;
+      Stack *stack_end = stack + STACK_COUNT;
+      Name *name = NULL;
+
+      while (stack < stack_end) {
+        bytes_read = read_code(buffer, &code);
+        if (bytes_read == 0) {
+          scanner_error(scanner, "Bad utf8 byte");
+          return;
+        }
+        buffer += bytes_read;
+        if (buffer > end) {
+          scanner_error(scanner, "Scanner truncated by length");
+          return;
+        }
+        if (code == STACK_SEP) {
+          ++stack;
+          name = NULL;
+          continue;
+        } else if (code == NAME_SEP) {
+          if (name == NULL) {
+            scanner_error(scanner, "Empty name read");
+            return;
+          }
+          name = NULL;
+        } else {
+          if (name == NULL) {
+            if (!check_name_start_char(code)) {
+              scanner_error(scanner, "Invalid name, must start with letter");
+              return;
+            }
+            array_push(stack, (Name) array_new());
+            name = array_back(stack);
+          } else if (!check_name_char(code)) {
+            scanner_error(scanner, "Invalid character in name");
+            return;
+          }
+          array_push(name, code);
+        }
+      }
+      if (buffer != end) {
+        scanner_error(scanner, "Scanner deserialization finished with left over length");
+      }
+    }
+  }
+}
+
+static bool read_name(TSLexer *const lexer, Name *const name) {
+  if (name->size == 0) {
+    if (!check_name_start_char(lexer->lookahead)) {
+      return false;
+    }
+    array_push(name, lexer->lookahead);
+    lexer->advance(lexer, false);
+  }
+  while (check_name_char(lexer->lookahead)) {
+    array_push(name, lexer->lookahead);
+    lexer->advance(lexer, false);
+  }
+  return true;
+}
+
+static unsigned check_name(TSLexer *const lexer, const Name *const name) {
+  for (unsigned i = 0; i < name->size; ++i) {
+    if (lexer->lookahead != name->contents[i]) {
+      return i;
+    }
+    lexer->advance(lexer, false);
+  }
+  return name->size;
+}
+
+static bool check_close_block(TSLexer *const lexer) {
+  while (true) {
+    switch (lexer->lookahead) {
+      case ' ':
+      case '\t':
+        lexer->advance(lexer, false);
+      case '%':
+        lexer->advance(lexer, false);
+        return lexer->lookahead == '}';
+      default:
+        return false;
+    }
+  }
+}
+
+const char inline_chars[] = "inline";
+
+static bool check_inline(TSLexer *const lexer) {
+  for (unsigned i = 0; i < sizeof(inline_chars) - 1; ++i) {
+    if (lexer->lookahead != inline_chars[i]) {
+      return false;
+    }
+    lexer->advance(lexer, false);
+  }
+  return check_close_block(lexer);
+}
+
+bool tree_sitter_django_external_scanner_scan(
+  void *payload,
+  TSLexer *lexer,
+  const bool *valid_symbols
+) {
+  struct Scanner *scanner = (struct Scanner*) payload;
+  if (scanner->has_error) {
+    lexer->log(lexer, "%s", scanner->error);
+    if (valid_symbols[MatcherError]) {
+      // scanner is an error state, flag problem by returning error token
+      lexer->result_symbol = MatcherError;
+      return true;
+    }
+    return false;
+  }
+  if (valid_symbols[MatcherError]) {
+    return false;
+  }
+  while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
+    lexer->advance(lexer, true);
+  }
+  bool is_empty = false;
+  if (lexer->lookahead == '%') {
+    lexer->mark_end(lexer);
+    lexer->advance(lexer, false);
+    if (lexer->lookahead != '}') {
+      return false;
+    }
+    is_empty = true;
+  }
+
+  // save most matched name in case we need to reread characters
+  Name *most_matched_name;
+  unsigned most_matched_amt = 0;
+  for (unsigned token = PopBlock; token < PushBlock; ++token) {
+    Stack *stack = get_stack_for_token(scanner, token);
+    if (valid_symbols[token] && stack->size > 0) {
+      Name *name = array_back(stack);
+      if (is_empty) {
+        // we matched a close block without an id
+        array_delete(name);
+        array_pop(stack);
+        lexer->result_symbol = token;
+        return true;
+      }
+      if (most_matched_amt > name->size) {
+        continue;
+      }
+      if (most_matched_amt > 0) {
+        for (int i = 0; i < most_matched_amt; ++i) {
+          if (*array_get(most_matched_name, i) != *array_get(name, i)) {
+            continue;
+          }
+        }
+      }
+      unsigned match_amt = check_name(lexer, name) + most_matched_amt;
+      bool has_next_char = name->size ? check_name_char(lexer->lookahead) : check_name_start_char(lexer->lookahead);
+      if (match_amt == name->size && !has_next_char) {
+        // name matches and there are no remaining name chars from lexer
+        lexer->mark_end(lexer);
+        if (check_close_block(lexer)) {
+          array_delete(name);
+          array_pop(stack);
+          lexer->result_symbol = token;
+          return true;
+        } else {
+          // bad close block means no matches
+          return false;
+        }
+      }
+      most_matched_amt = match_amt;
+      most_matched_name = name;
+    }
+  }
+  for (unsigned token = PushBlock; token < MatcherError; ++token) {
+    if (valid_symbols[token]) {
+      Stack *stack = get_stack_for_token(scanner, token);
+      if (is_empty && token == PushVerbatim) {
+        // use empty string for name to indicate empty push
+        array_push(stack, (Name) array_new());
+        lexer->result_symbol = token;
+        return true;
+      }
+      // do not push stack until we are done with most_matched_name
+      Name name = array_new();
+      if (most_matched_amt > 0) {
+        array_extend(&name, most_matched_amt, most_matched_name);
+      }
+      if (read_name(lexer, &name)) {
+        // validate tokens after name
+        lexer->mark_end(lexer);
+        if (check_close_block(lexer) || token == PushPartial && check_inline(lexer)) {
+          array_push(stack, name);
+          lexer->result_symbol = token;
+          return true;
+        }
+      }
+      // undo failed push
+      array_delete(&name);
+      array_pop(stack);
+      // single failed push implies failure to match any push
+      return false;
+    }
+  }
+  return false;
+}

src/tree_sitter/alloc.h

@@ -0,0 +1,54 @@
+#ifndef TREE_SITTER_ALLOC_H_
+#define TREE_SITTER_ALLOC_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+// Allow clients to override allocation functions
+#ifdef TREE_SITTER_REUSE_ALLOCATOR
+
+extern void *(*ts_current_malloc)(size_t size);
+extern void *(*ts_current_calloc)(size_t count, size_t size);
+extern void *(*ts_current_realloc)(void *ptr, size_t size);
+extern void (*ts_current_free)(void *ptr);
+
+#ifndef ts_malloc
+#define ts_malloc  ts_current_malloc
+#endif
+#ifndef ts_calloc
+#define ts_calloc  ts_current_calloc
+#endif
+#ifndef ts_realloc
+#define ts_realloc ts_current_realloc
+#endif
+#ifndef ts_free
+#define ts_free    ts_current_free
+#endif
+
+#else
+
+#ifndef ts_malloc
+#define ts_malloc  malloc
+#endif
+#ifndef ts_calloc
+#define ts_calloc  calloc
+#endif
+#ifndef ts_realloc
+#define ts_realloc realloc
+#endif
+#ifndef ts_free
+#define ts_free    free
+#endif
+
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // TREE_SITTER_ALLOC_H_

src/tree_sitter/array.h

@@ -0,0 +1,336 @@
+#ifndef TREE_SITTER_ARRAY_H_
+#define TREE_SITTER_ARRAY_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "./alloc.h"
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable : 4101)
+#elif defined(__GNUC__) || defined(__clang__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-variable"
+#endif
+
+#define Array(T)       \
+  struct {             \
+    T *contents;       \
+    uint32_t size;     \
+    uint32_t capacity; \
+  }
+
+/// Initialize an array.
+#define array_init(self) \
+  ((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL)
+
+/// Create an empty array.
+#define array_new() \
+  { NULL, 0, 0 }
+
+/// Get a pointer to the element at a given `index` in the array.
+#define array_get(self, _index) \
+  (assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index])
+
+/// Get a pointer to the first element in the array.
+#define array_front(self) array_get(self, 0)
+
+/// Get a pointer to the last element in the array.
+#define array_back(self) array_get(self, (self)->size - 1)
+
+/// Clear the array, setting its size to zero. Note that this does not free any
+/// memory allocated for the array's contents.
+#define array_clear(self) ((self)->size = 0)
+
+#ifdef __cplusplus
+#define _array__cast(self, expr) (decltype((self)->contents))(expr)
+#else
+#define _array__cast(self, expr) (expr)
+#endif
+
+/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
+/// less than the array's current capacity, this function has no effect.
+#define array_reserve(self, new_capacity)                 \
+  ((self)->contents = _array__cast(self, _array__reserve( \
+    (void *)(self)->contents, &(self)->capacity,          \
+    array_elem_size(self), new_capacity))                 \
+  )
+
+/// Free any memory allocated for this array. Note that this does not free any
+/// memory allocated for the array's contents.
+#define array_delete(self)                           \
+  do {                                               \
+    if ((self)->contents) ts_free((self)->contents); \
+    (self)->contents = NULL;                         \
+    (self)->size = 0;                                \
+    (self)->capacity = 0;                            \
+  } while (0)
+
+/// Push a new `element` onto the end of the array.
+#define array_push(self, element)                                 \
+  do {                                                            \
+    (self)->contents = _array__cast(self, _array__grow(           \
+      (void *)(self)->contents, (self)->size, &(self)->capacity,  \
+      1, array_elem_size(self)                                    \
+    ));                                                           \
+   (self)->contents[(self)->size++] = (element);                  \
+  } while(0)
+
+/// Increase the array's size by `count` elements.
+/// New elements are zero-initialized.
+#define array_grow_by(self, count)                                               \
+  do {                                                                           \
+    if ((count) == 0) break;                                                     \
+    (self)->contents = _array__cast(self, _array__grow(                          \
+      (self)->contents, (self)->size, &(self)->capacity,                         \
+      count, array_elem_size(self)                                               \
+    ));                                                                          \
+    memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
+    (self)->size += (count);                                                     \
+  } while (0)
+
+/// Append all elements from one array to the end of another.
+#define array_push_all(self, other) \
+  array_extend((self), (other)->size, (other)->contents)
+
+/// Append `count` elements to the end of the array, reading their values from the
+/// `contents` pointer.
+#define array_extend(self, count, other_contents)                 \
+  ((self)->contents = _array__cast(self, _array__splice(          \
+    (void*)(self)->contents, &(self)->size, &(self)->capacity,    \
+    array_elem_size(self), (self)->size, 0, count, other_contents \
+  )))
+
+/// Remove `old_count` elements from the array starting at the given `index`. At
+/// the same index, insert `new_count` new elements, reading their values from the
+/// `new_contents` pointer.
+#define array_splice(self, _index, old_count, new_count, new_contents) \
+  ((self)->contents = _array__cast(self, _array__splice(              \
+    (void *)(self)->contents, &(self)->size, &(self)->capacity,        \
+    array_elem_size(self), _index, old_count, new_count, new_contents  \
+  )))
+
+/// Insert one `element` into the array at the given `index`.
+#define array_insert(self, _index, element)                     \
+  ((self)->contents = _array__cast(self, _array__splice(        \
+    (void *)(self)->contents, &(self)->size, &(self)->capacity, \
+    array_elem_size(self), _index, 0, 1, &(element)             \
+  )))
+
+/// Remove one element from the array at the given `index`.
+#define array_erase(self, _index) \
+  _array__erase((void *)(self)->contents, &(self)->size, array_elem_size(self), _index)
+
+/// Pop the last element off the array, returning the element by value.
+#define array_pop(self) ((self)->contents[--(self)->size])
+
+/// Assign the contents of one array to another, reallocating if necessary.
+#define array_assign(self, other)                                   \
+  ((self)->contents = _array__cast(self, _array__assign(            \
+    (void *)(self)->contents, &(self)->size, &(self)->capacity,     \
+    (const void *)(other)->contents, (other)->size, array_elem_size(self) \
+  )))
+
+/// Swap one array with another
+#define array_swap(self, other)                                     \
+  do {                                                              \
+    void *_array_swap_tmp = (void *)(self)->contents;               \
+    (self)->contents = (other)->contents;                           \
+    (other)->contents = _array__cast(other, _array_swap_tmp);       \
+    _array__swap(&(self)->size, &(self)->capacity,                  \
+                 &(other)->size, &(other)->capacity);               \
+  } while (0)
+
+/// Get the size of the array contents
+#define array_elem_size(self) (sizeof *(self)->contents)
+
+/// Search a sorted array for a given `needle` value, using the given `compare`
+/// callback to determine the order.
+///
+/// If an existing element is found to be equal to `needle`, then the `index`
+/// out-parameter is set to the existing value's index, and the `exists`
+/// out-parameter is set to true. Otherwise, `index` is set to an index where
+/// `needle` should be inserted in order to preserve the sorting, and `exists`
+/// is set to false.
+#define array_search_sorted_with(self, compare, needle, _index, _exists) \
+  _array__search_sorted(self, 0, compare, , needle, _index, _exists)
+
+/// Search a sorted array for a given `needle` value, using integer comparisons
+/// of a given struct field (specified with a leading dot) to determine the order.
+///
+/// See also `array_search_sorted_with`.
+#define array_search_sorted_by(self, field, needle, _index, _exists) \
+  _array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists)
+
+/// Insert a given `value` into a sorted array, using the given `compare`
+/// callback to determine the order.
+#define array_insert_sorted_with(self, compare, value) \
+  do { \
+    unsigned _index, _exists; \
+    array_search_sorted_with(self, compare, &(value), &_index, &_exists); \
+    if (!_exists) array_insert(self, _index, value); \
+  } while (0)
+
+/// Insert a given `value` into a sorted array, using integer comparisons of
+/// a given struct field (specified with a leading dot) to determine the order.
+///
+/// See also `array_search_sorted_by`.
+#define array_insert_sorted_by(self, field, value) \
+  do { \
+    unsigned _index, _exists; \
+    array_search_sorted_by(self, field, (value) field, &_index, &_exists); \
+    if (!_exists) array_insert(self, _index, value); \
+  } while (0)
+
+// Private
+
+// Pointers to individual `Array` fields (rather than the entire `Array` itself)
+// are passed to the various `_array__*` functions below to address strict aliasing
+// violations that arises when the _entire_ `Array` struct is passed as `Array(void)*`.
+//
+// The `Array` type itself was not altered as a solution in order to avoid breakage
+// with existing consumers (in particular, parsers with external scanners).
+
+/// This is not what you're looking for, see `array_erase`.
+static inline void _array__erase(void* self_contents, uint32_t *size,
+                                size_t element_size, uint32_t index) {
+  assert(index < *size);
+  char *contents = (char *)self_contents;
+  memmove(contents + index * element_size, contents + (index + 1) * element_size,
+          (*size - index - 1) * element_size);
+  (*size)--;
+}
+
+/// This is not what you're looking for, see `array_reserve`.
+static inline void *_array__reserve(void *contents, uint32_t *capacity,
+                                  size_t element_size, uint32_t new_capacity) {
+  void *new_contents = contents;
+  if (new_capacity > *capacity) {
+    if (contents) {
+      new_contents = ts_realloc(contents, new_capacity * element_size);
+    } else {
+      new_contents = ts_malloc(new_capacity * element_size);
+    }
+    *capacity = new_capacity;
+  }
+  return new_contents;
+}
+
+/// This is not what you're looking for, see `array_assign`.
+static inline void *_array__assign(void* self_contents, uint32_t *self_size, uint32_t *self_capacity,
+                                 const void *other_contents, uint32_t other_size, size_t element_size) {
+  void *new_contents = _array__reserve(self_contents, self_capacity, element_size, other_size);
+  *self_size = other_size;
+  memcpy(new_contents, other_contents, *self_size * element_size);
+  return new_contents;
+}
+
+/// This is not what you're looking for, see `array_swap`.
+static inline void _array__swap(uint32_t *self_size, uint32_t *self_capacity,
+                               uint32_t *other_size, uint32_t *other_capacity) {
+  uint32_t tmp_size = *self_size;
+  uint32_t tmp_capacity = *self_capacity;
+  *self_size = *other_size;
+  *self_capacity = *other_capacity;
+  *other_size = tmp_size;
+  *other_capacity = tmp_capacity;
+}
+
+/// This is not what you're looking for, see `array_push` or `array_grow_by`.
+static inline void *_array__grow(void *contents, uint32_t size, uint32_t *capacity,
+                               uint32_t count, size_t element_size) {
+  void *new_contents = contents;
+  uint32_t new_size = size + count;
+  if (new_size > *capacity) {
+    uint32_t new_capacity = *capacity * 2;
+    if (new_capacity < 8) new_capacity = 8;
+    if (new_capacity < new_size) new_capacity = new_size;
+    new_contents = _array__reserve(contents, capacity, element_size, new_capacity);
+  }
+  return new_contents;
+}
+
+/// This is not what you're looking for, see `array_splice`.
+static inline void *_array__splice(void *self_contents, uint32_t *size, uint32_t *capacity,
+                                 size_t element_size,
+                                 uint32_t index, uint32_t old_count,
+                                 uint32_t new_count, const void *elements) {
+  uint32_t new_size = *size + new_count - old_count;
+  uint32_t old_end = index + old_count;
+  uint32_t new_end = index + new_count;
+  assert(old_end <= *size);
+
+  void *new_contents = _array__reserve(self_contents, capacity, element_size, new_size);
+
+  char *contents = (char *)new_contents;
+  if (*size > old_end) {
+    memmove(
+      contents + new_end * element_size,
+      contents + old_end * element_size,
+      (*size - old_end) * element_size
+    );
+  }
+  if (new_count > 0) {
+    if (elements) {
+      memcpy(
+        (contents + index * element_size),
+        elements,
+        new_count * element_size
+      );
+    } else {
+      memset(
+        (contents + index * element_size),
+        0,
+        new_count * element_size
+      );
+    }
+  }
+  *size += new_count - old_count;
+
+  return new_contents;
+}
+
+/// A binary search routine, based on Rust's `std::slice::binary_search_by`.
+/// This is not what you're looking for, see `array_search_sorted_with` or `array_search_sorted_by`.
+#define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \
+  do { \
+    *(_index) = start; \
+    *(_exists) = false; \
+    uint32_t size = (self)->size - *(_index); \
+    if (size == 0) break; \
+    int comparison; \
+    while (size > 1) { \
+      uint32_t half_size = size / 2; \
+      uint32_t mid_index = *(_index) + half_size; \
+      comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \
+      if (comparison <= 0) *(_index) = mid_index; \
+      size -= half_size; \
+    } \
+    comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \
+    if (comparison == 0) *(_exists) = true; \
+    else if (comparison < 0) *(_index) += 1; \
+  } while (0)
+
+/// Helper macro for the `_sorted_by` routines below. This takes the left (existing)
+/// parameter by reference in order to work with the generic sorting function above.
+#define _compare_int(a, b) ((int)*(a) - (int)(b))
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#elif defined(__GNUC__) || defined(__clang__)
+#pragma GCC diagnostic pop
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TREE_SITTER_ARRAY_H_

src/tree_sitter/parser.h

@@ -0,0 +1,286 @@
+#ifndef TREE_SITTER_PARSER_H_
+#define TREE_SITTER_PARSER_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#define ts_builtin_sym_error ((TSSymbol)-1)
+#define ts_builtin_sym_end 0
+#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
+
+#ifndef TREE_SITTER_API_H_
+typedef uint16_t TSStateId;
+typedef uint16_t TSSymbol;
+typedef uint16_t TSFieldId;
+typedef struct TSLanguage TSLanguage;
+typedef struct TSLanguageMetadata {
+  uint8_t major_version;
+  uint8_t minor_version;
+  uint8_t patch_version;
+} TSLanguageMetadata;
+#endif
+
+typedef struct {
+  TSFieldId field_id;
+  uint8_t child_index;
+  bool inherited;
+} TSFieldMapEntry;
+
+// Used to index the field and supertype maps.
+typedef struct {
+  uint16_t index;
+  uint16_t length;
+} TSMapSlice;
+
+typedef struct {
+  bool visible;
+  bool named;
+  bool supertype;
+} TSSymbolMetadata;
+
+typedef struct TSLexer TSLexer;
+
+struct TSLexer {
+  int32_t lookahead;
+  TSSymbol result_symbol;
+  void (*advance)(TSLexer *, bool);
+  void (*mark_end)(TSLexer *);
+  uint32_t (*get_column)(TSLexer *);
+  bool (*is_at_included_range_start)(const TSLexer *);
+  bool (*eof)(const TSLexer *);
+  void (*log)(const TSLexer *, const char *, ...);
+};
+
+typedef enum {
+  TSParseActionTypeShift,
+  TSParseActionTypeReduce,
+  TSParseActionTypeAccept,
+  TSParseActionTypeRecover,
+} TSParseActionType;
+
+typedef union {
+  struct {
+    uint8_t type;
+    TSStateId state;
+    bool extra;
+    bool repetition;
+  } shift;
+  struct {
+    uint8_t type;
+    uint8_t child_count;
+    TSSymbol symbol;
+    int16_t dynamic_precedence;
+    uint16_t production_id;
+  } reduce;
+  uint8_t type;
+} TSParseAction;
+
+typedef struct {
+  uint16_t lex_state;
+  uint16_t external_lex_state;
+} TSLexMode;
+
+typedef struct {
+  uint16_t lex_state;
+  uint16_t external_lex_state;
+  uint16_t reserved_word_set_id;
+} TSLexerMode;
+
+typedef union {
+  TSParseAction action;
+  struct {
+    uint8_t count;
+    bool reusable;
+  } entry;
+} TSParseActionEntry;
+
+typedef struct {
+  int32_t start;
+  int32_t end;
+} TSCharacterRange;
+
+struct TSLanguage {
+  uint32_t abi_version;
+  uint32_t symbol_count;
+  uint32_t alias_count;
+  uint32_t token_count;
+  uint32_t external_token_count;
+  uint32_t state_count;
+  uint32_t large_state_count;
+  uint32_t production_id_count;
+  uint32_t field_count;
+  uint16_t max_alias_sequence_length;
+  const uint16_t *parse_table;
+  const uint16_t *small_parse_table;
+  const uint32_t *small_parse_table_map;
+  const TSParseActionEntry *parse_actions;
+  const char * const *symbol_names;
+  const char * const *field_names;
+  const TSMapSlice *field_map_slices;
+  const TSFieldMapEntry *field_map_entries;
+  const TSSymbolMetadata *symbol_metadata;
+  const TSSymbol *public_symbol_map;
+  const uint16_t *alias_map;
+  const TSSymbol *alias_sequences;
+  const TSLexerMode *lex_modes;
+  bool (*lex_fn)(TSLexer *, TSStateId);
+  bool (*keyword_lex_fn)(TSLexer *, TSStateId);
+  TSSymbol keyword_capture_token;
+  struct {
+    const bool *states;
+    const TSSymbol *symbol_map;
+    void *(*create)(void);
+    void (*destroy)(void *);
+    bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist);
+    unsigned (*serialize)(void *, char *);
+    void (*deserialize)(void *, const char *, unsigned);
+  } external_scanner;
+  const TSStateId *primary_state_ids;
+  const char *name;
+  const TSSymbol *reserved_words;
+  uint16_t max_reserved_word_set_size;
+  uint32_t supertype_count;
+  const TSSymbol *supertype_symbols;
+  const TSMapSlice *supertype_map_slices;
+  const TSSymbol *supertype_map_entries;
+  TSLanguageMetadata metadata;
+};
+
+static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
+  uint32_t index = 0;
+  uint32_t size = len - index;
+  while (size > 1) {
+    uint32_t half_size = size / 2;
+    uint32_t mid_index = index + half_size;
+    const TSCharacterRange *range = &ranges[mid_index];
+    if (lookahead >= range->start && lookahead <= range->end) {
+      return true;
+    } else if (lookahead > range->end) {
+      index = mid_index;
+    }
+    size -= half_size;
+  }
+  const TSCharacterRange *range = &ranges[index];
+  return (lookahead >= range->start && lookahead <= range->end);
+}
+
+/*
+ *  Lexer Macros
+ */
+
+#ifdef _MSC_VER
+#define UNUSED __pragma(warning(suppress : 4101))
+#else
+#define UNUSED __attribute__((unused))
+#endif
+
+#define START_LEXER()           \
+  bool result = false;          \
+  bool skip = false;            \
+  UNUSED                        \
+  bool eof = false;             \
+  int32_t lookahead;            \
+  goto start;                   \
+  next_state:                   \
+  lexer->advance(lexer, skip);  \
+  start:                        \
+  skip = false;                 \
+  lookahead = lexer->lookahead;
+
+#define ADVANCE(state_value) \
+  {                          \
+    state = state_value;     \
+    goto next_state;         \
+  }
+
+#define ADVANCE_MAP(...)                                              \
+  {                                                                   \
+    static const uint16_t map[] = { __VA_ARGS__ };                    \
+    for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) {  \
+      if (map[i] == lookahead) {                                      \
+        state = map[i + 1];                                           \
+        goto next_state;                                              \
+      }                                                               \
+    }                                                                 \
+  }
+
+#define SKIP(state_value) \
+  {                       \
+    skip = true;          \
+    state = state_value;  \
+    goto next_state;      \
+  }
+
+#define ACCEPT_TOKEN(symbol_value)     \
+  result = true;                       \
+  lexer->result_symbol = symbol_value; \
+  lexer->mark_end(lexer);
+
+#define END_STATE() return result;
+
+/*
+ *  Parse Table Macros
+ */
+
+#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT)
+
+#define STATE(id) id
+
+#define ACTIONS(id) id
+
+#define SHIFT(state_value)            \
+  {{                                  \
+    .shift = {                        \
+      .type = TSParseActionTypeShift, \
+      .state = (state_value)          \
+    }                                 \
+  }}
+
+#define SHIFT_REPEAT(state_value)     \
+  {{                                  \
+    .shift = {                        \
+      .type = TSParseActionTypeShift, \
+      .state = (state_value),         \
+      .repetition = true              \
+    }                                 \
+  }}
+
+#define SHIFT_EXTRA()                 \
+  {{                                  \
+    .shift = {                        \
+      .type = TSParseActionTypeShift, \
+      .extra = true                   \
+    }                                 \
+  }}
+
+#define REDUCE(symbol_name, children, precedence, prod_id) \
+  {{                                                       \
+    .reduce = {                                            \
+      .type = TSParseActionTypeReduce,                     \
+      .symbol = symbol_name,                               \
+      .child_count = children,                             \
+      .dynamic_precedence = precedence,                    \
+      .production_id = prod_id                             \
+    },                                                     \
+  }}
+
+#define RECOVER()                    \
+  {{                                 \
+    .type = TSParseActionTypeRecover \
+  }}
+
+#define ACCEPT_INPUT()              \
+  {{                                \
+    .type = TSParseActionTypeAccept \
+  }}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TREE_SITTER_PARSER_H_