danhanson/tree-sitter-django
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

initial commit

Browse Source
This commit is contained in:
danhanson
2026-08-11 17:39:34 -05:00
commit 9b44006c18
22 changed files with 24580 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -0,0 +1 @@
node_modules
README.md
+13
View File
@@ -0,0 +1,13 @@
# tree-sitter-django
A [tree-sitter](https://tree-sitter.github.io/tree-sitter/) grammar for the Django Template Language (DTL) — the `{% %}` tags and `{{ }}` variables used in Django templates.
## Development
```sh
npm install
npm run parser-generate # regenerate the parser from grammar.js
npm run parser-test # run the corpus tests in test/corpus
npm run parser-build # build the WASM binary
npm run playground # interactively explore the grammar
```
eslint.config.mjs
+20
View File
@@ -0,0 +1,20 @@
import { defineConfig, globalIgnores } from "eslint/config";
import eslintConfigPrettier from "eslint-config-prettier";
const eslintConfig = defineConfig([
eslintConfigPrettier, // Disable ESLint rules that conflict with Prettier
{
rules: {
"@typescript-eslint/no-unused-vars": "error",
"react-hooks/exhaustive-deps": "error",
},
},
// Override default ignores of eslint-config-next.
globalIgnores([
// Default ignores of eslint-config-next:
"out/**",
"build/**",
]),
]);
export default eslintConfig;
grammar.js
+281
View File
@@ -0,0 +1,281 @@
/* eslint-disable @typescript-eslint/no-unused-vars */
/// <reference types="tree-sitter-cli/dsl" />
// @ts-check
/**
* @param {string} tag
* @param {...RuleOrLiteral} args
* @returns {SeqRule}
*/
function block(tag, ...args) {
return seq("{%", field("tag", tag), ...args, "%}");
}
const django = grammar({
name: "django",
extras: ($) => [/[ \t]/],
conflicts: ($) => [[$.template]],
supertypes: ($) => [$.template_tag, $.template_block_groups],
externals: ($) => [
$.pop_block,
$.pop_partial,
$.pop_verbatim,
$.push_block,
$.push_partial,
$.push_verbatim,
$.matcher_error,
],
reserved: {
global: ($) => ["not", "if", "in", "is", "as"],
},
rules: {
template: ($) => repeat1(choice($.template_tag, $.content)),
content: ($) => /(?:[^\{]|\{[^\{#%}])+/,
template_tag: ($) => choice($.template_block_groups, $.template_variable, $.template_comment),
filtered_value: ($) => seq(
$.value,
optional(
seq(
token.immediate("|"),
$.filter_expression
),
),
),
filter_expression: ($) => seq($.filter, repeat(seq(token.immediate("|"), $.filter))),
value: ($) => choice($.literal, $.variable_attribute),
literal: ($) => choice($.number, $.string),
number: ($) => /-?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE]-?[0-9]+)?/,
string: ($) => /"(?:[^"\\]|\\\\|\\")*"|'(?:[^'\\]|\\\\|\\')*'/,
attribute: ($) => /[a-zA-Z0-9_]+(?:\.[a-zA-Z0-9_]+)*/,
variable_attribute: ($) => seq($.identifier, optional(seq(token.immediate('.'), $.attribute))),
identifier: ($) => /[a-zA-Z][a-zA-Z0-9_]+/,
binaryOperator: ($) =>
choice(
"and",
"or",
"==",
"!=",
"<",
">",
"<=",
">=",
"in",
prec(1, seq("not", "in")),
"is",
prec(1, seq("is", "not"))
),
predicate: ($) =>
seq(
optional("not"),
$.filtered_value,
repeat(seq($.binaryOperator, optional("not"), $.filtered_value))
),
template_variable: ($) => seq("{{", $.filtered_value, "}}"),
template_comment: ($) => seq("{#", /(?:[^#]|#[^}])*/, "#}"),
template_block_groups: ($) =>
choice(
$.autoescape_group,
$.block_group,
$.csrf_token,
$.cycle,
$.debug,
$.extends,
$.filter_group,
$.firstof,
$.for_group,
$.if_group,
$.ifchanged_group,
$.include,
$.lorem,
$.now,
$.partial,
$.partialdef_group,
$.query_string,
$.regroup,
$.reset_cycle,
$.spaceless_group,
$.template_tag_block,
$.url_block,
$.verbatim_group,
$.with_group
),
filter: ($) => choice(
seq("add:", $.value),
"addslashes",
"capfirst",
seq("center:", $.value),
seq("cut:", $.value),
choice(seq("date:", $.value), "date"),
seq("default:", $.value),
seq("default_if_none:", $.value),
seq("dictsort:", $.value),
seq("dictsortreversed:", $.value),
seq("divisibleby:", $.value),
"escape",
"escapejs",
"filesizeformat",
"first",
choice(seq("floatformat:", $.value), "floatformat"),
"force_escape",
seq("get_digit:", $.value),
"iriencode",
seq("join:", $.value),
choice(seq("json_script:", $.value), "json_script"),
"last",
"length",
seq("length_is:", $.value),
"linebreaks",
"linebreaksbr",
"linenumbers",
seq("ljust:", $.value),
"lower",
"make_list",
"phone2numeric",
choice(seq("pluralize:", $.value), "pluralize"),
"pprint",
"random",
seq("rjust:", $.value),
"safe",
"safeseq",
seq("slice:", $.value),
"slugify",
seq("stringformat:", $.value),
"striptags",
choice(seq("time:", $.value), "time"),
choice(seq("timesince:", $.value), "timesince"),
choice(seq("timeuntil:", $.value), "timeuntil"),
"title",
seq("truncatechars:", $.value),
seq("truncatechars_html:", $.value),
seq("truncatewords:", $.value),
seq("truncatewords_html:", $.value),
"unordered_list",
choice(seq("urlencode:", $.value), "urlencode"),
"urlize",
seq("urlizetrunc:", $.value),
"wordcount",
seq("wordwrap:", $.value),
choice(seq("yesno:", $.value), "yesno"),
),
autoescape_group: ($) =>
seq(block("autoescape", choice("on", "off")), optional($.template), block("endautoescape")),
block_group: ($) =>
seq(
block("block", field("name", $.push_block)),
optional($.template),
block("endblock", $.pop_block)
),
comment: ($) => seq(block("comment", optional($.string)), /.*/, prec(1, block("endcomment"))),
csrf_token: ($) => block("csrf_token"),
cycle: ($) =>
block(
"cycle",
repeat1($.filtered_value),
optional(seq("as", field("name", $.identifier))),
optional("silent")
),
debug: ($) => block("debug"),
extends: ($) => block("extends", $.filtered_value),
filter_group: ($) =>
seq(block("filter", $.filter_expression), optional($.template), block("endfilter")),
firstof: ($) => block("firstof", repeat1($.filtered_value), optional(seq("as", $.identifier))),
for_group: ($) =>
seq(
block(
"for",
field("variables", $.identifier),
repeat(seq(",", field("variables", $.identifier))),
"in",
$.filtered_value
),
optional($.template),
optional(seq(block("empty"), optional($.template))),
block("endfor")
),
if_group: ($) =>
seq(
block("if", $.predicate),
optional($.template),
repeat(seq(block("elif", $.predicate), optional($.template))),
optional(seq(block("else"), optional($.template))),
block("endif")
),
ifchanged_group: ($) =>
seq(
block("ifchanged", repeat($.filtered_value)),
optional($.template),
block("endifchanged")
),
include: ($) => block("include", $.filtered_value),
load: ($) =>
block(
"load",
choice(
repeat1($.variable_attribute),
seq(repeat1($.identifier), "from", $.variable_attribute)
)
),
lorem: ($) => block("lorem", $.filtered_value, choice("w", "p", "b"), optional("random")),
now: ($) => block("now", $.string, optional(seq("as", field("variable", $.identifier)))),
partial: ($) => block("partial", field("name", $.identifier)),
partialdef_group: ($) =>
seq(
block("partialdef", field("name", $.push_partial), optional("inline")),
optional($.template),
block("endpartialdef", $.pop_partial)
),
query_string: ($) =>
block("querystring", repeat($.identifier), repeat(seq($.identifier, "=", $.filtered_value))),
regroup: ($) =>
block(
"regroup",
$.filtered_value,
"by",
$.attribute,
optional(seq("as", field("variable", $.identifier)))
),
reset_cycle: ($) => block("resetcycle", optional($.identifier)),
spaceless_group: ($) => seq(block("spaceless"), optional($.template), block("endspaceless")),
template_tag_block: ($) =>
block(
"templatetag",
choice(
"openblock",
"closeblock",
"openvariable",
"closevariable",
"openbrace",
"closebrace",
"opencomment",
"closecomment"
)
),
url_block: $ => block(
"url",
choice($.string, $.identifier),
optional(
choice(
repeat1($.filtered_value),
repeat1(seq($.identifier, '=', $.filtered_value)),
),
),
optional(
seq("as", field("variable", $.identifier)),
),
),
verbatim_group: ($) =>
seq(
block("verbatim", field("name", $.push_verbatim)),
/.*/,
prec(1, block("endverbatim", $.pop_verbatim))
),
with_group: ($) =>
seq(
block("with", field("variables", repeat1(seq(field("name", $.identifier), "=", $.filtered_value)))),
optional($.template),
block("endwith")
),
},
});
export default django;
package-lock.json
Generated
+1172
View File
File diff suppressed because it is too large Load Diff
package.json
+27
View File
@@ -0,0 +1,27 @@
{
"name": "tree-sitter-django",
"version": "0.1.0",
"private": true,
"license": "MIT",
"type": "module",
"scripts": {
"lint": "eslint",
"lint:fix": "eslint --fix",
"format": "prettier --write .",
"format:check": "prettier --check .",
"parser-test": "tree-sitter test",
"parser-generate": "tree-sitter generate",
"parser-build": "tree-sitter build --wasm",
"playground": "tree-sitter playground"
},
"dependencies": {
"web-tree-sitter": "^0.26.12"
},
"devDependencies": {
"eslint": "^9",
"eslint-config-prettier": "^10.1.8",
"prettier": "^3.9.6",
"tree-sitter-cli": "^0.26.12",
"typescript": "^5"
}
}
queries/highlights.scm
+29
View File
@@ -0,0 +1,29 @@
; Identifiers
(cycle name:
name: (identifier) @variable)
(block_group
name: (push_block) @block_name)
(partialdef_group
name: (push_partial) @partial_name)
(verbatim_group
name: (push_verbatim) @verbatim_name)
(with_group
variables: (_
name: (identifier) @variable))
; Literals
(number) @number
(string) @string
; Constants
(template_block_groups
tag: (_) @tag)
(filter . (_) @filter)
src/grammar.json
+2711
View File
File diff suppressed because it is too large Load Diff
src/node-types.json
+1694
View File
File diff suppressed because it is too large Load Diff
src/parser.c
+17128
View File
File diff suppressed because it is too large Load Diff
src/scanner.c
+430
View File
@@ -0,0 +1,430 @@
#include "tree_sitter/parser.h"
#include "tree_sitter/alloc.h"
#include "tree_sitter/array.h"
#define ERROR_SIZE 64
#define STACK_COUNT 3
#define NAME_SEP ' '
#define STACK_SEP '\n'
enum TokenType {
/* push name on stack to match with later */
PopBlock, // [ name ] %}
PopPartial, // [ name ] %}
PopVerbatim, // [ name ] %}
/* pop name when done with it */
PushBlock, // name %}
PushPartial, // name [ inline ] %}
PushVerbatim, // [ name ] %}
/* indicates that an error occurred */
MatcherError,
};
typedef Array(int32_t) Name;
typedef Array(Name) Stack;
struct Scanner {
bool has_error;
union {
Stack stacks [STACK_COUNT];
char error [ERROR_SIZE];
};
};
static Stack* get_stack_for_token(struct Scanner *scanner, enum TokenType token) {
if (token >= MatcherError) {
return NULL;
}
return &scanner->stacks[token % STACK_COUNT];
}
static void reset_scanner(struct Scanner *const scanner) {
if (scanner->has_error) {
scanner->error[0] = '\0';
scanner->has_error = false;
} else {
for (unsigned i = 0; i < STACK_COUNT; ++i) {
Stack *stack = &scanner->stacks[i];
for (unsigned i = 0; i < stack->size; ++i) {
array_delete(array_get(stack, i));
}
array_delete(stack);
}
}
}
#define scanner_error(scanner, string) do {\
struct Scanner *_scanner = scanner;\
reset_scanner(_scanner);\
_scanner->has_error = true;\
_scanner->error[0] = '\0';\
/*strncat(_scanner->error, string, ERROR_SIZE - 1);*/\
} while(0)
#define min(a, b)\
({\
typeof(a) _a = (a);\
typeof(b) _b = (b);\
_a > _b ? _a : _b;\
})
void * tree_sitter_django_external_scanner_create() {
return ts_calloc(1, sizeof(struct Scanner));
}
void tree_sitter_django_external_scanner_destroy(void *payload) {
struct Scanner *scanner = (struct Scanner*) payload;
reset_scanner(scanner);
ts_free(payload);
}
static unsigned write_code(int32_t code, char *out) {
if (code <= 0x7F) {
// 1 byte: 0xxxxxxx
out[0] = code;
return 1;
}
else if (code <= 0x7FF) {
// 2 bytes: 110xxxxx 10xxxxxx
out[0] = (code >> 6) | 0xC0;
out[1] = (code & 0x3F) | 0x80;
return 2;
}
else if (code <= 0xFFFF) {
// 3 bytes: 1110xxxx 10xxxxxx 10xxxxxx
// Note: Code points U+D800 to U+DFFF are reserved for UTF-16 surrogates and are invalid
if (code >= 0xD800 && code <= 0xDFFF) {
return 0;
}
out[0] = (code >> 12) | 0xE0;
out[1] = ((code >> 6) & 0x3F) | 0x80;
out[2] = (code & 0x3F) | 0x80;
return 3;
}
else if (code <= 0x10FFFF) {
// 4 bytes: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
out[0] = (code >> 18) | 0xF0;
out[1] = ((code >> 12) & 0x3F) | 0x80;
out[2] = ((code >> 6) & 0x3F) | 0x80;
out[3] = (code & 0x3F) | 0x80;
return 4;
}
return 0; // Out of Unicode range
}
unsigned tree_sitter_django_external_scanner_serialize(
void *payload,
char *const buffer
) {
struct Scanner *scanner = (struct Scanner*) payload;
write_serialization_error:;
char *iter = buffer;
iter += write_code(scanner->has_error + '0', iter);
if (scanner->has_error) {
for (int i = 0; i < ERROR_SIZE; ++i) {
char value = scanner->error[i];
*(iter++) = value;
if (value == '\0') {
break;
}
}
} else {
// Ok scanner requires copying its stack
for (unsigned i = 0; i < STACK_COUNT; ++i) {
Stack *stack = &scanner->stacks[i];
for (unsigned i = 0; i < stack->size; ++i) {
Name *name = array_get(stack, i);
for (unsigned j = 0; j < name->size; ++j) {
int32_t code = *array_get(name, j);
unsigned write_amt = write_code(code, iter);
if (write_amt == 0) {
scanner_error(scanner, "bad code from name");
goto write_serialization_error;
}
iter += write_amt;
}
iter += write_code(NAME_SEP, iter);
}
iter += write_code(STACK_SEP, iter);
}
}
return iter - buffer;
}
static unsigned read_code(const char *iter, int32_t *result) {
unsigned char byte0 = iter[0];
if (byte0 <= 0x7F) {
// 1 byte: 0xxxxxxx
*result = byte0;
return 1;
} else if ((byte0 & 0xE0) == 0xC0) {
// 2 bytes: 110xxxxx 10xxxxxx
*result = ((byte0 & 0x1F) << 6) | (iter[1] & 0x3F);
return 2;
} else if ((byte0 & 0xF0) == 0xE0) {
// 3 bytes: 1110xxxx 10xxxxxx 10xxxxxx
*result = ((byte0 & 0x0F) << 12) | ((iter[1] & 0x3F) << 6) | (iter[2] & 0x3F);
return 3;
} else if ((byte0 & 0xF8) == 0xF0) {
// 4 bytes: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
*result = ((byte0 & 0x07) << 18) | ((iter[1] & 0x3F) << 12) | ((iter[2] & 0x3F) << 6) | (iter[3] & 0x3F);
return 4;
}
return 0; // invalid UTF-8 leading byte
}
static bool check_name_start_char(const int32_t letter) {
return 'a' <= letter && letter <= 'z' || 'A' <= letter && letter <= 'Z';
}
static bool check_name_char(const int32_t letter) {
return check_name_start_char(letter) || '0' <= letter && letter <= '9' || letter == '_';
}
void tree_sitter_django_external_scanner_deserialize(
void *payload,
const char *buffer,
unsigned length
) {
struct Scanner *scanner = (struct Scanner*) payload;
reset_scanner(scanner);
if (length == 0) {
return;
}
int32_t code;
unsigned bytes_read = read_code(buffer, &code);
if (!bytes_read) {
scanner_error(scanner, "received invalid utf8 byte");
return;
}
if (bytes_read > length) {
scanner_error(scanner, "status code truncated by length");
return;
}
buffer += bytes_read;
switch (code) {
default:
scanner_error(scanner, "received unrecognized scanner status");
return;
case '1': {
scanner->has_error = true;
for (unsigned i = 0; i < length - bytes_read; ++i) {
char value = *(buffer++);
scanner->error[i] = value;
if (value == '\0') {
break;
}
}
return;
}
case '0': {
scanner->has_error = false;
const char *end = buffer + length - bytes_read;
Stack *stack = scanner->stacks;
Stack *stack_end = stack + STACK_COUNT;
Name *name = NULL;
while (stack < stack_end) {
bytes_read = read_code(buffer, &code);
if (bytes_read == 0) {
scanner_error(scanner, "Bad utf8 byte");
return;
}
buffer += bytes_read;
if (buffer > end) {
scanner_error(scanner, "Scanner truncated by length");
return;
}
if (code == STACK_SEP) {
++stack;
name = NULL;
continue;
} else if (code == NAME_SEP) {
if (name == NULL) {
scanner_error(scanner, "Empty name read");
return;
}
name = NULL;
} else {
if (name == NULL) {
if (!check_name_start_char(code)) {
scanner_error(scanner, "Invalid name, must start with letter");
return;
}
array_push(stack, (Name) array_new());
name = array_back(stack);
} else if (!check_name_char(code)) {
scanner_error(scanner, "Invalid character in name");
return;
}
array_push(name, code);
}
}
if (buffer != end) {
scanner_error(scanner, "Scanner deserialization finished with left over length");
}
}
}
}
static bool read_name(TSLexer *const lexer, Name *const name) {
if (name->size == 0) {
if (!check_name_start_char(lexer->lookahead)) {
return false;
}
array_push(name, lexer->lookahead);
lexer->advance(lexer, false);
}
while (check_name_char(lexer->lookahead)) {
array_push(name, lexer->lookahead);
lexer->advance(lexer, false);
}
return true;
}
static unsigned check_name(TSLexer *const lexer, const Name *const name) {
for (unsigned i = 0; i < name->size; ++i) {
if (lexer->lookahead != name->contents[i]) {
return i;
}
lexer->advance(lexer, false);
}
return name->size;
}
static bool check_close_block(TSLexer *const lexer) {
while (true) {
switch (lexer->lookahead) {
case ' ':
case '\t':
lexer->advance(lexer, false);
case '%':
lexer->advance(lexer, false);
return lexer->lookahead == '}';
default:
return false;
}
}
}
const char inline_chars[] = "inline";
static bool check_inline(TSLexer *const lexer) {
for (unsigned i = 0; i < sizeof(inline_chars) - 1; ++i) {
if (lexer->lookahead != inline_chars[i]) {
return false;
}
lexer->advance(lexer, false);
}
return check_close_block(lexer);
}
bool tree_sitter_django_external_scanner_scan(
void *payload,
TSLexer *lexer,
const bool *valid_symbols
) {
struct Scanner *scanner = (struct Scanner*) payload;
if (scanner->has_error) {
lexer->log(lexer, "%s", scanner->error);
if (valid_symbols[MatcherError]) {
// scanner is an error state, flag problem by returning error token
lexer->result_symbol = MatcherError;
return true;
}
return false;
}
if (valid_symbols[MatcherError]) {
return false;
}
while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
lexer->advance(lexer, true);
}
bool is_empty = false;
if (lexer->lookahead == '%') {
lexer->mark_end(lexer);
lexer->advance(lexer, false);
if (lexer->lookahead != '}') {
return false;
}
is_empty = true;
}
// save most matched name in case we need to reread characters
Name *most_matched_name;
unsigned most_matched_amt = 0;
for (unsigned token = PopBlock; token < PushBlock; ++token) {
Stack *stack = get_stack_for_token(scanner, token);
if (valid_symbols[token] && stack->size > 0) {
Name *name = array_back(stack);
if (is_empty) {
// we matched a close block without an id
array_delete(name);
array_pop(stack);
lexer->result_symbol = token;
return true;
}
if (most_matched_amt > name->size) {
continue;
}
if (most_matched_amt > 0) {
for (int i = 0; i < most_matched_amt; ++i) {
if (*array_get(most_matched_name, i) != *array_get(name, i)) {
continue;
}
}
}
unsigned match_amt = check_name(lexer, name) + most_matched_amt;
bool has_next_char = name->size ? check_name_char(lexer->lookahead) : check_name_start_char(lexer->lookahead);
if (match_amt == name->size && !has_next_char) {
// name matches and there are no remaining name chars from lexer
lexer->mark_end(lexer);
if (check_close_block(lexer)) {
array_delete(name);
array_pop(stack);
lexer->result_symbol = token;
return true;
} else {
// bad close block means no matches
return false;
}
}
most_matched_amt = match_amt;
most_matched_name = name;
}
}
for (unsigned token = PushBlock; token < MatcherError; ++token) {
if (valid_symbols[token]) {
Stack *stack = get_stack_for_token(scanner, token);
if (is_empty && token == PushVerbatim) {
// use empty string for name to indicate empty push
array_push(stack, (Name) array_new());
lexer->result_symbol = token;
return true;
}
// do not push stack until we are done with most_matched_name
Name name = array_new();
if (most_matched_amt > 0) {
array_extend(&name, most_matched_amt, most_matched_name);
}
if (read_name(lexer, &name)) {
// validate tokens after name
lexer->mark_end(lexer);
if (check_close_block(lexer) || token == PushPartial && check_inline(lexer)) {
array_push(stack, name);
lexer->result_symbol = token;
return true;
}
}
// undo failed push
array_delete(&name);
array_pop(stack);
// single failed push implies failure to match any push
return false;
}
}
return false;
}
src/tree_sitter/alloc.h
+54
View File
@@ -0,0 +1,54 @@
#ifndef TREE_SITTER_ALLOC_H_
#define TREE_SITTER_ALLOC_H_
#ifdef __cplusplus
extern "C" {
#endif
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
// Allow clients to override allocation functions
#ifdef TREE_SITTER_REUSE_ALLOCATOR
extern void *(*ts_current_malloc)(size_t size);
extern void *(*ts_current_calloc)(size_t count, size_t size);
extern void *(*ts_current_realloc)(void *ptr, size_t size);
extern void (*ts_current_free)(void *ptr);
#ifndef ts_malloc
#define ts_malloc ts_current_malloc
#endif
#ifndef ts_calloc
#define ts_calloc ts_current_calloc
#endif
#ifndef ts_realloc
#define ts_realloc ts_current_realloc
#endif
#ifndef ts_free
#define ts_free ts_current_free
#endif
#else
#ifndef ts_malloc
#define ts_malloc malloc
#endif
#ifndef ts_calloc
#define ts_calloc calloc
#endif
#ifndef ts_realloc
#define ts_realloc realloc
#endif
#ifndef ts_free
#define ts_free free
#endif
#endif
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_ALLOC_H_
src/tree_sitter/array.h
+336
View File
@@ -0,0 +1,336 @@
#ifndef TREE_SITTER_ARRAY_H_
#define TREE_SITTER_ARRAY_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "./alloc.h"
#include <assert.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#ifdef _MSC_VER
#pragma warning(push)
#pragma warning(disable : 4101)
#elif defined(__GNUC__) || defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-variable"
#endif
#define Array(T) \
struct { \
T *contents; \
uint32_t size; \
uint32_t capacity; \
}
/// Initialize an array.
#define array_init(self) \
((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL)
/// Create an empty array.
#define array_new() \
{ NULL, 0, 0 }
/// Get a pointer to the element at a given `index` in the array.
#define array_get(self, _index) \
(assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index])
/// Get a pointer to the first element in the array.
#define array_front(self) array_get(self, 0)
/// Get a pointer to the last element in the array.
#define array_back(self) array_get(self, (self)->size - 1)
/// Clear the array, setting its size to zero. Note that this does not free any
/// memory allocated for the array's contents.
#define array_clear(self) ((self)->size = 0)
#ifdef __cplusplus
#define _array__cast(self, expr) (decltype((self)->contents))(expr)
#else
#define _array__cast(self, expr) (expr)
#endif
/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
/// less than the array's current capacity, this function has no effect.
#define array_reserve(self, new_capacity) \
((self)->contents = _array__cast(self, _array__reserve( \
(void *)(self)->contents, &(self)->capacity, \
array_elem_size(self), new_capacity)) \
)
/// Free any memory allocated for this array. Note that this does not free any
/// memory allocated for the array's contents.
#define array_delete(self) \
do { \
if ((self)->contents) ts_free((self)->contents); \
(self)->contents = NULL; \
(self)->size = 0; \
(self)->capacity = 0; \
} while (0)
/// Push a new `element` onto the end of the array.
#define array_push(self, element) \
do { \
(self)->contents = _array__cast(self, _array__grow( \
(void *)(self)->contents, (self)->size, &(self)->capacity, \
1, array_elem_size(self) \
)); \
(self)->contents[(self)->size++] = (element); \
} while(0)
/// Increase the array's size by `count` elements.
/// New elements are zero-initialized.
#define array_grow_by(self, count) \
do { \
if ((count) == 0) break; \
(self)->contents = _array__cast(self, _array__grow( \
(self)->contents, (self)->size, &(self)->capacity, \
count, array_elem_size(self) \
)); \
memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
(self)->size += (count); \
} while (0)
/// Append all elements from one array to the end of another.
#define array_push_all(self, other) \
array_extend((self), (other)->size, (other)->contents)
/// Append `count` elements to the end of the array, reading their values from the
/// `contents` pointer.
#define array_extend(self, count, other_contents) \
((self)->contents = _array__cast(self, _array__splice( \
(void*)(self)->contents, &(self)->size, &(self)->capacity, \
array_elem_size(self), (self)->size, 0, count, other_contents \
)))
/// Remove `old_count` elements from the array starting at the given `index`. At
/// the same index, insert `new_count` new elements, reading their values from the
/// `new_contents` pointer.
#define array_splice(self, _index, old_count, new_count, new_contents) \
((self)->contents = _array__cast(self, _array__splice( \
(void *)(self)->contents, &(self)->size, &(self)->capacity, \
array_elem_size(self), _index, old_count, new_count, new_contents \
)))
/// Insert one `element` into the array at the given `index`.
#define array_insert(self, _index, element) \
((self)->contents = _array__cast(self, _array__splice( \
(void *)(self)->contents, &(self)->size, &(self)->capacity, \
array_elem_size(self), _index, 0, 1, &(element) \
)))
/// Remove one element from the array at the given `index`.
#define array_erase(self, _index) \
_array__erase((void *)(self)->contents, &(self)->size, array_elem_size(self), _index)
/// Pop the last element off the array, returning the element by value.
#define array_pop(self) ((self)->contents[--(self)->size])
/// Assign the contents of one array to another, reallocating if necessary.
#define array_assign(self, other) \
((self)->contents = _array__cast(self, _array__assign( \
(void *)(self)->contents, &(self)->size, &(self)->capacity, \
(const void *)(other)->contents, (other)->size, array_elem_size(self) \
)))
/// Swap one array with another
#define array_swap(self, other) \
do { \
void *_array_swap_tmp = (void *)(self)->contents; \
(self)->contents = (other)->contents; \
(other)->contents = _array__cast(other, _array_swap_tmp); \
_array__swap(&(self)->size, &(self)->capacity, \
&(other)->size, &(other)->capacity); \
} while (0)
/// Get the size of the array contents
#define array_elem_size(self) (sizeof *(self)->contents)
/// Search a sorted array for a given `needle` value, using the given `compare`
/// callback to determine the order.
///
/// If an existing element is found to be equal to `needle`, then the `index`
/// out-parameter is set to the existing value's index, and the `exists`
/// out-parameter is set to true. Otherwise, `index` is set to an index where
/// `needle` should be inserted in order to preserve the sorting, and `exists`
/// is set to false.
#define array_search_sorted_with(self, compare, needle, _index, _exists) \
_array__search_sorted(self, 0, compare, , needle, _index, _exists)
/// Search a sorted array for a given `needle` value, using integer comparisons
/// of a given struct field (specified with a leading dot) to determine the order.
///
/// See also `array_search_sorted_with`.
#define array_search_sorted_by(self, field, needle, _index, _exists) \
_array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists)
/// Insert a given `value` into a sorted array, using the given `compare`
/// callback to determine the order.
#define array_insert_sorted_with(self, compare, value) \
do { \
unsigned _index, _exists; \
array_search_sorted_with(self, compare, &(value), &_index, &_exists); \
if (!_exists) array_insert(self, _index, value); \
} while (0)
/// Insert a given `value` into a sorted array, using integer comparisons of
/// a given struct field (specified with a leading dot) to determine the order.
///
/// See also `array_search_sorted_by`.
#define array_insert_sorted_by(self, field, value) \
do { \
unsigned _index, _exists; \
array_search_sorted_by(self, field, (value) field, &_index, &_exists); \
if (!_exists) array_insert(self, _index, value); \
} while (0)
// Private
// Pointers to individual `Array` fields (rather than the entire `Array` itself)
// are passed to the various `_array__*` functions below to address strict aliasing
// violations that arises when the _entire_ `Array` struct is passed as `Array(void)*`.
//
// The `Array` type itself was not altered as a solution in order to avoid breakage
// with existing consumers (in particular, parsers with external scanners).
/// This is not what you're looking for, see `array_erase`.
static inline void _array__erase(void* self_contents, uint32_t *size,
size_t element_size, uint32_t index) {
assert(index < *size);
char *contents = (char *)self_contents;
memmove(contents + index * element_size, contents + (index + 1) * element_size,
(*size - index - 1) * element_size);
(*size)--;
}
/// This is not what you're looking for, see `array_reserve`.
static inline void *_array__reserve(void *contents, uint32_t *capacity,
size_t element_size, uint32_t new_capacity) {
void *new_contents = contents;
if (new_capacity > *capacity) {
if (contents) {
new_contents = ts_realloc(contents, new_capacity * element_size);
} else {
new_contents = ts_malloc(new_capacity * element_size);
}
*capacity = new_capacity;
}
return new_contents;
}
/// This is not what you're looking for, see `array_assign`.
static inline void *_array__assign(void* self_contents, uint32_t *self_size, uint32_t *self_capacity,
const void *other_contents, uint32_t other_size, size_t element_size) {
void *new_contents = _array__reserve(self_contents, self_capacity, element_size, other_size);
*self_size = other_size;
memcpy(new_contents, other_contents, *self_size * element_size);
return new_contents;
}
/// This is not what you're looking for, see `array_swap`.
static inline void _array__swap(uint32_t *self_size, uint32_t *self_capacity,
uint32_t *other_size, uint32_t *other_capacity) {
uint32_t tmp_size = *self_size;
uint32_t tmp_capacity = *self_capacity;
*self_size = *other_size;
*self_capacity = *other_capacity;
*other_size = tmp_size;
*other_capacity = tmp_capacity;
}
/// This is not what you're looking for, see `array_push` or `array_grow_by`.
static inline void *_array__grow(void *contents, uint32_t size, uint32_t *capacity,
uint32_t count, size_t element_size) {
void *new_contents = contents;
uint32_t new_size = size + count;
if (new_size > *capacity) {
uint32_t new_capacity = *capacity * 2;
if (new_capacity < 8) new_capacity = 8;
if (new_capacity < new_size) new_capacity = new_size;
new_contents = _array__reserve(contents, capacity, element_size, new_capacity);
}
return new_contents;
}
/// This is not what you're looking for, see `array_splice`.
static inline void *_array__splice(void *self_contents, uint32_t *size, uint32_t *capacity,
size_t element_size,
uint32_t index, uint32_t old_count,
uint32_t new_count, const void *elements) {
uint32_t new_size = *size + new_count - old_count;
uint32_t old_end = index + old_count;
uint32_t new_end = index + new_count;
assert(old_end <= *size);
void *new_contents = _array__reserve(self_contents, capacity, element_size, new_size);
char *contents = (char *)new_contents;
if (*size > old_end) {
memmove(
contents + new_end * element_size,
contents + old_end * element_size,
(*size - old_end) * element_size
);
}
if (new_count > 0) {
if (elements) {
memcpy(
(contents + index * element_size),
elements,
new_count * element_size
);
} else {
memset(
(contents + index * element_size),
0,
new_count * element_size
);
}
}
*size += new_count - old_count;
return new_contents;
}
/// A binary search routine, based on Rust's `std::slice::binary_search_by`.
/// This is not what you're looking for, see `array_search_sorted_with` or `array_search_sorted_by`.
#define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \
do { \
*(_index) = start; \
*(_exists) = false; \
uint32_t size = (self)->size - *(_index); \
if (size == 0) break; \
int comparison; \
while (size > 1) { \
uint32_t half_size = size / 2; \
uint32_t mid_index = *(_index) + half_size; \
comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \
if (comparison <= 0) *(_index) = mid_index; \
size -= half_size; \
} \
comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \
if (comparison == 0) *(_exists) = true; \
else if (comparison < 0) *(_index) += 1; \
} while (0)
/// Helper macro for the `_sorted_by` routines below. This takes the left (existing)
/// parameter by reference in order to work with the generic sorting function above.
#define _compare_int(a, b) ((int)*(a) - (int)(b))
#ifdef _MSC_VER
#pragma warning(pop)
#elif defined(__GNUC__) || defined(__clang__)
#pragma GCC diagnostic pop
#endif
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_ARRAY_H_
src/tree_sitter/parser.h
+286
View File
@@ -0,0 +1,286 @@
#ifndef TREE_SITTER_PARSER_H_
#define TREE_SITTER_PARSER_H_
#ifdef __cplusplus
extern "C" {
#endif
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#define ts_builtin_sym_error ((TSSymbol)-1)
#define ts_builtin_sym_end 0
#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
#ifndef TREE_SITTER_API_H_
typedef uint16_t TSStateId;
typedef uint16_t TSSymbol;
typedef uint16_t TSFieldId;
typedef struct TSLanguage TSLanguage;
typedef struct TSLanguageMetadata {
uint8_t major_version;
uint8_t minor_version;
uint8_t patch_version;
} TSLanguageMetadata;
#endif
typedef struct {
TSFieldId field_id;
uint8_t child_index;
bool inherited;
} TSFieldMapEntry;
// Used to index the field and supertype maps.
typedef struct {
uint16_t index;
uint16_t length;
} TSMapSlice;
typedef struct {
bool visible;
bool named;
bool supertype;
} TSSymbolMetadata;
typedef struct TSLexer TSLexer;
struct TSLexer {
int32_t lookahead;
TSSymbol result_symbol;
void (*advance)(TSLexer *, bool);
void (*mark_end)(TSLexer *);
uint32_t (*get_column)(TSLexer *);
bool (*is_at_included_range_start)(const TSLexer *);
bool (*eof)(const TSLexer *);
void (*log)(const TSLexer *, const char *, ...);
};
typedef enum {
TSParseActionTypeShift,
TSParseActionTypeReduce,
TSParseActionTypeAccept,
TSParseActionTypeRecover,
} TSParseActionType;
typedef union {
struct {
uint8_t type;
TSStateId state;
bool extra;
bool repetition;
} shift;
struct {
uint8_t type;
uint8_t child_count;
TSSymbol symbol;
int16_t dynamic_precedence;
uint16_t production_id;
} reduce;
uint8_t type;
} TSParseAction;
typedef struct {
uint16_t lex_state;
uint16_t external_lex_state;
} TSLexMode;
typedef struct {
uint16_t lex_state;
uint16_t external_lex_state;
uint16_t reserved_word_set_id;
} TSLexerMode;
typedef union {
TSParseAction action;
struct {
uint8_t count;
bool reusable;
} entry;
} TSParseActionEntry;
typedef struct {
int32_t start;
int32_t end;
} TSCharacterRange;
struct TSLanguage {
uint32_t abi_version;
uint32_t symbol_count;
uint32_t alias_count;
uint32_t token_count;
uint32_t external_token_count;
uint32_t state_count;
uint32_t large_state_count;
uint32_t production_id_count;
uint32_t field_count;
uint16_t max_alias_sequence_length;
const uint16_t *parse_table;
const uint16_t *small_parse_table;
const uint32_t *small_parse_table_map;
const TSParseActionEntry *parse_actions;
const char * const *symbol_names;
const char * const *field_names;
const TSMapSlice *field_map_slices;
const TSFieldMapEntry *field_map_entries;
const TSSymbolMetadata *symbol_metadata;
const TSSymbol *public_symbol_map;
const uint16_t *alias_map;
const TSSymbol *alias_sequences;
const TSLexerMode *lex_modes;
bool (*lex_fn)(TSLexer *, TSStateId);
bool (*keyword_lex_fn)(TSLexer *, TSStateId);
TSSymbol keyword_capture_token;
struct {
const bool *states;
const TSSymbol *symbol_map;
void *(*create)(void);
void (*destroy)(void *);
bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist);
unsigned (*serialize)(void *, char *);
void (*deserialize)(void *, const char *, unsigned);
} external_scanner;
const TSStateId *primary_state_ids;
const char *name;
const TSSymbol *reserved_words;
uint16_t max_reserved_word_set_size;
uint32_t supertype_count;
const TSSymbol *supertype_symbols;
const TSMapSlice *supertype_map_slices;
const TSSymbol *supertype_map_entries;
TSLanguageMetadata metadata;
};
static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
uint32_t index = 0;
uint32_t size = len - index;
while (size > 1) {
uint32_t half_size = size / 2;
uint32_t mid_index = index + half_size;
const TSCharacterRange *range = &ranges[mid_index];
if (lookahead >= range->start && lookahead <= range->end) {
return true;
} else if (lookahead > range->end) {
index = mid_index;
}
size -= half_size;
}
const TSCharacterRange *range = &ranges[index];
return (lookahead >= range->start && lookahead <= range->end);
}
/*
* Lexer Macros
*/
#ifdef _MSC_VER
#define UNUSED __pragma(warning(suppress : 4101))
#else
#define UNUSED __attribute__((unused))
#endif
#define START_LEXER() \
bool result = false; \
bool skip = false; \
UNUSED \
bool eof = false; \
int32_t lookahead; \
goto start; \
next_state: \
lexer->advance(lexer, skip); \
start: \
skip = false; \
lookahead = lexer->lookahead;
#define ADVANCE(state_value) \
{ \
state = state_value; \
goto next_state; \
}
#define ADVANCE_MAP(...) \
{ \
static const uint16_t map[] = { __VA_ARGS__ }; \
for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) { \
if (map[i] == lookahead) { \
state = map[i + 1]; \
goto next_state; \
} \
} \
}
#define SKIP(state_value) \
{ \
skip = true; \
state = state_value; \
goto next_state; \
}
#define ACCEPT_TOKEN(symbol_value) \
result = true; \
lexer->result_symbol = symbol_value; \
lexer->mark_end(lexer);
#define END_STATE() return result;
/*
* Parse Table Macros
*/
#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT)
#define STATE(id) id
#define ACTIONS(id) id
#define SHIFT(state_value) \
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.state = (state_value) \
} \
}}
#define SHIFT_REPEAT(state_value) \
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.state = (state_value), \
.repetition = true \
} \
}}
#define SHIFT_EXTRA() \
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.extra = true \
} \
}}
#define REDUCE(symbol_name, children, precedence, prod_id) \
{{ \
.reduce = { \
.type = TSParseActionTypeReduce, \
.symbol = symbol_name, \
.child_count = children, \
.dynamic_precedence = precedence, \
.production_id = prod_id \
}, \
}}
#define RECOVER() \
{{ \
.type = TSParseActionTypeRecover \
}}
#define ACCEPT_INPUT() \
{{ \
.type = TSParseActionTypeAccept \
}}
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_PARSER_H_
test/corpus/autoescape.txt
+22
View File
@@ -0,0 +1,22 @@
==================
Autoescape
==================
<div>
<ul>
<li>one</li>
{% autoescape off %}
<li>two</li>
{% endautoescape %}
<li>tree</li>
</ul>
</div>
---
(template
(content)
(autoescape_group
(template
(content)))
(content))
test/corpus/block.txt
+165
View File
@@ -0,0 +1,165 @@
==================
Block
==================
<section>
<h1>This is a block</h1>
{% block test %}
<main>
<h2>Hello World</h2>
</main>
{% endblock %}
<footer>Goodbye</footer>
</section>
---
(template
(content)
(block_group
(push_block)
(template
(content))
(pop_block))
(content))
==================
Block Matching End Tag
==================
<section>
<h1>This is a block</h1>
{% block test %}
<main>
<h2>Hello World</h2>
</main>
{% endblock test %}
<footer>Goodbye</footer>
</section>
---
(template
(content)
(block_group
(push_block)
(template
(content))
(pop_block))
(content))
==================
Nested Blocks
==================
<section>
<h1>This is a block</h1>
{% block test %}
<main>
{% block hello %}
<h2>Hello World</h2>
{% endblock %}
</main>
{% endblock %}
<footer>Goodbye</footer>
</section>
---
(template
(content)
(block_group
(push_block)
(template
(content)
(block_group
(push_block)
(template
(content))
(pop_block))
(content))
(pop_block))
(content))
---
(template
(content)
(block_group
(push_block)
(template
(content)
(block_group
(push_block)
(template
(content))
(pop_block))
(content))
(pop_block))
(content))
==================
Nested Blocks End Tag
==================
<section>
<h1>This is a block</h1>
{% block test %}
<main>
{% block hello %}
<h2>Hello World</h2>
{% endblock hello %}
</main>
{% endblock test %}
<footer>Goodbye</footer>
</section>
---
(template
(content)
(block_group
(push_block)
(template
(content)
(block_group
(push_block)
(template
(content))
(pop_block))
(content))
(pop_block))
(content))
==================
Nested Blocks Mixed Tags
==================
<section>
<h1>This is a block</h1>
{% block test %}
<main>
{% block hello %}
<h2>Hello World</h2>
{% endblock %}
</main>
{% endblock test %}
<footer>Goodbye</footer>
</section>
---
(template
(content)
(block_group
(push_block)
(template
(content)
(block_group
(push_block)
(template
(content))
(pop_block))
(content))
(pop_block))
(content))
test/corpus/csrf_token.txt
+14
View File
@@ -0,0 +1,14 @@
==================
Csrf Token
==================
<form method="post">
{% csrf_token %}
</form>
---
(template
(content)
(csrf_token)
(content))
test/corpus/cycle.txt
+93
View File
@@ -0,0 +1,93 @@
==================
Cycle Basic
==================
<table>
<tr class="{% cycle 'row1' 'row2' %}">
</tr>
</table>
---
(template
(content)
(cycle
(filtered_value
(value
(literal
(string))))
(filtered_value
(value
(literal
(string)))))
(content))
==================
Cycle With As
==================
<table>
<tr class="{% cycle 'row1' 'row2' as rowcolors %}">
</tr>
</table>
---
(template
(content)
(cycle
(filtered_value
(value
(literal
(string))))
(filtered_value
(value
(literal
(string))))
name: (identifier))
(content))
==================
Cycle With As And Silent
==================
<table>
<tr class="{% cycle 'row1' 'row2' as rowcolors silent %}">
</tr>
</table>
---
(template
(content)
(cycle
(filtered_value
(value
(literal
(string))))
(filtered_value
(value
(literal
(string))))
name: (identifier))
(content))
==================
Cycle Referencing Existing Variable
==================
<table>
<tr class="{% cycle rowcolors %}">
</tr>
</table>
---
(template
(content)
(cycle
(filtered_value
(value
(variable_attribute
(identifier)))))
(content))
test/corpus/debug.txt
+14
View File
@@ -0,0 +1,14 @@
==================
Debug
==================
<p>
{% debug %}
</p>
---
(template
(content)
(debug)
(content))
test/corpus/extends.txt
+35
View File
@@ -0,0 +1,35 @@
==================
Extends With String
==================
{% extends "base.html" %}
<h1>Hello</h1>
---
(template
(content)
(extends
(filtered_value
(value
(literal
(string)))))
(content))
==================
Extends With Variable
==================
{% extends base_template %}
<h1>Hello</h1>
---
(template
(content)
(extends
(filtered_value
(value
(variable_attribute
(identifier)))))
(content))
test/highlights/example.html
+27
View File
@@ -0,0 +1,27 @@
<body>
{# hello this is an example file #}
<header>
<h1>{{ page.title|title }}</h1>
<nav>
{% if page.links|length > 0 %}
<ul>
{% for label, link in page.links %}
<li>
{% url 'nav-pages' link as href %}
<a href="{{href}}">{{label}}</a>
</li>
{% endfor %}
</ul>
{% endif %}
</nav>
</header>
{% block content %}
{% endblock %}
<footer>
{% for foot in page.footer %}
{% endfor %}
</footer>
</body>
tree-sitter.json
+28
View File
@@ -0,0 +1,28 @@
{
"$schema": "https://tree-sitter.github.io/tree-sitter/assets/schemas/config.schema.json",
"grammars": [
{
"name": "django",
"camelcase": "django",
"title": "Django Parser",
"scope": "source.django",
"file-types": ["html", "txt", "md"],
"injection-regex": "^django$",
"class-name": "TreeSitterDjango"
}
],
"bindings": {
"node": true
},
"metadata": {
"version": "0.0.1",
"description": "DJango grammar for tree-sitter",
"authors": [{
"name": "Daniel Hanson",
"url": "https://github.com/danhanson"
}],
"links": {
"repository": "https://github.com/danhanson/tree-sitter-django"
}
}
}