initial commit
This commit is contained in:
@@ -0,0 +1 @@
|
||||
node_modules
|
||||
@@ -0,0 +1,13 @@
|
||||
# tree-sitter-django
|
||||
|
||||
A [tree-sitter](https://tree-sitter.github.io/tree-sitter/) grammar for the Django Template Language (DTL) — the `{% %}` tags and `{{ }}` variables used in Django templates.
|
||||
|
||||
## Development
|
||||
|
||||
```sh
|
||||
npm install
|
||||
npm run parser-generate # regenerate the parser from grammar.js
|
||||
npm run parser-test # run the corpus tests in test/corpus
|
||||
npm run parser-build # build the WASM binary
|
||||
npm run playground # interactively explore the grammar
|
||||
```
|
||||
@@ -0,0 +1,20 @@
|
||||
import { defineConfig, globalIgnores } from "eslint/config";
|
||||
import eslintConfigPrettier from "eslint-config-prettier";
|
||||
|
||||
const eslintConfig = defineConfig([
|
||||
eslintConfigPrettier, // Disable ESLint rules that conflict with Prettier
|
||||
{
|
||||
rules: {
|
||||
"@typescript-eslint/no-unused-vars": "error",
|
||||
"react-hooks/exhaustive-deps": "error",
|
||||
},
|
||||
},
|
||||
// Override default ignores of eslint-config-next.
|
||||
globalIgnores([
|
||||
// Default ignores of eslint-config-next:
|
||||
"out/**",
|
||||
"build/**",
|
||||
]),
|
||||
]);
|
||||
|
||||
export default eslintConfig;
|
||||
+281
@@ -0,0 +1,281 @@
|
||||
/* eslint-disable @typescript-eslint/no-unused-vars */
|
||||
/// <reference types="tree-sitter-cli/dsl" />
|
||||
// @ts-check
|
||||
|
||||
/**
|
||||
* @param {string} tag
|
||||
* @param {...RuleOrLiteral} args
|
||||
* @returns {SeqRule}
|
||||
*/
|
||||
function block(tag, ...args) {
|
||||
return seq("{%", field("tag", tag), ...args, "%}");
|
||||
}
|
||||
|
||||
const django = grammar({
|
||||
name: "django",
|
||||
extras: ($) => [/[ \t]/],
|
||||
conflicts: ($) => [[$.template]],
|
||||
supertypes: ($) => [$.template_tag, $.template_block_groups],
|
||||
externals: ($) => [
|
||||
$.pop_block,
|
||||
$.pop_partial,
|
||||
$.pop_verbatim,
|
||||
$.push_block,
|
||||
$.push_partial,
|
||||
$.push_verbatim,
|
||||
$.matcher_error,
|
||||
],
|
||||
reserved: {
|
||||
global: ($) => ["not", "if", "in", "is", "as"],
|
||||
},
|
||||
rules: {
|
||||
template: ($) => repeat1(choice($.template_tag, $.content)),
|
||||
content: ($) => /(?:[^\{]|\{[^\{#%}])+/,
|
||||
template_tag: ($) => choice($.template_block_groups, $.template_variable, $.template_comment),
|
||||
filtered_value: ($) => seq(
|
||||
$.value,
|
||||
optional(
|
||||
seq(
|
||||
token.immediate("|"),
|
||||
$.filter_expression
|
||||
),
|
||||
),
|
||||
),
|
||||
filter_expression: ($) => seq($.filter, repeat(seq(token.immediate("|"), $.filter))),
|
||||
value: ($) => choice($.literal, $.variable_attribute),
|
||||
literal: ($) => choice($.number, $.string),
|
||||
number: ($) => /-?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE]-?[0-9]+)?/,
|
||||
string: ($) => /"(?:[^"\\]|\\\\|\\")*"|'(?:[^'\\]|\\\\|\\')*'/,
|
||||
attribute: ($) => /[a-zA-Z0-9_]+(?:\.[a-zA-Z0-9_]+)*/,
|
||||
variable_attribute: ($) => seq($.identifier, optional(seq(token.immediate('.'), $.attribute))),
|
||||
identifier: ($) => /[a-zA-Z][a-zA-Z0-9_]+/,
|
||||
binaryOperator: ($) =>
|
||||
choice(
|
||||
"and",
|
||||
"or",
|
||||
"==",
|
||||
"!=",
|
||||
"<",
|
||||
">",
|
||||
"<=",
|
||||
">=",
|
||||
"in",
|
||||
prec(1, seq("not", "in")),
|
||||
"is",
|
||||
prec(1, seq("is", "not"))
|
||||
),
|
||||
predicate: ($) =>
|
||||
seq(
|
||||
optional("not"),
|
||||
$.filtered_value,
|
||||
repeat(seq($.binaryOperator, optional("not"), $.filtered_value))
|
||||
),
|
||||
template_variable: ($) => seq("{{", $.filtered_value, "}}"),
|
||||
template_comment: ($) => seq("{#", /(?:[^#]|#[^}])*/, "#}"),
|
||||
template_block_groups: ($) =>
|
||||
choice(
|
||||
$.autoescape_group,
|
||||
$.block_group,
|
||||
$.csrf_token,
|
||||
$.cycle,
|
||||
$.debug,
|
||||
$.extends,
|
||||
$.filter_group,
|
||||
$.firstof,
|
||||
$.for_group,
|
||||
$.if_group,
|
||||
$.ifchanged_group,
|
||||
$.include,
|
||||
$.lorem,
|
||||
$.now,
|
||||
$.partial,
|
||||
$.partialdef_group,
|
||||
$.query_string,
|
||||
$.regroup,
|
||||
$.reset_cycle,
|
||||
$.spaceless_group,
|
||||
$.template_tag_block,
|
||||
$.url_block,
|
||||
$.verbatim_group,
|
||||
$.with_group
|
||||
),
|
||||
filter: ($) => choice(
|
||||
seq("add:", $.value),
|
||||
"addslashes",
|
||||
"capfirst",
|
||||
seq("center:", $.value),
|
||||
seq("cut:", $.value),
|
||||
choice(seq("date:", $.value), "date"),
|
||||
seq("default:", $.value),
|
||||
seq("default_if_none:", $.value),
|
||||
seq("dictsort:", $.value),
|
||||
seq("dictsortreversed:", $.value),
|
||||
seq("divisibleby:", $.value),
|
||||
"escape",
|
||||
"escapejs",
|
||||
"filesizeformat",
|
||||
"first",
|
||||
choice(seq("floatformat:", $.value), "floatformat"),
|
||||
"force_escape",
|
||||
seq("get_digit:", $.value),
|
||||
"iriencode",
|
||||
seq("join:", $.value),
|
||||
choice(seq("json_script:", $.value), "json_script"),
|
||||
"last",
|
||||
"length",
|
||||
seq("length_is:", $.value),
|
||||
"linebreaks",
|
||||
"linebreaksbr",
|
||||
"linenumbers",
|
||||
seq("ljust:", $.value),
|
||||
"lower",
|
||||
"make_list",
|
||||
"phone2numeric",
|
||||
choice(seq("pluralize:", $.value), "pluralize"),
|
||||
"pprint",
|
||||
"random",
|
||||
seq("rjust:", $.value),
|
||||
"safe",
|
||||
"safeseq",
|
||||
seq("slice:", $.value),
|
||||
"slugify",
|
||||
seq("stringformat:", $.value),
|
||||
"striptags",
|
||||
choice(seq("time:", $.value), "time"),
|
||||
choice(seq("timesince:", $.value), "timesince"),
|
||||
choice(seq("timeuntil:", $.value), "timeuntil"),
|
||||
"title",
|
||||
seq("truncatechars:", $.value),
|
||||
seq("truncatechars_html:", $.value),
|
||||
seq("truncatewords:", $.value),
|
||||
seq("truncatewords_html:", $.value),
|
||||
"unordered_list",
|
||||
choice(seq("urlencode:", $.value), "urlencode"),
|
||||
"urlize",
|
||||
seq("urlizetrunc:", $.value),
|
||||
"wordcount",
|
||||
seq("wordwrap:", $.value),
|
||||
choice(seq("yesno:", $.value), "yesno"),
|
||||
),
|
||||
autoescape_group: ($) =>
|
||||
seq(block("autoescape", choice("on", "off")), optional($.template), block("endautoescape")),
|
||||
block_group: ($) =>
|
||||
seq(
|
||||
block("block", field("name", $.push_block)),
|
||||
optional($.template),
|
||||
block("endblock", $.pop_block)
|
||||
),
|
||||
comment: ($) => seq(block("comment", optional($.string)), /.*/, prec(1, block("endcomment"))),
|
||||
csrf_token: ($) => block("csrf_token"),
|
||||
cycle: ($) =>
|
||||
block(
|
||||
"cycle",
|
||||
repeat1($.filtered_value),
|
||||
optional(seq("as", field("name", $.identifier))),
|
||||
optional("silent")
|
||||
),
|
||||
debug: ($) => block("debug"),
|
||||
extends: ($) => block("extends", $.filtered_value),
|
||||
filter_group: ($) =>
|
||||
seq(block("filter", $.filter_expression), optional($.template), block("endfilter")),
|
||||
firstof: ($) => block("firstof", repeat1($.filtered_value), optional(seq("as", $.identifier))),
|
||||
for_group: ($) =>
|
||||
seq(
|
||||
block(
|
||||
"for",
|
||||
field("variables", $.identifier),
|
||||
repeat(seq(",", field("variables", $.identifier))),
|
||||
"in",
|
||||
$.filtered_value
|
||||
),
|
||||
optional($.template),
|
||||
optional(seq(block("empty"), optional($.template))),
|
||||
block("endfor")
|
||||
),
|
||||
if_group: ($) =>
|
||||
seq(
|
||||
block("if", $.predicate),
|
||||
optional($.template),
|
||||
repeat(seq(block("elif", $.predicate), optional($.template))),
|
||||
optional(seq(block("else"), optional($.template))),
|
||||
block("endif")
|
||||
),
|
||||
ifchanged_group: ($) =>
|
||||
seq(
|
||||
block("ifchanged", repeat($.filtered_value)),
|
||||
optional($.template),
|
||||
block("endifchanged")
|
||||
),
|
||||
include: ($) => block("include", $.filtered_value),
|
||||
load: ($) =>
|
||||
block(
|
||||
"load",
|
||||
choice(
|
||||
repeat1($.variable_attribute),
|
||||
seq(repeat1($.identifier), "from", $.variable_attribute)
|
||||
)
|
||||
),
|
||||
lorem: ($) => block("lorem", $.filtered_value, choice("w", "p", "b"), optional("random")),
|
||||
now: ($) => block("now", $.string, optional(seq("as", field("variable", $.identifier)))),
|
||||
partial: ($) => block("partial", field("name", $.identifier)),
|
||||
partialdef_group: ($) =>
|
||||
seq(
|
||||
block("partialdef", field("name", $.push_partial), optional("inline")),
|
||||
optional($.template),
|
||||
block("endpartialdef", $.pop_partial)
|
||||
),
|
||||
query_string: ($) =>
|
||||
block("querystring", repeat($.identifier), repeat(seq($.identifier, "=", $.filtered_value))),
|
||||
regroup: ($) =>
|
||||
block(
|
||||
"regroup",
|
||||
$.filtered_value,
|
||||
"by",
|
||||
$.attribute,
|
||||
optional(seq("as", field("variable", $.identifier)))
|
||||
),
|
||||
reset_cycle: ($) => block("resetcycle", optional($.identifier)),
|
||||
spaceless_group: ($) => seq(block("spaceless"), optional($.template), block("endspaceless")),
|
||||
template_tag_block: ($) =>
|
||||
block(
|
||||
"templatetag",
|
||||
choice(
|
||||
"openblock",
|
||||
"closeblock",
|
||||
"openvariable",
|
||||
"closevariable",
|
||||
"openbrace",
|
||||
"closebrace",
|
||||
"opencomment",
|
||||
"closecomment"
|
||||
)
|
||||
),
|
||||
url_block: $ => block(
|
||||
"url",
|
||||
choice($.string, $.identifier),
|
||||
optional(
|
||||
choice(
|
||||
repeat1($.filtered_value),
|
||||
repeat1(seq($.identifier, '=', $.filtered_value)),
|
||||
),
|
||||
),
|
||||
optional(
|
||||
seq("as", field("variable", $.identifier)),
|
||||
),
|
||||
),
|
||||
verbatim_group: ($) =>
|
||||
seq(
|
||||
block("verbatim", field("name", $.push_verbatim)),
|
||||
/.*/,
|
||||
prec(1, block("endverbatim", $.pop_verbatim))
|
||||
),
|
||||
with_group: ($) =>
|
||||
seq(
|
||||
block("with", field("variables", repeat1(seq(field("name", $.identifier), "=", $.filtered_value)))),
|
||||
optional($.template),
|
||||
block("endwith")
|
||||
),
|
||||
},
|
||||
});
|
||||
|
||||
export default django;
|
||||
Generated
+1172
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"name": "tree-sitter-django",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"license": "MIT",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"lint": "eslint",
|
||||
"lint:fix": "eslint --fix",
|
||||
"format": "prettier --write .",
|
||||
"format:check": "prettier --check .",
|
||||
"parser-test": "tree-sitter test",
|
||||
"parser-generate": "tree-sitter generate",
|
||||
"parser-build": "tree-sitter build --wasm",
|
||||
"playground": "tree-sitter playground"
|
||||
},
|
||||
"dependencies": {
|
||||
"web-tree-sitter": "^0.26.12"
|
||||
},
|
||||
"devDependencies": {
|
||||
"eslint": "^9",
|
||||
"eslint-config-prettier": "^10.1.8",
|
||||
"prettier": "^3.9.6",
|
||||
"tree-sitter-cli": "^0.26.12",
|
||||
"typescript": "^5"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
; Identifiers
|
||||
|
||||
(cycle name:
|
||||
name: (identifier) @variable)
|
||||
|
||||
(block_group
|
||||
name: (push_block) @block_name)
|
||||
|
||||
(partialdef_group
|
||||
name: (push_partial) @partial_name)
|
||||
|
||||
(verbatim_group
|
||||
name: (push_verbatim) @verbatim_name)
|
||||
|
||||
(with_group
|
||||
variables: (_
|
||||
name: (identifier) @variable))
|
||||
|
||||
; Literals
|
||||
|
||||
(number) @number
|
||||
(string) @string
|
||||
|
||||
; Constants
|
||||
|
||||
(template_block_groups
|
||||
tag: (_) @tag)
|
||||
|
||||
(filter . (_) @filter)
|
||||
+2711
File diff suppressed because it is too large
Load Diff
+1694
File diff suppressed because it is too large
Load Diff
+17128
File diff suppressed because it is too large
Load Diff
+430
@@ -0,0 +1,430 @@
|
||||
#include "tree_sitter/parser.h"
|
||||
#include "tree_sitter/alloc.h"
|
||||
#include "tree_sitter/array.h"
|
||||
|
||||
#define ERROR_SIZE 64
|
||||
#define STACK_COUNT 3
|
||||
#define NAME_SEP ' '
|
||||
#define STACK_SEP '\n'
|
||||
|
||||
enum TokenType {
|
||||
/* push name on stack to match with later */
|
||||
PopBlock, // [ name ] %}
|
||||
PopPartial, // [ name ] %}
|
||||
PopVerbatim, // [ name ] %}
|
||||
/* pop name when done with it */
|
||||
PushBlock, // name %}
|
||||
PushPartial, // name [ inline ] %}
|
||||
PushVerbatim, // [ name ] %}
|
||||
/* indicates that an error occurred */
|
||||
MatcherError,
|
||||
};
|
||||
|
||||
typedef Array(int32_t) Name;
|
||||
typedef Array(Name) Stack;
|
||||
|
||||
struct Scanner {
|
||||
bool has_error;
|
||||
union {
|
||||
Stack stacks [STACK_COUNT];
|
||||
char error [ERROR_SIZE];
|
||||
};
|
||||
};
|
||||
|
||||
static Stack* get_stack_for_token(struct Scanner *scanner, enum TokenType token) {
|
||||
if (token >= MatcherError) {
|
||||
return NULL;
|
||||
}
|
||||
return &scanner->stacks[token % STACK_COUNT];
|
||||
}
|
||||
|
||||
static void reset_scanner(struct Scanner *const scanner) {
|
||||
if (scanner->has_error) {
|
||||
scanner->error[0] = '\0';
|
||||
scanner->has_error = false;
|
||||
} else {
|
||||
for (unsigned i = 0; i < STACK_COUNT; ++i) {
|
||||
Stack *stack = &scanner->stacks[i];
|
||||
for (unsigned i = 0; i < stack->size; ++i) {
|
||||
array_delete(array_get(stack, i));
|
||||
}
|
||||
array_delete(stack);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#define scanner_error(scanner, string) do {\
|
||||
struct Scanner *_scanner = scanner;\
|
||||
reset_scanner(_scanner);\
|
||||
_scanner->has_error = true;\
|
||||
_scanner->error[0] = '\0';\
|
||||
/*strncat(_scanner->error, string, ERROR_SIZE - 1);*/\
|
||||
} while(0)
|
||||
|
||||
#define min(a, b)\
|
||||
({\
|
||||
typeof(a) _a = (a);\
|
||||
typeof(b) _b = (b);\
|
||||
_a > _b ? _a : _b;\
|
||||
})
|
||||
|
||||
void * tree_sitter_django_external_scanner_create() {
|
||||
return ts_calloc(1, sizeof(struct Scanner));
|
||||
}
|
||||
|
||||
void tree_sitter_django_external_scanner_destroy(void *payload) {
|
||||
struct Scanner *scanner = (struct Scanner*) payload;
|
||||
reset_scanner(scanner);
|
||||
ts_free(payload);
|
||||
}
|
||||
|
||||
static unsigned write_code(int32_t code, char *out) {
|
||||
if (code <= 0x7F) {
|
||||
// 1 byte: 0xxxxxxx
|
||||
out[0] = code;
|
||||
return 1;
|
||||
}
|
||||
else if (code <= 0x7FF) {
|
||||
// 2 bytes: 110xxxxx 10xxxxxx
|
||||
out[0] = (code >> 6) | 0xC0;
|
||||
out[1] = (code & 0x3F) | 0x80;
|
||||
return 2;
|
||||
}
|
||||
else if (code <= 0xFFFF) {
|
||||
// 3 bytes: 1110xxxx 10xxxxxx 10xxxxxx
|
||||
// Note: Code points U+D800 to U+DFFF are reserved for UTF-16 surrogates and are invalid
|
||||
if (code >= 0xD800 && code <= 0xDFFF) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
out[0] = (code >> 12) | 0xE0;
|
||||
out[1] = ((code >> 6) & 0x3F) | 0x80;
|
||||
out[2] = (code & 0x3F) | 0x80;
|
||||
return 3;
|
||||
}
|
||||
else if (code <= 0x10FFFF) {
|
||||
// 4 bytes: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
|
||||
out[0] = (code >> 18) | 0xF0;
|
||||
out[1] = ((code >> 12) & 0x3F) | 0x80;
|
||||
out[2] = ((code >> 6) & 0x3F) | 0x80;
|
||||
out[3] = (code & 0x3F) | 0x80;
|
||||
return 4;
|
||||
}
|
||||
return 0; // Out of Unicode range
|
||||
}
|
||||
|
||||
unsigned tree_sitter_django_external_scanner_serialize(
|
||||
void *payload,
|
||||
char *const buffer
|
||||
) {
|
||||
struct Scanner *scanner = (struct Scanner*) payload;
|
||||
|
||||
write_serialization_error:;
|
||||
char *iter = buffer;
|
||||
|
||||
iter += write_code(scanner->has_error + '0', iter);
|
||||
if (scanner->has_error) {
|
||||
for (int i = 0; i < ERROR_SIZE; ++i) {
|
||||
char value = scanner->error[i];
|
||||
*(iter++) = value;
|
||||
if (value == '\0') {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Ok scanner requires copying its stack
|
||||
for (unsigned i = 0; i < STACK_COUNT; ++i) {
|
||||
Stack *stack = &scanner->stacks[i];
|
||||
for (unsigned i = 0; i < stack->size; ++i) {
|
||||
Name *name = array_get(stack, i);
|
||||
for (unsigned j = 0; j < name->size; ++j) {
|
||||
int32_t code = *array_get(name, j);
|
||||
unsigned write_amt = write_code(code, iter);
|
||||
if (write_amt == 0) {
|
||||
scanner_error(scanner, "bad code from name");
|
||||
goto write_serialization_error;
|
||||
}
|
||||
iter += write_amt;
|
||||
}
|
||||
iter += write_code(NAME_SEP, iter);
|
||||
}
|
||||
iter += write_code(STACK_SEP, iter);
|
||||
}
|
||||
}
|
||||
return iter - buffer;
|
||||
}
|
||||
|
||||
static unsigned read_code(const char *iter, int32_t *result) {
|
||||
unsigned char byte0 = iter[0];
|
||||
if (byte0 <= 0x7F) {
|
||||
// 1 byte: 0xxxxxxx
|
||||
*result = byte0;
|
||||
return 1;
|
||||
} else if ((byte0 & 0xE0) == 0xC0) {
|
||||
// 2 bytes: 110xxxxx 10xxxxxx
|
||||
*result = ((byte0 & 0x1F) << 6) | (iter[1] & 0x3F);
|
||||
return 2;
|
||||
} else if ((byte0 & 0xF0) == 0xE0) {
|
||||
// 3 bytes: 1110xxxx 10xxxxxx 10xxxxxx
|
||||
*result = ((byte0 & 0x0F) << 12) | ((iter[1] & 0x3F) << 6) | (iter[2] & 0x3F);
|
||||
return 3;
|
||||
} else if ((byte0 & 0xF8) == 0xF0) {
|
||||
// 4 bytes: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
|
||||
*result = ((byte0 & 0x07) << 18) | ((iter[1] & 0x3F) << 12) | ((iter[2] & 0x3F) << 6) | (iter[3] & 0x3F);
|
||||
return 4;
|
||||
}
|
||||
return 0; // invalid UTF-8 leading byte
|
||||
}
|
||||
|
||||
static bool check_name_start_char(const int32_t letter) {
|
||||
return 'a' <= letter && letter <= 'z' || 'A' <= letter && letter <= 'Z';
|
||||
}
|
||||
|
||||
static bool check_name_char(const int32_t letter) {
|
||||
return check_name_start_char(letter) || '0' <= letter && letter <= '9' || letter == '_';
|
||||
}
|
||||
|
||||
void tree_sitter_django_external_scanner_deserialize(
|
||||
void *payload,
|
||||
const char *buffer,
|
||||
unsigned length
|
||||
) {
|
||||
struct Scanner *scanner = (struct Scanner*) payload;
|
||||
reset_scanner(scanner);
|
||||
if (length == 0) {
|
||||
return;
|
||||
}
|
||||
int32_t code;
|
||||
unsigned bytes_read = read_code(buffer, &code);
|
||||
if (!bytes_read) {
|
||||
scanner_error(scanner, "received invalid utf8 byte");
|
||||
return;
|
||||
}
|
||||
if (bytes_read > length) {
|
||||
scanner_error(scanner, "status code truncated by length");
|
||||
return;
|
||||
}
|
||||
buffer += bytes_read;
|
||||
switch (code) {
|
||||
default:
|
||||
scanner_error(scanner, "received unrecognized scanner status");
|
||||
return;
|
||||
case '1': {
|
||||
scanner->has_error = true;
|
||||
for (unsigned i = 0; i < length - bytes_read; ++i) {
|
||||
char value = *(buffer++);
|
||||
scanner->error[i] = value;
|
||||
if (value == '\0') {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
case '0': {
|
||||
scanner->has_error = false;
|
||||
const char *end = buffer + length - bytes_read;
|
||||
Stack *stack = scanner->stacks;
|
||||
Stack *stack_end = stack + STACK_COUNT;
|
||||
Name *name = NULL;
|
||||
|
||||
while (stack < stack_end) {
|
||||
bytes_read = read_code(buffer, &code);
|
||||
if (bytes_read == 0) {
|
||||
scanner_error(scanner, "Bad utf8 byte");
|
||||
return;
|
||||
}
|
||||
buffer += bytes_read;
|
||||
if (buffer > end) {
|
||||
scanner_error(scanner, "Scanner truncated by length");
|
||||
return;
|
||||
}
|
||||
if (code == STACK_SEP) {
|
||||
++stack;
|
||||
name = NULL;
|
||||
continue;
|
||||
} else if (code == NAME_SEP) {
|
||||
if (name == NULL) {
|
||||
scanner_error(scanner, "Empty name read");
|
||||
return;
|
||||
}
|
||||
name = NULL;
|
||||
} else {
|
||||
if (name == NULL) {
|
||||
if (!check_name_start_char(code)) {
|
||||
scanner_error(scanner, "Invalid name, must start with letter");
|
||||
return;
|
||||
}
|
||||
array_push(stack, (Name) array_new());
|
||||
name = array_back(stack);
|
||||
} else if (!check_name_char(code)) {
|
||||
scanner_error(scanner, "Invalid character in name");
|
||||
return;
|
||||
}
|
||||
array_push(name, code);
|
||||
}
|
||||
}
|
||||
if (buffer != end) {
|
||||
scanner_error(scanner, "Scanner deserialization finished with left over length");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static bool read_name(TSLexer *const lexer, Name *const name) {
|
||||
if (name->size == 0) {
|
||||
if (!check_name_start_char(lexer->lookahead)) {
|
||||
return false;
|
||||
}
|
||||
array_push(name, lexer->lookahead);
|
||||
lexer->advance(lexer, false);
|
||||
}
|
||||
while (check_name_char(lexer->lookahead)) {
|
||||
array_push(name, lexer->lookahead);
|
||||
lexer->advance(lexer, false);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static unsigned check_name(TSLexer *const lexer, const Name *const name) {
|
||||
for (unsigned i = 0; i < name->size; ++i) {
|
||||
if (lexer->lookahead != name->contents[i]) {
|
||||
return i;
|
||||
}
|
||||
lexer->advance(lexer, false);
|
||||
}
|
||||
return name->size;
|
||||
}
|
||||
|
||||
static bool check_close_block(TSLexer *const lexer) {
|
||||
while (true) {
|
||||
switch (lexer->lookahead) {
|
||||
case ' ':
|
||||
case '\t':
|
||||
lexer->advance(lexer, false);
|
||||
case '%':
|
||||
lexer->advance(lexer, false);
|
||||
return lexer->lookahead == '}';
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const char inline_chars[] = "inline";
|
||||
|
||||
static bool check_inline(TSLexer *const lexer) {
|
||||
for (unsigned i = 0; i < sizeof(inline_chars) - 1; ++i) {
|
||||
if (lexer->lookahead != inline_chars[i]) {
|
||||
return false;
|
||||
}
|
||||
lexer->advance(lexer, false);
|
||||
}
|
||||
return check_close_block(lexer);
|
||||
}
|
||||
|
||||
bool tree_sitter_django_external_scanner_scan(
|
||||
void *payload,
|
||||
TSLexer *lexer,
|
||||
const bool *valid_symbols
|
||||
) {
|
||||
struct Scanner *scanner = (struct Scanner*) payload;
|
||||
if (scanner->has_error) {
|
||||
lexer->log(lexer, "%s", scanner->error);
|
||||
if (valid_symbols[MatcherError]) {
|
||||
// scanner is an error state, flag problem by returning error token
|
||||
lexer->result_symbol = MatcherError;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
if (valid_symbols[MatcherError]) {
|
||||
return false;
|
||||
}
|
||||
while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
|
||||
lexer->advance(lexer, true);
|
||||
}
|
||||
bool is_empty = false;
|
||||
if (lexer->lookahead == '%') {
|
||||
lexer->mark_end(lexer);
|
||||
lexer->advance(lexer, false);
|
||||
if (lexer->lookahead != '}') {
|
||||
return false;
|
||||
}
|
||||
is_empty = true;
|
||||
}
|
||||
|
||||
// save most matched name in case we need to reread characters
|
||||
Name *most_matched_name;
|
||||
unsigned most_matched_amt = 0;
|
||||
for (unsigned token = PopBlock; token < PushBlock; ++token) {
|
||||
Stack *stack = get_stack_for_token(scanner, token);
|
||||
if (valid_symbols[token] && stack->size > 0) {
|
||||
Name *name = array_back(stack);
|
||||
if (is_empty) {
|
||||
// we matched a close block without an id
|
||||
array_delete(name);
|
||||
array_pop(stack);
|
||||
lexer->result_symbol = token;
|
||||
return true;
|
||||
}
|
||||
if (most_matched_amt > name->size) {
|
||||
continue;
|
||||
}
|
||||
if (most_matched_amt > 0) {
|
||||
for (int i = 0; i < most_matched_amt; ++i) {
|
||||
if (*array_get(most_matched_name, i) != *array_get(name, i)) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
unsigned match_amt = check_name(lexer, name) + most_matched_amt;
|
||||
bool has_next_char = name->size ? check_name_char(lexer->lookahead) : check_name_start_char(lexer->lookahead);
|
||||
if (match_amt == name->size && !has_next_char) {
|
||||
// name matches and there are no remaining name chars from lexer
|
||||
lexer->mark_end(lexer);
|
||||
if (check_close_block(lexer)) {
|
||||
array_delete(name);
|
||||
array_pop(stack);
|
||||
lexer->result_symbol = token;
|
||||
return true;
|
||||
} else {
|
||||
// bad close block means no matches
|
||||
return false;
|
||||
}
|
||||
}
|
||||
most_matched_amt = match_amt;
|
||||
most_matched_name = name;
|
||||
}
|
||||
}
|
||||
for (unsigned token = PushBlock; token < MatcherError; ++token) {
|
||||
if (valid_symbols[token]) {
|
||||
Stack *stack = get_stack_for_token(scanner, token);
|
||||
if (is_empty && token == PushVerbatim) {
|
||||
// use empty string for name to indicate empty push
|
||||
array_push(stack, (Name) array_new());
|
||||
lexer->result_symbol = token;
|
||||
return true;
|
||||
}
|
||||
// do not push stack until we are done with most_matched_name
|
||||
Name name = array_new();
|
||||
if (most_matched_amt > 0) {
|
||||
array_extend(&name, most_matched_amt, most_matched_name);
|
||||
}
|
||||
if (read_name(lexer, &name)) {
|
||||
// validate tokens after name
|
||||
lexer->mark_end(lexer);
|
||||
if (check_close_block(lexer) || token == PushPartial && check_inline(lexer)) {
|
||||
array_push(stack, name);
|
||||
lexer->result_symbol = token;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
// undo failed push
|
||||
array_delete(&name);
|
||||
array_pop(stack);
|
||||
// single failed push implies failure to match any push
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
#ifndef TREE_SITTER_ALLOC_H_
|
||||
#define TREE_SITTER_ALLOC_H_
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
// Allow clients to override allocation functions
|
||||
#ifdef TREE_SITTER_REUSE_ALLOCATOR
|
||||
|
||||
extern void *(*ts_current_malloc)(size_t size);
|
||||
extern void *(*ts_current_calloc)(size_t count, size_t size);
|
||||
extern void *(*ts_current_realloc)(void *ptr, size_t size);
|
||||
extern void (*ts_current_free)(void *ptr);
|
||||
|
||||
#ifndef ts_malloc
|
||||
#define ts_malloc ts_current_malloc
|
||||
#endif
|
||||
#ifndef ts_calloc
|
||||
#define ts_calloc ts_current_calloc
|
||||
#endif
|
||||
#ifndef ts_realloc
|
||||
#define ts_realloc ts_current_realloc
|
||||
#endif
|
||||
#ifndef ts_free
|
||||
#define ts_free ts_current_free
|
||||
#endif
|
||||
|
||||
#else
|
||||
|
||||
#ifndef ts_malloc
|
||||
#define ts_malloc malloc
|
||||
#endif
|
||||
#ifndef ts_calloc
|
||||
#define ts_calloc calloc
|
||||
#endif
|
||||
#ifndef ts_realloc
|
||||
#define ts_realloc realloc
|
||||
#endif
|
||||
#ifndef ts_free
|
||||
#define ts_free free
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif // TREE_SITTER_ALLOC_H_
|
||||
@@ -0,0 +1,336 @@
|
||||
#ifndef TREE_SITTER_ARRAY_H_
|
||||
#define TREE_SITTER_ARRAY_H_
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "./alloc.h"
|
||||
|
||||
#include <assert.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#pragma warning(push)
|
||||
#pragma warning(disable : 4101)
|
||||
#elif defined(__GNUC__) || defined(__clang__)
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wunused-variable"
|
||||
#endif
|
||||
|
||||
#define Array(T) \
|
||||
struct { \
|
||||
T *contents; \
|
||||
uint32_t size; \
|
||||
uint32_t capacity; \
|
||||
}
|
||||
|
||||
/// Initialize an array.
|
||||
#define array_init(self) \
|
||||
((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL)
|
||||
|
||||
/// Create an empty array.
|
||||
#define array_new() \
|
||||
{ NULL, 0, 0 }
|
||||
|
||||
/// Get a pointer to the element at a given `index` in the array.
|
||||
#define array_get(self, _index) \
|
||||
(assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index])
|
||||
|
||||
/// Get a pointer to the first element in the array.
|
||||
#define array_front(self) array_get(self, 0)
|
||||
|
||||
/// Get a pointer to the last element in the array.
|
||||
#define array_back(self) array_get(self, (self)->size - 1)
|
||||
|
||||
/// Clear the array, setting its size to zero. Note that this does not free any
|
||||
/// memory allocated for the array's contents.
|
||||
#define array_clear(self) ((self)->size = 0)
|
||||
|
||||
#ifdef __cplusplus
|
||||
#define _array__cast(self, expr) (decltype((self)->contents))(expr)
|
||||
#else
|
||||
#define _array__cast(self, expr) (expr)
|
||||
#endif
|
||||
|
||||
/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
|
||||
/// less than the array's current capacity, this function has no effect.
|
||||
#define array_reserve(self, new_capacity) \
|
||||
((self)->contents = _array__cast(self, _array__reserve( \
|
||||
(void *)(self)->contents, &(self)->capacity, \
|
||||
array_elem_size(self), new_capacity)) \
|
||||
)
|
||||
|
||||
/// Free any memory allocated for this array. Note that this does not free any
|
||||
/// memory allocated for the array's contents.
|
||||
#define array_delete(self) \
|
||||
do { \
|
||||
if ((self)->contents) ts_free((self)->contents); \
|
||||
(self)->contents = NULL; \
|
||||
(self)->size = 0; \
|
||||
(self)->capacity = 0; \
|
||||
} while (0)
|
||||
|
||||
/// Push a new `element` onto the end of the array.
|
||||
#define array_push(self, element) \
|
||||
do { \
|
||||
(self)->contents = _array__cast(self, _array__grow( \
|
||||
(void *)(self)->contents, (self)->size, &(self)->capacity, \
|
||||
1, array_elem_size(self) \
|
||||
)); \
|
||||
(self)->contents[(self)->size++] = (element); \
|
||||
} while(0)
|
||||
|
||||
/// Increase the array's size by `count` elements.
|
||||
/// New elements are zero-initialized.
|
||||
#define array_grow_by(self, count) \
|
||||
do { \
|
||||
if ((count) == 0) break; \
|
||||
(self)->contents = _array__cast(self, _array__grow( \
|
||||
(self)->contents, (self)->size, &(self)->capacity, \
|
||||
count, array_elem_size(self) \
|
||||
)); \
|
||||
memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
|
||||
(self)->size += (count); \
|
||||
} while (0)
|
||||
|
||||
/// Append all elements from one array to the end of another.
|
||||
#define array_push_all(self, other) \
|
||||
array_extend((self), (other)->size, (other)->contents)
|
||||
|
||||
/// Append `count` elements to the end of the array, reading their values from the
|
||||
/// `contents` pointer.
|
||||
#define array_extend(self, count, other_contents) \
|
||||
((self)->contents = _array__cast(self, _array__splice( \
|
||||
(void*)(self)->contents, &(self)->size, &(self)->capacity, \
|
||||
array_elem_size(self), (self)->size, 0, count, other_contents \
|
||||
)))
|
||||
|
||||
/// Remove `old_count` elements from the array starting at the given `index`. At
|
||||
/// the same index, insert `new_count` new elements, reading their values from the
|
||||
/// `new_contents` pointer.
|
||||
#define array_splice(self, _index, old_count, new_count, new_contents) \
|
||||
((self)->contents = _array__cast(self, _array__splice( \
|
||||
(void *)(self)->contents, &(self)->size, &(self)->capacity, \
|
||||
array_elem_size(self), _index, old_count, new_count, new_contents \
|
||||
)))
|
||||
|
||||
/// Insert one `element` into the array at the given `index`.
|
||||
#define array_insert(self, _index, element) \
|
||||
((self)->contents = _array__cast(self, _array__splice( \
|
||||
(void *)(self)->contents, &(self)->size, &(self)->capacity, \
|
||||
array_elem_size(self), _index, 0, 1, &(element) \
|
||||
)))
|
||||
|
||||
/// Remove one element from the array at the given `index`.
|
||||
#define array_erase(self, _index) \
|
||||
_array__erase((void *)(self)->contents, &(self)->size, array_elem_size(self), _index)
|
||||
|
||||
/// Pop the last element off the array, returning the element by value.
|
||||
#define array_pop(self) ((self)->contents[--(self)->size])
|
||||
|
||||
/// Assign the contents of one array to another, reallocating if necessary.
|
||||
#define array_assign(self, other) \
|
||||
((self)->contents = _array__cast(self, _array__assign( \
|
||||
(void *)(self)->contents, &(self)->size, &(self)->capacity, \
|
||||
(const void *)(other)->contents, (other)->size, array_elem_size(self) \
|
||||
)))
|
||||
|
||||
/// Swap one array with another
|
||||
#define array_swap(self, other) \
|
||||
do { \
|
||||
void *_array_swap_tmp = (void *)(self)->contents; \
|
||||
(self)->contents = (other)->contents; \
|
||||
(other)->contents = _array__cast(other, _array_swap_tmp); \
|
||||
_array__swap(&(self)->size, &(self)->capacity, \
|
||||
&(other)->size, &(other)->capacity); \
|
||||
} while (0)
|
||||
|
||||
/// Get the size of the array contents
|
||||
#define array_elem_size(self) (sizeof *(self)->contents)
|
||||
|
||||
/// Search a sorted array for a given `needle` value, using the given `compare`
|
||||
/// callback to determine the order.
|
||||
///
|
||||
/// If an existing element is found to be equal to `needle`, then the `index`
|
||||
/// out-parameter is set to the existing value's index, and the `exists`
|
||||
/// out-parameter is set to true. Otherwise, `index` is set to an index where
|
||||
/// `needle` should be inserted in order to preserve the sorting, and `exists`
|
||||
/// is set to false.
|
||||
#define array_search_sorted_with(self, compare, needle, _index, _exists) \
|
||||
_array__search_sorted(self, 0, compare, , needle, _index, _exists)
|
||||
|
||||
/// Search a sorted array for a given `needle` value, using integer comparisons
|
||||
/// of a given struct field (specified with a leading dot) to determine the order.
|
||||
///
|
||||
/// See also `array_search_sorted_with`.
|
||||
#define array_search_sorted_by(self, field, needle, _index, _exists) \
|
||||
_array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists)
|
||||
|
||||
/// Insert a given `value` into a sorted array, using the given `compare`
|
||||
/// callback to determine the order.
|
||||
#define array_insert_sorted_with(self, compare, value) \
|
||||
do { \
|
||||
unsigned _index, _exists; \
|
||||
array_search_sorted_with(self, compare, &(value), &_index, &_exists); \
|
||||
if (!_exists) array_insert(self, _index, value); \
|
||||
} while (0)
|
||||
|
||||
/// Insert a given `value` into a sorted array, using integer comparisons of
|
||||
/// a given struct field (specified with a leading dot) to determine the order.
|
||||
///
|
||||
/// See also `array_search_sorted_by`.
|
||||
#define array_insert_sorted_by(self, field, value) \
|
||||
do { \
|
||||
unsigned _index, _exists; \
|
||||
array_search_sorted_by(self, field, (value) field, &_index, &_exists); \
|
||||
if (!_exists) array_insert(self, _index, value); \
|
||||
} while (0)
|
||||
|
||||
// Private
|
||||
|
||||
// Pointers to individual `Array` fields (rather than the entire `Array` itself)
|
||||
// are passed to the various `_array__*` functions below to address strict aliasing
|
||||
// violations that arises when the _entire_ `Array` struct is passed as `Array(void)*`.
|
||||
//
|
||||
// The `Array` type itself was not altered as a solution in order to avoid breakage
|
||||
// with existing consumers (in particular, parsers with external scanners).
|
||||
|
||||
/// This is not what you're looking for, see `array_erase`.
|
||||
static inline void _array__erase(void* self_contents, uint32_t *size,
|
||||
size_t element_size, uint32_t index) {
|
||||
assert(index < *size);
|
||||
char *contents = (char *)self_contents;
|
||||
memmove(contents + index * element_size, contents + (index + 1) * element_size,
|
||||
(*size - index - 1) * element_size);
|
||||
(*size)--;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_reserve`.
|
||||
static inline void *_array__reserve(void *contents, uint32_t *capacity,
|
||||
size_t element_size, uint32_t new_capacity) {
|
||||
void *new_contents = contents;
|
||||
if (new_capacity > *capacity) {
|
||||
if (contents) {
|
||||
new_contents = ts_realloc(contents, new_capacity * element_size);
|
||||
} else {
|
||||
new_contents = ts_malloc(new_capacity * element_size);
|
||||
}
|
||||
*capacity = new_capacity;
|
||||
}
|
||||
return new_contents;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_assign`.
|
||||
static inline void *_array__assign(void* self_contents, uint32_t *self_size, uint32_t *self_capacity,
|
||||
const void *other_contents, uint32_t other_size, size_t element_size) {
|
||||
void *new_contents = _array__reserve(self_contents, self_capacity, element_size, other_size);
|
||||
*self_size = other_size;
|
||||
memcpy(new_contents, other_contents, *self_size * element_size);
|
||||
return new_contents;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_swap`.
|
||||
static inline void _array__swap(uint32_t *self_size, uint32_t *self_capacity,
|
||||
uint32_t *other_size, uint32_t *other_capacity) {
|
||||
uint32_t tmp_size = *self_size;
|
||||
uint32_t tmp_capacity = *self_capacity;
|
||||
*self_size = *other_size;
|
||||
*self_capacity = *other_capacity;
|
||||
*other_size = tmp_size;
|
||||
*other_capacity = tmp_capacity;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_push` or `array_grow_by`.
|
||||
static inline void *_array__grow(void *contents, uint32_t size, uint32_t *capacity,
|
||||
uint32_t count, size_t element_size) {
|
||||
void *new_contents = contents;
|
||||
uint32_t new_size = size + count;
|
||||
if (new_size > *capacity) {
|
||||
uint32_t new_capacity = *capacity * 2;
|
||||
if (new_capacity < 8) new_capacity = 8;
|
||||
if (new_capacity < new_size) new_capacity = new_size;
|
||||
new_contents = _array__reserve(contents, capacity, element_size, new_capacity);
|
||||
}
|
||||
return new_contents;
|
||||
}
|
||||
|
||||
/// This is not what you're looking for, see `array_splice`.
|
||||
static inline void *_array__splice(void *self_contents, uint32_t *size, uint32_t *capacity,
|
||||
size_t element_size,
|
||||
uint32_t index, uint32_t old_count,
|
||||
uint32_t new_count, const void *elements) {
|
||||
uint32_t new_size = *size + new_count - old_count;
|
||||
uint32_t old_end = index + old_count;
|
||||
uint32_t new_end = index + new_count;
|
||||
assert(old_end <= *size);
|
||||
|
||||
void *new_contents = _array__reserve(self_contents, capacity, element_size, new_size);
|
||||
|
||||
char *contents = (char *)new_contents;
|
||||
if (*size > old_end) {
|
||||
memmove(
|
||||
contents + new_end * element_size,
|
||||
contents + old_end * element_size,
|
||||
(*size - old_end) * element_size
|
||||
);
|
||||
}
|
||||
if (new_count > 0) {
|
||||
if (elements) {
|
||||
memcpy(
|
||||
(contents + index * element_size),
|
||||
elements,
|
||||
new_count * element_size
|
||||
);
|
||||
} else {
|
||||
memset(
|
||||
(contents + index * element_size),
|
||||
0,
|
||||
new_count * element_size
|
||||
);
|
||||
}
|
||||
}
|
||||
*size += new_count - old_count;
|
||||
|
||||
return new_contents;
|
||||
}
|
||||
|
||||
/// A binary search routine, based on Rust's `std::slice::binary_search_by`.
|
||||
/// This is not what you're looking for, see `array_search_sorted_with` or `array_search_sorted_by`.
|
||||
#define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \
|
||||
do { \
|
||||
*(_index) = start; \
|
||||
*(_exists) = false; \
|
||||
uint32_t size = (self)->size - *(_index); \
|
||||
if (size == 0) break; \
|
||||
int comparison; \
|
||||
while (size > 1) { \
|
||||
uint32_t half_size = size / 2; \
|
||||
uint32_t mid_index = *(_index) + half_size; \
|
||||
comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \
|
||||
if (comparison <= 0) *(_index) = mid_index; \
|
||||
size -= half_size; \
|
||||
} \
|
||||
comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \
|
||||
if (comparison == 0) *(_exists) = true; \
|
||||
else if (comparison < 0) *(_index) += 1; \
|
||||
} while (0)
|
||||
|
||||
/// Helper macro for the `_sorted_by` routines below. This takes the left (existing)
|
||||
/// parameter by reference in order to work with the generic sorting function above.
|
||||
#define _compare_int(a, b) ((int)*(a) - (int)(b))
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#pragma warning(pop)
|
||||
#elif defined(__GNUC__) || defined(__clang__)
|
||||
#pragma GCC diagnostic pop
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif // TREE_SITTER_ARRAY_H_
|
||||
@@ -0,0 +1,286 @@
|
||||
#ifndef TREE_SITTER_PARSER_H_
|
||||
#define TREE_SITTER_PARSER_H_
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#define ts_builtin_sym_error ((TSSymbol)-1)
|
||||
#define ts_builtin_sym_end 0
|
||||
#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
|
||||
|
||||
#ifndef TREE_SITTER_API_H_
|
||||
typedef uint16_t TSStateId;
|
||||
typedef uint16_t TSSymbol;
|
||||
typedef uint16_t TSFieldId;
|
||||
typedef struct TSLanguage TSLanguage;
|
||||
typedef struct TSLanguageMetadata {
|
||||
uint8_t major_version;
|
||||
uint8_t minor_version;
|
||||
uint8_t patch_version;
|
||||
} TSLanguageMetadata;
|
||||
#endif
|
||||
|
||||
typedef struct {
|
||||
TSFieldId field_id;
|
||||
uint8_t child_index;
|
||||
bool inherited;
|
||||
} TSFieldMapEntry;
|
||||
|
||||
// Used to index the field and supertype maps.
|
||||
typedef struct {
|
||||
uint16_t index;
|
||||
uint16_t length;
|
||||
} TSMapSlice;
|
||||
|
||||
typedef struct {
|
||||
bool visible;
|
||||
bool named;
|
||||
bool supertype;
|
||||
} TSSymbolMetadata;
|
||||
|
||||
typedef struct TSLexer TSLexer;
|
||||
|
||||
struct TSLexer {
|
||||
int32_t lookahead;
|
||||
TSSymbol result_symbol;
|
||||
void (*advance)(TSLexer *, bool);
|
||||
void (*mark_end)(TSLexer *);
|
||||
uint32_t (*get_column)(TSLexer *);
|
||||
bool (*is_at_included_range_start)(const TSLexer *);
|
||||
bool (*eof)(const TSLexer *);
|
||||
void (*log)(const TSLexer *, const char *, ...);
|
||||
};
|
||||
|
||||
typedef enum {
|
||||
TSParseActionTypeShift,
|
||||
TSParseActionTypeReduce,
|
||||
TSParseActionTypeAccept,
|
||||
TSParseActionTypeRecover,
|
||||
} TSParseActionType;
|
||||
|
||||
typedef union {
|
||||
struct {
|
||||
uint8_t type;
|
||||
TSStateId state;
|
||||
bool extra;
|
||||
bool repetition;
|
||||
} shift;
|
||||
struct {
|
||||
uint8_t type;
|
||||
uint8_t child_count;
|
||||
TSSymbol symbol;
|
||||
int16_t dynamic_precedence;
|
||||
uint16_t production_id;
|
||||
} reduce;
|
||||
uint8_t type;
|
||||
} TSParseAction;
|
||||
|
||||
typedef struct {
|
||||
uint16_t lex_state;
|
||||
uint16_t external_lex_state;
|
||||
} TSLexMode;
|
||||
|
||||
typedef struct {
|
||||
uint16_t lex_state;
|
||||
uint16_t external_lex_state;
|
||||
uint16_t reserved_word_set_id;
|
||||
} TSLexerMode;
|
||||
|
||||
typedef union {
|
||||
TSParseAction action;
|
||||
struct {
|
||||
uint8_t count;
|
||||
bool reusable;
|
||||
} entry;
|
||||
} TSParseActionEntry;
|
||||
|
||||
typedef struct {
|
||||
int32_t start;
|
||||
int32_t end;
|
||||
} TSCharacterRange;
|
||||
|
||||
struct TSLanguage {
|
||||
uint32_t abi_version;
|
||||
uint32_t symbol_count;
|
||||
uint32_t alias_count;
|
||||
uint32_t token_count;
|
||||
uint32_t external_token_count;
|
||||
uint32_t state_count;
|
||||
uint32_t large_state_count;
|
||||
uint32_t production_id_count;
|
||||
uint32_t field_count;
|
||||
uint16_t max_alias_sequence_length;
|
||||
const uint16_t *parse_table;
|
||||
const uint16_t *small_parse_table;
|
||||
const uint32_t *small_parse_table_map;
|
||||
const TSParseActionEntry *parse_actions;
|
||||
const char * const *symbol_names;
|
||||
const char * const *field_names;
|
||||
const TSMapSlice *field_map_slices;
|
||||
const TSFieldMapEntry *field_map_entries;
|
||||
const TSSymbolMetadata *symbol_metadata;
|
||||
const TSSymbol *public_symbol_map;
|
||||
const uint16_t *alias_map;
|
||||
const TSSymbol *alias_sequences;
|
||||
const TSLexerMode *lex_modes;
|
||||
bool (*lex_fn)(TSLexer *, TSStateId);
|
||||
bool (*keyword_lex_fn)(TSLexer *, TSStateId);
|
||||
TSSymbol keyword_capture_token;
|
||||
struct {
|
||||
const bool *states;
|
||||
const TSSymbol *symbol_map;
|
||||
void *(*create)(void);
|
||||
void (*destroy)(void *);
|
||||
bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist);
|
||||
unsigned (*serialize)(void *, char *);
|
||||
void (*deserialize)(void *, const char *, unsigned);
|
||||
} external_scanner;
|
||||
const TSStateId *primary_state_ids;
|
||||
const char *name;
|
||||
const TSSymbol *reserved_words;
|
||||
uint16_t max_reserved_word_set_size;
|
||||
uint32_t supertype_count;
|
||||
const TSSymbol *supertype_symbols;
|
||||
const TSMapSlice *supertype_map_slices;
|
||||
const TSSymbol *supertype_map_entries;
|
||||
TSLanguageMetadata metadata;
|
||||
};
|
||||
|
||||
static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
|
||||
uint32_t index = 0;
|
||||
uint32_t size = len - index;
|
||||
while (size > 1) {
|
||||
uint32_t half_size = size / 2;
|
||||
uint32_t mid_index = index + half_size;
|
||||
const TSCharacterRange *range = &ranges[mid_index];
|
||||
if (lookahead >= range->start && lookahead <= range->end) {
|
||||
return true;
|
||||
} else if (lookahead > range->end) {
|
||||
index = mid_index;
|
||||
}
|
||||
size -= half_size;
|
||||
}
|
||||
const TSCharacterRange *range = &ranges[index];
|
||||
return (lookahead >= range->start && lookahead <= range->end);
|
||||
}
|
||||
|
||||
/*
|
||||
* Lexer Macros
|
||||
*/
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#define UNUSED __pragma(warning(suppress : 4101))
|
||||
#else
|
||||
#define UNUSED __attribute__((unused))
|
||||
#endif
|
||||
|
||||
#define START_LEXER() \
|
||||
bool result = false; \
|
||||
bool skip = false; \
|
||||
UNUSED \
|
||||
bool eof = false; \
|
||||
int32_t lookahead; \
|
||||
goto start; \
|
||||
next_state: \
|
||||
lexer->advance(lexer, skip); \
|
||||
start: \
|
||||
skip = false; \
|
||||
lookahead = lexer->lookahead;
|
||||
|
||||
#define ADVANCE(state_value) \
|
||||
{ \
|
||||
state = state_value; \
|
||||
goto next_state; \
|
||||
}
|
||||
|
||||
#define ADVANCE_MAP(...) \
|
||||
{ \
|
||||
static const uint16_t map[] = { __VA_ARGS__ }; \
|
||||
for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) { \
|
||||
if (map[i] == lookahead) { \
|
||||
state = map[i + 1]; \
|
||||
goto next_state; \
|
||||
} \
|
||||
} \
|
||||
}
|
||||
|
||||
#define SKIP(state_value) \
|
||||
{ \
|
||||
skip = true; \
|
||||
state = state_value; \
|
||||
goto next_state; \
|
||||
}
|
||||
|
||||
#define ACCEPT_TOKEN(symbol_value) \
|
||||
result = true; \
|
||||
lexer->result_symbol = symbol_value; \
|
||||
lexer->mark_end(lexer);
|
||||
|
||||
#define END_STATE() return result;
|
||||
|
||||
/*
|
||||
* Parse Table Macros
|
||||
*/
|
||||
|
||||
#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT)
|
||||
|
||||
#define STATE(id) id
|
||||
|
||||
#define ACTIONS(id) id
|
||||
|
||||
#define SHIFT(state_value) \
|
||||
{{ \
|
||||
.shift = { \
|
||||
.type = TSParseActionTypeShift, \
|
||||
.state = (state_value) \
|
||||
} \
|
||||
}}
|
||||
|
||||
#define SHIFT_REPEAT(state_value) \
|
||||
{{ \
|
||||
.shift = { \
|
||||
.type = TSParseActionTypeShift, \
|
||||
.state = (state_value), \
|
||||
.repetition = true \
|
||||
} \
|
||||
}}
|
||||
|
||||
#define SHIFT_EXTRA() \
|
||||
{{ \
|
||||
.shift = { \
|
||||
.type = TSParseActionTypeShift, \
|
||||
.extra = true \
|
||||
} \
|
||||
}}
|
||||
|
||||
#define REDUCE(symbol_name, children, precedence, prod_id) \
|
||||
{{ \
|
||||
.reduce = { \
|
||||
.type = TSParseActionTypeReduce, \
|
||||
.symbol = symbol_name, \
|
||||
.child_count = children, \
|
||||
.dynamic_precedence = precedence, \
|
||||
.production_id = prod_id \
|
||||
}, \
|
||||
}}
|
||||
|
||||
#define RECOVER() \
|
||||
{{ \
|
||||
.type = TSParseActionTypeRecover \
|
||||
}}
|
||||
|
||||
#define ACCEPT_INPUT() \
|
||||
{{ \
|
||||
.type = TSParseActionTypeAccept \
|
||||
}}
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif // TREE_SITTER_PARSER_H_
|
||||
@@ -0,0 +1,22 @@
|
||||
==================
|
||||
Autoescape
|
||||
==================
|
||||
|
||||
<div>
|
||||
<ul>
|
||||
<li>one</li>
|
||||
{% autoescape off %}
|
||||
<li>two</li>
|
||||
{% endautoescape %}
|
||||
<li>tree</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
(template
|
||||
(content)
|
||||
(autoescape_group
|
||||
(template
|
||||
(content)))
|
||||
(content))
|
||||
@@ -0,0 +1,165 @@
|
||||
==================
|
||||
Block
|
||||
==================
|
||||
|
||||
<section>
|
||||
<h1>This is a block</h1>
|
||||
{% block test %}
|
||||
<main>
|
||||
<h2>Hello World</h2>
|
||||
</main>
|
||||
{% endblock %}
|
||||
<footer>Goodbye</footer>
|
||||
</section>
|
||||
|
||||
---
|
||||
|
||||
(template
|
||||
(content)
|
||||
(block_group
|
||||
(push_block)
|
||||
(template
|
||||
(content))
|
||||
(pop_block))
|
||||
(content))
|
||||
|
||||
==================
|
||||
Block Matching End Tag
|
||||
==================
|
||||
|
||||
<section>
|
||||
<h1>This is a block</h1>
|
||||
{% block test %}
|
||||
<main>
|
||||
<h2>Hello World</h2>
|
||||
</main>
|
||||
{% endblock test %}
|
||||
<footer>Goodbye</footer>
|
||||
</section>
|
||||
|
||||
---
|
||||
|
||||
(template
|
||||
(content)
|
||||
(block_group
|
||||
(push_block)
|
||||
(template
|
||||
(content))
|
||||
(pop_block))
|
||||
(content))
|
||||
|
||||
==================
|
||||
Nested Blocks
|
||||
==================
|
||||
|
||||
<section>
|
||||
<h1>This is a block</h1>
|
||||
{% block test %}
|
||||
<main>
|
||||
{% block hello %}
|
||||
<h2>Hello World</h2>
|
||||
{% endblock %}
|
||||
</main>
|
||||
{% endblock %}
|
||||
<footer>Goodbye</footer>
|
||||
</section>
|
||||
|
||||
---
|
||||
|
||||
(template
|
||||
(content)
|
||||
(block_group
|
||||
(push_block)
|
||||
(template
|
||||
(content)
|
||||
(block_group
|
||||
(push_block)
|
||||
(template
|
||||
(content))
|
||||
(pop_block))
|
||||
(content))
|
||||
(pop_block))
|
||||
(content))
|
||||
|
||||
---
|
||||
|
||||
(template
|
||||
(content)
|
||||
(block_group
|
||||
(push_block)
|
||||
(template
|
||||
(content)
|
||||
(block_group
|
||||
(push_block)
|
||||
(template
|
||||
(content))
|
||||
(pop_block))
|
||||
(content))
|
||||
(pop_block))
|
||||
(content))
|
||||
|
||||
==================
|
||||
Nested Blocks End Tag
|
||||
==================
|
||||
|
||||
<section>
|
||||
<h1>This is a block</h1>
|
||||
{% block test %}
|
||||
<main>
|
||||
{% block hello %}
|
||||
<h2>Hello World</h2>
|
||||
{% endblock hello %}
|
||||
</main>
|
||||
{% endblock test %}
|
||||
<footer>Goodbye</footer>
|
||||
</section>
|
||||
|
||||
---
|
||||
|
||||
(template
|
||||
(content)
|
||||
(block_group
|
||||
(push_block)
|
||||
(template
|
||||
(content)
|
||||
(block_group
|
||||
(push_block)
|
||||
(template
|
||||
(content))
|
||||
(pop_block))
|
||||
(content))
|
||||
(pop_block))
|
||||
(content))
|
||||
|
||||
==================
|
||||
Nested Blocks Mixed Tags
|
||||
==================
|
||||
|
||||
<section>
|
||||
<h1>This is a block</h1>
|
||||
{% block test %}
|
||||
<main>
|
||||
{% block hello %}
|
||||
<h2>Hello World</h2>
|
||||
{% endblock %}
|
||||
</main>
|
||||
{% endblock test %}
|
||||
<footer>Goodbye</footer>
|
||||
</section>
|
||||
|
||||
---
|
||||
|
||||
(template
|
||||
(content)
|
||||
(block_group
|
||||
(push_block)
|
||||
(template
|
||||
(content)
|
||||
(block_group
|
||||
(push_block)
|
||||
(template
|
||||
(content))
|
||||
(pop_block))
|
||||
(content))
|
||||
(pop_block))
|
||||
(content))
|
||||
@@ -0,0 +1,14 @@
|
||||
==================
|
||||
Csrf Token
|
||||
==================
|
||||
|
||||
<form method="post">
|
||||
{% csrf_token %}
|
||||
</form>
|
||||
|
||||
---
|
||||
|
||||
(template
|
||||
(content)
|
||||
(csrf_token)
|
||||
(content))
|
||||
@@ -0,0 +1,93 @@
|
||||
==================
|
||||
Cycle Basic
|
||||
==================
|
||||
|
||||
<table>
|
||||
<tr class="{% cycle 'row1' 'row2' %}">
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
---
|
||||
|
||||
(template
|
||||
(content)
|
||||
(cycle
|
||||
(filtered_value
|
||||
(value
|
||||
(literal
|
||||
(string))))
|
||||
(filtered_value
|
||||
(value
|
||||
(literal
|
||||
(string)))))
|
||||
(content))
|
||||
|
||||
==================
|
||||
Cycle With As
|
||||
==================
|
||||
|
||||
<table>
|
||||
<tr class="{% cycle 'row1' 'row2' as rowcolors %}">
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
---
|
||||
|
||||
(template
|
||||
(content)
|
||||
(cycle
|
||||
(filtered_value
|
||||
(value
|
||||
(literal
|
||||
(string))))
|
||||
(filtered_value
|
||||
(value
|
||||
(literal
|
||||
(string))))
|
||||
name: (identifier))
|
||||
(content))
|
||||
|
||||
==================
|
||||
Cycle With As And Silent
|
||||
==================
|
||||
|
||||
<table>
|
||||
<tr class="{% cycle 'row1' 'row2' as rowcolors silent %}">
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
---
|
||||
|
||||
(template
|
||||
(content)
|
||||
(cycle
|
||||
(filtered_value
|
||||
(value
|
||||
(literal
|
||||
(string))))
|
||||
(filtered_value
|
||||
(value
|
||||
(literal
|
||||
(string))))
|
||||
name: (identifier))
|
||||
(content))
|
||||
|
||||
==================
|
||||
Cycle Referencing Existing Variable
|
||||
==================
|
||||
|
||||
<table>
|
||||
<tr class="{% cycle rowcolors %}">
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
---
|
||||
|
||||
(template
|
||||
(content)
|
||||
(cycle
|
||||
(filtered_value
|
||||
(value
|
||||
(variable_attribute
|
||||
(identifier)))))
|
||||
(content))
|
||||
@@ -0,0 +1,14 @@
|
||||
==================
|
||||
Debug
|
||||
==================
|
||||
|
||||
<p>
|
||||
{% debug %}
|
||||
</p>
|
||||
|
||||
---
|
||||
|
||||
(template
|
||||
(content)
|
||||
(debug)
|
||||
(content))
|
||||
@@ -0,0 +1,35 @@
|
||||
==================
|
||||
Extends With String
|
||||
==================
|
||||
|
||||
{% extends "base.html" %}
|
||||
<h1>Hello</h1>
|
||||
|
||||
---
|
||||
|
||||
(template
|
||||
(content)
|
||||
(extends
|
||||
(filtered_value
|
||||
(value
|
||||
(literal
|
||||
(string)))))
|
||||
(content))
|
||||
|
||||
==================
|
||||
Extends With Variable
|
||||
==================
|
||||
|
||||
{% extends base_template %}
|
||||
<h1>Hello</h1>
|
||||
|
||||
---
|
||||
|
||||
(template
|
||||
(content)
|
||||
(extends
|
||||
(filtered_value
|
||||
(value
|
||||
(variable_attribute
|
||||
(identifier)))))
|
||||
(content))
|
||||
@@ -0,0 +1,27 @@
|
||||
<body>
|
||||
{# hello this is an example file #}
|
||||
<header>
|
||||
<h1>{{ page.title|title }}</h1>
|
||||
<nav>
|
||||
{% if page.links|length > 0 %}
|
||||
<ul>
|
||||
{% for label, link in page.links %}
|
||||
<li>
|
||||
{% url 'nav-pages' link as href %}
|
||||
<a href="{{href}}">{{label}}</a>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
{% endif %}
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
{% block content %}
|
||||
|
||||
{% endblock %}
|
||||
|
||||
<footer>
|
||||
{% for foot in page.footer %}
|
||||
{% endfor %}
|
||||
</footer>
|
||||
</body>
|
||||
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"$schema": "https://tree-sitter.github.io/tree-sitter/assets/schemas/config.schema.json",
|
||||
"grammars": [
|
||||
{
|
||||
"name": "django",
|
||||
"camelcase": "django",
|
||||
"title": "Django Parser",
|
||||
"scope": "source.django",
|
||||
"file-types": ["html", "txt", "md"],
|
||||
"injection-regex": "^django$",
|
||||
"class-name": "TreeSitterDjango"
|
||||
}
|
||||
],
|
||||
"bindings": {
|
||||
"node": true
|
||||
},
|
||||
"metadata": {
|
||||
"version": "0.0.1",
|
||||
"description": "DJango grammar for tree-sitter",
|
||||
"authors": [{
|
||||
"name": "Daniel Hanson",
|
||||
"url": "https://github.com/danhanson"
|
||||
}],
|
||||
"links": {
|
||||
"repository": "https://github.com/danhanson/tree-sitter-django"
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user