danhanson/tree-sitter-django
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add more tests

Browse Source
This commit is contained in:
danhanson
2026-08-11 17:41:09 -05:00
parent 9b44006c18
commit 7a32f9c8d3
14 changed files with 7023 additions and 6345 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -1 +1,2 @@
node_modules
tree-sitter-django.wasm
grammar.js
+5 -3
View File
@@ -23,6 +23,7 @@ const django = grammar({
$.push_block,
$.push_partial,
$.push_verbatim,
$.verbatim_content,
$.matcher_error,
],
reserved: {
@@ -48,7 +49,7 @@ const django = grammar({
string: ($) => /"(?:[^"\\]|\\\\|\\")*"|'(?:[^'\\]|\\\\|\\')*'/,
attribute: ($) => /[a-zA-Z0-9_]+(?:\.[a-zA-Z0-9_]+)*/,
variable_attribute: ($) => seq($.identifier, optional(seq(token.immediate('.'), $.attribute))),
identifier: ($) => /[a-zA-Z][a-zA-Z0-9_]+/,
identifier: ($) => /[a-zA-Z][a-zA-Z0-9_]*/,
binaryOperator: ($) =>
choice(
"and",
@@ -150,6 +151,7 @@ const django = grammar({
seq("truncatewords:", $.value),
seq("truncatewords_html:", $.value),
"unordered_list",
"upper",
choice(seq("urlencode:", $.value), "urlencode"),
"urlize",
seq("urlizetrunc:", $.value),
@@ -266,8 +268,8 @@ const django = grammar({
verbatim_group: ($) =>
seq(
block("verbatim", field("name", $.push_verbatim)),
/.*/,
prec(1, block("endverbatim", $.pop_verbatim))
optional($.verbatim_content),
block("endverbatim", $.pop_verbatim),
),
with_group: ($) =>
seq(
src/grammar.json
+19 -7
View File
@@ -173,7 +173,7 @@
},
"identifier": {
"type": "PATTERN",
"value": "[a-zA-Z][a-zA-Z0-9_]+"
"value": "[a-zA-Z][a-zA-Z0-9_]*"
},
"binaryOperator": {
"type": "CHOICE",
@@ -938,6 +938,10 @@
"type": "STRING",
"value": "unordered_list"
},
{
"type": "STRING",
"value": "upper"
},
{
"type": "CHOICE",
"members": [
@@ -2515,13 +2519,18 @@
]
},
{
"type": "PATTERN",
"value": ".*"
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "verbatim_content"
},
{
"type": "BLANK"
}
]
},
{
"type": "PREC",
"value": 1,
"content": {
"type": "SEQ",
"members": [
{
@@ -2546,7 +2555,6 @@
}
]
}
}
]
},
"with_group": {
@@ -2674,6 +2682,10 @@
"type": "SYMBOL",
"name": "push_verbatim"
},
{
"type": "SYMBOL",
"name": "verbatim_content"
},
{
"type": "SYMBOL",
"name": "matcher_error"
src/node-types.json
+13 -1
View File
@@ -1019,12 +1019,16 @@
}
},
"children": {
"multiple": false,
"multiple": true,
"required": true,
"types": [
{
"type": "pop_verbatim",
"named": true
},
{
"type": "verbatim_content",
"named": true
}
]
}
@@ -1623,6 +1627,10 @@
"type": "unordered_list",
"named": false
},
{
"type": "upper",
"named": false
},
{
"type": "url",
"named": false
@@ -1647,6 +1655,10 @@
"type": "verbatim",
"named": false
},
{
"type": "verbatim_content",
"named": true
},
{
"type": "w",
"named": false
src/parser.c
+6410 -6310
View File
File diff suppressed because it is too large Load Diff
src/scanner.c
+93 -3
View File
@@ -16,6 +16,8 @@ enum TokenType {
PushBlock, // name %}
PushPartial, // name [ inline ] %}
PushVerbatim, // [ name ] %}
/* raw text inside a verbatim block, up through (but excluding) the matching endverbatim tag */
VerbatimContent,
/* indicates that an error occurred */
MatcherError,
};
@@ -244,10 +246,11 @@ void tree_sitter_django_external_scanner_deserialize(
continue;
} else if (code == NAME_SEP) {
if (name == NULL) {
scanner_error(scanner, "Empty name read");
return;
}
// an empty name was pushed (e.g. an anonymous verbatim block)
array_push(stack, (Name) array_new());
} else {
name = NULL;
}
} else {
if (name == NULL) {
if (!check_name_start_char(code)) {
@@ -322,6 +325,89 @@ static bool check_inline(TSLexer *const lexer) {
return check_close_block(lexer);
}
static void skip_horizontal_whitespace(TSLexer *const lexer) {
while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
lexer->advance(lexer, false);
}
}
const char endverbatim_chars[] = "endverbatim";
/* Called right after consuming "{%"; checks whether what follows closes the
* verbatim block identified by `name` (matching a name on the stack, or, if
* `name` is empty, an unnamed "{% endverbatim %}"). Consumes input either
* way, since a failed match still belongs in the verbatim block's raw text. */
static bool check_endverbatim_close(TSLexer *const lexer, const Name *const name) {
skip_horizontal_whitespace(lexer);
for (unsigned i = 0; i < sizeof(endverbatim_chars) - 1; ++i) {
if (lexer->lookahead != endverbatim_chars[i]) {
return false;
}
lexer->advance(lexer, false);
}
skip_horizontal_whitespace(lexer);
if (name->size > 0) {
for (unsigned i = 0; i < name->size; ++i) {
if (lexer->lookahead != name->contents[i]) {
return false;
}
lexer->advance(lexer, false);
}
if (check_name_char(lexer->lookahead)) {
return false;
}
skip_horizontal_whitespace(lexer);
}
if (lexer->lookahead != '%') {
return false;
}
lexer->advance(lexer, false);
return lexer->lookahead == '}';
}
/* Consumes raw verbatim content up to (but not including) the next tag that
* closes the innermost open verbatim block, since that content must not be
* parsed as template syntax. A verbatim block opened with a name is only
* closed by an "endverbatim" tag bearing the same name, which lets an
* unrelated (e.g. unnamed) "{% verbatim %}...{% endverbatim %}" pair appear
* unparsed inside a named verbatim block. */
static bool scan_verbatim_content(struct Scanner *const scanner, TSLexer *const lexer) {
Stack *stack = get_stack_for_token(scanner, PopVerbatim);
if (stack->size == 0) {
return false;
}
Name *name = array_back(stack);
bool consumed_any = false;
while (true) {
if (lexer->eof(lexer)) {
if (!consumed_any) {
return false;
}
lexer->mark_end(lexer);
lexer->result_symbol = VerbatimContent;
return true;
}
if (lexer->lookahead == '{') {
lexer->mark_end(lexer);
lexer->advance(lexer, false);
if (lexer->lookahead == '%') {
lexer->advance(lexer, false);
if (check_endverbatim_close(lexer, name)) {
if (!consumed_any) {
return false;
}
lexer->result_symbol = VerbatimContent;
return true;
}
}
consumed_any = true;
continue;
}
lexer->advance(lexer, false);
consumed_any = true;
}
}
bool tree_sitter_django_external_scanner_scan(
void *payload,
TSLexer *lexer,
@@ -340,6 +426,9 @@ bool tree_sitter_django_external_scanner_scan(
if (valid_symbols[MatcherError]) {
return false;
}
if (valid_symbols[VerbatimContent]) {
return scan_verbatim_content(scanner, lexer);
}
while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
lexer->advance(lexer, true);
}
@@ -426,5 +515,6 @@ bool tree_sitter_django_external_scanner_scan(
return false;
}
}
return false;
}
test/corpus/filter.txt
+42
View File
@@ -0,0 +1,42 @@
==================
Filter Basic
==================
<p>
{% filter upper %}
hello
{% endfilter %}
</p>
---
(template
(content)
(filter_group
(filter_expression
(filter))
(template
(content)))
(content))
==================
Filter With Chain
==================
<p>
{% filter lower|title %}
HELLO WORLD
{% endfilter %}
</p>
---
(template
(content)
(filter_group
(filter_expression
(filter)
(filter))
(template
(content)))
(content))
test/corpus/firstof.txt
+46
View File
@@ -0,0 +1,46 @@
==================
Firstof Basic
==================
{% firstof var1 var2 "default" %}
---
(template
(content)
(firstof
(filtered_value
(value
(variable_attribute
(identifier))))
(filtered_value
(value
(variable_attribute
(identifier))))
(filtered_value
(value
(literal
(string)))))
(content))
==================
Firstof With As
==================
{% firstof var1 var2 as result %}
---
(template
(content)
(firstof
(filtered_value
(value
(variable_attribute
(identifier))))
(filtered_value
(value
(variable_attribute
(identifier))))
(identifier))
(content))
test/corpus/for.txt
+94
View File
@@ -0,0 +1,94 @@
==================
For Basic
==================
<ul>
{% for item in items %}
<li>{{ item }}</li>
{% endfor %}
</ul>
---
(template
(content)
(for_group
variables: (identifier)
(filtered_value
(value
(variable_attribute
(identifier))))
(template
(content)
(template_variable
(filtered_value
(value
(variable_attribute
(identifier)))))
(content)))
(content))
==================
For With Multiple Variables
==================
<dl>
{% for key, value in items %}
<dt>{{ key }}</dt>
{% endfor %}
</dl>
---
(template
(content)
(for_group
variables: (identifier)
variables: (identifier)
(filtered_value
(value
(variable_attribute
(identifier))))
(template
(content)
(template_variable
(filtered_value
(value
(variable_attribute
(identifier)))))
(content)))
(content))
==================
For With Empty
==================
<ul>
{% for item in items %}
<li>{{ item }}</li>
{% empty %}
<li>Empty</li>
{% endfor %}
</ul>
---
(template
(content)
(for_group
variables: (identifier)
(filtered_value
(value
(variable_attribute
(identifier))))
(template
(content)
(template_variable
(filtered_value
(value
(variable_attribute
(identifier)))))
(content))
(template
(content)))
(content))
test/corpus/if.txt
+89
View File
@@ -0,0 +1,89 @@
==================
If Basic
==================
{% if user.is_authenticated %}
Hi
{% endif %}
---
(template
(content)
(if_group
(predicate
(filtered_value
(value
(variable_attribute
(identifier)
(attribute)))))
(template
(content)))
(content))
==================
If With Not
==================
{% if not done %}
working
{% endif %}
---
(template
(content)
(if_group
(predicate
(filtered_value
(value
(variable_attribute
(identifier)))))
(template
(content)))
(content))
==================
If With Elif And Else
==================
{% if count == 1 %}
one
{% elif count == 2 %}
two
{% else %}
other
{% endif %}
---
(template
(content)
(if_group
(predicate
(filtered_value
(value
(variable_attribute
(identifier))))
(binaryOperator)
(filtered_value
(value
(literal
(number)))))
(template
(content))
(predicate
(filtered_value
(value
(variable_attribute
(identifier))))
(binaryOperator)
(filtered_value
(value
(literal
(number)))))
(template
(content))
(template
(content)))
(content))
test/corpus/ifchanged.txt
+41
View File
@@ -0,0 +1,41 @@
==================
Ifchanged Basic
==================
{% ifchanged %}
content
{% endifchanged %}
---
(template
(content)
(ifchanged_group
(template
(content)))
(content))
==================
Ifchanged With Values
==================
{% ifchanged var1 var2 %}
content
{% endifchanged %}
---
(template
(content)
(ifchanged_group
(filtered_value
(value
(variable_attribute
(identifier))))
(filtered_value
(value
(variable_attribute
(identifier))))
(template
(content)))
(content))
test/corpus/include.txt
+33
View File
@@ -0,0 +1,33 @@
==================
Include With String
==================
{% include "sidebar.html" %}
---
(template
(content)
(include
(filtered_value
(value
(literal
(string)))))
(content))
==================
Include With Variable
==================
{% include template_name %}
---
(template
(content)
(include
(filtered_value
(value
(variable_attribute
(identifier)))))
(content))
test/corpus/lorem.txt
+33
View File
@@ -0,0 +1,33 @@
==================
Lorem Words
==================
{% lorem 3 w %}
---
(template
(content)
(lorem
(filtered_value
(value
(literal
(number)))))
(content))
==================
Lorem Paragraphs Random
==================
{% lorem 2 p random %}
---
(template
(content)
(lorem
(filtered_value
(value
(literal
(number)))))
(content))
test/corpus/verbatim.txt
+83
View File
@@ -0,0 +1,83 @@
==================
Verbatim Basic
==================
{% verbatim %}{{ hello }}{% endverbatim %}
---
(template
(content)
(verbatim_group
name: (push_verbatim)
(verbatim_content)
(pop_verbatim))
(content))
==================
Verbatim Empty
==================
{% verbatim %}{% endverbatim %}
---
(template
(content)
(verbatim_group
name: (push_verbatim)
(pop_verbatim))
(content))
==================
Verbatim Ignores Tags
==================
{% verbatim %}
{{ hello }}
{% for x in y %}
{% endverbatim %}
---
(template
(content)
(verbatim_group
name: (push_verbatim)
(verbatim_content)
(pop_verbatim))
(content))
==================
Verbatim With Name
==================
{% verbatim myblock %}{{ hello }}{% endverbatim myblock %}
---
(template
(content)
(verbatim_group
name: (push_verbatim)
(verbatim_content)
(pop_verbatim))
(content))
==================
Verbatim With Name Allows Nested Unnamed Verbatim
==================
{% verbatim myblock %}
Avoid template rendering via the {% verbatim %}{% endverbatim %} block.
{% endverbatim myblock %}
---
(template
(content)
(verbatim_group
name: (push_verbatim)
(verbatim_content)
(pop_verbatim))
(content))