danhanson/tree-sitter-django
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixed whitespace issue

Browse Source
This commit is contained in:
danhanson committed 2026-09-05 22:28:36 -05:00
1 parent d087c70b46
commit 05465cd528
9 files changed
+11888 -10652
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

grammar.js
+73 -19
View File
@@ -11,12 +11,33 @@ function block(tag, ...args) {
return seq("{%", field("tag", tag), ...args, "%}");
}
/**
* A keyword that must be separated from the token before it by whitespace.
* Django splits tag contents on whitespace, so "{% cycle 1as x %}" is one
* token and an error, not "1" followed by "as". Nothing in the lexer enforces
* that on its own: two keywords cannot run together (the lexer would read one
* longer identifier), but a literal can run into the keyword after it, and
* extras can never be made mandatory. The separator is therefore a zero-width
* external token that the scanner only emits when whitespace really precedes
* its keyword.
*
* @param {RuleOrLiteral} separator
* @param {string} keyword
* @param {...RuleOrLiteral} args
* @returns {SeqRule}
*/
function spaced(separator, keyword, ...args) {
return seq(separator, keyword, ...args);
}
const django = grammar({
name: "django",
extras: ($) => [/[ \t]/],
word: ($) => $.identifier,
conflicts: ($) => [[$.template]],
supertypes: ($) => [$.template_tag, $.template_block_groups],
externals: ($) => [
$.matcher_error,
$.pop_block,
$.pop_partial,
$.pop_verbatim,
@@ -25,7 +46,18 @@ const django = grammar({
$.push_verbatim,
$.verbatim_content,
$.comment_content,
$.matcher_error,
$._before_and,
$._before_as,
$._before_b,
$._before_by,
$._before_in,
$._before_is,
$._before_not,
$._before_or,
$._before_p,
$._before_random,
$._before_w,
$._after_literal,
],
reserved: {
global: ($) => ["not", "if", "in", "is", "as", "for", "from"],
@@ -40,7 +72,7 @@ const django = grammar({
filter_expression: ($) =>
seq($.filter, repeat(seq(token.immediate("|"), $.filter))),
value: ($) => choice($.literal, $.variable_attribute),
literal: ($) => choice($.number, $.string),
literal: ($) => seq(choice($.number, $.string), $._after_literal),
number: ($) => /-?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE]-?[0-9]+)?/,
string: ($) => /"(?:[^"\\]|\\\\|\\")*"|'(?:[^'\\]|\\\\|\\')*'/,
attribute: ($) => /[a-zA-Z0-9_]+(?:\.[a-zA-Z0-9_]+)*/,
@@ -49,24 +81,30 @@ const django = grammar({
identifier: ($) => /[a-zA-Z][a-zA-Z0-9_]*/,
binaryOperator: ($) =>
choice(
"and",
"or",
spaced($._before_and, "and"),
spaced($._before_or, "or"),
"==",
"!=",
"<",
">",
"<=",
">=",
"in",
prec(1, seq("not", "in")),
"is",
prec(1, seq("is", "not")),
spaced($._before_in, "in"),
prec(1, spaced($._before_not, "not", "in")),
spaced($._before_is, "is"),
prec(1, spaced($._before_is, "is", "not")),
),
predicate: ($) =>
seq(
optional("not"),
optional(spaced($._before_not, "not")),
$.filtered_value,
repeat(seq($.binaryOperator, optional("not"), $.filtered_value)),
repeat(
seq(
$.binaryOperator,
optional(spaced($._before_not, "not")),
$.filtered_value,
),
),
),
template_variable: ($) => seq("{{", $.filtered_value, "}}"),
template_comment: ($) => seq("{#", /(?:[^#]|#[^}])*/, "#}"),
@@ -183,9 +221,17 @@ const django = grammar({
block(
"cycle",
repeat1($.filtered_value),
optional(seq("as", field("variable", $.identifier))),
optional(
spaced(
$._before_as,
"as",
field("variable", $.identifier),
// only the as-form takes the flag, and an identifier can never run
// into it, so "silent" needs no separator of its own
optional("silent"),
),
),
),
debug: ($) => block("debug"),
extends: ($) => block("extends", $.filtered_value),
filter_group: ($) =>
@@ -198,7 +244,7 @@ const django = grammar({
block(
"firstof",
repeat1($.filtered_value),
optional(seq("as", $.identifier)),
optional(spaced($._before_as, "as", $.identifier)),
),
for_scope: ($) =>
prec.left(
@@ -251,11 +297,19 @@ const django = grammar({
block(
"lorem",
$.filtered_value,
choice("w", "p", "b"),
optional("random"),
choice(
spaced($._before_w, "w"),
spaced($._before_p, "p"),
spaced($._before_b, "b"),
),
optional(spaced($._before_random, "random")),
),
now: ($) =>
block("now", $.string, optional(seq("as", field("name", $.identifier)))),
block(
"now",
$.string,
optional(spaced($._before_as, "as", field("name", $.identifier))),
),
partial: ($) => block("partial", $.identifier),
partialdef_group: ($) =>
seq(
@@ -273,9 +327,9 @@ const django = grammar({
block(
"regroup",
$.filtered_value,
"by",
spaced($._before_by, "by"),
$.attribute,
optional(seq("as", field("variable", $.identifier))),
optional(spaced($._before_as, "as", field("variable", $.identifier))),
),
reset_cycle: ($) => block("resetcycle", optional($.identifier)),
spaceless_group: ($) =>
@@ -304,7 +358,7 @@ const django = grammar({
repeat1(seq($.identifier, "=", $.filtered_value)),
),
),
optional(seq("as", field("variable", $.identifier))),
optional(spaced($._before_as, "as", field("variable", $.identifier))),
),
verbatim_group: ($) =>
seq(
@@ -318,7 +372,7 @@ const django = grammar({
$.filtered_value,
$.filtered_value,
$.filtered_value,
optional(seq("as", field("variable", $.identifier))),
optional(spaced($._before_as, "as", field("variable", $.identifier))),
),
with_group: ($) =>
seq(
src/grammar.json
+197 -8
View File
@@ -1,6 +1,7 @@
{
"$schema": "https://tree-sitter.github.io/tree-sitter/assets/schemas/grammar.schema.json",
"name": "django",
"word": "identifier",
"rules": {
"template": {
"type": "REPEAT1",
@@ -114,6 +115,9 @@
]
},
"literal": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
@@ -126,6 +130,12 @@
}
]
},
{
"type": "SYMBOL",
"name": "_after_literal"
}
]
},
"number": {
"type": "PATTERN",
"value": "-?(?:[0-9]+(?:\\.[0-9]*)?|\\.[0-9]+)(?:[eE]-?[0-9]+)?"
@@ -178,13 +188,31 @@
"binaryOperator": {
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_before_and"
},
{
"type": "STRING",
"value": "and"
}
]
},
{
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_before_or"
},
{
"type": "STRING",
"value": "or"
}
]
},
{
"type": "STRING",
@@ -210,9 +238,18 @@
"type": "STRING",
"value": ">="
},
{
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_before_in"
},
{
"type": "STRING",
"value": "in"
}
]
},
{
"type": "PREC",
@@ -220,6 +257,10 @@
"content": {
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_before_not"
},
{
"type": "STRING",
"value": "not"
@@ -231,9 +272,18 @@
]
}
},
{
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_before_is"
},
{
"type": "STRING",
"value": "is"
}
]
},
{
"type": "PREC",
@@ -241,6 +291,10 @@
"content": {
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_before_is"
},
{
"type": "STRING",
"value": "is"
@@ -260,9 +314,18 @@
{
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_before_not"
},
{
"type": "STRING",
"value": "not"
}
]
},
{
"type": "BLANK"
@@ -285,9 +348,18 @@
{
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_before_not"
},
{
"type": "STRING",
"value": "not"
}
]
},
{
"type": "BLANK"
@@ -1297,6 +1369,10 @@
{
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_before_as"
},
{
"type": "STRING",
"value": "as"
@@ -1308,13 +1384,6 @@
"type": "SYMBOL",
"name": "identifier"
}
}
]
},
{
"type": "BLANK"
}
]
},
{
"type": "CHOICE",
@@ -1327,6 +1396,13 @@
"type": "BLANK"
}
]
}
]
},
{
"type": "BLANK"
}
]
},
{
"type": "STRING",
@@ -1471,6 +1547,10 @@
{
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_before_as"
},
{
"type": "STRING",
"value": "as"
@@ -2003,26 +2083,62 @@
{
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_before_w"
},
{
"type": "STRING",
"value": "w"
}
]
},
{
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_before_p"
},
{
"type": "STRING",
"value": "p"
}
]
},
{
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_before_b"
},
{
"type": "STRING",
"value": "b"
}
]
}
]
},
{
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_before_random"
},
{
"type": "STRING",
"value": "random"
}
]
},
{
"type": "BLANK"
@@ -2060,6 +2176,10 @@
{
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_before_as"
},
{
"type": "STRING",
"value": "as"
@@ -2256,9 +2376,18 @@
"type": "SYMBOL",
"name": "filtered_value"
},
{
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_before_by"
},
{
"type": "STRING",
"value": "by"
}
]
},
{
"type": "SYMBOL",
@@ -2270,6 +2399,10 @@
{
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_before_as"
},
{
"type": "STRING",
"value": "as"
@@ -2519,6 +2652,10 @@
{
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_before_as"
},
{
"type": "STRING",
"value": "as"
@@ -2644,6 +2781,10 @@
{
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_before_as"
},
{
"type": "STRING",
"value": "as"
@@ -2766,6 +2907,10 @@
],
"precedences": [],
"externals": [
{
"type": "SYMBOL",
"name": "matcher_error"
},
{
"type": "SYMBOL",
"name": "pop_block"
@@ -2800,7 +2945,51 @@
},
{
"type": "SYMBOL",
"name": "matcher_error"
"name": "_before_and"
},
{
"type": "SYMBOL",
"name": "_before_as"
},
{
"type": "SYMBOL",
"name": "_before_b"
},
{
"type": "SYMBOL",
"name": "_before_by"
},
{
"type": "SYMBOL",
"name": "_before_in"
},
{
"type": "SYMBOL",
"name": "_before_is"
},
{
"type": "SYMBOL",
"name": "_before_not"
},
{
"type": "SYMBOL",
"name": "_before_or"
},
{
"type": "SYMBOL",
"name": "_before_p"
},
{
"type": "SYMBOL",
"name": "_before_random"
},
{
"type": "SYMBOL",
"name": "_before_w"
},
{
"type": "SYMBOL",
"name": "_after_literal"
}
],
"inline": [],
src/parser.c
+11443 -10620
View File
File diff suppressed because it is too large. Load diff
src/scanner.c
+109 -5
View File
@@ -1,6 +1,7 @@
#include "tree_sitter/parser.h"
#include "tree_sitter/alloc.h"
#include "tree_sitter/array.h"
#include <string.h>
#define ERROR_SIZE 64
#define STACK_COUNT 3
@@ -8,6 +9,8 @@
#define STACK_SEP '\n'
enum TokenType {
/* indicates that an error occurred */
MatcherError,
/* push name on stack to match with later */
PopBlock, // [ name ] %}
PopPartial, // [ name ] %}
@@ -20,10 +23,32 @@ enum TokenType {
VerbatimContent,
/* raw text inside a comment block, up through (but excluding) the endcomment tag */
CommentContent,
/* indicates that an error occurred */
MatcherError,
/* zero-width markers for the whitespace preceding a keyword, in the same
* order as separated_keywords below */
BeforeAnd,
BeforeAs,
BeforeB,
BeforeBy,
BeforeIn,
BeforeIs,
BeforeNot,
BeforeOr,
BeforeP,
BeforeRandom,
BeforeW,
/* zero-width marker for the end of a literal */
AfterLiteral,
};
/* Keywords Django only reads as keywords when whitespace separates them from
* the token before, listed in BeforeAnd..BeforeW order. */
static const char *const separated_keywords[] = {
"and", "as", "b", "by", "in", "is", "not", "or", "p", "random", "w",
};
#define KEYWORD_COUNT (sizeof(separated_keywords) / sizeof(*separated_keywords))
#define KEYWORD_SIZE 6 // "random", the longest of them
typedef Array(int32_t) Name;
typedef Array(Name) Stack;
@@ -36,10 +61,11 @@ struct Scanner {
};
static Stack* get_stack_for_token(struct Scanner *scanner, enum TokenType token) {
if (token >= MatcherError) {
if (token < PopBlock || token > PushVerbatim) {
return NULL;
}
return &scanner->stacks[token % STACK_COUNT];
// the pop and push tokens are in the same order, so each pair shares a stack
return &scanner->stacks[(token - PopBlock) % STACK_COUNT];
}
static void reset_scanner(struct Scanner *const scanner) {
@@ -406,6 +432,66 @@ static bool scan_verbatim_content(struct Scanner *const scanner, TSLexer *const
}
}
/* Emits a zero-width token at the whitespace that separates one of the
* separated_keywords from the token before it. Django splits tag contents on
* whitespace, so those words are only keywords when something separates them;
* without this, "{% cycle 1as x %}" would parse as a cycle over 1 bound to x.
* Two keywords cannot run together (the lexer reads one longer identifier
* instead), but a literal can run into the keyword after it, and the grammar's
* extras can never be made mandatory.
*
* Reads the whole word that follows and only fires when it is a keyword the
* parser is currently expecting, since the whitespace after a value is equally
* the whitespace before another value ("{% cycle a b %}"). */
/* Emits a zero-width token at the end of a literal, and fails when the
* literal runs straight into whatever follows it. Django splits tag contents
* on whitespace, so "{% cycle 1a %}" is the single token "1a" and an error,
* not the number 1 followed by the variable a. A literal ends wherever its
* own pattern ends, so unlike an identifier it cannot swallow the text after
* it, which leaves this the only place to catch the run-on. */
static bool scan_after_literal(TSLexer *const lexer) {
int32_t next = lexer->lookahead;
if (check_name_char(next) || next == '\'' || next == '"' || next == '-') {
return false;
}
lexer->mark_end(lexer);
lexer->result_symbol = AfterLiteral;
return true;
}
static bool scan_separator(TSLexer *const lexer, const bool *const valid_symbols) {
bool separated = false;
while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
lexer->advance(lexer, true);
separated = true;
}
if (!separated) {
return false;
}
// the token ends here, before the keyword itself, which the grammar lexes
lexer->mark_end(lexer);
char word[KEYWORD_SIZE + 1];
unsigned length = 0;
while (check_name_char(lexer->lookahead)) {
if (length == KEYWORD_SIZE) {
// too long to be one of them, and a prefix match would be wrong anyway
return false;
}
word[length++] = (char) lexer->lookahead;
lexer->advance(lexer, false);
}
word[length] = '\0';
for (unsigned i = 0; i < KEYWORD_COUNT; ++i) {
if (valid_symbols[BeforeAnd + i] && strcmp(word, separated_keywords[i]) == 0) {
lexer->result_symbol = BeforeAnd + i;
return true;
}
}
return false;
}
const char endcomment_chars[] = "endcomment";
/* Called right after consuming "{%"; checks whether what follows closes the
@@ -486,6 +572,24 @@ bool tree_sitter_django_external_scanner_scan(
if (valid_symbols[CommentContent]) {
return scan_comment_content(lexer);
}
if (valid_symbols[AfterLiteral]) {
return scan_after_literal(lexer);
}
bool separator = false;
for (unsigned i = 0; i < KEYWORD_COUNT; ++i) {
separator |= valid_symbols[BeforeAnd + i];
}
if (separator) {
bool matching_name = false;
for (unsigned token = PopBlock; token <= PushVerbatim; ++token) {
matching_name |= valid_symbols[token];
}
/* these keywords never follow a tag name that opens or closes a named
* block, so the two scans never compete over the same position */
if (!matching_name) {
return scan_separator(lexer, valid_symbols);
}
}
while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
lexer->advance(lexer, true);
}
@@ -542,7 +646,7 @@ bool tree_sitter_django_external_scanner_scan(
most_matched_name = name;
}
}
for (unsigned token = PushBlock; token < MatcherError; ++token) {
for (unsigned token = PushBlock; token <= PushVerbatim; ++token) {
if (valid_symbols[token]) {
Stack *stack = get_stack_for_token(scanner, token);
if (is_empty && token == PushVerbatim) {
test/corpus/cycle.txt
+22
View File
@@ -91,3 +91,25 @@ Cycle Referencing Existing Variable
(variable_attribute
(identifier)))))
(content))
==================
Cycle With As Adjacent To Value
:error
==================
{% cycle 1 2as rowcolors %}
---
(ERROR)
==================
Cycle Value Adjacent To Value
:error
==================
{% cycle 'row1'rowcolors %}
---
(ERROR)
test/corpus/if.txt
+11
View File
@@ -141,3 +141,14 @@ yes
(template
(content)))
(content))
==================
If Operator Adjacent To Value
:error
==================
{% if 1and 2 %}yes{% endif %}
---
(ERROR)
test/corpus/lorem.txt
+11
View File
@@ -31,3 +31,14 @@ Lorem Paragraphs Random
(literal
(number)))))
(content))
==================
Lorem Method Adjacent To Count
:error
==================
{% lorem 5w %}
---
(ERROR)
test/corpus/regroup.txt
+11
View File
@@ -16,3 +16,14 @@ Regroup Basic
(attribute)
variable: (identifier))
(content))
==================
Regroup By Adjacent To Value
:error
==================
{% regroup 1by gender as grouped %}
---
(ERROR)
test/corpus/template_variable.txt
+11
View File
@@ -200,3 +200,14 @@ Template Variable Number With Filter Argument
(literal
(number)))))))
(content))
==================
Template Variable Literal Adjacent To Name
:error
==================
{{ 1x }}
---
(ERROR)