danhanson/tree-sitter-django
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

directly handle white space and support multiline tags

Browse Source
This commit is contained in:
danhanson committed 2026-09-06 15:56:24 -05:00
1 parent 25c4be568f
commit e357dc17ad
8 files changed
+40453 -12738
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

grammar.js
+159 -108
View File
@@ -3,31 +3,55 @@
// @ts-check // @ts-check
/** /**
* The whitespace between two parts of a tag. Django splits a tag's contents on
* whitespace, so a separator is part of the syntax and not something to skip:
* "{% cycle 1as x %}" is the single token "1as", never "1" followed by "as".
* Leaving it out of `extras` is what lets the grammar say so.
*
* A newline separates parts like any other whitespace, since Django matches a
* tag with re.DOTALL and splits its contents on whitespace of any kind. The
* scanner skips the same characters when it matches a name of its own.
*/
const SEP = /[ \t\r\n]+/;
/**
* Parts of a tag, each separated from the one before it by whitespace.
* @param {RuleOrLiteral} first
* @param {...RuleOrLiteral} rest
* @returns {RuleOrLiteral}
*/
function joined(first, ...rest) {
return rest.length === 0
? first
: seq(first, ...rest.flatMap((arg) => [SEP, arg]));
}
/**
* One more part of a tag, taking the whitespace that separates it from the
* part before it.
* @param {RuleOrLiteral} first
* @param {...RuleOrLiteral} rest
* @returns {SeqRule}
*/
function part(first, ...rest) {
return seq(SEP, joined(first, ...rest));
}
/**
* A tag, which may hug its delimiters ("{%cycle 1%}") or not.
* @param {string} tag * @param {string} tag
* @param {...RuleOrLiteral} args * @param {...RuleOrLiteral} args
* @returns {SeqRule} * @returns {SeqRule}
*/ */
function block(tag, ...args) { function block(tag, ...args) {
return seq("{%", field("tag", tag), ...args, "%}"); return seq(
} "{%",
optional(SEP),
/** field("tag", tag),
* A keyword that must be separated from the token before it by whitespace. ...args,
* Django splits tag contents on whitespace, so "{% cycle 1as x %}" is one optional(SEP),
* token and an error, not "1" followed by "as". Nothing in the lexer enforces "%}",
* that on its own: two keywords cannot run together (the lexer would read one );
* longer identifier), but a literal can run into the keyword after it, and
* extras can never be made mandatory. The separator is therefore a zero-width
* external token that the scanner only emits when whitespace really precedes
* its keyword.
*
* @param {RuleOrLiteral} separator
* @param {string} keyword
* @param {...RuleOrLiteral} args
* @returns {SeqRule}
*/
function spaced(separator, keyword, ...args) {
return seq(separator, keyword, ...args);
} }
/** Django's string constant, which takes any escape but a line break. */ /** Django's string constant, which takes any escape but a line break. */
@@ -35,9 +59,17 @@ const STRING = /"(?:[^"\\]|\\[^\n])*"|'(?:[^'\\]|\\[^\n])*'/;
const django = grammar({ const django = grammar({
name: "django", name: "django",
extras: ($) => [/[ \t]/], extras: ($) => [],
word: ($) => $.identifier, word: ($) => $.identifier,
conflicts: ($) => [[$.template]], // a separator can end a part or start the next one, which only the token
// after it settles
conflicts: ($) => [
[$.template],
[$.predicate],
[$.library, $.load],
[$._filtered_value_spaced],
[$._filter_expression_spaced],
],
supertypes: ($) => [$.template_tag, $.template_block_groups], supertypes: ($) => [$.template_tag, $.template_block_groups],
externals: ($) => [ externals: ($) => [
$.matcher_error, $.matcher_error,
@@ -49,18 +81,6 @@ const django = grammar({
$.push_verbatim, $.push_verbatim,
$.verbatim_content, $.verbatim_content,
$.comment_content, $.comment_content,
$._before_and,
$._before_as,
$._before_b,
$._before_by,
$._before_in,
$._before_is,
$._before_not,
$._before_or,
$._before_p,
$._before_random,
$._before_w,
$._after_literal,
], ],
reserved: { reserved: {
global: ($) => ["not", "if", "in", "is", "as", "for", "from"], global: ($) => ["not", "if", "in", "is", "as", "for", "from"],
@@ -71,12 +91,10 @@ const django = grammar({
template_tag: ($) => template_tag: ($) =>
choice($.template_block_groups, $.template_variable, $.template_comment), choice($.template_block_groups, $.template_variable, $.template_comment),
filtered_value: ($) => filtered_value: ($) =>
seq($.value, optional(seq(token.immediate("|"), $.filter_expression))), seq($.value, optional(seq("|", $.filter_expression))),
filter_expression: ($) => filter_expression: ($) => seq($.filter, repeat(seq("|", $.filter))),
seq($.filter, repeat(seq(token.immediate("|"), $.filter))),
value: ($) => choice($.literal, $.variable_attribute), value: ($) => choice($.literal, $.variable_attribute),
literal: ($) => literal: ($) => choice($.number, $.string, $.translated_string),
seq(choice($.number, $.string, $.translated_string), $._after_literal),
// digits, with single underscores allowed between them, as Python's // digits, with single underscores allowed between them, as Python's
// int() and float() accept ("1_000") and Django's Variable relies on // int() and float() accept ("1_000") and Django's Variable relies on
number: ($) => number: ($) =>
@@ -87,40 +105,66 @@ const django = grammar({
* part of the constant itself, so nothing may come between them and the * part of the constant itself, so nothing may come between them and the
* string: "_( 'a' )" is a syntax error there, not a translated literal. * string: "_( 'a' )" is a syntax error there, not a translated literal.
*/ */
translated_string: ($) => translated_string: ($) => seq("_(", $.string, ")"),
seq("_(", alias(token.immediate(STRING), $.string), token.immediate(")")),
attribute: ($) => /[a-zA-Z0-9][a-zA-Z0-9_]*(?:\.[a-zA-Z0-9][a-zA-Z0-9_]*)*/, attribute: ($) => /[a-zA-Z0-9][a-zA-Z0-9_]*(?:\.[a-zA-Z0-9][a-zA-Z0-9_]*)*/,
variable_attribute: ($) => variable_attribute: ($) =>
seq($.identifier, optional(seq(token.immediate("."), $.attribute))), seq($.identifier, optional(seq(".", $.attribute))),
identifier: ($) => /[a-zA-Z][a-zA-Z0-9_]*/, identifier: ($) => /[a-zA-Z][a-zA-Z0-9_]*/,
binaryOperator: ($) => binaryOperator: ($) =>
choice( choice(
spaced($._before_and, "and"), "and",
spaced($._before_or, "or"), "or",
"==", "==",
"!=", "!=",
"<", "<",
">", ">",
"<=", "<=",
">=", ">=",
spaced($._before_in, "in"), "in",
prec(1, spaced($._before_not, "not", "in")), prec(1, joined("not", "in")),
spaced($._before_is, "is"), "is",
prec(1, spaced($._before_is, "is", "not")), prec(1, joined("is", "not")),
), ),
predicate: ($) => predicate: ($) =>
seq( seq(
optional(spaced($._before_not, "not")), optional(seq("not", SEP)),
$.filtered_value, $.filtered_value,
repeat( repeat(
seq( seq(
$.binaryOperator, part($.binaryOperator),
optional(spaced($._before_not, "not")), optional(part("not")),
$.filtered_value, part($.filtered_value),
), ),
), ),
), ),
template_variable: ($) => seq("{{", $.filtered_value, "}}"), template_variable: ($) =>
seq(
"{{",
optional(SEP),
alias($._filtered_value_spaced, $.filtered_value),
optional(SEP),
"}}",
),
/**
* Everything between "{{" and "}}" reaches Django as a single expression,
* so whitespace may surround a pipe there. Inside a tag it may not: the
* contents are split on whitespace before each argument is parsed, which
* would leave a bare "|" as an argument of its own.
*/
_filtered_value_spaced: ($) =>
seq(
$.value,
optional(
seq(
optional(SEP),
"|",
optional(SEP),
alias($._filter_expression_spaced, $.filter_expression),
),
),
),
_filter_expression_spaced: ($) =>
seq($.filter, repeat(seq(optional(SEP), "|", optional(SEP), $.filter))),
template_comment: ($) => seq("{#", /(?:[^#]|#[^}])*/, "#}"), template_comment: ($) => seq("{#", /(?:[^#]|#[^}])*/, "#}"),
template_block_groups: ($) => template_block_groups: ($) =>
choice( choice(
@@ -214,10 +258,11 @@ const django = grammar({
), ),
autoescape_group: ($) => autoescape_group: ($) =>
seq( seq(
block("autoescape", choice("on", "off")), block("autoescape", part(choice("on", "off"))),
optional($.template), optional($.template),
block("endautoescape"), block("endautoescape"),
), ),
// the scanner takes the whitespace before a name it has to match itself
block_group: ($) => block_group: ($) =>
seq( seq(
block("block", field("name", $.push_block)), block("block", field("name", $.push_block)),
@@ -226,7 +271,7 @@ const django = grammar({
), ),
comment_group: ($) => comment_group: ($) =>
seq( seq(
block("comment", optional($.string)), block("comment", optional(part($.string))),
optional($.comment_content), optional($.comment_content),
block("endcomment"), block("endcomment"),
), ),
@@ -234,41 +279,51 @@ const django = grammar({
cycle: ($) => cycle: ($) =>
block( block(
"cycle", "cycle",
repeat1($.filtered_value), repeat1(part($.filtered_value)),
optional( optional(
spaced( seq(
$._before_as, part("as", field("variable", $.identifier)),
"as", // only the as-form takes the flag
field("variable", $.identifier), optional(part("silent")),
// only the as-form takes the flag, and an identifier can never run
// into it, so "silent" needs no separator of its own
optional("silent"),
), ),
), ),
), ),
debug: ($) => block("debug"), debug: ($) => block("debug"),
extends: ($) => block("extends", $.filtered_value), extends: ($) => block("extends", part($.filtered_value)),
filter_group: ($) => filter_group: ($) =>
seq( seq(
block("filter", $.filter_expression), // the filter tag hands Django the rest of its text as one expression
// rather than as split arguments, so pipes here take whitespace just
// as they do between "{{" and "}}"
block(
"filter",
part(alias($._filter_expression_spaced, $.filter_expression)),
),
optional($.template), optional($.template),
block("endfilter"), block("endfilter"),
), ),
firstof: ($) => firstof: ($) =>
block( block(
"firstof", "firstof",
repeat1($.filtered_value), repeat1(part($.filtered_value)),
optional(spaced($._before_as, "as", $.identifier)), optional(part("as", $.identifier)),
), ),
for_scope: ($) => for_scope: ($) =>
prec.left( prec.left(
seq( seq(
block( block(
"for", "for",
part(field("variable", $.identifier)),
repeat(
seq(
optional(SEP),
",",
optional(SEP),
field("variable", $.identifier), field("variable", $.identifier),
repeat(seq(",", field("variable", $.identifier))), ),
"in", ),
$.filtered_value, part("in"),
part($.filtered_value),
), ),
optional($.template), optional($.template),
), ),
@@ -281,76 +336,69 @@ const django = grammar({
), ),
if_group: ($) => if_group: ($) =>
seq( seq(
block("if", $.predicate), block("if", part($.predicate)),
optional($.template), optional($.template),
repeat(seq(block("elif", $.predicate), optional($.template))), repeat(seq(block("elif", part($.predicate)), optional($.template))),
optional(seq(block("else"), optional($.template))), optional(seq(block("else"), optional($.template))),
block("endif"), block("endif"),
), ),
ifchanged_group: ($) => ifchanged_group: ($) =>
seq( seq(
block("ifchanged", repeat($.filtered_value)), block("ifchanged", repeat(part($.filtered_value))),
optional($.template), optional($.template),
block("endifchanged"), block("endifchanged"),
), ),
include: ($) => block("include", $.filtered_value), include: ($) => block("include", part($.filtered_value)),
library: ($) => library: ($) => seq($.identifier, optional(seq(".", $.identifier))),
prec(
-1,
seq($.identifier, optional(seq(token.immediate("."), $.identifier))),
),
load: ($) => load: ($) =>
block( block(
"load", "load",
choice( choice(
seq(repeat1($.identifier), "from", $.library), seq(repeat1(part($.identifier)), part("from"), part($.library)),
repeat1($.library), repeat1(part($.library)),
), ),
), ),
lorem: ($) => lorem: ($) =>
block( block(
"lorem", "lorem",
$.filtered_value, part($.filtered_value),
choice( part(choice("w", "p", "b")),
spaced($._before_w, "w"), optional(part("random")),
spaced($._before_p, "p"),
spaced($._before_b, "b"),
),
optional(spaced($._before_random, "random")),
), ),
now: ($) => now: ($) =>
block( block(
"now", "now",
$.string, part($.string),
optional(spaced($._before_as, "as", field("name", $.identifier))), optional(part("as", field("name", $.identifier))),
), ),
partial: ($) => block("partial", $.identifier), partial: ($) => block("partial", part($.identifier)),
partialdef_group: ($) => partialdef_group: ($) =>
seq( seq(
block("partialdef", $.push_partial, optional("inline")), block("partialdef", $.push_partial, optional(part("inline"))),
optional($.template), optional($.template),
block("endpartialdef", $.pop_partial), block("endpartialdef", $.pop_partial),
), ),
query_string: ($) => query_string: ($) =>
block( block(
"querystring", "querystring",
repeat($.identifier), repeat(part($.identifier)),
repeat(seq($.identifier, "=", $.filtered_value)), repeat(part(seq($.identifier, "=", $.filtered_value))),
), ),
regroup: ($) => regroup: ($) =>
block( block(
"regroup", "regroup",
$.filtered_value, part($.filtered_value),
spaced($._before_by, "by"), part("by"),
$.attribute, part($.attribute),
optional(spaced($._before_as, "as", field("variable", $.identifier))), optional(part("as", field("variable", $.identifier))),
), ),
reset_cycle: ($) => block("resetcycle", optional($.identifier)), reset_cycle: ($) => block("resetcycle", optional(part($.identifier))),
spaceless_group: ($) => spaceless_group: ($) =>
seq(block("spaceless"), optional($.template), block("endspaceless")), seq(block("spaceless"), optional($.template), block("endspaceless")),
template_tag_block: ($) => template_tag_block: ($) =>
block( block(
"templatetag", "templatetag",
part(
choice( choice(
"openblock", "openblock",
"closeblock", "closeblock",
@@ -362,17 +410,18 @@ const django = grammar({
"closecomment", "closecomment",
), ),
), ),
),
url_block: ($) => url_block: ($) =>
block( block(
"url", "url",
choice($.string, $.identifier), part(choice($.string, $.identifier)),
optional( optional(
choice( choice(
repeat1($.filtered_value), repeat1(part($.filtered_value)),
repeat1(seq($.identifier, "=", $.filtered_value)), repeat1(part(seq($.identifier, "=", $.filtered_value))),
), ),
), ),
optional(spaced($._before_as, "as", field("variable", $.identifier))), optional(part("as", field("variable", $.identifier))),
), ),
verbatim_group: ($) => verbatim_group: ($) =>
seq( seq(
@@ -383,16 +432,18 @@ const django = grammar({
width_ratio: ($) => width_ratio: ($) =>
block( block(
"widthratio", "widthratio",
$.filtered_value, part($.filtered_value),
$.filtered_value, part($.filtered_value),
$.filtered_value, part($.filtered_value),
optional(spaced($._before_as, "as", field("variable", $.identifier))), optional(part("as", field("variable", $.identifier))),
), ),
with_group: ($) => with_group: ($) =>
seq( seq(
block( block(
"with", "with",
repeat1(seq(field("variable", $.identifier), "=", $.filtered_value)), repeat1(
part(seq(field("variable", $.identifier), "=", $.filtered_value)),
),
), ),
optional($.template), optional($.template),
block("endwith"), block("endwith"),
src/grammar.json
+1599 -187
View File
File diff suppressed because it is too large. Load diff
src/parser.c
+38525 -12330
View File
File diff suppressed because it is too large. Load diff
src/scanner.c
+16 -113
View File
@@ -21,32 +21,8 @@ enum TokenType {
VerbatimContent, VerbatimContent,
/* raw text inside a comment block, up through (but excluding) the endcomment tag */ /* raw text inside a comment block, up through (but excluding) the endcomment tag */
CommentContent, CommentContent,
/* zero-width markers for the whitespace preceding a keyword, in the same
* order as separated_keywords below */
BeforeAnd,
BeforeAs,
BeforeB,
BeforeBy,
BeforeIn,
BeforeIs,
BeforeNot,
BeforeOr,
BeforeP,
BeforeRandom,
BeforeW,
/* zero-width marker for the end of a literal */
AfterLiteral,
}; };
/* Keywords Django only reads as keywords when whitespace separates them from
* the token before, listed in BeforeAnd..BeforeW order. */
static const char *const separated_keywords[] = {
"and", "as", "b", "by", "in", "is", "not", "or", "p", "random", "w",
};
#define KEYWORD_COUNT (sizeof(separated_keywords) / sizeof(*separated_keywords))
#define KEYWORD_SIZE 6 // "random", the longest of them
typedef Array(int32_t) Name; typedef Array(int32_t) Name;
/* The kinds of tag that open a named region. PushBlock..PushVerbatim and /* The kinds of tag that open a named region. PushBlock..PushVerbatim and
@@ -215,6 +191,13 @@ static unsigned read_code(const char *iter, int32_t *result) {
return 0; // invalid UTF-8 leading byte return 0; // invalid UTF-8 leading byte
} }
/* The characters SEP matches in the grammar: whitespace separating the parts
* of a tag, a newline among them. */
static bool check_space(const int32_t character) {
return character == ' ' || character == '\t'
|| character == '\n' || character == '\r';
}
static bool check_name_start_char(const int32_t letter) { static bool check_name_start_char(const int32_t letter) {
return 'a' <= letter && letter <= 'z' || 'A' <= letter && letter <= 'Z'; return 'a' <= letter && letter <= 'z' || 'A' <= letter && letter <= 'Z';
} }
@@ -331,7 +314,7 @@ static unsigned check_name(TSLexer *const lexer, const Name *const name) {
} }
static bool check_close_block(TSLexer *const lexer) { static bool check_close_block(TSLexer *const lexer) {
while (lexer->lookahead == ' ' || lexer->lookahead == '\t') { while (check_space(lexer->lookahead)) {
lexer->advance(lexer, false); lexer->advance(lexer, false);
} }
if (lexer->lookahead != '%') { if (lexer->lookahead != '%') {
@@ -359,8 +342,8 @@ static bool check_inline(TSLexer *const lexer) {
return check_close_block(lexer); return check_close_block(lexer);
} }
static void skip_horizontal_whitespace(TSLexer *const lexer) { static void skip_whitespace(TSLexer *const lexer) {
while (lexer->lookahead == ' ' || lexer->lookahead == '\t') { while (check_space(lexer->lookahead)) {
lexer->advance(lexer, false); lexer->advance(lexer, false);
} }
} }
@@ -372,14 +355,14 @@ const char endverbatim_chars[] = "endverbatim";
* `name` is empty, an unnamed "{% endverbatim %}"). Consumes input either * `name` is empty, an unnamed "{% endverbatim %}"). Consumes input either
* way, since a failed match still belongs in the verbatim block's raw text. */ * way, since a failed match still belongs in the verbatim block's raw text. */
static bool check_endverbatim_close(TSLexer *const lexer, const Name *const name) { static bool check_endverbatim_close(TSLexer *const lexer, const Name *const name) {
skip_horizontal_whitespace(lexer); skip_whitespace(lexer);
for (unsigned i = 0; i < sizeof(endverbatim_chars) - 1; ++i) { for (unsigned i = 0; i < sizeof(endverbatim_chars) - 1; ++i) {
if (lexer->lookahead != endverbatim_chars[i]) { if (lexer->lookahead != endverbatim_chars[i]) {
return false; return false;
} }
lexer->advance(lexer, false); lexer->advance(lexer, false);
} }
skip_horizontal_whitespace(lexer); skip_whitespace(lexer);
if (name->size > 0) { if (name->size > 0) {
for (unsigned i = 0; i < name->size; ++i) { for (unsigned i = 0; i < name->size; ++i) {
if (lexer->lookahead != name->contents[i]) { if (lexer->lookahead != name->contents[i]) {
@@ -390,7 +373,7 @@ static bool check_endverbatim_close(TSLexer *const lexer, const Name *const name
if (check_name_char(lexer->lookahead)) { if (check_name_char(lexer->lookahead)) {
return false; return false;
} }
skip_horizontal_whitespace(lexer); skip_whitespace(lexer);
} }
if (lexer->lookahead != '%') { if (lexer->lookahead != '%') {
return false; return false;
@@ -445,82 +428,20 @@ static bool scan_verbatim_content(struct Scanner *const scanner, TSLexer *const
} }
} }
/* Emits a zero-width token at the whitespace that separates one of the
* separated_keywords from the token before it. Django splits tag contents on
* whitespace, so those words are only keywords when something separates them;
* without this, "{% cycle 1as x %}" would parse as a cycle over 1 bound to x.
* Two keywords cannot run together (the lexer reads one longer identifier
* instead), but a literal can run into the keyword after it, and the grammar's
* extras can never be made mandatory.
*
* Reads the whole word that follows and only fires when it is a keyword the
* parser is currently expecting, since the whitespace after a value is equally
* the whitespace before another value ("{% cycle a b %}"). */
/* Emits a zero-width token at the end of a literal, and fails when the
* literal runs straight into whatever follows it. Django splits tag contents
* on whitespace, so "{% cycle 1a %}" is the single token "1a" and an error,
* not the number 1 followed by the variable a. A literal ends wherever its
* own pattern ends, so unlike an identifier it cannot swallow the text after
* it, which leaves this the only place to catch the run-on. */
static bool scan_after_literal(TSLexer *const lexer) {
int32_t next = lexer->lookahead;
// anything that could begin another value: a name, a quote, or a sign or
// point that would start a second number ("{% cycle 1.2.3 %}")
if (check_name_char(next) || next == '\'' || next == '"' || next == '-' || next == '+' || next == '.') {
return false;
}
lexer->mark_end(lexer);
lexer->result_symbol = AfterLiteral;
return true;
}
static bool scan_separator(TSLexer *const lexer, const bool *const valid_symbols) {
bool separated = false;
while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
lexer->advance(lexer, true);
separated = true;
}
if (!separated) {
return false;
}
// the token ends here, before the keyword itself, which the grammar lexes
lexer->mark_end(lexer);
char word[KEYWORD_SIZE + 1];
unsigned length = 0;
while (check_name_char(lexer->lookahead)) {
if (length == KEYWORD_SIZE) {
// too long to be one of them, and a prefix match would be wrong anyway
return false;
}
word[length++] = (char) lexer->lookahead;
lexer->advance(lexer, false);
}
word[length] = '\0';
for (unsigned i = 0; i < KEYWORD_COUNT; ++i) {
if (valid_symbols[BeforeAnd + i] && strcmp(word, separated_keywords[i]) == 0) {
lexer->result_symbol = BeforeAnd + i;
return true;
}
}
return false;
}
const char endcomment_chars[] = "endcomment"; const char endcomment_chars[] = "endcomment";
/* Called right after consuming "{%"; checks whether what follows closes the /* Called right after consuming "{%"; checks whether what follows closes the
* comment block. Unlike verbatim, comment blocks have no name to match and * comment block. Unlike verbatim, comment blocks have no name to match and
* cannot be nested, so any "{% endcomment %}" closes the innermost one. */ * cannot be nested, so any "{% endcomment %}" closes the innermost one. */
static bool check_endcomment_close(TSLexer *const lexer) { static bool check_endcomment_close(TSLexer *const lexer) {
skip_horizontal_whitespace(lexer); skip_whitespace(lexer);
for (unsigned i = 0; i < sizeof(endcomment_chars) - 1; ++i) { for (unsigned i = 0; i < sizeof(endcomment_chars) - 1; ++i) {
if (lexer->lookahead != endcomment_chars[i]) { if (lexer->lookahead != endcomment_chars[i]) {
return false; return false;
} }
lexer->advance(lexer, false); lexer->advance(lexer, false);
} }
skip_horizontal_whitespace(lexer); skip_whitespace(lexer);
if (lexer->lookahead != '%') { if (lexer->lookahead != '%') {
return false; return false;
} }
@@ -587,25 +508,7 @@ bool tree_sitter_django_external_scanner_scan(
if (valid_symbols[CommentContent]) { if (valid_symbols[CommentContent]) {
return scan_comment_content(lexer); return scan_comment_content(lexer);
} }
if (valid_symbols[AfterLiteral]) { while (check_space(lexer->lookahead)) {
return scan_after_literal(lexer);
}
bool separator = false;
for (unsigned i = 0; i < KEYWORD_COUNT; ++i) {
separator |= valid_symbols[BeforeAnd + i];
}
if (separator) {
bool matching_name = false;
for (unsigned token = PopBlock; token <= PushVerbatim; ++token) {
matching_name |= valid_symbols[token];
}
/* these keywords never follow a tag name that opens or closes a named
* block, so the two scans never compete over the same position */
if (!matching_name) {
return scan_separator(lexer, valid_symbols);
}
}
while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
lexer->advance(lexer, true); lexer->advance(lexer, true);
} }
bool is_empty = false; bool is_empty = false;
test/corpus/cycle.txt
+24
View File
@@ -124,3 +124,27 @@ Cycle Number Run On To Number
--- ---
(ERROR) (ERROR)
==================
Cycle Spanning Lines
==================
{% cycle 1
2 as x %}
---
(template
(content)
(cycle
(filtered_value
(value
(literal
(number))))
(filtered_value
(value
(literal
(number))))
variable: (identifier))
(content))
test/corpus/filter.txt
+24
View File
@@ -40,3 +40,27 @@ HELLO WORLD
(template (template
(content))) (content)))
(content)) (content))
==================
Filter With Spaced Chain
==================
<p>
{% filter lower
| title %}
HELLO WORLD
{% endfilter %}
</p>
---
(template
(content)
(filter_group
(filter_expression
(filter)
(filter))
(template
(content)))
(content))
test/corpus/load.txt
+64
View File
@@ -0,0 +1,64 @@
==================
Load Single Library
==================
{% load static %}
---
(template
(content)
(load
(library
(identifier)))
(content))
==================
Load Multiple Libraries
==================
{% load i18n static %}
---
(template
(content)
(load
(library
(identifier))
(library
(identifier)))
(content))
==================
Load Dotted Library
==================
{% load django.contrib %}
---
(template
(content)
(load
(library
(identifier)
(identifier)))
(content))
==================
Load Names From Library
==================
{% load humanize naturaltime from django %}
---
(template
(content)
(load
(identifier)
(identifier)
(library
(identifier)))
(content))
test/corpus/template_variable.txt
+42
View File
@@ -323,3 +323,45 @@ Template Variable Translated String With Spaces
(ERROR) (ERROR)
==================
Template Variable Spanning Lines
==================
{{ value
|floatformat:2 }}
---
(template
(content)
(template_variable
(filtered_value
(value
(variable_attribute
(identifier)))
(filter_expression
(filter
(value
(literal
(number)))))))
(content))
==================
Template Variable With Spaced Filter
==================
{{ value | upper }}
---
(template
(content)
(template_variable
(filtered_value
(value
(variable_attribute
(identifier)))
(filter_expression
(filter))))
(content))