handle content edge cases

CLAUDE.md

@@ -282,7 +282,7 @@ README nothing checks at all. The library-contents differential below, run again
 
 ### External scanner (`src/scanner.c`)
 
-Eighteen external tokens, for the constraints a context-free grammar cannot express:
+Nineteen external tokens, for the constraints a context-free grammar cannot express:
 
 - `push_block`/`push_partial`/`push_verbatim` and the matching `pop_*` — the names in
   `{% block a %}…{% endblock a %}`. A push token reads the name into `pending_name`, which the group takes
@@ -338,6 +338,12 @@ Eighteen external tokens, for the constraints a context-free grammar cannot expr
   `{% if %}…{% else %}{% if %}…{% else %}`, because the outer group, having held an `else`, claimed the
   inner one's.
 
+- `_text_brace` — a single `{` that is text although the content regex cannot tell: before `{%` or `{#`,
+  and at the end of input. Django splits a template with `({%.*?%}|{{.*?}}|{#.*?#})`, so a `{` is text
+  unless one of those starts at it; `{{{` is not a text brace, since `{{` starts a variable at its first
+  brace. `content` is a right-associative run of `_text` and `_text_brace`, so it stays one node. Inside a
+  group's body the missing-tag check reads past `{` first, so it scans the brace itself
+  (`scan_text_brace_rest`) rather than give up the position.
 - `matcher_error` — used by no rule; returned only if the scanner reaches an error state.
 
 `check_space()` defines whitespace for the scanner and **must stay in sync with `_sep`**, because the

grammar.js

@@ -565,6 +565,7 @@ const django = grammar({
     $._group_repeat,
     $._group_close,
     $._group_held,
+    $._text_brace,
   ],
   reserved: {
     global: ($) => ["not", "if", "in", "is", "as", "for", "from"],
@@ -603,7 +604,14 @@ const django = grammar({
     endTagOpen: ($) => seq("{%", $._group_open_tag_read, optional(SEP)),
     endTagClose: ($) =>
       seq(optional($.unexpected_argument), $._group_close, optional(SEP), "%}"),
-    content: ($) => /(?:[^\{]|\{[^\{#%])+/,
+    content: ($) => prec.right(repeat1(choice($._text, $._text_brace))),
+    /**
+     * Template text up to a "{{", "{%" or "{#". A "{" that is text only because
+     * of what comes after the next character, as in "a{{% if x %}" or a "{" at
+     * the end of input, is _text_brace from the scanner; see
+     * scan_text_brace_rest.
+     */
+    _text: ($) => /(?:[^\{]|\{[^\{#%])+/,
     // the separator between the parts of a tag; see SEP
     _sep: ($) => /[ \t\r\n]+/,
     template_node: ($) =>

src/grammar.json

@@ -326,6 +326,26 @@
       ]
     },
     "content": {
+      "type": "PREC_RIGHT",
+      "value": 0,
+      "content": {
+        "type": "REPEAT1",
+        "content": {
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "SYMBOL",
+              "name": "_text"
+            },
+            {
+              "type": "SYMBOL",
+              "name": "_text_brace"
+            }
+          ]
+        }
+      }
+    },
+    "_text": {
       "type": "PATTERN",
       "value": "(?:[^\\{]|\\{[^\\{#%])+"
     },
@@ -8107,6 +8127,10 @@
     {
       "type": "SYMBOL",
       "name": "_group_held"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_text_brace"
     }
   ],
   "inline": [

src/node-types.json

@@ -712,6 +712,11 @@
       ]
     }
   },
+  {
+    "type": "content",
+    "named": true,
+    "fields": {}
+  },
   {
     "type": "csp_nonce_attr_tag",
     "named": true,
@@ -4045,10 +4050,6 @@
     "type": "comment_content",
     "named": true
   },
-  {
-    "type": "content",
-    "named": true
-  },
   {
     "type": "context",
     "named": false

src/scanner.c

@@ -57,6 +57,9 @@ enum TokenType {
    * the innermost group already holds a tag by that name, which makes this the
    * likelier stray of the two */
   GroupHeld,
+  /* a "{" that is text although another "{" follows it, because the second
+   * one opens a tag or a comment, or although nothing follows it at all */
+  TextBrace,
 };
 
 /* The options blocktranslate takes, in the order of the bits recording them.
@@ -773,8 +776,33 @@ typedef enum {
   /* no marker, but input was read to decide that, so nothing else may be
    * scanned from here */
   NotMissingAfterReading,
+  /* no marker, but the "{" read to decide that was text, and was scanned */
+  TextBraceScanned,
 } MissingResult;
 
+/* Scans TextBrace once its "{" has been read. Django's lexer takes a template
+ * apart with ({%.*?%}|{{.*?}}|{#.*?#}), so a "{" is text unless one of those
+ * starts at it: before "{%" or "{#" the tag or comment starts one character
+ * later, and at the end of input nothing starts at all. The content regex
+ * cannot look that far ahead, so these are scanned here; "{{{" is not one of
+ * them, since "{{" does start a variable at its first brace. */
+static bool scan_text_brace_rest(TSLexer *const lexer) {
+  lexer->mark_end(lexer);
+  if (lexer->eof(lexer)) {
+    lexer->result_symbol = TextBrace;
+    return true;
+  }
+  if (lexer->lookahead != '{') {
+    return false;
+  }
+  lexer->advance(lexer, false);
+  if (lexer->lookahead == '%' || lexer->lookahead == '#') {
+    lexer->result_symbol = TextBrace;
+    return true;
+  }
+  return false;
+}
+
 /* Scans the zero-width marker for the innermost group's missing tag. The
  * marker is valid only where that group could close, which is after a body,
  * where nothing else of this scanner is valid but the raw-text tokens, and
@@ -809,6 +837,10 @@ static MissingResult scan_missing_tag(
     }
     lexer->advance(lexer, false);
     if (lexer->lookahead != '%') {
+      // not a tag, but the "{" may be text the grammar still has to take
+      if (valid_symbols[TextBrace] && scan_text_brace_rest(lexer)) {
+        return TextBraceScanned;
+      }
       return NotMissingAfterReading;
     }
     lexer->advance(lexer, false);
@@ -950,6 +982,7 @@ bool tree_sitter_django_external_scanner_scan(
   if (valid_symbols[MissingTag]) {
     switch (scan_missing_tag(scanner, lexer, valid_symbols)) {
       case Missing:
+      case TextBraceScanned:
         return true;
       case NotMissingAfterReading:
         return false;
@@ -957,6 +990,10 @@ bool tree_sitter_django_external_scanner_scan(
         break;
     }
   }
+  if (valid_symbols[TextBrace] && lexer->lookahead == '{') {
+    lexer->advance(lexer, false);
+    return scan_text_brace_rest(lexer);
+  }
   if (valid_symbols[GroupOpenTagRead]) {
     scan_group_open_tag_read(scanner, lexer);
     return true;

test/corpus/template_variable.txt

@@ -527,3 +527,84 @@ Braces That Open Nothing Are Text
         (content)))
     (endif_tag))
   (content))
+
+==================
+A Brace Before A Tag Is Text
+==================
+
+Hello {{% if x %}a{% endif %}
+
+---
+
+(template
+  (content)
+  (if_group
+    (if_block
+      (if_tag
+        (predicate
+          (filtered_value
+            (value
+              (variable_attribute
+                (identifier))))))
+      (template
+        (content)))
+    (endif_tag))
+  (content))
+
+==================
+A Brace Before A Comment Is Text
+==================
+
+a{{# c #}
+
+---
+
+(template
+  (content)
+  (template_comment)
+  (content))
+
+==================
+A Brace At The End Is Text
+==================
+
+a{
+
+---
+
+(template
+  (content))
+
+==================
+A Brace Before The End Tag Of A Group Is Text
+==================
+
+{% if x %}Hello {{% endif %}
+
+---
+
+(template
+  (content)
+  (if_group
+    (if_block
+      (if_tag
+        (predicate
+          (filtered_value
+            (value
+              (variable_attribute
+                (identifier))))))
+      (template
+        (content)))
+    (endif_tag))
+  (content))
+
+==================
+Three Braces Open A Variable
+:error
+==================
+
+a{{{ x }}
+
+---
+
+