add more tests

.gitignore

@@ -1 +1,2 @@
 node_modules
+tree-sitter-django.wasm

grammar.js

@@ -23,6 +23,7 @@ const django = grammar({
     $.push_block,
     $.push_partial,
     $.push_verbatim,
+    $.verbatim_content,
     $.matcher_error,
   ],
   reserved: {
@@ -48,7 +49,7 @@ const django = grammar({
     string: ($) => /"(?:[^"\\]|\\\\|\\")*"|'(?:[^'\\]|\\\\|\\')*'/,
     attribute: ($) => /[a-zA-Z0-9_]+(?:\.[a-zA-Z0-9_]+)*/,
     variable_attribute: ($) => seq($.identifier, optional(seq(token.immediate('.'), $.attribute))),
-    identifier: ($) => /[a-zA-Z][a-zA-Z0-9_]+/,
+    identifier: ($) => /[a-zA-Z][a-zA-Z0-9_]*/,
     binaryOperator: ($) =>
       choice(
         "and",
@@ -150,6 +151,7 @@ const django = grammar({
       seq("truncatewords:", $.value),
       seq("truncatewords_html:", $.value),
       "unordered_list",
+      "upper",
       choice(seq("urlencode:", $.value), "urlencode"),
       "urlize",
       seq("urlizetrunc:", $.value),
@@ -266,8 +268,8 @@ const django = grammar({
     verbatim_group: ($) =>
       seq(
         block("verbatim", field("name", $.push_verbatim)),
-        /.*/,
+        optional($.verbatim_content),
-        prec(1, block("endverbatim", $.pop_verbatim))
+        block("endverbatim", $.pop_verbatim),
       ),
     with_group: ($) =>
       seq(

src/grammar.json

@@ -173,7 +173,7 @@
     },
     "identifier": {
       "type": "PATTERN",
-      "value": "[a-zA-Z][a-zA-Z0-9_]+"
+      "value": "[a-zA-Z][a-zA-Z0-9_]*"
     },
     "binaryOperator": {
       "type": "CHOICE",
@@ -938,6 +938,10 @@
           "type": "STRING",
           "value": "unordered_list"
         },
+        {
+          "type": "STRING",
+          "value": "upper"
+        },
         {
           "type": "CHOICE",
           "members": [
@@ -2515,37 +2519,41 @@
           ]
         },
         {
-          "type": "PATTERN",
+          "type": "CHOICE",
-          "value": ".*"
+          "members": [
+            {
+              "type": "SYMBOL",
+              "name": "verbatim_content"
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
         },
         {
-          "type": "PREC",
+          "type": "SEQ",
-          "value": 1,
+          "members": [
-          "content": {
+            {
-            "type": "SEQ",
+              "type": "STRING",
-            "members": [
+              "value": "{%"
-              {
+            },
+            {
+              "type": "FIELD",
+              "name": "tag",
+              "content": {
                 "type": "STRING",
-                "value": "{%"
+                "value": "endverbatim"
-              },
-              {
-                "type": "FIELD",
-                "name": "tag",
-                "content": {
-                  "type": "STRING",
-                  "value": "endverbatim"
-                }
-              },
-              {
-                "type": "SYMBOL",
-                "name": "pop_verbatim"
-              },
-              {
-                "type": "STRING",
-                "value": "%}"
               }
-            ]
+            },
-          }
+            {
+              "type": "SYMBOL",
+              "name": "pop_verbatim"
+            },
+            {
+              "type": "STRING",
+              "value": "%}"
+            }
+          ]
         }
       ]
     },
@@ -2674,6 +2682,10 @@
       "type": "SYMBOL",
       "name": "push_verbatim"
     },
+    {
+      "type": "SYMBOL",
+      "name": "verbatim_content"
+    },
     {
       "type": "SYMBOL",
       "name": "matcher_error"

src/node-types.json

@@ -1019,12 +1019,16 @@
       }
     },
     "children": {
-      "multiple": false,
+      "multiple": true,
       "required": true,
       "types": [
         {
           "type": "pop_verbatim",
           "named": true
+        },
+        {
+          "type": "verbatim_content",
+          "named": true
         }
       ]
     }
@@ -1623,6 +1627,10 @@
     "type": "unordered_list",
     "named": false
   },
+  {
+    "type": "upper",
+    "named": false
+  },
   {
     "type": "url",
     "named": false
@@ -1647,6 +1655,10 @@
     "type": "verbatim",
     "named": false
   },
+  {
+    "type": "verbatim_content",
+    "named": true
+  },
   {
     "type": "w",
     "named": false

src/scanner.c

@@ -16,6 +16,8 @@ enum TokenType {
   PushBlock,    // name %}
   PushPartial,  // name [ inline ] %}
   PushVerbatim, // [ name ] %}
+  /* raw text inside a verbatim block, up through (but excluding) the matching endverbatim tag */
+  VerbatimContent,
   /* indicates that an error occurred */
   MatcherError,
 };
@@ -244,10 +246,11 @@ void tree_sitter_django_external_scanner_deserialize(
           continue;
         } else if (code == NAME_SEP) {
           if (name == NULL) {
-            scanner_error(scanner, "Empty name read");
+            // an empty name was pushed (e.g. an anonymous verbatim block)
-            return;
+            array_push(stack, (Name) array_new());
+          } else {
+            name = NULL;
           }
-          name = NULL;
         } else {
           if (name == NULL) {
             if (!check_name_start_char(code)) {
@@ -322,6 +325,89 @@ static bool check_inline(TSLexer *const lexer) {
   return check_close_block(lexer);
 }
 
+static void skip_horizontal_whitespace(TSLexer *const lexer) {
+  while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
+    lexer->advance(lexer, false);
+  }
+}
+
+const char endverbatim_chars[] = "endverbatim";
+
+/* Called right after consuming "{%"; checks whether what follows closes the
+ * verbatim block identified by `name` (matching a name on the stack, or, if
+ * `name` is empty, an unnamed "{% endverbatim %}"). Consumes input either
+ * way, since a failed match still belongs in the verbatim block's raw text. */
+static bool check_endverbatim_close(TSLexer *const lexer, const Name *const name) {
+  skip_horizontal_whitespace(lexer);
+  for (unsigned i = 0; i < sizeof(endverbatim_chars) - 1; ++i) {
+    if (lexer->lookahead != endverbatim_chars[i]) {
+      return false;
+    }
+    lexer->advance(lexer, false);
+  }
+  skip_horizontal_whitespace(lexer);
+  if (name->size > 0) {
+    for (unsigned i = 0; i < name->size; ++i) {
+      if (lexer->lookahead != name->contents[i]) {
+        return false;
+      }
+      lexer->advance(lexer, false);
+    }
+    if (check_name_char(lexer->lookahead)) {
+      return false;
+    }
+    skip_horizontal_whitespace(lexer);
+  }
+  if (lexer->lookahead != '%') {
+    return false;
+  }
+  lexer->advance(lexer, false);
+  return lexer->lookahead == '}';
+}
+
+/* Consumes raw verbatim content up to (but not including) the next tag that
+ * closes the innermost open verbatim block, since that content must not be
+ * parsed as template syntax. A verbatim block opened with a name is only
+ * closed by an "endverbatim" tag bearing the same name, which lets an
+ * unrelated (e.g. unnamed) "{% verbatim %}...{% endverbatim %}" pair appear
+ * unparsed inside a named verbatim block. */
+static bool scan_verbatim_content(struct Scanner *const scanner, TSLexer *const lexer) {
+  Stack *stack = get_stack_for_token(scanner, PopVerbatim);
+  if (stack->size == 0) {
+    return false;
+  }
+  Name *name = array_back(stack);
+  bool consumed_any = false;
+  while (true) {
+    if (lexer->eof(lexer)) {
+      if (!consumed_any) {
+        return false;
+      }
+      lexer->mark_end(lexer);
+      lexer->result_symbol = VerbatimContent;
+      return true;
+    }
+    if (lexer->lookahead == '{') {
+      lexer->mark_end(lexer);
+      lexer->advance(lexer, false);
+      if (lexer->lookahead == '%') {
+        lexer->advance(lexer, false);
+        if (check_endverbatim_close(lexer, name)) {
+          if (!consumed_any) {
+            return false;
+          }
+          lexer->result_symbol = VerbatimContent;
+          return true;
+        }
+      }
+      consumed_any = true;
+      continue;
+    }
+    lexer->advance(lexer, false);
+    consumed_any = true;
+  }
+}
+
 bool tree_sitter_django_external_scanner_scan(
   void *payload,
   TSLexer *lexer,
@@ -340,6 +426,9 @@ bool tree_sitter_django_external_scanner_scan(
   if (valid_symbols[MatcherError]) {
     return false;
   }
+  if (valid_symbols[VerbatimContent]) {
+    return scan_verbatim_content(scanner, lexer);
+  }
   while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
     lexer->advance(lexer, true);
   }
@@ -426,5 +515,6 @@ bool tree_sitter_django_external_scanner_scan(
       return false;
     }
   }
+
   return false;
 }

test/corpus/filter.txt

@@ -0,0 +1,42 @@
+==================
+Filter Basic
+==================
+
+<p>
+{% filter upper %}
+hello
+{% endfilter %}
+</p>
+
+---
+
+(template
+  (content)
+  (filter_group
+    (filter_expression
+      (filter))
+    (template
+      (content)))
+  (content))
+
+==================
+Filter With Chain
+==================
+
+<p>
+{% filter lower|title %}
+HELLO WORLD
+{% endfilter %}
+</p>
+
+---
+
+(template
+  (content)
+  (filter_group
+    (filter_expression
+      (filter)
+      (filter))
+    (template
+      (content)))
+  (content))

test/corpus/firstof.txt

@@ -0,0 +1,46 @@
+==================
+Firstof Basic
+==================
+
+{% firstof var1 var2 "default" %}
+
+---
+
+(template
+  (content)
+  (firstof
+    (filtered_value
+      (value
+        (variable_attribute
+          (identifier))))
+    (filtered_value
+      (value
+        (variable_attribute
+          (identifier))))
+    (filtered_value
+      (value
+        (literal
+          (string)))))
+  (content))
+
+==================
+Firstof With As
+==================
+
+{% firstof var1 var2 as result %}
+
+---
+
+(template
+  (content)
+  (firstof
+    (filtered_value
+      (value
+        (variable_attribute
+          (identifier))))
+    (filtered_value
+      (value
+        (variable_attribute
+          (identifier))))
+    (identifier))
+  (content))

test/corpus/for.txt

@@ -0,0 +1,94 @@
+==================
+For Basic
+==================
+
+<ul>
+{% for item in items %}
+<li>{{ item }}</li>
+{% endfor %}
+</ul>
+
+---
+
+(template
+  (content)
+  (for_group
+    variables: (identifier)
+    (filtered_value
+      (value
+        (variable_attribute
+          (identifier))))
+    (template
+      (content)
+      (template_variable
+        (filtered_value
+          (value
+            (variable_attribute
+              (identifier)))))
+      (content)))
+  (content))
+
+==================
+For With Multiple Variables
+==================
+
+<dl>
+{% for key, value in items %}
+<dt>{{ key }}</dt>
+{% endfor %}
+</dl>
+
+---
+
+(template
+  (content)
+  (for_group
+    variables: (identifier)
+    variables: (identifier)
+    (filtered_value
+      (value
+        (variable_attribute
+          (identifier))))
+    (template
+      (content)
+      (template_variable
+        (filtered_value
+          (value
+            (variable_attribute
+              (identifier)))))
+      (content)))
+  (content))
+
+==================
+For With Empty
+==================
+
+<ul>
+{% for item in items %}
+<li>{{ item }}</li>
+{% empty %}
+<li>Empty</li>
+{% endfor %}
+</ul>
+
+---
+
+(template
+  (content)
+  (for_group
+    variables: (identifier)
+    (filtered_value
+      (value
+        (variable_attribute
+          (identifier))))
+    (template
+      (content)
+      (template_variable
+        (filtered_value
+          (value
+            (variable_attribute
+              (identifier)))))
+      (content))
+    (template
+      (content)))
+  (content))

test/corpus/if.txt

@@ -0,0 +1,89 @@
+==================
+If Basic
+==================
+
+{% if user.is_authenticated %}
+Hi
+{% endif %}
+
+---
+
+(template
+  (content)
+  (if_group
+    (predicate
+      (filtered_value
+        (value
+          (variable_attribute
+            (identifier)
+            (attribute)))))
+    (template
+      (content)))
+  (content))
+
+==================
+If With Not
+==================
+
+{% if not done %}
+working
+{% endif %}
+
+---
+
+(template
+  (content)
+  (if_group
+    (predicate
+      (filtered_value
+        (value
+          (variable_attribute
+            (identifier)))))
+    (template
+      (content)))
+  (content))
+
+==================
+If With Elif And Else
+==================
+
+{% if count == 1 %}
+one
+{% elif count == 2 %}
+two
+{% else %}
+other
+{% endif %}
+
+---
+
+(template
+  (content)
+  (if_group
+    (predicate
+      (filtered_value
+        (value
+          (variable_attribute
+            (identifier))))
+      (binaryOperator)
+      (filtered_value
+        (value
+          (literal
+            (number)))))
+    (template
+      (content))
+    (predicate
+      (filtered_value
+        (value
+          (variable_attribute
+            (identifier))))
+      (binaryOperator)
+      (filtered_value
+        (value
+          (literal
+            (number)))))
+    (template
+      (content))
+    (template
+      (content)))
+  (content))

test/corpus/ifchanged.txt

@@ -0,0 +1,41 @@
+==================
+Ifchanged Basic
+==================
+
+{% ifchanged %}
+content
+{% endifchanged %}
+
+---
+
+(template
+  (content)
+  (ifchanged_group
+    (template
+      (content)))
+  (content))
+
+==================
+Ifchanged With Values
+==================
+
+{% ifchanged var1 var2 %}
+content
+{% endifchanged %}
+
+---
+
+(template
+  (content)
+  (ifchanged_group
+    (filtered_value
+      (value
+        (variable_attribute
+          (identifier))))
+    (filtered_value
+      (value
+        (variable_attribute
+          (identifier))))
+    (template
+      (content)))
+  (content))

test/corpus/include.txt

@@ -0,0 +1,33 @@
+==================
+Include With String
+==================
+
+{% include "sidebar.html" %}
+
+---
+
+(template
+  (content)
+  (include
+    (filtered_value
+      (value
+        (literal
+          (string)))))
+  (content))
+
+==================
+Include With Variable
+==================
+
+{% include template_name %}
+
+---
+
+(template
+  (content)
+  (include
+    (filtered_value
+      (value
+        (variable_attribute
+          (identifier)))))
+  (content))

test/corpus/lorem.txt

@@ -0,0 +1,33 @@
+==================
+Lorem Words
+==================
+
+{% lorem 3 w %}
+
+---
+
+(template
+  (content)
+  (lorem
+    (filtered_value
+      (value
+        (literal
+          (number)))))
+  (content))
+
+==================
+Lorem Paragraphs Random
+==================
+
+{% lorem 2 p random %}
+
+---
+
+(template
+  (content)
+  (lorem
+    (filtered_value
+      (value
+        (literal
+          (number)))))
+  (content))

test/corpus/verbatim.txt

@@ -0,0 +1,83 @@
+==================
+Verbatim Basic
+==================
+
+{% verbatim %}{{ hello }}{% endverbatim %}
+
+---
+
+(template
+  (content)
+  (verbatim_group
+    name: (push_verbatim)
+    (verbatim_content)
+    (pop_verbatim))
+  (content))
+
+==================
+Verbatim Empty
+==================
+
+{% verbatim %}{% endverbatim %}
+
+---
+
+(template
+  (content)
+  (verbatim_group
+    name: (push_verbatim)
+    (pop_verbatim))
+  (content))
+
+==================
+Verbatim Ignores Tags
+==================
+
+{% verbatim %}
+{{ hello }}
+{% for x in y %}
+{% endverbatim %}
+
+---
+
+(template
+  (content)
+  (verbatim_group
+    name: (push_verbatim)
+    (verbatim_content)
+    (pop_verbatim))
+  (content))
+
+==================
+Verbatim With Name
+==================
+
+{% verbatim myblock %}{{ hello }}{% endverbatim myblock %}
+
+---
+
+(template
+  (content)
+  (verbatim_group
+    name: (push_verbatim)
+    (verbatim_content)
+    (pop_verbatim))
+  (content))
+
+==================
+Verbatim With Name Allows Nested Unnamed Verbatim
+==================
+
+{% verbatim myblock %}
+Avoid template rendering via the {% verbatim %}{% endverbatim %} block.
+{% endverbatim myblock %}
+
+---
+
+(template
+  (content)
+  (verbatim_group
+    name: (push_verbatim)
+    (verbatim_content)
+    (pop_verbatim))
+  (content))