add more tests

src/grammar.json

@@ -173,7 +173,7 @@
     },
     "identifier": {
       "type": "PATTERN",
-      "value": "[a-zA-Z][a-zA-Z0-9_]+"
+      "value": "[a-zA-Z][a-zA-Z0-9_]*"
     },
     "binaryOperator": {
       "type": "CHOICE",
@@ -938,6 +938,10 @@
           "type": "STRING",
           "value": "unordered_list"
         },
+        {
+          "type": "STRING",
+          "value": "upper"
+        },
         {
           "type": "CHOICE",
           "members": [
@@ -2515,37 +2519,41 @@
           ]
         },
         {
-          "type": "PATTERN",
-          "value": ".*"
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "SYMBOL",
+              "name": "verbatim_content"
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
         },
         {
-          "type": "PREC",
-          "value": 1,
-          "content": {
-            "type": "SEQ",
-            "members": [
-              {
+          "type": "SEQ",
+          "members": [
+            {
+              "type": "STRING",
+              "value": "{%"
+            },
+            {
+              "type": "FIELD",
+              "name": "tag",
+              "content": {
                 "type": "STRING",
-                "value": "{%"
-              },
-              {
-                "type": "FIELD",
-                "name": "tag",
-                "content": {
-                  "type": "STRING",
-                  "value": "endverbatim"
-                }
-              },
-              {
-                "type": "SYMBOL",
-                "name": "pop_verbatim"
-              },
-              {
-                "type": "STRING",
-                "value": "%}"
+                "value": "endverbatim"
               }
-            ]
-          }
+            },
+            {
+              "type": "SYMBOL",
+              "name": "pop_verbatim"
+            },
+            {
+              "type": "STRING",
+              "value": "%}"
+            }
+          ]
         }
       ]
     },
@@ -2674,6 +2682,10 @@
       "type": "SYMBOL",
       "name": "push_verbatim"
     },
+    {
+      "type": "SYMBOL",
+      "name": "verbatim_content"
+    },
     {
       "type": "SYMBOL",
       "name": "matcher_error"

src/node-types.json

@@ -1019,12 +1019,16 @@
       }
     },
     "children": {
-      "multiple": false,
+      "multiple": true,
       "required": true,
       "types": [
         {
           "type": "pop_verbatim",
           "named": true
+        },
+        {
+          "type": "verbatim_content",
+          "named": true
         }
       ]
     }
@@ -1623,6 +1627,10 @@
     "type": "unordered_list",
     "named": false
   },
+  {
+    "type": "upper",
+    "named": false
+  },
   {
     "type": "url",
     "named": false
@@ -1647,6 +1655,10 @@
     "type": "verbatim",
     "named": false
   },
+  {
+    "type": "verbatim_content",
+    "named": true
+  },
   {
     "type": "w",
     "named": false

src/scanner.c

@@ -16,6 +16,8 @@ enum TokenType {
   PushBlock,    // name %}
   PushPartial,  // name [ inline ] %}
   PushVerbatim, // [ name ] %}
+  /* raw text inside a verbatim block, up through (but excluding) the matching endverbatim tag */
+  VerbatimContent,
   /* indicates that an error occurred */
   MatcherError,
 };
@@ -244,10 +246,11 @@ void tree_sitter_django_external_scanner_deserialize(
           continue;
         } else if (code == NAME_SEP) {
           if (name == NULL) {
-            scanner_error(scanner, "Empty name read");
-            return;
+            // an empty name was pushed (e.g. an anonymous verbatim block)
+            array_push(stack, (Name) array_new());
+          } else {
+            name = NULL;
           }
-          name = NULL;
         } else {
           if (name == NULL) {
             if (!check_name_start_char(code)) {
@@ -322,6 +325,89 @@ static bool check_inline(TSLexer *const lexer) {
   return check_close_block(lexer);
 }
 
+static void skip_horizontal_whitespace(TSLexer *const lexer) {
+  while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
+    lexer->advance(lexer, false);
+  }
+}
+
+const char endverbatim_chars[] = "endverbatim";
+
+/* Called right after consuming "{%"; checks whether what follows closes the
+ * verbatim block identified by `name` (matching a name on the stack, or, if
+ * `name` is empty, an unnamed "{% endverbatim %}"). Consumes input either
+ * way, since a failed match still belongs in the verbatim block's raw text. */
+static bool check_endverbatim_close(TSLexer *const lexer, const Name *const name) {
+  skip_horizontal_whitespace(lexer);
+  for (unsigned i = 0; i < sizeof(endverbatim_chars) - 1; ++i) {
+    if (lexer->lookahead != endverbatim_chars[i]) {
+      return false;
+    }
+    lexer->advance(lexer, false);
+  }
+  skip_horizontal_whitespace(lexer);
+  if (name->size > 0) {
+    for (unsigned i = 0; i < name->size; ++i) {
+      if (lexer->lookahead != name->contents[i]) {
+        return false;
+      }
+      lexer->advance(lexer, false);
+    }
+    if (check_name_char(lexer->lookahead)) {
+      return false;
+    }
+    skip_horizontal_whitespace(lexer);
+  }
+  if (lexer->lookahead != '%') {
+    return false;
+  }
+  lexer->advance(lexer, false);
+  return lexer->lookahead == '}';
+}
+
+/* Consumes raw verbatim content up to (but not including) the next tag that
+ * closes the innermost open verbatim block, since that content must not be
+ * parsed as template syntax. A verbatim block opened with a name is only
+ * closed by an "endverbatim" tag bearing the same name, which lets an
+ * unrelated (e.g. unnamed) "{% verbatim %}...{% endverbatim %}" pair appear
+ * unparsed inside a named verbatim block. */
+static bool scan_verbatim_content(struct Scanner *const scanner, TSLexer *const lexer) {
+  Stack *stack = get_stack_for_token(scanner, PopVerbatim);
+  if (stack->size == 0) {
+    return false;
+  }
+  Name *name = array_back(stack);
+  bool consumed_any = false;
+  while (true) {
+    if (lexer->eof(lexer)) {
+      if (!consumed_any) {
+        return false;
+      }
+      lexer->mark_end(lexer);
+      lexer->result_symbol = VerbatimContent;
+      return true;
+    }
+    if (lexer->lookahead == '{') {
+      lexer->mark_end(lexer);
+      lexer->advance(lexer, false);
+      if (lexer->lookahead == '%') {
+        lexer->advance(lexer, false);
+        if (check_endverbatim_close(lexer, name)) {
+          if (!consumed_any) {
+            return false;
+          }
+          lexer->result_symbol = VerbatimContent;
+          return true;
+        }
+      }
+      consumed_any = true;
+      continue;
+    }
+    lexer->advance(lexer, false);
+    consumed_any = true;
+  }
+}
+
 bool tree_sitter_django_external_scanner_scan(
   void *payload,
   TSLexer *lexer,
@@ -340,6 +426,9 @@ bool tree_sitter_django_external_scanner_scan(
   if (valid_symbols[MatcherError]) {
     return false;
   }
+  if (valid_symbols[VerbatimContent]) {
+    return scan_verbatim_content(scanner, lexer);
+  }
   while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
     lexer->advance(lexer, true);
   }
@@ -426,5 +515,6 @@ bool tree_sitter_django_external_scanner_scan(
       return false;
     }
   }
+
   return false;
 }