more tests

grammar.js

@@ -24,6 +24,7 @@ const django = grammar({
     $.push_partial,
     $.push_verbatim,
     $.verbatim_content,
+    $.comment_content,
     $.matcher_error,
   ],
   reserved: {
@@ -77,6 +78,7 @@ const django = grammar({
       choice(
         $.autoescape_group,
         $.block_group,
+        $.comment_group,
         $.csrf_token,
         $.cycle,
         $.debug,
@@ -167,7 +169,12 @@ const django = grammar({
         optional($.template),
         block("endblock", $.pop_block)
       ),
-    comment: ($) => seq(block("comment", optional($.string)), /.*/, prec(1, block("endcomment"))),
+    comment_group: ($) =>
+      seq(
+        block("comment", optional($.string)),
+        optional($.comment_content),
+        prec(1, block("endcomment"))
+      ),
     csrf_token: ($) => block("csrf_token"),
     cycle: ($) =>
       block(

src/grammar.json

@@ -348,6 +348,10 @@
           "type": "SYMBOL",
           "name": "block_group"
         },
+        {
+          "type": "SYMBOL",
+          "name": "comment_group"
+        },
         {
           "type": "SYMBOL",
           "name": "csrf_token"
@@ -1165,7 +1169,7 @@
         }
       ]
     },
-    "comment": {
+    "comment_group": {
       "type": "SEQ",
       "members": [
         {
@@ -1202,8 +1206,16 @@
           ]
         },
         {
-          "type": "PATTERN",
+          "type": "CHOICE",
-          "value": ".*"
+          "members": [
+            {
+              "type": "SYMBOL",
+              "name": "comment_content"
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
         },
         {
           "type": "PREC",
@@ -2686,6 +2698,10 @@
       "type": "SYMBOL",
       "name": "verbatim_content"
     },
+    {
+      "type": "SYMBOL",
+      "name": "comment_content"
+    },
     {
       "type": "SYMBOL",
       "name": "matcher_error"

src/node-types.json

@@ -11,6 +11,10 @@
         "type": "block_group",
         "named": true
       },
+      {
+        "type": "comment_group",
+        "named": true
+      },
       {
         "type": "csrf_token",
         "named": true
@@ -198,6 +202,40 @@
       ]
     }
   },
+  {
+    "type": "comment_group",
+    "named": true,
+    "fields": {
+      "tag": {
+        "multiple": true,
+        "required": true,
+        "types": [
+          {
+            "type": "comment",
+            "named": false
+          },
+          {
+            "type": "endcomment",
+            "named": false
+          }
+        ]
+      }
+    },
+    "children": {
+      "multiple": true,
+      "required": false,
+      "types": [
+        {
+          "type": "comment_content",
+          "named": true
+        },
+        {
+          "type": "string",
+          "named": true
+        }
+      ]
+    }
+  },
   {
     "type": "csrf_token",
     "named": true,
@@ -1195,6 +1233,14 @@
     "type": "closevariable",
     "named": false
   },
+  {
+    "type": "comment",
+    "named": false
+  },
+  {
+    "type": "comment_content",
+    "named": true
+  },
   {
     "type": "content",
     "named": true
@@ -1263,6 +1309,10 @@
     "type": "endblock",
     "named": false
   },
+  {
+    "type": "endcomment",
+    "named": false
+  },
   {
     "type": "endfilter",
     "named": false

src/scanner.c

@@ -18,6 +18,8 @@ enum TokenType {
   PushVerbatim, // [ name ] %}
   /* raw text inside a verbatim block, up through (but excluding) the matching endverbatim tag */
   VerbatimContent,
+  /* raw text inside a comment block, up through (but excluding) the endcomment tag */
+  CommentContent,
   /* indicates that an error occurred */
   MatcherError,
 };
@@ -299,18 +301,14 @@ static unsigned check_name(TSLexer *const lexer, const Name *const name) {
 }
 
 static bool check_close_block(TSLexer *const lexer) {
-  while (true) {
+  while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
-    switch (lexer->lookahead) {
+    lexer->advance(lexer, false);
-      case ' ':
-      case '\t':
-        lexer->advance(lexer, false);
-      case '%':
-        lexer->advance(lexer, false);
-        return lexer->lookahead == '}';
-      default:
-        return false;
-    }
   }
+  if (lexer->lookahead != '%') {
+    return false;
+  }
+  lexer->advance(lexer, false);
+  return lexer->lookahead == '}';
 }
 
 const char inline_chars[] = "inline";
@@ -408,6 +406,62 @@ static bool scan_verbatim_content(struct Scanner *const scanner, TSLexer *const
   }
 }
 
+const char endcomment_chars[] = "endcomment";
+
+/* Called right after consuming "{%"; checks whether what follows closes the
+ * comment block. Unlike verbatim, comment blocks have no name to match and
+ * cannot be nested, so any "{% endcomment %}" closes the innermost one. */
+static bool check_endcomment_close(TSLexer *const lexer) {
+  skip_horizontal_whitespace(lexer);
+  for (unsigned i = 0; i < sizeof(endcomment_chars) - 1; ++i) {
+    if (lexer->lookahead != endcomment_chars[i]) {
+      return false;
+    }
+    lexer->advance(lexer, false);
+  }
+  skip_horizontal_whitespace(lexer);
+  if (lexer->lookahead != '%') {
+    return false;
+  }
+  lexer->advance(lexer, false);
+  return lexer->lookahead == '}';
+}
+
+/* Consumes raw comment content up to (but not including) the next
+ * "{% endcomment %}", since comment bodies must not be parsed as template
+ * syntax even when they happen to contain tag-like text. */
+static bool scan_comment_content(TSLexer *const lexer) {
+  bool consumed_any = false;
+  while (true) {
+    if (lexer->eof(lexer)) {
+      if (!consumed_any) {
+        return false;
+      }
+      lexer->mark_end(lexer);
+      lexer->result_symbol = CommentContent;
+      return true;
+    }
+    if (lexer->lookahead == '{') {
+      lexer->mark_end(lexer);
+      lexer->advance(lexer, false);
+      if (lexer->lookahead == '%') {
+        lexer->advance(lexer, false);
+        if (check_endcomment_close(lexer)) {
+          if (!consumed_any) {
+            return false;
+          }
+          lexer->result_symbol = CommentContent;
+          return true;
+        }
+      }
+      consumed_any = true;
+      continue;
+    }
+    lexer->advance(lexer, false);
+    consumed_any = true;
+  }
+}
+
 bool tree_sitter_django_external_scanner_scan(
   void *payload,
   TSLexer *lexer,
@@ -429,6 +483,9 @@ bool tree_sitter_django_external_scanner_scan(
   if (valid_symbols[VerbatimContent]) {
     return scan_verbatim_content(scanner, lexer);
   }
+  if (valid_symbols[CommentContent]) {
+    return scan_comment_content(lexer);
+  }
   while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
     lexer->advance(lexer, true);
   }
@@ -508,9 +565,8 @@ bool tree_sitter_django_external_scanner_scan(
           return true;
         }
       }
-      // undo failed push
+      // name was never pushed onto the stack, just free the local copy
       array_delete(&name);
-      array_pop(stack);
       // single failed push implies failure to match any push
       return false;
     }

test/corpus/comment.txt

@@ -0,0 +1,61 @@
+==================
+Comment Basic
+==================
+
+{% comment %}
+This is ignored
+{% endcomment %}
+
+---
+
+(template
+  (content)
+  (comment_group
+    (comment_content))
+  (content))
+
+==================
+Comment Empty
+==================
+
+{% comment %}{% endcomment %}
+
+---
+
+(template
+  (content)
+  (comment_group)
+  (content))
+
+==================
+Comment With Note
+==================
+
+{% comment "why" %}
+This is ignored
+{% endcomment %}
+
+---
+
+(template
+  (content)
+  (comment_group
+    (string)
+    (comment_content))
+  (content))
+
+==================
+Comment Ignores Tag Like Text
+==================
+
+{% comment %}
+Contains {% if x %} tag-like text
+{% endcomment %}
+
+---
+
+(template
+  (content)
+  (comment_group
+    (comment_content))
+  (content))

test/corpus/if.txt

@@ -87,3 +87,57 @@ other
     (template
       (content)))
   (content))
+
+==================
+If With Complex Predicate
+==================
+
+{% if not done and count in valid_counts and status is not none or tags|length >= 3 %}
+yes
+{% endif %}
+
+---
+
+(template
+  (content)
+  (if_group
+    (predicate
+      (filtered_value
+        (value
+          (variable_attribute
+            (identifier))))
+      (binaryOperator)
+      (filtered_value
+        (value
+          (variable_attribute
+            (identifier))))
+      (binaryOperator)
+      (filtered_value
+        (value
+          (variable_attribute
+            (identifier))))
+      (binaryOperator)
+      (filtered_value
+        (value
+          (variable_attribute
+            (identifier))))
+      (binaryOperator)
+      (filtered_value
+        (value
+          (variable_attribute
+            (identifier))))
+      (binaryOperator)
+      (filtered_value
+        (value
+          (variable_attribute
+            (identifier)))
+        (filter_expression
+          (filter)))
+      (binaryOperator)
+      (filtered_value
+        (value
+          (literal
+            (number)))))
+    (template
+      (content)))
+  (content))

test/corpus/partial.txt

@@ -0,0 +1,13 @@
+==================
+Partial Basic
+==================
+
+{% partial header %}
+
+---
+
+(template
+  (content)
+  (partial
+    name: (identifier))
+  (content))

test/corpus/partialdef.txt

@@ -0,0 +1,41 @@
+==================
+Partialdef Basic
+==================
+
+<div>
+{% partialdef header %}
+<h1>Title</h1>
+{% endpartialdef %}
+</div>
+
+---
+
+(template
+  (content)
+  (partialdef_group
+    name: (push_partial)
+    (template
+      (content))
+    (pop_partial))
+  (content))
+
+==================
+Partialdef Inline
+==================
+
+<div>
+{% partialdef header inline %}
+<h1>Title</h1>
+{% endpartialdef %}
+</div>
+
+---
+
+(template
+  (content)
+  (partialdef_group
+    name: (push_partial)
+    (template
+      (content))
+    (pop_partial))
+  (content))

test/corpus/querystring.txt

@@ -0,0 +1,49 @@
+==================
+Querystring Empty
+==================
+
+{% querystring %}
+
+---
+
+(template
+  (content)
+  (query_string)
+  (content))
+
+==================
+Querystring With Key Value
+==================
+
+{% querystring page=2 %}
+
+---
+
+(template
+  (content)
+  (query_string
+    (identifier)
+    (filtered_value
+      (value
+        (literal
+          (number)))))
+  (content))
+
+==================
+Querystring With Identifier And Key Value
+==================
+
+{% querystring remove_key page=2 %}
+
+---
+
+(template
+  (content)
+  (query_string
+    (identifier)
+    (identifier)
+    (filtered_value
+      (value
+        (literal
+          (number)))))
+  (content))

test/corpus/regroup.txt

@@ -0,0 +1,18 @@
+==================
+Regroup Basic
+==================
+
+{% regroup people by gender as grouped %}
+
+---
+
+(template
+  (content)
+  (regroup
+    (filtered_value
+      (value
+        (variable_attribute
+          (identifier))))
+    (attribute)
+    variable: (identifier))
+  (content))

test/corpus/resetcycle.txt

@@ -0,0 +1,26 @@
+==================
+Resetcycle Bare
+==================
+
+{% resetcycle %}
+
+---
+
+(template
+  (content)
+  (reset_cycle)
+  (content))
+
+==================
+Resetcycle With Name
+==================
+
+{% resetcycle rowcolors %}
+
+---
+
+(template
+  (content)
+  (reset_cycle
+    (identifier))
+  (content))

test/corpus/spaceless.txt

@@ -0,0 +1,18 @@
+==================
+Spaceless Basic
+==================
+
+<div>
+{% spaceless %}
+<p>  hi  </p>
+{% endspaceless %}
+</div>
+
+---
+
+(template
+  (content)
+  (spaceless_group
+    (template
+      (content)))
+  (content))

test/corpus/template_variable.txt

@@ -0,0 +1,202 @@
+==================
+Template Variable Integer
+==================
+
+{{ 42 }}
+
+---
+
+(template
+  (content)
+  (template_variable
+    (filtered_value
+      (value
+        (literal
+          (number)))))
+  (content))
+
+==================
+Template Variable Negative Float
+==================
+
+{{ -3.14 }}
+
+---
+
+(template
+  (content)
+  (template_variable
+    (filtered_value
+      (value
+        (literal
+          (number)))))
+  (content))
+
+==================
+Template Variable Scientific Notation
+==================
+
+{{ 1.5e10 }}
+
+---
+
+(template
+  (content)
+  (template_variable
+    (filtered_value
+      (value
+        (literal
+          (number)))))
+  (content))
+
+==================
+Template Variable Double Quoted String
+==================
+
+{{ "hello world" }}
+
+---
+
+(template
+  (content)
+  (template_variable
+    (filtered_value
+      (value
+        (literal
+          (string)))))
+  (content))
+
+==================
+Template Variable Single Quoted String With Escape
+==================
+
+{{ 'it\'s ok' }}
+
+---
+
+(template
+  (content)
+  (template_variable
+    (filtered_value
+      (value
+        (literal
+          (string)))))
+  (content))
+
+==================
+Template Variable Identifier
+==================
+
+{{ name }}
+
+---
+
+(template
+  (content)
+  (template_variable
+    (filtered_value
+      (value
+        (variable_attribute
+          (identifier)))))
+  (content))
+
+==================
+Template Variable Attribute Access
+==================
+
+{{ user.profile.email }}
+
+---
+
+(template
+  (content)
+  (template_variable
+    (filtered_value
+      (value
+        (variable_attribute
+          (identifier)
+          (attribute)))))
+  (content))
+
+==================
+Template Variable With Single Filter
+==================
+
+{{ name|upper }}
+
+---
+
+(template
+  (content)
+  (template_variable
+    (filtered_value
+      (value
+        (variable_attribute
+          (identifier)))
+      (filter_expression
+        (filter))))
+  (content))
+
+==================
+Template Variable With Filter Chain
+==================
+
+{{ name|lower|title }}
+
+---
+
+(template
+  (content)
+  (template_variable
+    (filtered_value
+      (value
+        (variable_attribute
+          (identifier)))
+      (filter_expression
+        (filter)
+        (filter))))
+  (content))
+
+==================
+Template Variable With Filter Argument
+==================
+
+{{ price|add:"5" }}
+
+---
+
+(template
+  (content)
+  (template_variable
+    (filtered_value
+      (value
+        (variable_attribute
+          (identifier)))
+      (filter_expression
+        (filter
+          (value
+            (literal
+              (string)))))))
+  (content))
+
+==================
+Template Variable Number With Filter Argument
+==================
+
+{{ 3.14|floatformat:2 }}
+
+---
+
+(template
+  (content)
+  (template_variable
+    (filtered_value
+      (value
+        (literal
+          (number)))
+      (filter_expression
+        (filter
+          (value
+            (literal
+              (number)))))))
+  (content))

test/corpus/templatetag.txt

@@ -0,0 +1,25 @@
+==================
+Templatetag Openblock
+==================
+
+{% templatetag openblock %}
+
+---
+
+(template
+  (content)
+  (template_tag_block)
+  (content))
+
+==================
+Templatetag Closevariable
+==================
+
+{% templatetag closevariable %}
+
+---
+
+(template
+  (content)
+  (template_tag_block)
+  (content))

test/corpus/url.txt

@@ -0,0 +1,69 @@
+==================
+Url Basic
+==================
+
+{% url 'view-name' %}
+
+---
+
+(template
+  (content)
+  (url_block
+    (string))
+  (content))
+
+==================
+Url With Positional Args
+==================
+
+{% url 'view-name' arg1 arg2 %}
+
+---
+
+(template
+  (content)
+  (url_block
+    (string)
+    (filtered_value
+      (value
+        (variable_attribute
+          (identifier))))
+    (filtered_value
+      (value
+        (variable_attribute
+          (identifier)))))
+  (content))
+
+==================
+Url With Keyword Args
+==================
+
+{% url 'view-name' key=val %}
+
+---
+
+(template
+  (content)
+  (url_block
+    (string)
+    (identifier)
+    (filtered_value
+      (value
+        (variable_attribute
+          (identifier)))))
+  (content))
+
+==================
+Url With As
+==================
+
+{% url 'view-name' as myurl %}
+
+---
+
+(template
+  (content)
+  (url_block
+    (string)
+    variable: (identifier))
+  (content))

test/corpus/with.txt

@@ -0,0 +1,61 @@
+==================
+With Basic
+==================
+
+{% with total=business.employees.count %}
+{{ total }}
+{% endwith %}
+
+---
+
+(template
+  (content)
+  (with_group
+    name: (identifier)
+    variables: (filtered_value
+      (value
+        (variable_attribute
+          (identifier)
+          (attribute))))
+    (template
+      (content)
+      (template_variable
+        (filtered_value
+          (value
+            (variable_attribute
+              (identifier)))))
+      (content)))
+  (content))
+
+==================
+With Multiple Variables
+==================
+
+{% with a=1 b=2 %}
+{{ a }}
+{% endwith %}
+
+---
+
+(template
+  (content)
+  (with_group
+    name: (identifier)
+    variables: (filtered_value
+      (value
+        (literal
+          (number))))
+    name: (identifier)
+    variables: (filtered_value
+      (value
+        (literal
+          (number))))
+    (template
+      (content)
+      (template_variable
+        (filtered_value
+          (value
+            (variable_attribute
+              (identifier)))))
+      (content)))
+  (content))