add Django 6 support and simpleTag

Pipfile

@@ -0,0 +1,12 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+django = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3.13"

Pipfile.lock

@@ -0,0 +1,46 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "1d824dfb7e1efaa95eefc090ec048bd1e901b57997b98d5b8319268f04f72176"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "3.13"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "asgiref": {
+            "hashes": [
+                "sha256:59dcb51c272ad209d59bed5708a64a333083e86017d7fcdd67498eeab7784340",
+                "sha256:fe386d1c2bff7259ea95929266d12a8cf9a8b5a1c2598402967d8792e7a7c094"
+            ],
+            "markers": "python_version >= '3.10'",
+            "version": "==3.12.1"
+        },
+        "django": {
+            "hashes": [
+                "sha256:585fb82bf15053c42cf52e67c2f1b54a032dca384759315fd6678ab1870d1d72",
+                "sha256:7ab93536d8e677aa14554c56426290c69480bf2ee68d7513d4a22f57d6bfeb84"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.12'",
+            "version": "==6.1.1"
+        },
+        "sqlparse": {
+            "hashes": [
+                "sha256:113c35c75365ab9cc9c7231d68c6428fb11c085fc8e9eb1ad659b7ddbf6cd2b9",
+                "sha256:b861c0288ce2fa56209a9a6412d2e066ac664b3873b89c26c9d8415e8e32996f"
+            ],
+            "markers": "python_version >= '3.10'",
+            "version": "==0.6.0"
+        }
+    },
+    "develop": {}
+}

grammar.js

@@ -54,6 +54,101 @@ function block(tag, ...args) {
   );
 }
 
+/**
+ * The "as name" clause that binds a tag's result in the context. Django spells
+ * it the same way in every tag that takes one, so the grammar names the target
+ * the same way too, matching the bindings in "for" and "with".
+ * @param {GrammarSymbols<string>} $
+ * @returns {SeqRule}
+ */
+function asVariable($) {
+  return part("as", field("variable", $.identifier));
+}
+
+/**
+ * @param {RuleOrLiteral[]} chosen
+ * @param {RuleOrLiteral[]} rest
+ * @param {RuleOrLiteral[]} spellings
+ */
+function walk(chosen, rest, spellings) {
+  if (chosen.length > 0) {
+    spellings.push(chosen.length === 1 ? chosen[0] : seq(...chosen));
+  }
+  rest.forEach((item, i) =>
+    walk(
+      [...chosen, item],
+      rest.filter((_, j) => j !== i),
+      spellings,
+    ),
+  );
+};
+
+/**
+ * Every way of writing some of `items` in any order, using each at most once,
+ * which is how a tag's keyword arguments reach parse_bits(). There is no
+ * interleaving operator to say that directly, so the orderings are spelled
+ * out; a signature names a handful of arguments at most, and one that takes
+ * more takes them as **kwargs instead.
+ * @param {RuleOrLiteral[]} items
+ * @returns {ChoiceRule|RuleOrLiteral}
+ */
+function arrangements(items) {
+  if (items.length >= 5) {
+    throw new Error(`Too many items for arrangement: ${items.length}`);
+  }
+  /**
+   * @type {RuleOrLiteral[]}
+   */
+  const spellings = [];
+  walk([], items, spellings);
+  return spellings.length === 1 ? spellings[0] : choice(...spellings);
+}
+
+/**
+ * A tag registered with Django's simple_tag helper: positional filter
+ * expressions first, then "name=value" keyword arguments, then the "as name"
+ * clause that Library.simple_tag gives every such tag. The helper takes the
+ * whitespace between the parts itself, so a caller only describes what the
+ * tag's Python signature accepts.
+ *
+ * Django checks the rest of the signature when it parses the tag, so what is
+ * spelled here is what parse_bits() would allow: too many arguments, an
+ * unknown keyword or a repeated one are all syntax errors there.
+ *
+ * @param {GrammarSymbols<string>} $
+ * @param {boolean|number} args how many positional arguments the signature
+ *   takes: true for any number (*args), a count for at most that many, or
+ *   false for none
+ * @param {boolean|Record<string, RuleOrLiteral>} kwargs true for any
+ *   "name=value" pair (**kwargs), false for none, or the accepted names mapped
+ *   to their value rules, each of which may be given at most once
+ * @param {string} tag
+ * @returns {SeqRule}
+ */
+function simpleTag($, tag, args = true, kwargs = true) {
+  const parts = [];
+  if (args === true) {
+    parts.push(repeat(part($.filtered_value)));
+  } else if (typeof args === "number" && args > 0) {
+    // at most `args` of them, so each further one nests inside the last
+    let rule = part($.filtered_value);
+    for (let i = 1; i < args; ++i) {
+      rule = seq(part($.filtered_value), optional(rule));
+    }
+    parts.push(optional(rule));
+  }
+  if (kwargs === true) {
+    parts.push(repeat(part(seq($.identifier, "=", $.filtered_value))));
+  } else if (kwargs) {
+    // a known signature: any of the names it accepts, in any order, none twice
+    const named = Object.entries(kwargs).map(([name, value]) =>
+      part(seq(name, "=", value)),
+    );
+    parts.push(optional(arrangements(named)));
+  }
+  return block(tag, ...parts, optional(asVariable($)));
+}
+
 /** Django's string constant, which takes any escape but a line break. */
 const STRING = /"(?:[^"\\]|\\[^\n])*"|'(?:[^'\\]|\\[^\n])*'/;
 
@@ -171,6 +266,7 @@ const django = grammar({
         $.autoescape_group,
         $.block_group,
         $.comment_group,
+        $.csp_nonce_attr,
         $.csrf_token,
         $.cycle,
         $.debug,
@@ -211,6 +307,7 @@ const django = grammar({
         seq("divisibleby:", $.value),
         "escape",
         "escapejs",
+        "escapeseq",
         "filesizeformat",
         "first",
         choice(seq("floatformat:", $.value), "floatformat"),
@@ -221,7 +318,6 @@ const django = grammar({
         choice(seq("json_script:", $.value), "json_script"),
         "last",
         "length",
-        seq("length_is:", $.value),
         "linebreaks",
         "linebreaksbr",
         "linenumbers",
@@ -275,6 +371,9 @@ const django = grammar({
         optional($.comment_content),
         block("endcomment"),
       ),
+    // csp_nonce_attr(context, media=None)
+    csp_nonce_attr: ($) =>
+      simpleTag($, "csp_nonce_attr", 1, { media: $.filtered_value }),
     csrf_token: ($) => block("csrf_token"),
     cycle: ($) =>
       block(
@@ -282,7 +381,7 @@ const django = grammar({
         repeat1(part($.filtered_value)),
         optional(
           seq(
-            part("as", field("variable", $.identifier)),
+            asVariable($),
             // only the as-form takes the flag
             optional(part("silent")),
           ),
@@ -306,7 +405,7 @@ const django = grammar({
       block(
         "firstof",
         repeat1(part($.filtered_value)),
-        optional(part("as", $.identifier)),
+        optional(asVariable($)),
       ),
     for_scope: ($) =>
       prec.left(
@@ -365,12 +464,7 @@ const django = grammar({
         part(choice("w", "p", "b")),
         optional(part("random")),
       ),
-    now: ($) =>
+    now: ($) => block("now", part($.string), optional(asVariable($))),
-      block(
-        "now",
-        part($.string),
-        optional(part("as", field("name", $.identifier))),
-      ),
     partial: ($) => block("partial", part($.identifier)),
     partialdef_group: ($) =>
       seq(
@@ -378,19 +472,15 @@ const django = grammar({
         optional($.template),
         block("endpartialdef", $.pop_partial),
       ),
-    query_string: ($) =>
+    // querystring(context, *args, **kwargs)
-      block(
+    query_string: ($) => simpleTag($, "querystring"),
-        "querystring",
-        repeat(part($.identifier)),
-        repeat(part(seq($.identifier, "=", $.filtered_value))),
-      ),
     regroup: ($) =>
       block(
         "regroup",
         part($.filtered_value),
         part("by"),
         part($.attribute),
-        optional(part("as", field("variable", $.identifier))),
+        optional(asVariable($)),
       ),
     reset_cycle: ($) => block("resetcycle", optional(part($.identifier))),
     spaceless_group: ($) =>
@@ -421,7 +511,7 @@ const django = grammar({
             repeat1(part(seq($.identifier, "=", $.filtered_value))),
           ),
         ),
-        optional(part("as", field("variable", $.identifier))),
+        optional(asVariable($)),
       ),
     verbatim_group: ($) =>
       seq(
@@ -435,7 +525,7 @@ const django = grammar({
         part($.filtered_value),
         part($.filtered_value),
         part($.filtered_value),
-        optional(part("as", field("variable", $.identifier))),
+        optional(asVariable($)),
       ),
     with_group: ($) =>
       seq(

queries/highlights.scm

@@ -51,6 +51,7 @@ tag: [
     ("endblock")
     ("comment")
     ("endcomment")
+    ("csp_nonce_attr")
     ("csrf_token")
     ("cycle")
     ("debug")
@@ -105,6 +106,7 @@ tag: [
         ("divisibleby:")
         ("escape")
         ("escapejs")
+        ("escapeseq")
         ("filesizeformat")
         ("first")
         ("floatformat")
@@ -117,7 +119,6 @@ tag: [
         ("json_script:")
         ("last")
         ("length")
-        ("length_is:")
         ("linebreaks")
         ("linebreaksbr")
         ("linenumbers")

src/grammar.json

@@ -546,6 +546,10 @@
           "type": "SYMBOL",
           "name": "comment_group"
         },
+        {
+          "type": "SYMBOL",
+          "name": "csp_nonce_attr"
+        },
         {
           "type": "SYMBOL",
           "name": "csrf_token"
@@ -789,6 +793,10 @@
           "type": "STRING",
           "value": "escapejs"
         },
+        {
+          "type": "STRING",
+          "value": "escapeseq"
+        },
         {
           "type": "STRING",
           "value": "filesizeformat"
@@ -883,19 +891,6 @@
           "type": "STRING",
           "value": "length"
         },
-        {
-          "type": "SEQ",
-          "members": [
-            {
-              "type": "STRING",
-              "value": "length_is:"
-            },
-            {
-              "type": "SYMBOL",
-              "name": "value"
-            }
-          ]
-        },
         {
           "type": "STRING",
           "value": "linebreaks"
@@ -1604,6 +1599,144 @@
         }
       ]
     },
+    "csp_nonce_attr": {
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "STRING",
+          "value": "{%"
+        },
+        {
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "PATTERN",
+              "value": "[ \\t\\r\\n]+"
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
+        },
+        {
+          "type": "FIELD",
+          "name": "tag",
+          "content": {
+            "type": "STRING",
+            "value": "csp_nonce_attr"
+          }
+        },
+        {
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "SEQ",
+              "members": [
+                {
+                  "type": "PATTERN",
+                  "value": "[ \\t\\r\\n]+"
+                },
+                {
+                  "type": "SYMBOL",
+                  "name": "filtered_value"
+                }
+              ]
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
+        },
+        {
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "SEQ",
+              "members": [
+                {
+                  "type": "PATTERN",
+                  "value": "[ \\t\\r\\n]+"
+                },
+                {
+                  "type": "SEQ",
+                  "members": [
+                    {
+                      "type": "STRING",
+                      "value": "media"
+                    },
+                    {
+                      "type": "STRING",
+                      "value": "="
+                    },
+                    {
+                      "type": "SYMBOL",
+                      "name": "filtered_value"
+                    }
+                  ]
+                }
+              ]
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
+        },
+        {
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "SEQ",
+              "members": [
+                {
+                  "type": "PATTERN",
+                  "value": "[ \\t\\r\\n]+"
+                },
+                {
+                  "type": "SEQ",
+                  "members": [
+                    {
+                      "type": "STRING",
+                      "value": "as"
+                    },
+                    {
+                      "type": "PATTERN",
+                      "value": "[ \\t\\r\\n]+"
+                    },
+                    {
+                      "type": "FIELD",
+                      "name": "variable",
+                      "content": {
+                        "type": "SYMBOL",
+                        "name": "identifier"
+                      }
+                    }
+                  ]
+                }
+              ]
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
+        },
+        {
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "PATTERN",
+              "value": "[ \\t\\r\\n]+"
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
+        },
+        {
+          "type": "STRING",
+          "value": "%}"
+        }
+      ]
+    },
     "csrf_token": {
       "type": "SEQ",
       "members": [
@@ -2067,8 +2200,12 @@
                       "value": "[ \\t\\r\\n]+"
                     },
                     {
-                      "type": "SYMBOL",
+                      "type": "FIELD",
-                      "name": "identifier"
+                      "name": "variable",
+                      "content": {
+                        "type": "SYMBOL",
+                        "name": "identifier"
+                      }
                     }
                   ]
                 }
@@ -3134,7 +3271,7 @@
                     },
                     {
                       "type": "FIELD",
-                      "name": "name",
+                      "name": "variable",
                       "content": {
                         "type": "SYMBOL",
                         "name": "identifier"
@@ -3399,7 +3536,7 @@
               },
               {
                 "type": "SYMBOL",
-                "name": "identifier"
+                "name": "filtered_value"
               }
             ]
           }
@@ -3433,6 +3570,44 @@
             ]
           }
         },
+        {
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "SEQ",
+              "members": [
+                {
+                  "type": "PATTERN",
+                  "value": "[ \\t\\r\\n]+"
+                },
+                {
+                  "type": "SEQ",
+                  "members": [
+                    {
+                      "type": "STRING",
+                      "value": "as"
+                    },
+                    {
+                      "type": "PATTERN",
+                      "value": "[ \\t\\r\\n]+"
+                    },
+                    {
+                      "type": "FIELD",
+                      "name": "variable",
+                      "content": {
+                        "type": "SYMBOL",
+                        "name": "identifier"
+                      }
+                    }
+                  ]
+                }
+              ]
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
+        },
         {
           "type": "CHOICE",
           "members": [

src/node-types.json

@@ -15,6 +15,10 @@
         "type": "comment_group",
         "named": true
       },
+      {
+        "type": "csp_nonce_attr",
+        "named": true
+      },
       {
         "type": "csrf_token",
         "named": true
@@ -244,6 +248,42 @@
       ]
     }
   },
+  {
+    "type": "csp_nonce_attr",
+    "named": true,
+    "fields": {
+      "tag": {
+        "multiple": false,
+        "required": true,
+        "types": [
+          {
+            "type": "csp_nonce_attr",
+            "named": false
+          }
+        ]
+      },
+      "variable": {
+        "multiple": false,
+        "required": false,
+        "types": [
+          {
+            "type": "identifier",
+            "named": true
+          }
+        ]
+      }
+    },
+    "children": {
+      "multiple": true,
+      "required": false,
+      "types": [
+        {
+          "type": "filtered_value",
+          "named": true
+        }
+      ]
+    }
+  },
   {
     "type": "csrf_token",
     "named": true,
@@ -434,6 +474,16 @@
             "named": false
           }
         ]
+      },
+      "variable": {
+        "multiple": false,
+        "required": false,
+        "types": [
+          {
+            "type": "identifier",
+            "named": true
+          }
+        ]
       }
     },
     "children": {
@@ -443,10 +493,6 @@
         {
           "type": "filtered_value",
           "named": true
-        },
-        {
-          "type": "identifier",
-          "named": true
         }
       ]
     }
@@ -725,16 +771,6 @@
     "type": "now",
     "named": true,
     "fields": {
-      "name": {
-        "multiple": false,
-        "required": false,
-        "types": [
-          {
-            "type": "identifier",
-            "named": true
-          }
-        ]
-      },
       "tag": {
         "multiple": false,
         "required": true,
@@ -744,6 +780,16 @@
             "named": false
           }
         ]
+      },
+      "variable": {
+        "multiple": false,
+        "required": false,
+        "types": [
+          {
+            "type": "identifier",
+            "named": true
+          }
+        ]
       }
     },
     "children": {
@@ -853,6 +899,16 @@
             "named": false
           }
         ]
+      },
+      "variable": {
+        "multiple": false,
+        "required": false,
+        "types": [
+          {
+            "type": "identifier",
+            "named": true
+          }
+        ]
       }
     },
     "children": {
@@ -1361,6 +1417,10 @@
     "type": "content",
     "named": true
   },
+  {
+    "type": "csp_nonce_attr",
+    "named": false
+  },
   {
     "type": "csrf_token",
     "named": false
@@ -1469,6 +1529,10 @@
     "type": "escapejs",
     "named": false
   },
+  {
+    "type": "escapeseq",
+    "named": false
+  },
   {
     "type": "extends",
     "named": false
@@ -1565,10 +1629,6 @@
     "type": "length",
     "named": false
   },
-  {
-    "type": "length_is:",
-    "named": false
-  },
   {
     "type": "linebreaks",
     "named": false
@@ -1601,6 +1661,10 @@
     "type": "make_list",
     "named": false
   },
+  {
+    "type": "media",
+    "named": false
+  },
   {
     "type": "not",
     "named": false

test/corpus/csp_nonce_attr.txt

@@ -0,0 +1,81 @@
+==================
+Csp Nonce Attr
+==================
+
+{% csp_nonce_attr %}
+
+---
+
+(template
+  (content)
+  (csp_nonce_attr)
+  (content))
+
+==================
+Csp Nonce Attr With Argument And As
+==================
+
+{% csp_nonce_attr "script" as nonce %}
+
+---
+
+(template
+  (content)
+  (csp_nonce_attr
+    (filtered_value
+      (value
+        (literal
+          (string))))
+    variable: (identifier))
+  (content))
+
+==================
+Csp Nonce Attr With Media Keyword
+==================
+
+{% csp_nonce_attr media="script" %}
+
+---
+
+(template
+  (content)
+  (csp_nonce_attr
+    (filtered_value
+      (value
+        (literal
+          (string)))))
+  (content))
+
+==================
+Csp Nonce Attr Takes One Argument
+:error
+==================
+
+{% csp_nonce_attr "a" "b" %}
+
+---
+
+(ERROR)
+
+==================
+Csp Nonce Attr Rejects Unknown Keyword
+:error
+==================
+
+{% csp_nonce_attr other="b" %}
+
+---
+
+(ERROR)
+
+==================
+Csp Nonce Attr Rejects Repeated Keyword
+:error
+==================
+
+{% csp_nonce_attr media=1 media=2 %}
+
+---
+
+(ERROR)
+

test/corpus/firstof.txt

@@ -42,5 +42,5 @@ Firstof With As
       (value
         (variable_attribute
           (identifier))))
-    (identifier))
+    variable: (identifier))
   (content))

test/corpus/now.txt

@@ -0,0 +1,28 @@
+==================
+Now Basic
+==================
+
+{% now "Y-m-d" %}
+
+---
+
+(template
+  (content)
+  (now
+    (string))
+  (content))
+
+==================
+Now With As
+==================
+
+{% now "Y" as today %}
+
+---
+
+(template
+  (content)
+  (now
+    (string)
+    variable: (identifier))
+  (content))

test/corpus/querystring.txt

@@ -40,10 +40,37 @@ Querystring With Identifier And Key Value
 (template
   (content)
   (query_string
-    (identifier)
+    (filtered_value
+      (value
+        (variable_attribute
+          (identifier))))
     (identifier)
     (filtered_value
       (value
         (literal
           (number)))))
   (content))
+
+==================
+Querystring With Expression Argument And As
+==================
+
+{% querystring request.GET page=2 as qs %}
+
+---
+
+(template
+  (content)
+  (query_string
+    (filtered_value
+      (value
+        (variable_attribute
+          (identifier)
+          (attribute))))
+    (identifier)
+    (filtered_value
+      (value
+        (literal
+          (number))))
+    variable: (identifier))
+  (content))